Add configurable severity mapping for Sophora server alerts

charts/sophora-server/Chart.yaml

@@ -15,11 +15,11 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 3.8.6
+version: 3.9.0
 annotations:
   artifacthub.io/changes: |
-    - kind: fixed
-      description: "Removed an invalid document separator when using gcp.backendPolicy.enabled:true"
+    - kind: added
+      description: "Added prometheusRule.severityMapping to globally override the severity of the default alerts per severity class"
 
 
 # This is the version number of the application being deployed. This version number should be

charts/sophora-server/alerting-runbook.md

@@ -2,6 +2,9 @@
 
 This document is a reference to the alerts this Helm chart can fire.
 
+The severity labels shown below are the defaults. They can be overridden globally per severity class with
+`prometheusRule.severityMapping` in the chart's `values.yaml`.
+
 ## Sophora Server: General
 
 ### SophoraServerOffline

charts/sophora-server/templates/prometheusrule.yaml

@@ -1,5 +1,7 @@
 {{- if .Values.prometheusRule.enabled }}
 {{- $defaultRulesEnabled := .Values.prometheusRule.defaultRulesEnabled }}
+{{- $highSeverity := .Values.prometheusRule.severityMapping.high }}
+{{- $mediumSeverity := .Values.prometheusRule.severityMapping.medium }}
 apiVersion: monitoring.coreos.com/v1
 kind: PrometheusRule
 metadata:
@@ -14,7 +16,7 @@ spec:
           for: 10m
           expr: 'up{container="sophora-server", job="{{ include "sophora-server.fullname" . }}"} != 1'
           labels:
-            severity: high
+            severity: {{ $highSeverity }}
             namespace: "{{ .Release.Namespace }}"
           annotations:
             summary: Sophora Server offline.
@@ -24,7 +26,7 @@ spec:
           for: 15m
           expr: 'histogram_quantile(0.95, sum(rate(sophora_server_contentmanager_call_duration_seconds_bucket{job="{{ include "sophora-server.fullname" . }}"}[1m])) by (pod, le)) > 0.5'
           labels:
-            severity: high
+            severity: {{ $highSeverity }}
             namespace: "{{ .Release.Namespace }}"
           annotations:
             summary: Sophora Server API is slow
@@ -34,7 +36,7 @@ spec:
           for: 10m
           expr: 'sophora_server_events_number_of_asyc_events_waiting_to_be_processed{job="{{ include "sophora-server.fullname" . }}"} > 10000'
           labels:
-            severity: high
+            severity: {{ $highSeverity }}
             namespace: "{{ .Release.Namespace }}"
           annotations:
             summary: Sophora Server event processing is slow or blocked
@@ -45,7 +47,7 @@ spec:
           for: 5m
           expr: 'sophora_server_state{job="{{ include "sophora-server.fullname" . }}"} == -1'
           labels:
-            severity: medium
+            severity: {{ $mediumSeverity }}
             namespace: "{{ .Release.Namespace }}"
           annotations:
             summary: State of Sophora server is unknown
@@ -55,7 +57,7 @@ spec:
           for: 10m
           expr: 'sophora_server_state{job="{{ include "sophora-server.fullname" . }}"} == 3'
           labels:
-            severity: medium
+            severity: {{ $mediumSeverity }}
             namespace: "{{ .Release.Namespace }}"
           annotations:
             summary: Sophora server's synchronization is delayed
@@ -65,7 +67,7 @@ spec:
           for: 10m
           expr: 'sophora_server_state{job="{{ include "sophora-server.fullname" . }}"} == 4'
           labels:
-            severity: medium
+            severity: {{ $mediumSeverity }}
             namespace: "{{ .Release.Namespace }}"
           annotations:
             summary: Sophora server's queue is too long
@@ -75,7 +77,7 @@ spec:
           for: 10m
           expr: 'sophora_server_state{job="{{ include "sophora-server.fullname" . }}"} == 5'
           labels:
-            severity: high
+            severity: {{ $highSeverity }}
             namespace: "{{ .Release.Namespace }}"
           annotations:
             summary: Sophora server unavailable
@@ -85,7 +87,7 @@ spec:
           for: 10m
           expr: 'sophora_server_state{job="{{ include "sophora-server.fullname" . }}"} == 6'
           labels:
-            severity: high
+            severity: {{ $highSeverity }}
             namespace: "{{ .Release.Namespace }}"
           annotations:
             summary: Sophora server lost connection to primary

charts/sophora-server/values.yaml

@@ -494,6 +494,9 @@ serviceMonitor:
 prometheusRule:
   enabled: false
   defaultRulesEnabled: true
+  severityMapping:
+    high: high
+    medium: medium
   rules: []
 
 resources: