diff --git a/charts/sophora-server/Chart.yaml b/charts/sophora-server/Chart.yaml index e6318e0b..4a80d472 100644 --- a/charts/sophora-server/Chart.yaml +++ b/charts/sophora-server/Chart.yaml @@ -15,11 +15,11 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 3.8.6 +version: 3.9.0 annotations: artifacthub.io/changes: | - - kind: fixed - description: "Removed an invalid document separator when using gcp.backendPolicy.enabled:true" + - kind: added + description: "Added prometheusRule.severityMapping to globally override the severity of the default alerts per severity class" # This is the version number of the application being deployed. This version number should be diff --git a/charts/sophora-server/alerting-runbook.md b/charts/sophora-server/alerting-runbook.md index 3c15e7a6..1aa4653a 100644 --- a/charts/sophora-server/alerting-runbook.md +++ b/charts/sophora-server/alerting-runbook.md @@ -2,6 +2,9 @@ This document is a reference to the alerts this Helm chart can fire. +The severity labels shown below are the defaults. They can be overridden globally per severity class with +`prometheusRule.severityMapping` in the chart's `values.yaml`. + ## Sophora Server: General ### SophoraServerOffline diff --git a/charts/sophora-server/templates/prometheusrule.yaml b/charts/sophora-server/templates/prometheusrule.yaml index e7552905..3ffd81a7 100644 --- a/charts/sophora-server/templates/prometheusrule.yaml +++ b/charts/sophora-server/templates/prometheusrule.yaml @@ -1,5 +1,7 @@ {{- if .Values.prometheusRule.enabled }} {{- $defaultRulesEnabled := .Values.prometheusRule.defaultRulesEnabled }} +{{- $highSeverity := .Values.prometheusRule.severityMapping.high }} +{{- $mediumSeverity := .Values.prometheusRule.severityMapping.medium }} apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule metadata: @@ -14,7 +16,7 @@ spec: for: 10m expr: 'up{container="sophora-server", job="{{ include "sophora-server.fullname" . }}"} != 1' labels: - severity: high + severity: {{ $highSeverity }} namespace: "{{ .Release.Namespace }}" annotations: summary: Sophora Server offline. @@ -24,7 +26,7 @@ spec: for: 15m expr: 'histogram_quantile(0.95, sum(rate(sophora_server_contentmanager_call_duration_seconds_bucket{job="{{ include "sophora-server.fullname" . }}"}[1m])) by (pod, le)) > 0.5' labels: - severity: high + severity: {{ $highSeverity }} namespace: "{{ .Release.Namespace }}" annotations: summary: Sophora Server API is slow @@ -34,7 +36,7 @@ spec: for: 10m expr: 'sophora_server_events_number_of_asyc_events_waiting_to_be_processed{job="{{ include "sophora-server.fullname" . }}"} > 10000' labels: - severity: high + severity: {{ $highSeverity }} namespace: "{{ .Release.Namespace }}" annotations: summary: Sophora Server event processing is slow or blocked @@ -45,7 +47,7 @@ spec: for: 5m expr: 'sophora_server_state{job="{{ include "sophora-server.fullname" . }}"} == -1' labels: - severity: medium + severity: {{ $mediumSeverity }} namespace: "{{ .Release.Namespace }}" annotations: summary: State of Sophora server is unknown @@ -55,7 +57,7 @@ spec: for: 10m expr: 'sophora_server_state{job="{{ include "sophora-server.fullname" . }}"} == 3' labels: - severity: medium + severity: {{ $mediumSeverity }} namespace: "{{ .Release.Namespace }}" annotations: summary: Sophora server's synchronization is delayed @@ -65,7 +67,7 @@ spec: for: 10m expr: 'sophora_server_state{job="{{ include "sophora-server.fullname" . }}"} == 4' labels: - severity: medium + severity: {{ $mediumSeverity }} namespace: "{{ .Release.Namespace }}" annotations: summary: Sophora server's queue is too long @@ -75,7 +77,7 @@ spec: for: 10m expr: 'sophora_server_state{job="{{ include "sophora-server.fullname" . }}"} == 5' labels: - severity: high + severity: {{ $highSeverity }} namespace: "{{ .Release.Namespace }}" annotations: summary: Sophora server unavailable @@ -85,7 +87,7 @@ spec: for: 10m expr: 'sophora_server_state{job="{{ include "sophora-server.fullname" . }}"} == 6' labels: - severity: high + severity: {{ $highSeverity }} namespace: "{{ .Release.Namespace }}" annotations: summary: Sophora server lost connection to primary diff --git a/charts/sophora-server/values.yaml b/charts/sophora-server/values.yaml index 9b90e360..7da3b0f0 100644 --- a/charts/sophora-server/values.yaml +++ b/charts/sophora-server/values.yaml @@ -494,6 +494,9 @@ serviceMonitor: prometheusRule: enabled: false defaultRulesEnabled: true + severityMapping: + high: high + medium: medium rules: [] resources: