Add set of basic operations alerts

.gitignore

@@ -13,3 +13,4 @@
 /_public
 
 # Additional entries
+jsonnetfile.json

class/defaults.yml

@@ -32,3 +32,8 @@ parameters:
     external_secrets: {}
 
     secrets: {}
+
+    alerts:
+      ignoreNames: []
+      patches: {}
+      additionalRules: {}

class/external-secrets-operator.yml

@@ -14,6 +14,7 @@ parameters:
       - input_paths:
           - ${_base_directory}/component/main.jsonnet
           - ${_base_directory}/component/resources.jsonnet
+          - ${_base_directory}/component/alerts.jsonnet
         input_type: jsonnet
         output_path: external-secrets-operator/
       - input_type: jsonnet

component/alerts.jsonnet

@@ -0,0 +1,137 @@
+local alertpatching = import 'lib/alert-patching.libsonnet';
+local com = import 'lib/commodore.libjsonnet';
+local kap = import 'lib/kapitan.libjsonnet';
+local prom = import 'lib/prom.libsonnet';
+
+
+local inv = kap.inventory();
+local params = inv.parameters.external_secrets_operator;
+
+local ignoreNames = com.renderArray(params.alerts.ignoreNames);
+
+local isOpenshift = std.member([ 'openshift4', 'oke' ], inv.parameters.facts.distribution);
+local has_monitoring = isOpenshift || std.member(inv.applications, 'prometheus');
+
+local eso_group = {
+  name: 'external-secrets-operator.rules',
+  rules: [
+    {
+      local this = self,
+      alert: 'ExternalSecretNotReady',
+      expr: |||
+        max by (namespace, name) (externalsecret_status_condition{condition="Ready",status="False"}) == 1
+      |||,
+      'for': '10m',
+      labels: {
+        severity: 'warning',
+      },
+      annotations: {
+        summary: 'ExternalSecret not ready: {{ $labels.namespace }}/{{ $labels.name }}',
+        description: |||
+          ExternalSecret {{ $labels.namespace }}/{{ $labels.name }} has been Ready=False for %s.
+          Check status conditions and provider connectivity.
+        ||| % this['for'],
+      },
+    },
+    {
+      local this = self,
+      alert: 'ExternalSecretsWebhookErrors',
+      expr: |||
+        sum(increase(controller_runtime_webhook_requests_total{service=~".*external-secrets.*",code="500"}[5m])) > 0
+      |||,
+      'for': '5m',
+      labels: {
+        severity: 'critical',
+      },
+      annotations: {
+        summary: 'external-secrets admission webhook returning HTTP 500',
+        description: |||
+          Webhook has returned {{ $value }} HTTP 500s in %s.
+          If failurePolicy=Fail, creating/updating ExternalSecrets and SecretStores is blocked cluster-wide.
+        ||| % this['for'],
+      },
+    },
+    {
+      local this = self,
+      alert: 'ExternalSecretsReconcileErrorRateHigh',
+      expr: |||
+        (
+          sum by (controller) (increase(controller_runtime_reconcile_total{service=~".*external-secrets.*",result="error"}[15m]))
+          /
+          sum by (controller) (increase(controller_runtime_reconcile_total{service=~".*external-secrets.*"}[15m]))
+        ) > 0.05
+        and
+        sum by (controller) (increase(controller_runtime_reconcile_total{service=~".*external-secrets.*"}[15m])) > 1
+      |||,
+      'for': '15m',
+      labels: {
+        severity: 'warning',
+      },
+      annotations: {
+        summary: 'High reconcile error rate: {{ $labels.controller }}',
+        description: |||
+          Controller {{ $labels.controller }} error rate is {{ $value | humanizePercentage }} over %s.
+        ||| % this['for'],
+      },
+    },
+    {
+      local this = self,
+      alert: 'ExternalSecretsProviderAPIErrorRateHigh',
+      expr: |||
+        sum by (provider) (increase(externalsecret_provider_api_calls_count{service=~".*external-secrets.*",status="error"}[15m]))
+        /
+        sum by (provider) (increase(externalsecret_provider_api_calls_count{service=~".*external-secrets.*"}[15m])) > 0.10
+      |||,
+      'for': '15m',
+      labels: {
+        severity: 'warning',
+      },
+      annotations: {
+        summary: 'High provider API error rate: {{ $labels.provider }}',
+        description: |||
+          More than 10 percent of calls to {{ $labels.provider }} failing for %s. Check provider availability, credentials and rate limits.
+        ||| % this['for'],
+      },
+    },
+  ],
+};
+
+local eso_alerts = prom.PrometheusRule('external-secrets-operator') {
+  spec+: {
+    groups: [
+      alertpatching.filterPatchRules(
+        eso_group,
+        ignoreNames=ignoreNames,
+        patches=params.alerts.patches,
+        preserveRecordingRules=true,
+        patchNames=false,
+      ),
+    ],
+  },
+};
+
+local additional_groups = alertpatching.renderGroups({
+  'external-secrets-operator-custom.rules': params.alerts.additionalRules,
+});
+
+local additional_alerts = prom.PrometheusRule('external-secrets-operator-custom') {
+  spec+: {
+    groups: std.map(
+      function(g) alertpatching.filterPatchRules(
+        g,
+        ignoreNames=ignoreNames,
+        patches=params.alerts.patches,
+        preserveRecordingRules=true,
+        patchNames=false,
+      ),
+      additional_groups,
+    ),
+  },
+};
+
+if has_monitoring then {
+  [if std.length(eso_alerts.spec.groups[0].rules) > 0 then
+    '10_eso_alerts']: eso_alerts,
+  [if std.length(additional_groups[0].rules) > 0 then
+    '10_custom_alerts']: additional_alerts,
+} else std.trace('Component ESO compiled for cluster that does not have monitoring, skipping alert rules generation.', {})

docs/modules/ROOT/pages/references/parameters.adoc

@@ -90,3 +90,43 @@ The value is used as is and setting `metadata.name` or `metadata.namespace` over
 == `secrets`
 A dict of secrets to create in the namespace.
 The key is the name of the secret, the value is the content of the secret.
+
+== `alerts`
+
+This section allows users to configure alerts for Cilium.
+The component expects that an externally-managed Prometheus stack is running on the target cluster.
+It renders the alerts through the `alert-patching.libsonnet` and `prom.libsonnet` library aliases, which are provided by both https://github.com/appuio/component-openshift4-monitoring.git[component `openshift4-monitoring`] and https://github.com/projectsyn/component-prometheus.git[component `prometheus`].
+If neither component is present on the target cluster, no alerts are rendered.
+
+
+=== `alerts.ignoreNames`
+
+[horizontal]
+type:: list
+default:: `[]`
+
+Alerts which shouldn't be deployed.
+The list supports removal of entries by prefixing them with `~`.
+
+=== `alerts.patches`
+
+[horizontal]
+type:: dict
+default:: `{}`
+
+Patches for alerts managed by the component.
+The component expects that keys in this object match the name of an alert managed through the component.
+The value of each entry is expected to be a valid partial Prometheus rule definition.
+
+=== `alerts.additionalRules`
+
+[horizontal]
+type:: dict
+default:: `{}`
+
+This parameter allows users to configure additional Prometheus recording and alerting rules.
+The component expects that keys of this object are prefixed with either `alert:` or `record:` and will use these prefixes to create alerting or recording rules.
+The component will take the suffix of the key as the alerting or recording rule name and will set field `alert` or `record` of the rule to the suffix.
+The value of each entry will be used as the base Prometheus rule in which the `alert` or `record` field will be injected.
+
+NOTE: Parameters `alerts.ignoreNames` and `alerts.patches` are also applied to alerts defined through this parameter.

jsonnetfile.jsonnet

@@ -0,0 +1,15 @@
+{
+  version: 1,
+  dependencies: [
+    {
+      source: {
+        git: {
+          remote: 'https://github.com/projectsyn/jsonnet-libs',
+          subdir: '',
+        },
+      },
+      version: 'main',
+      name: 'syn',
+    },
+  ],
+}

tests/golden/openshift/external-secrets-operator/external-secrets-operator/10_custom_alerts.yaml

@@ -0,0 +1,17 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  labels:
+    name: external-secrets-operator-custom
+  name: external-secrets-operator-custom
+spec:
+  groups:
+    - name: external-secrets-operator-custom.rules
+      rules:
+        - alert: FooTesting
+          annotations:
+            summary: footest
+          expr: vector(1)
+          labels:
+            syn: 'true'
+            syn_component: external-secrets-operator

tests/golden/openshift/external-secrets-operator/external-secrets-operator/10_eso_alerts.yaml

@@ -0,0 +1,69 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  labels:
+    name: external-secrets-operator
+  name: external-secrets-operator
+spec:
+  groups:
+    - name: external-secrets-operator.rules
+      rules:
+        - alert: ExternalSecretNotReady
+          annotations:
+            description: |
+              ExternalSecret {{ $labels.namespace }}/{{ $labels.name }} has been Ready=False for 10m.
+              Check status conditions and provider connectivity.
+            summary: 'ExternalSecret not ready: {{ $labels.namespace }}/{{ $labels.name
+              }}'
+          expr: |
+            max by (namespace, name) (externalsecret_status_condition{condition="Ready",status="False"}) == 1
+          for: 10m
+          labels:
+            severity: warning
+            syn: 'true'
+            syn_component: external-secrets-operator
+        - alert: ExternalSecretsWebhookErrors
+          annotations:
+            description: |
+              Webhook has returned {{ $value }} HTTP 500s in 5m.
+              If failurePolicy=Fail, creating/updating ExternalSecrets and SecretStores is blocked cluster-wide.
+            summary: external-secrets admission webhook returning HTTP 500
+          expr: |
+            sum(increase(controller_runtime_webhook_requests_total{service=~".*external-secrets.*",code="500"}[5m])) > 0
+          for: 5m
+          labels:
+            severity: critical
+            syn: 'true'
+            syn_component: external-secrets-operator
+        - alert: ExternalSecretsReconcileErrorRateHigh
+          annotations:
+            description: |
+              Controller {{ $labels.controller }} error rate is {{ $value | humanizePercentage }} over 15m.
+            summary: 'High reconcile error rate: {{ $labels.controller }}'
+          expr: |
+            (
+              sum by (controller) (increase(controller_runtime_reconcile_total{service=~".*external-secrets.*",result="error"}[15m]))
+              /
+              sum by (controller) (increase(controller_runtime_reconcile_total{service=~".*external-secrets.*"}[15m]))
+            ) > 0.05
+            and
+            sum by (controller) (increase(controller_runtime_reconcile_total{service=~".*external-secrets.*"}[15m])) > 1
+          for: 15m
+          labels:
+            severity: warning
+            syn: 'true'
+            syn_component: external-secrets-operator
+        - alert: ExternalSecretsProviderAPIErrorRateHigh
+          annotations:
+            description: |
+              More than 10 percent of calls to {{ $labels.provider }} failing for 15m. Check provider availability, credentials and rate limits.
+            summary: 'High provider API error rate: {{ $labels.provider }}'
+          expr: |
+            sum by (provider) (increase(externalsecret_provider_api_calls_count{service=~".*external-secrets.*",status="error"}[15m]))
+            /
+            sum by (provider) (increase(externalsecret_provider_api_calls_count{service=~".*external-secrets.*"}[15m])) > 0.10
+          for: 15m
+          labels:
+            severity: warning
+            syn: 'true'
+            syn_component: external-secrets-operator

tests/openshift.yml

@@ -2,8 +2,24 @@ parameters:
   facts:
     distribution: openshift4
 
+  kapitan:
+    dependencies:
+      - type: https
+        source: https://raw.githubusercontent.com/appuio/component-openshift4-monitoring/master/lib/openshift4-monitoring-prom.libsonnet
+        output_path: vendor/lib/prom.libsonnet
+      - type: https
+        source: https://raw.githubusercontent.com/appuio/component-openshift4-monitoring/master/lib/openshift4-monitoring-alert-patching.libsonnet
+        output_path: vendor/lib/alert-patching.libsonnet
+
   external_secrets_operator:
     namespaceAnnotations:
       test: "1"
     namespaceLabels:
       test: "1"
+
+    alerts:
+      additionalRules:
+        alert:FooTesting:
+          expr: vector(1)
+          annotations:
+            summary: footest