diff --git a/.gitignore b/.gitignore index f6ff082..2b89758 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ /_public # Additional entries +jsonnetfile.json diff --git a/class/defaults.yml b/class/defaults.yml index 1b5df88..30c4412 100644 --- a/class/defaults.yml +++ b/class/defaults.yml @@ -32,3 +32,8 @@ parameters: external_secrets: {} secrets: {} + + alerts: + ignoreNames: [] + patches: {} + additionalRules: {} diff --git a/class/external-secrets-operator.yml b/class/external-secrets-operator.yml index f4d6a30..0079e10 100644 --- a/class/external-secrets-operator.yml +++ b/class/external-secrets-operator.yml @@ -14,6 +14,7 @@ parameters: - input_paths: - ${_base_directory}/component/main.jsonnet - ${_base_directory}/component/resources.jsonnet + - ${_base_directory}/component/alerts.jsonnet input_type: jsonnet output_path: external-secrets-operator/ - input_type: jsonnet diff --git a/component/alerts.jsonnet b/component/alerts.jsonnet new file mode 100644 index 0000000..d5d16de --- /dev/null +++ b/component/alerts.jsonnet @@ -0,0 +1,137 @@ +local alertpatching = import 'lib/alert-patching.libsonnet'; +local com = import 'lib/commodore.libjsonnet'; +local kap = import 'lib/kapitan.libjsonnet'; +local prom = import 'lib/prom.libsonnet'; + + +local inv = kap.inventory(); +local params = inv.parameters.external_secrets_operator; + +local ignoreNames = com.renderArray(params.alerts.ignoreNames); + +local isOpenshift = std.member([ 'openshift4', 'oke' ], inv.parameters.facts.distribution); +local has_monitoring = isOpenshift || std.member(inv.applications, 'prometheus'); + +local eso_group = { + name: 'external-secrets-operator.rules', + rules: [ + { + local this = self, + alert: 'ExternalSecretNotReady', + expr: ||| + max by (namespace, name) (externalsecret_status_condition{condition="Ready",status="False"}) == 1 + |||, + 'for': '10m', + labels: { + severity: 'warning', + }, + annotations: { + summary: 'ExternalSecret not ready: {{ $labels.namespace }}/{{ $labels.name }}', + description: ||| + ExternalSecret {{ $labels.namespace }}/{{ $labels.name }} has been Ready=False for %s. + Check status conditions and provider connectivity. + ||| % this['for'], + }, + }, + { + local this = self, + alert: 'ExternalSecretsWebhookErrors', + expr: ||| + sum(increase(controller_runtime_webhook_requests_total{service=~".*external-secrets.*",code="500"}[5m])) > 0 + |||, + 'for': '5m', + labels: { + severity: 'critical', + }, + annotations: { + summary: 'external-secrets admission webhook returning HTTP 500', + description: ||| + Webhook has returned {{ $value }} HTTP 500s in %s. + If failurePolicy=Fail, creating/updating ExternalSecrets and SecretStores is blocked cluster-wide. + ||| % this['for'], + }, + }, + { + local this = self, + alert: 'ExternalSecretsReconcileErrorRateHigh', + expr: ||| + ( + sum by (controller) (increase(controller_runtime_reconcile_total{service=~".*external-secrets.*",result="error"}[15m])) + / + sum by (controller) (increase(controller_runtime_reconcile_total{service=~".*external-secrets.*"}[15m])) + ) > 0.05 + and + sum by (controller) (increase(controller_runtime_reconcile_total{service=~".*external-secrets.*"}[15m])) > 1 + |||, + 'for': '15m', + labels: { + severity: 'warning', + }, + annotations: { + summary: 'High reconcile error rate: {{ $labels.controller }}', + description: ||| + Controller {{ $labels.controller }} error rate is {{ $value | humanizePercentage }} over %s. + ||| % this['for'], + }, + }, + { + local this = self, + alert: 'ExternalSecretsProviderAPIErrorRateHigh', + expr: ||| + sum by (provider) (increase(externalsecret_provider_api_calls_count{service=~".*external-secrets.*",status="error"}[15m])) + / + sum by (provider) (increase(externalsecret_provider_api_calls_count{service=~".*external-secrets.*"}[15m])) > 0.10 + |||, + 'for': '15m', + labels: { + severity: 'warning', + }, + annotations: { + summary: 'High provider API error rate: {{ $labels.provider }}', + description: ||| + More than 10 percent of calls to {{ $labels.provider }} failing for %s. Check provider availability, credentials and rate limits. + ||| % this['for'], + }, + }, + ], +}; + +local eso_alerts = prom.PrometheusRule('external-secrets-operator') { + spec+: { + groups: [ + alertpatching.filterPatchRules( + eso_group, + ignoreNames=ignoreNames, + patches=params.alerts.patches, + preserveRecordingRules=true, + patchNames=false, + ), + ], + }, +}; + +local additional_groups = alertpatching.renderGroups({ + 'external-secrets-operator-custom.rules': params.alerts.additionalRules, +}); + +local additional_alerts = prom.PrometheusRule('external-secrets-operator-custom') { + spec+: { + groups: std.map( + function(g) alertpatching.filterPatchRules( + g, + ignoreNames=ignoreNames, + patches=params.alerts.patches, + preserveRecordingRules=true, + patchNames=false, + ), + additional_groups, + ), + }, +}; + +if has_monitoring then { + [if std.length(eso_alerts.spec.groups[0].rules) > 0 then + '10_eso_alerts']: eso_alerts, + [if std.length(additional_groups[0].rules) > 0 then + '10_custom_alerts']: additional_alerts, +} else std.trace('Component ESO compiled for cluster that does not have monitoring, skipping alert rules generation.', {}) diff --git a/docs/modules/ROOT/pages/references/parameters.adoc b/docs/modules/ROOT/pages/references/parameters.adoc index 71e606f..aaf37c8 100644 --- a/docs/modules/ROOT/pages/references/parameters.adoc +++ b/docs/modules/ROOT/pages/references/parameters.adoc @@ -90,3 +90,43 @@ The value is used as is and setting `metadata.name` or `metadata.namespace` over == `secrets` A dict of secrets to create in the namespace. The key is the name of the secret, the value is the content of the secret. + +== `alerts` + +This section allows users to configure alerts for Cilium. +The component expects that an externally-managed Prometheus stack is running on the target cluster. +It renders the alerts through the `alert-patching.libsonnet` and `prom.libsonnet` library aliases, which are provided by both https://github.com/appuio/component-openshift4-monitoring.git[component `openshift4-monitoring`] and https://github.com/projectsyn/component-prometheus.git[component `prometheus`]. +If neither component is present on the target cluster, no alerts are rendered. + + +=== `alerts.ignoreNames` + +[horizontal] +type:: list +default:: `[]` + +Alerts which shouldn't be deployed. +The list supports removal of entries by prefixing them with `~`. + +=== `alerts.patches` + +[horizontal] +type:: dict +default:: `{}` + +Patches for alerts managed by the component. +The component expects that keys in this object match the name of an alert managed through the component. +The value of each entry is expected to be a valid partial Prometheus rule definition. + +=== `alerts.additionalRules` + +[horizontal] +type:: dict +default:: `{}` + +This parameter allows users to configure additional Prometheus recording and alerting rules. +The component expects that keys of this object are prefixed with either `alert:` or `record:` and will use these prefixes to create alerting or recording rules. +The component will take the suffix of the key as the alerting or recording rule name and will set field `alert` or `record` of the rule to the suffix. +The value of each entry will be used as the base Prometheus rule in which the `alert` or `record` field will be injected. + +NOTE: Parameters `alerts.ignoreNames` and `alerts.patches` are also applied to alerts defined through this parameter. diff --git a/jsonnetfile.jsonnet b/jsonnetfile.jsonnet new file mode 100644 index 0000000..882ced5 --- /dev/null +++ b/jsonnetfile.jsonnet @@ -0,0 +1,15 @@ +{ + version: 1, + dependencies: [ + { + source: { + git: { + remote: 'https://github.com/projectsyn/jsonnet-libs', + subdir: '', + }, + }, + version: 'main', + name: 'syn', + }, + ], +} diff --git a/tests/golden/openshift/external-secrets-operator/external-secrets-operator/10_custom_alerts.yaml b/tests/golden/openshift/external-secrets-operator/external-secrets-operator/10_custom_alerts.yaml new file mode 100644 index 0000000..2dd2f1d --- /dev/null +++ b/tests/golden/openshift/external-secrets-operator/external-secrets-operator/10_custom_alerts.yaml @@ -0,0 +1,17 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + labels: + name: external-secrets-operator-custom + name: external-secrets-operator-custom +spec: + groups: + - name: external-secrets-operator-custom.rules + rules: + - alert: FooTesting + annotations: + summary: footest + expr: vector(1) + labels: + syn: 'true' + syn_component: external-secrets-operator diff --git a/tests/golden/openshift/external-secrets-operator/external-secrets-operator/10_eso_alerts.yaml b/tests/golden/openshift/external-secrets-operator/external-secrets-operator/10_eso_alerts.yaml new file mode 100644 index 0000000..4f0e0db --- /dev/null +++ b/tests/golden/openshift/external-secrets-operator/external-secrets-operator/10_eso_alerts.yaml @@ -0,0 +1,69 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + labels: + name: external-secrets-operator + name: external-secrets-operator +spec: + groups: + - name: external-secrets-operator.rules + rules: + - alert: ExternalSecretNotReady + annotations: + description: | + ExternalSecret {{ $labels.namespace }}/{{ $labels.name }} has been Ready=False for 10m. + Check status conditions and provider connectivity. + summary: 'ExternalSecret not ready: {{ $labels.namespace }}/{{ $labels.name + }}' + expr: | + max by (namespace, name) (externalsecret_status_condition{condition="Ready",status="False"}) == 1 + for: 10m + labels: + severity: warning + syn: 'true' + syn_component: external-secrets-operator + - alert: ExternalSecretsWebhookErrors + annotations: + description: | + Webhook has returned {{ $value }} HTTP 500s in 5m. + If failurePolicy=Fail, creating/updating ExternalSecrets and SecretStores is blocked cluster-wide. + summary: external-secrets admission webhook returning HTTP 500 + expr: | + sum(increase(controller_runtime_webhook_requests_total{service=~".*external-secrets.*",code="500"}[5m])) > 0 + for: 5m + labels: + severity: critical + syn: 'true' + syn_component: external-secrets-operator + - alert: ExternalSecretsReconcileErrorRateHigh + annotations: + description: | + Controller {{ $labels.controller }} error rate is {{ $value | humanizePercentage }} over 15m. + summary: 'High reconcile error rate: {{ $labels.controller }}' + expr: | + ( + sum by (controller) (increase(controller_runtime_reconcile_total{service=~".*external-secrets.*",result="error"}[15m])) + / + sum by (controller) (increase(controller_runtime_reconcile_total{service=~".*external-secrets.*"}[15m])) + ) > 0.05 + and + sum by (controller) (increase(controller_runtime_reconcile_total{service=~".*external-secrets.*"}[15m])) > 1 + for: 15m + labels: + severity: warning + syn: 'true' + syn_component: external-secrets-operator + - alert: ExternalSecretsProviderAPIErrorRateHigh + annotations: + description: | + More than 10 percent of calls to {{ $labels.provider }} failing for 15m. Check provider availability, credentials and rate limits. + summary: 'High provider API error rate: {{ $labels.provider }}' + expr: | + sum by (provider) (increase(externalsecret_provider_api_calls_count{service=~".*external-secrets.*",status="error"}[15m])) + / + sum by (provider) (increase(externalsecret_provider_api_calls_count{service=~".*external-secrets.*"}[15m])) > 0.10 + for: 15m + labels: + severity: warning + syn: 'true' + syn_component: external-secrets-operator diff --git a/tests/openshift.yml b/tests/openshift.yml index 2fdf3a3..1c29c34 100644 --- a/tests/openshift.yml +++ b/tests/openshift.yml @@ -2,8 +2,24 @@ parameters: facts: distribution: openshift4 + kapitan: + dependencies: + - type: https + source: https://raw.githubusercontent.com/appuio/component-openshift4-monitoring/master/lib/openshift4-monitoring-prom.libsonnet + output_path: vendor/lib/prom.libsonnet + - type: https + source: https://raw.githubusercontent.com/appuio/component-openshift4-monitoring/master/lib/openshift4-monitoring-alert-patching.libsonnet + output_path: vendor/lib/alert-patching.libsonnet + external_secrets_operator: namespaceAnnotations: test: "1" namespaceLabels: test: "1" + + alerts: + additionalRules: + alert:FooTesting: + expr: vector(1) + annotations: + summary: footest