From 0d0dbe93143c49efd32098d8e92b128558ea2d50 Mon Sep 17 00:00:00 2001 From: haasad Date: Fri, 23 Oct 2026 15:47:51 +0200 Subject: [PATCH 1/3] Add a basic ExternalSecretNotReady alert --- .gitignore | 1 + class/defaults.yml | 5 ++ class/external-secrets-operator.yml | 1 + component/alerts.jsonnet | 89 +++++++++++++++++++ .../ROOT/pages/references/parameters.adoc | 40 +++++++++ jsonnetfile.jsonnet | 15 ++++ .../10_custom_alerts.yaml | 17 ++++ .../10_eso_alerts.yaml | 25 ++++++ tests/openshift.yml | 16 ++++ 9 files changed, 209 insertions(+) create mode 100644 component/alerts.jsonnet create mode 100644 jsonnetfile.jsonnet create mode 100644 tests/golden/openshift/external-secrets-operator/external-secrets-operator/10_custom_alerts.yaml create mode 100644 tests/golden/openshift/external-secrets-operator/external-secrets-operator/10_eso_alerts.yaml diff --git a/.gitignore b/.gitignore index f6ff082..2b89758 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ /_public # Additional entries +jsonnetfile.json diff --git a/class/defaults.yml b/class/defaults.yml index 1b5df88..30c4412 100644 --- a/class/defaults.yml +++ b/class/defaults.yml @@ -32,3 +32,8 @@ parameters: external_secrets: {} secrets: {} + + alerts: + ignoreNames: [] + patches: {} + additionalRules: {} diff --git a/class/external-secrets-operator.yml b/class/external-secrets-operator.yml index f4d6a30..0079e10 100644 --- a/class/external-secrets-operator.yml +++ b/class/external-secrets-operator.yml @@ -14,6 +14,7 @@ parameters: - input_paths: - ${_base_directory}/component/main.jsonnet - ${_base_directory}/component/resources.jsonnet + - ${_base_directory}/component/alerts.jsonnet input_type: jsonnet output_path: external-secrets-operator/ - input_type: jsonnet diff --git a/component/alerts.jsonnet b/component/alerts.jsonnet new file mode 100644 index 0000000..915cf14 --- /dev/null +++ b/component/alerts.jsonnet @@ -0,0 +1,89 @@ +local alertpatching = import 'lib/alert-patching.libsonnet'; +local com = import 'lib/commodore.libjsonnet'; +local kap = import 'lib/kapitan.libjsonnet'; +local prom = import 'lib/prom.libsonnet'; + + +local inv = kap.inventory(); +local params = inv.parameters.external_secrets_operator; + +local ignoreNames = com.renderArray(params.alerts.ignoreNames); + +local isOpenshift = std.member([ 'openshift4', 'oke' ], inv.parameters.facts.distribution); +local has_monitoring = isOpenshift || std.member(inv.applications, 'prometheus'); + +local eso_group = { + name: 'external-secrets-operator.rules', + rules: [ + { + local this = self, + alert: 'ExternalSecretNotReady', + expr: 'max by (namespace, name) (externalsecret_status_condition{condition="Ready",status="False"}) == 1', + 'for': '10m', + labels: { + severity: 'warning', + }, + annotations: { + summary: 'ExternalSecret not ready: {{ $labels.namespace }}/{{ $labels.name }}', + description: ||| + ExternalSecret {{ $labels.namespace }}/{{ $labels.name }} has been + Ready=False for %s. Check status conditions and provider + connectivity. + ||| % this['for'], + }, + }, + ], +}; + +local eso_alerts = prom.PrometheusRule('external-secrets-operator') { + spec+: { + groups: [ + alertpatching.filterPatchRules( + eso_group, + ignoreNames=ignoreNames, + patches=params.alerts.patches, + preserveRecordingRules=true, + patchNames=false, + ), + ], + }, +}; + +local additional_group = + local parseRuleName(rname) = + local rparts = std.splitLimit(rname, ':', 1); + assert + std.length(rparts) == 2 : + 'Expected custom alert rule to be prefixed with `record:` or `alert:`.'; + assert + std.member([ 'alert', 'record' ], rparts[0]) : + 'Expected custom alert rule to be prefixed with `record:` or `alert:`, got `%s:`.' % rparts[0]; + { [rparts[0]]: rparts[1] }; + { + name: 'external-secrets-operator-custom.rules', + rules: [ + params.alerts.additionalRules[rname] + parseRuleName(rname) + for rname in std.objectFields(params.alerts.additionalRules) + ], + }; + +local additional_alerts = prom.PrometheusRule('external-secrets-operator-custom') { + spec+: { + groups: [ + alertpatching.filterPatchRules( + additional_group, + ignoreNames=ignoreNames, + patches=params.alerts.patches, + preserveRecordingRules=true, + patchNames=false, + ), + ], + }, +}; + +if has_monitoring then { + [if std.length(eso_alerts.spec.groups[0].rules) > 0 then + '10_eso_alerts']: eso_alerts, + [if std.length(additional_group.rules) > 0 then + '10_custom_alerts']: additional_alerts, +} else std.trace('Component ESO compiled for cluster that does not have monitoring, skipping alert rules generation.', {}) diff --git a/docs/modules/ROOT/pages/references/parameters.adoc b/docs/modules/ROOT/pages/references/parameters.adoc index 71e606f..aaf37c8 100644 --- a/docs/modules/ROOT/pages/references/parameters.adoc +++ b/docs/modules/ROOT/pages/references/parameters.adoc @@ -90,3 +90,43 @@ The value is used as is and setting `metadata.name` or `metadata.namespace` over == `secrets` A dict of secrets to create in the namespace. The key is the name of the secret, the value is the content of the secret. + +== `alerts` + +This section allows users to configure alerts for Cilium. +The component expects that an externally-managed Prometheus stack is running on the target cluster. +It renders the alerts through the `alert-patching.libsonnet` and `prom.libsonnet` library aliases, which are provided by both https://github.com/appuio/component-openshift4-monitoring.git[component `openshift4-monitoring`] and https://github.com/projectsyn/component-prometheus.git[component `prometheus`]. +If neither component is present on the target cluster, no alerts are rendered. + + +=== `alerts.ignoreNames` + +[horizontal] +type:: list +default:: `[]` + +Alerts which shouldn't be deployed. +The list supports removal of entries by prefixing them with `~`. + +=== `alerts.patches` + +[horizontal] +type:: dict +default:: `{}` + +Patches for alerts managed by the component. +The component expects that keys in this object match the name of an alert managed through the component. +The value of each entry is expected to be a valid partial Prometheus rule definition. + +=== `alerts.additionalRules` + +[horizontal] +type:: dict +default:: `{}` + +This parameter allows users to configure additional Prometheus recording and alerting rules. +The component expects that keys of this object are prefixed with either `alert:` or `record:` and will use these prefixes to create alerting or recording rules. +The component will take the suffix of the key as the alerting or recording rule name and will set field `alert` or `record` of the rule to the suffix. +The value of each entry will be used as the base Prometheus rule in which the `alert` or `record` field will be injected. + +NOTE: Parameters `alerts.ignoreNames` and `alerts.patches` are also applied to alerts defined through this parameter. diff --git a/jsonnetfile.jsonnet b/jsonnetfile.jsonnet new file mode 100644 index 0000000..882ced5 --- /dev/null +++ b/jsonnetfile.jsonnet @@ -0,0 +1,15 @@ +{ + version: 1, + dependencies: [ + { + source: { + git: { + remote: 'https://github.com/projectsyn/jsonnet-libs', + subdir: '', + }, + }, + version: 'main', + name: 'syn', + }, + ], +} diff --git a/tests/golden/openshift/external-secrets-operator/external-secrets-operator/10_custom_alerts.yaml b/tests/golden/openshift/external-secrets-operator/external-secrets-operator/10_custom_alerts.yaml new file mode 100644 index 0000000..2dd2f1d --- /dev/null +++ b/tests/golden/openshift/external-secrets-operator/external-secrets-operator/10_custom_alerts.yaml @@ -0,0 +1,17 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + labels: + name: external-secrets-operator-custom + name: external-secrets-operator-custom +spec: + groups: + - name: external-secrets-operator-custom.rules + rules: + - alert: FooTesting + annotations: + summary: footest + expr: vector(1) + labels: + syn: 'true' + syn_component: external-secrets-operator diff --git a/tests/golden/openshift/external-secrets-operator/external-secrets-operator/10_eso_alerts.yaml b/tests/golden/openshift/external-secrets-operator/external-secrets-operator/10_eso_alerts.yaml new file mode 100644 index 0000000..2018856 --- /dev/null +++ b/tests/golden/openshift/external-secrets-operator/external-secrets-operator/10_eso_alerts.yaml @@ -0,0 +1,25 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + labels: + name: external-secrets-operator + name: external-secrets-operator +spec: + groups: + - name: external-secrets-operator.rules + rules: + - alert: ExternalSecretNotReady + annotations: + description: | + ExternalSecret {{ $labels.namespace }}/{{ $labels.name }} has been + Ready=False for 10m. Check status conditions and provider + connectivity. + summary: 'ExternalSecret not ready: {{ $labels.namespace }}/{{ $labels.name + }}' + expr: max by (namespace, name) (externalsecret_status_condition{condition="Ready",status="False"}) + == 1 + for: 10m + labels: + severity: warning + syn: 'true' + syn_component: external-secrets-operator diff --git a/tests/openshift.yml b/tests/openshift.yml index 2fdf3a3..1c29c34 100644 --- a/tests/openshift.yml +++ b/tests/openshift.yml @@ -2,8 +2,24 @@ parameters: facts: distribution: openshift4 + kapitan: + dependencies: + - type: https + source: https://raw.githubusercontent.com/appuio/component-openshift4-monitoring/master/lib/openshift4-monitoring-prom.libsonnet + output_path: vendor/lib/prom.libsonnet + - type: https + source: https://raw.githubusercontent.com/appuio/component-openshift4-monitoring/master/lib/openshift4-monitoring-alert-patching.libsonnet + output_path: vendor/lib/alert-patching.libsonnet + external_secrets_operator: namespaceAnnotations: test: "1" namespaceLabels: test: "1" + + alerts: + additionalRules: + alert:FooTesting: + expr: vector(1) + annotations: + summary: footest From f8b60b4922be42fe785a44d7ff6ea81716572d33 Mon Sep 17 00:00:00 2001 From: haasad Date: Fri, 23 Oct 2026 16:40:52 +0200 Subject: [PATCH 2/3] Add more alerts --- component/alerts.jsonnet | 69 +++++++++++++++++-- .../10_eso_alerts.yaml | 54 +++++++++++++-- 2 files changed, 114 insertions(+), 9 deletions(-) diff --git a/component/alerts.jsonnet b/component/alerts.jsonnet index 915cf14..24048d2 100644 --- a/component/alerts.jsonnet +++ b/component/alerts.jsonnet @@ -18,7 +18,9 @@ local eso_group = { { local this = self, alert: 'ExternalSecretNotReady', - expr: 'max by (namespace, name) (externalsecret_status_condition{condition="Ready",status="False"}) == 1', + expr: ||| + max by (namespace, name) (externalsecret_status_condition{condition="Ready",status="False"}) == 1 + |||, 'for': '10m', labels: { severity: 'warning', @@ -26,9 +28,68 @@ local eso_group = { annotations: { summary: 'ExternalSecret not ready: {{ $labels.namespace }}/{{ $labels.name }}', description: ||| - ExternalSecret {{ $labels.namespace }}/{{ $labels.name }} has been - Ready=False for %s. Check status conditions and provider - connectivity. + ExternalSecret {{ $labels.namespace }}/{{ $labels.name }} has been Ready=False for %s. + Check status conditions and provider connectivity. + ||| % this['for'], + }, + }, + { + local this = self, + alert: 'ExternalSecretsWebhookErrors', + expr: ||| + sum(increase(controller_runtime_webhook_requests_total{service=~".*external-secrets.*",code="500"}[5m])) > 0 + |||, + 'for': '5m', + labels: { + severity: 'critical', + }, + annotations: { + summary: 'external-secrets admission webhook returning HTTP 500', + description: ||| + Webhook has returned {{ $value }} HTTP 500s in %s. + If failurePolicy=Fail, creating/updating ExternalSecrets and SecretStores is blocked cluster-wide. + ||| % this['for'], + }, + }, + { + local this = self, + alert: 'ExternalSecretsReconcileErrorRateHigh', + expr: ||| + ( + sum by (controller) (increase(controller_runtime_reconcile_total{service=~".*external-secrets.*",result="error"}[15m])) + / + sum by (controller) (increase(controller_runtime_reconcile_total{service=~".*external-secrets.*"}[15m])) + ) > 0.05 + and + sum by (controller) (increase(controller_runtime_reconcile_total{service=~".*external-secrets.*"}[15m])) > 1 + |||, + 'for': '15m', + labels: { + severity: 'warning', + }, + annotations: { + summary: 'High reconcile error rate: {{ $labels.controller }}', + description: ||| + Controller {{ $labels.controller }} error rate is {{ $value | humanizePercentage }} over %s. + ||| % this['for'], + }, + }, + { + local this = self, + alert: 'ExternalSecretsProviderAPIErrorRateHigh', + expr: ||| + sum by (provider) (increase(externalsecret_provider_api_calls_count{service=~".*external-secrets.*",status="error"}[15m])) + / + sum by (provider) (increase(externalsecret_provider_api_calls_count{service=~".*external-secrets.*"}[15m])) > 0.10 + |||, + 'for': '15m', + labels: { + severity: 'warning', + }, + annotations: { + summary: 'High provider API error rate: {{ $labels.provider }}', + description: ||| + More than 10 percent of calls to {{ $labels.provider }} failing for %s. Check provider availability, credentials and rate limits. ||| % this['for'], }, }, diff --git a/tests/golden/openshift/external-secrets-operator/external-secrets-operator/10_eso_alerts.yaml b/tests/golden/openshift/external-secrets-operator/external-secrets-operator/10_eso_alerts.yaml index 2018856..4f0e0db 100644 --- a/tests/golden/openshift/external-secrets-operator/external-secrets-operator/10_eso_alerts.yaml +++ b/tests/golden/openshift/external-secrets-operator/external-secrets-operator/10_eso_alerts.yaml @@ -11,15 +11,59 @@ spec: - alert: ExternalSecretNotReady annotations: description: | - ExternalSecret {{ $labels.namespace }}/{{ $labels.name }} has been - Ready=False for 10m. Check status conditions and provider - connectivity. + ExternalSecret {{ $labels.namespace }}/{{ $labels.name }} has been Ready=False for 10m. + Check status conditions and provider connectivity. summary: 'ExternalSecret not ready: {{ $labels.namespace }}/{{ $labels.name }}' - expr: max by (namespace, name) (externalsecret_status_condition{condition="Ready",status="False"}) - == 1 + expr: | + max by (namespace, name) (externalsecret_status_condition{condition="Ready",status="False"}) == 1 for: 10m labels: severity: warning syn: 'true' syn_component: external-secrets-operator + - alert: ExternalSecretsWebhookErrors + annotations: + description: | + Webhook has returned {{ $value }} HTTP 500s in 5m. + If failurePolicy=Fail, creating/updating ExternalSecrets and SecretStores is blocked cluster-wide. + summary: external-secrets admission webhook returning HTTP 500 + expr: | + sum(increase(controller_runtime_webhook_requests_total{service=~".*external-secrets.*",code="500"}[5m])) > 0 + for: 5m + labels: + severity: critical + syn: 'true' + syn_component: external-secrets-operator + - alert: ExternalSecretsReconcileErrorRateHigh + annotations: + description: | + Controller {{ $labels.controller }} error rate is {{ $value | humanizePercentage }} over 15m. + summary: 'High reconcile error rate: {{ $labels.controller }}' + expr: | + ( + sum by (controller) (increase(controller_runtime_reconcile_total{service=~".*external-secrets.*",result="error"}[15m])) + / + sum by (controller) (increase(controller_runtime_reconcile_total{service=~".*external-secrets.*"}[15m])) + ) > 0.05 + and + sum by (controller) (increase(controller_runtime_reconcile_total{service=~".*external-secrets.*"}[15m])) > 1 + for: 15m + labels: + severity: warning + syn: 'true' + syn_component: external-secrets-operator + - alert: ExternalSecretsProviderAPIErrorRateHigh + annotations: + description: | + More than 10 percent of calls to {{ $labels.provider }} failing for 15m. Check provider availability, credentials and rate limits. + summary: 'High provider API error rate: {{ $labels.provider }}' + expr: | + sum by (provider) (increase(externalsecret_provider_api_calls_count{service=~".*external-secrets.*",status="error"}[15m])) + / + sum by (provider) (increase(externalsecret_provider_api_calls_count{service=~".*external-secrets.*"}[15m])) > 0.10 + for: 15m + labels: + severity: warning + syn: 'true' + syn_component: external-secrets-operator From 72794ca06d0db382b39315007ebf2a07862906cd Mon Sep 17 00:00:00 2001 From: Adrian Haas <11636405+haasad@users.noreply.github.com> Date: Fri, 30 Oct 2026 16:25:01 +0100 Subject: [PATCH 3/3] Apply suggestions from review Co-authored-by: Simon Gerber --- component/alerts.jsonnet | 31 +++++++++---------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/component/alerts.jsonnet b/component/alerts.jsonnet index 24048d2..d5d16de 100644 --- a/component/alerts.jsonnet +++ b/component/alerts.jsonnet @@ -110,41 +110,28 @@ local eso_alerts = prom.PrometheusRule('external-secrets-operator') { }, }; -local additional_group = - local parseRuleName(rname) = - local rparts = std.splitLimit(rname, ':', 1); - assert - std.length(rparts) == 2 : - 'Expected custom alert rule to be prefixed with `record:` or `alert:`.'; - assert - std.member([ 'alert', 'record' ], rparts[0]) : - 'Expected custom alert rule to be prefixed with `record:` or `alert:`, got `%s:`.' % rparts[0]; - { [rparts[0]]: rparts[1] }; - { - name: 'external-secrets-operator-custom.rules', - rules: [ - params.alerts.additionalRules[rname] + parseRuleName(rname) - for rname in std.objectFields(params.alerts.additionalRules) - ], - }; +local additional_groups = alertpatching.renderGroups({ + 'external-secrets-operator-custom.rules': params.alerts.additionalRules, +}); local additional_alerts = prom.PrometheusRule('external-secrets-operator-custom') { spec+: { - groups: [ - alertpatching.filterPatchRules( - additional_group, + groups: std.map( + function(g) alertpatching.filterPatchRules( + g, ignoreNames=ignoreNames, patches=params.alerts.patches, preserveRecordingRules=true, patchNames=false, ), - ], + additional_groups, + ), }, }; if has_monitoring then { [if std.length(eso_alerts.spec.groups[0].rules) > 0 then '10_eso_alerts']: eso_alerts, - [if std.length(additional_group.rules) > 0 then + [if std.length(additional_groups[0].rules) > 0 then '10_custom_alerts']: additional_alerts, } else std.trace('Component ESO compiled for cluster that does not have monitoring, skipping alert rules generation.', {})