Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -20,82 +20,71 @@
# PolicyReports for a human to assess. After the chart-PDB sweep this policy
# ships with, the cluster reports zero violations -- so any future violation is a
# new chart or operator reintroducing the landmine.
apiVersion: kyverno.io/v1
kind: ClusterPolicy
apiVersion: policies.kyverno.io/v1
kind: ValidatingPolicy
metadata:
name: validate-pdb-drain-safe
annotations:
policies.kyverno.io/title: Validate Drain-Safe PodDisruptionBudgets
policies.kyverno.io/category: Best Practices
policies.kyverno.io/severity: medium
policies.kyverno.io/subject: PodDisruptionBudget
policies.kyverno.io/minversion: 1.13.0
policies.kyverno.io/minversion: 1.17.0
policies.kyverno.io/description: >-
Flags PodDisruptionBudgets that set spec.minAvailable. The platform
standard is spec.maxUnavailable, which keeps nodes drainable at any
replica count; a minAvailable PDB over a low-replica workload can make its
node undrainable. CloudNativePG-managed PDBs are excluded.
spec:
background: true
rules:
- name: require-maxunavailable
match:
any:
- resources:
kinds:
- PodDisruptionBudget
exclude:
any:
# CloudNativePG owns drain-safety for its clusters via switchover and
# scales minAvailable to (instances - 1); don't flag operator-managed
# PDBs (labelled cnpg.io/cluster).
- resources:
selector:
matchExpressions:
- key: cnpg.io/cluster
operator: Exists
# ksail bootstrap-installs cert-manager, metrics-server and kyverno
# (ksail.yaml certManager/metricsServer/policyEngine) with their
# default minAvailable PDBs BEFORE Flux. Flipping those to
# maxUnavailable conflicts at server-side apply with ksail's field
# manager, so they intentionally stay minAvailable: 1 -- drain-safe at
# their 2-replica floors. Don't flag them.
- resources:
names:
- cert-manager
- cert-manager-webhook
- cert-manager-cainjector
- metrics-server
- kyverno-admission-controller
# hcloud-csi-controller is also ksail-installed (Hetzner CSI) and
# keeps minAvailable: 1; its drain-safety comes from the HA PR's
# replica bump to 2, not maxUnavailable.
- hcloud-csi-controller
# Longhorn manages these PDBs itself: longhorn-manager creates a
# minAvailable: 1 PDB per instance-manager that still hosts replicas
# or engines (and deletes it once the node is safe to drain), plus
# PDBs for its csi-attacher/csi-provisioner deployments. The
# minAvailable shape IS the operator's eviction interlock — not a
# chart value anyone can flip — so flagging them is permanent Audit
# noise (15 objects on prod). They carry no labels or ownerReferences,
# so exclude by namespace + the operator's fixed names.
- resources:
namespaces:
- longhorn-system
names:
- instance-manager-?*
- csi-attacher
- csi-provisioner
validate:
failureAction: Audit
message: >-
PodDisruptionBudget '{{ request.object.metadata.name }}' sets
spec.minAvailable. Use spec.maxUnavailable instead (e.g.
maxUnavailable: 1) so node drains are never deadlocked at low replica
counts -- see issues #1880 and #1882.
deny:
conditions:
any:
- key: "{{ request.object.spec.minAvailable || '' }}"
operator: NotEquals
value: ""
validationActions:
- Audit
evaluation:
background:
enabled: true
matchConstraints:
resourceRules:
- apiGroups: ["policy"]
apiVersions: ["v1"]
operations: ["CREATE", "UPDATE"]
resources: ["poddisruptionbudgets"]
matchConditions:
# CloudNativePG owns drain-safety for its clusters via switchover and
# scales minAvailable to (instances - 1); don't flag operator-managed
# PDBs (labelled cnpg.io/cluster).
- name: not-cnpg-managed
expression: >-
!('cnpg.io/cluster' in object.metadata.?labels.orValue({}))
# ksail bootstrap-installs cert-manager, metrics-server and kyverno
# (ksail.yaml certManager/metricsServer/policyEngine) with their
# default minAvailable PDBs BEFORE Flux. Flipping those to
# maxUnavailable conflicts at server-side apply with ksail's field
# manager, so they intentionally stay minAvailable: 1 -- drain-safe at
# their 2-replica floors. hcloud-csi-controller is also ksail-installed
# (Hetzner CSI) and keeps minAvailable: 1; its drain-safety comes from the
# HA PR's replica bump to 2, not maxUnavailable. Don't flag them.
- name: not-ksail-installed
expression: >-
!(object.metadata.name in [
'cert-manager', 'cert-manager-webhook', 'cert-manager-cainjector',
'metrics-server', 'kyverno-admission-controller',
'hcloud-csi-controller'])
# Longhorn manages these PDBs itself: longhorn-manager creates a
# minAvailable: 1 PDB per instance-manager that still hosts replicas
# or engines (and deletes it once the node is safe to drain), plus
# PDBs for its csi-attacher/csi-provisioner deployments. The
# minAvailable shape IS the operator's eviction interlock — not a
# chart value anyone can flip — so flagging them is permanent Audit
# noise (15 objects on prod). They carry no labels or ownerReferences,
# so exclude by namespace + the operator's fixed names.
- name: not-longhorn-managed
expression: >-
!(object.metadata.?namespace.orValue('') == 'longhorn-system' &&
(object.metadata.name.matches('^instance-manager-.+$') ||
object.metadata.name in ['csi-attacher', 'csi-provisioner']))
validations:
- expression: "!has(object.spec.minAvailable)"
messageExpression: >-
"PodDisruptionBudget '" + object.metadata.name + "' sets
spec.minAvailable. Use spec.maxUnavailable instead (e.g.
maxUnavailable: 1) so node drains are never deadlocked at low replica
counts -- see issues #1880 and #1882."
2 changes: 1 addition & 1 deletion tests/policy-failure-actions.json
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
"validate-host-restrictions/disallow-host-path": "Enforce",
"validate-host-restrictions/disallow-host-namespaces": "Enforce",
"validate-host-restrictions/disallow-host-users": "Enforce",
"validate-pdb-drain-safe/require-maxunavailable": "Audit",
"validate-pdb-drain-safe": "Audit",
"validate-pod-security/validate-container-security": "Enforce",
"validate-pod-security/require-seccomp-profile": "Enforce",
"validate-pod-security/deny-container-seccomp-unconfined": "Enforce",
Expand Down
5 changes: 1 addition & 4 deletions tests/validate-pdb-drain-safe/kyverno-test.yaml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Pins the ClusterPolicy's behaviour before its ValidatingPolicy rewrite (#4996).
# Pins the policy's behaviour across its ValidatingPolicy rewrite (#4996).
# A maxUnavailable PDB passes and a minAvailable PDB fails. Each exclusion is
# pinned: a CloudNativePG-labelled PDB, a KSail-installed PDB by name, and a
# Longhorn-managed PDB, which is exempt only inside longhorn-system.
Expand All @@ -12,20 +12,17 @@ resources:
- resources.yaml
results:
- policy: validate-pdb-drain-safe
rule: require-maxunavailable
resources:
- apps/max-unavailable
kind: PodDisruptionBudget
result: pass
- policy: validate-pdb-drain-safe
rule: require-maxunavailable
resources:
- apps/min-available
- apps/instance-manager-outside-longhorn
kind: PodDisruptionBudget
result: fail
- policy: validate-pdb-drain-safe
rule: require-maxunavailable
resources:
- apps/cnpg-cluster
- cert-manager/cert-manager
Expand Down
Loading