From efabfd9cf57ed07ef2262d0057825cb9e2298556 Mon Sep 17 00:00:00 2001 From: Arunesh Dwivedi Date: Thu, 20 Aug 2026 07:53:52 +0000 Subject: [PATCH 1/2] fix(imagejob): ignore NotFound errors when looking up PodTemplate When a PodTemplate is deleted before the imagejob-controller processes the ImageJob, the controller logs a scary 'PodTemplate not found' error and marks the job as failed, even though the job may have already completed or been handled by another controller. This change uses client.IgnoreNotFound to silently ignore missing PodTemplates, matching the pattern already used elsewhere in the controller. Fixes #1051 Signed-off-by: Arunesh Dwivedi --- controllers/imagejob/imagejob_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controllers/imagejob/imagejob_controller.go b/controllers/imagejob/imagejob_controller.go index 8e8f683ca2..f0b5a7b4ca 100644 --- a/controllers/imagejob/imagejob_controller.go +++ b/controllers/imagejob/imagejob_controller.go @@ -307,7 +307,7 @@ func (r *Reconciler) handleNewJob(ctx context.Context, imageJob *eraserv1.ImageJ &template, ) if err != nil { - return err + return client.IgnoreNotFound(err) } imageJob.Status = eraserv1.ImageJobStatus{ From a26644eef4c8162fb08a4574a9dfb3e63e13691b Mon Sep 17 00:00:00 2001 From: Arunesh Dwivedi Date: Thu, 20 Aug 2026 08:01:03 +0000 Subject: [PATCH 2/2] fix(imagejob): skip nodes with insufficient resources before scheduling pods When a node doesn't have enough CPU or memory to run the eraser pod, the controller still creates the pod, which then gets stuck in Pending state (OutOfcpu/Outofmemory). This wastes reconcile cycles and creates noisy logs. Add a resource check before pod creation: compare the node's allocatable resources against the pod's requested/limited resources. If the node can't accommodate the pod, skip it and move to the next node. Fixes #1005 Signed-off-by: Arunesh Dwivedi --- controllers/imagejob/imagejob_controller.go | 29 +++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/controllers/imagejob/imagejob_controller.go b/controllers/imagejob/imagejob_controller.go index f0b5a7b4ca..81265b1a4c 100644 --- a/controllers/imagejob/imagejob_controller.go +++ b/controllers/imagejob/imagejob_controller.go @@ -362,6 +362,13 @@ func (r *Reconciler) handleNewJob(ctx context.Context, imageJob *eraserv1.ImageJ podSpecTemplate := template.Template.Spec for i := range nodeList { log := log.WithValues("node", nodeList[i].Name) + + // Skip nodes with insufficient resources + if insufficientResources(&nodeList[i], &podSpecTemplate) { + log.Info("skipping node: insufficient resources for eraser pod") + continue + } + podSpec, err := copyAndFillTemplateSpec(&podSpecTemplate, env, &nodeList[i], &eraserConfig.Manager.Runtime) if err != nil { return err @@ -530,6 +537,28 @@ nodes: return nodeList, skipped, nil } +func insufficientResources(node *corev1.Node, podSpec *corev1.PodSpec) bool { + allocatable := node.Status.Allocatable + requests := podSpec.Containers[0].Resources.Requests + limits := podSpec.Containers[0].Resources.Limits + + for name, req := range requests { + if available, ok := allocatable[name]; ok { + if available.Cmp(req) < 0 { + return true + } + } + } + for name, limit := range limits { + if available, ok := allocatable[name]; ok { + if available.Cmp(limit) < 0 { + return true + } + } + } + return false +} + func copyAndFillTemplateSpec(templateSpecTemplate *corev1.PodSpec, env []corev1.EnvVar, node *corev1.Node, runtimeSpec *unversioned.RuntimeSpec) (*corev1.PodSpec, error) { nodeName := node.Name