diff --git a/Modules/Numerics/Optimizersv4/include/itkExhaustiveOptimizerv4.hxx b/Modules/Numerics/Optimizersv4/include/itkExhaustiveOptimizerv4.hxx
index e7a1c84abce..3b1f32b3f0b 100644
--- a/Modules/Numerics/Optimizersv4/include/itkExhaustiveOptimizerv4.hxx
+++ b/Modules/Numerics/Optimizersv4/include/itkExhaustiveOptimizerv4.hxx
@@ -134,6 +134,10 @@ ExhaustiveOptimizerv4<TInternalComputationValueType>::ResumeWalking()
                                << "@ index " << this->GetCurrentIndex() << " value is " << m_CurrentValue;
 
     this->InvokeEvent(IterationEvent());
+    if (m_Stop)
+    {
+      break;
+    }
     this->AdvanceOneStep();
     this->m_CurrentIteration++;
   }
diff --git a/Modules/Numerics/Optimizersv4/include/itkOnePlusOneEvolutionaryOptimizerv4.hxx b/Modules/Numerics/Optimizersv4/include/itkOnePlusOneEvolutionaryOptimizerv4.hxx
index 7d464af752b..bc12b7e7951 100644
--- a/Modules/Numerics/Optimizersv4/include/itkOnePlusOneEvolutionaryOptimizerv4.hxx
+++ b/Modules/Numerics/Optimizersv4/include/itkOnePlusOneEvolutionaryOptimizerv4.hxx
@@ -257,6 +257,13 @@ OnePlusOneEvolutionaryOptimizerv4<TInternalComputationValueType>::StartOptimizat
     }
 
     this->InvokeEvent(IterationEvent());
+    if (m_Stop)
+    {
+      m_StopConditionDescription.str("");
+      m_StopConditionDescription << this->GetNameOfClass() << ": "
+                                 << "StopOptimization() called";
+      break;
+    }
     itkDebugMacro("Current position: " << this->GetCurrentPosition());
   }
   if (this->m_CurrentIteration >= m_MaximumIteration)
diff --git a/Modules/Numerics/Optimizersv4/include/itkPowellOptimizerv4.hxx b/Modules/Numerics/Optimizersv4/include/itkPowellOptimizerv4.hxx
index a511067cf84..9804b4be4ff 100644
--- a/Modules/Numerics/Optimizersv4/include/itkPowellOptimizerv4.hxx
+++ b/Modules/Numerics/Optimizersv4/include/itkPowellOptimizerv4.hxx
@@ -498,6 +498,12 @@ PowellOptimizerv4<TInternalComputationValueType>::StartOptimization(bool /* doOn
     }
 
     this->InvokeEvent(IterationEvent());
+    if (m_Stop)
+    {
+      m_StopConditionDescription << "StopOptimization() called";
+      this->InvokeEvent(EndEvent());
+      return;
+    }
   }
 
   m_StopConditionDescription << "Maximum number of iterations exceeded. "