fix: prevent CNClaim Finalize stuck and scale-in race during migration

Dockerfile

@@ -4,8 +4,6 @@ ARG GOPROXY="https://proxy.golang.org,direct"
 
 WORKDIR /workspace
 
-RUN go env -w GOMODCACHE=/root/.cache/go-build
-
 RUN go env -w GOPROXY=${GOPROXY}
 
 COPY go.mod go.mod
@@ -15,12 +13,14 @@ COPY api/go.sum api/go.sum
 
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
-RUN --mount=type=cache,target=/root/.cache/go-build go mod download
+RUN --mount=type=cache,id=gomodcache,target=/go/pkg/mod go mod download
 
 COPY . .
 
 # Build
-RUN --mount=type=cache,target=/root/.cache/go-build CGO_ENABLED=0 go build -a -o manager cmd/operator/main.go
+RUN --mount=type=cache,id=gomodcache,target=/go/pkg/mod \
+    --mount=type=cache,id=gobuildcache,target=/root/.cache/go-build \
+    CGO_ENABLED=0 go build -a -o manager cmd/operator/main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details

api/go.mod

@@ -5,6 +5,7 @@ go 1.19
 require (
 	github.com/blang/semver/v4 v4.0.0
 	github.com/go-errors/errors v1.5.1
+	github.com/google/go-cmp v0.6.0
 	github.com/matrixorigin/controller-runtime v0.0.0-20240909085031-5f706d779ec6
 	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826
 	github.com/onsi/gomega v1.27.7
@@ -31,7 +32,6 @@ require (
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/gnostic v0.5.7-v3refs // indirect
-	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/pprof v0.0.0-20230510103437-eeec1cb781c3 // indirect
 	github.com/google/uuid v1.6.0 // indirect

hack/lib.sh

@@ -83,14 +83,21 @@ function kind::ensure-kind() {
 
 function kind::load-image() {
     local kruise_image
+    local kruise_hook_image
     kruise_image=$(helm template kruise "${ROOT}/charts/kruise" | awk '/^[[:space:]]+image:.*kruise-manager/ {print $2; exit}')
     if [[ -z "${kruise_image}" ]]; then
         echo "error: failed to resolve Kruise manager image from local chart"
         return 1
     fi
+    kruise_hook_image=$(helm template kruise "${ROOT}/charts/kruise" | awk '/^[[:space:]]+image:.*kruise-helm-hook/ {print $2; exit}')
+    if [[ -z "${kruise_hook_image}" ]]; then
+        echo "error: failed to resolve Kruise Helm hook image from local chart"
+        return 1
+    fi
 
     kind::prepare_image ${CLUSTER} ${MO_IMAGE_REPO}:${MO_VERSION}
     kind::prepare_image ${CLUSTER} "${kruise_image}"
+    kind::prepare_image ${CLUSTER} "${kruise_hook_image}"
     kind::prepare_image ${CLUSTER} minio/minio:RELEASE.2023-11-01T01-57-10Z
 }
 
@@ -111,9 +118,10 @@ function e2e::check() {
 }
 
 function e2e::run() {
+    local nodes="${E2E_NODES:-4}"
     echo "> Run e2e test"
     make ginkgo
-    ./bin/ginkgo -nodes=4 -stream=true -slowSpecThreshold=3000 ./test/e2e/... -- \
+    ./bin/ginkgo -nodes="${nodes}" -stream=true -slowSpecThreshold=3000 ./test/e2e/... -- \
                 -mo-version="${MO_VERSION}" \
                 -mo-image-repo="${MO_IMAGE_REPO}"
 
@@ -140,16 +148,62 @@ function e2e::install() {
   fi
   rm -rf -- "${chart_root}"
 
-  echo "> Wait webhook certificate injected"
-  sleep 30
+  e2e::wait-webhook-ready
+}
+
+function e2e::wait-webhook-ready() {
+  local selector="app.kubernetes.io/name=matrixone-operator,app.kubernetes.io/instance=mo"
+  local mutating="matrixone-operator-mutating-webhook-mo"
+  local validating="matrixone-operator-validating-webhook-mo"
+  local timeout_seconds=300
+  local deadline
+
+  echo "> Wait for operator deployment"
+  if ! kubectl -n "${OPNAMESPACE}" wait deployment \
+    -l "${selector}" \
+    --for=condition=Available \
+    --timeout="${timeout_seconds}s"; then
+    kubectl -n "${OPNAMESPACE}" get pods -o wide || true
+    return 1
+  fi
+
+  echo "> Wait for webhook CA injection"
+  deadline=$((SECONDS + timeout_seconds))
+  while ((SECONDS < deadline)); do
+    local mutating_ca
+    local validating_ca
+    mutating_ca=$(kubectl get mutatingwebhookconfiguration "${mutating}" \
+      -o jsonpath='{.webhooks[0].clientConfig.caBundle}' 2>/dev/null || true)
+    validating_ca=$(kubectl get validatingwebhookconfiguration "${validating}" \
+      -o jsonpath='{.webhooks[0].clientConfig.caBundle}' 2>/dev/null || true)
+    if [[ -n "${mutating_ca}" && "${mutating_ca}" != "Cg==" && \
+          -n "${validating_ca}" && "${validating_ca}" != "Cg==" ]]; then
+      echo "> Webhook CA injection completed"
+      return 0
+    fi
+    sleep 2
+  done
+
+  echo "error: webhook CA injection did not complete within ${timeout_seconds}s"
+  kubectl get mutatingwebhookconfiguration "${mutating}" -o yaml || true
+  kubectl get validatingwebhookconfiguration "${validating}" -o yaml || true
+  kubectl -n "${OPNAMESPACE}" logs deployment/mo-matrixone-operator --tail=100 || true
+  return 1
 }
 
 function e2e::cleanup() {
     echo "Delete e2e test namespace"
-    kubectl get ns --all-namespaces --no-headers=true | awk '/^e2e/{print $1}' | xargs kubectl delete ns
+    if ! kubectl delete namespace -l managed-by=e2e-suite \
+      --ignore-not-found --wait=true --timeout=600s; then
+      kubectl get namespace -l managed-by=e2e-suite -o yaml || true
+      return 1
+    fi
     # Uninstall helm charts
     echo "Uninstall helm charts..."
-    helm uninstall mo -n "${OPNAMESPACE}"
+    if ! helm uninstall mo -n "${OPNAMESPACE}"; then
+      kubectl -n kruise-system logs job/mo-finalizer --all-containers=true || true
+      return 1
+    fi
     echo "Wait for charts uninstall"
     sleep 10
     echo "Delete operator namespace"
@@ -160,5 +214,9 @@ function e2e::workflow() {
   e2e::check
   trap "e2e::cleanup" EXIT
   e2e::install || return 1
-  e2e::run
+  local run_status=0
+  e2e::run || run_status=$?
+  trap - EXIT
+  e2e::cleanup || return 1
+  return "${run_status}"
 }