feat(mysql): A2 engine split — 8.4 opt-in for fresh installs + datadir format guard (backend#723 PR-2)

client/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: client
 description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift
 type: application
-version: 1.9.20
-appVersion: "1.9.20"
+version: 1.9.21
+appVersion: "1.9.21"
 keywords:
   - tracebloc
   - kubernetes

client/templates/_helpers.tpl

@@ -397,3 +397,36 @@ Usage inside a container's env: list:
   value: {{ $noProxy | quote }}
 {{- end }}
 {{- end -}}
+
+{{/*
+tracebloc.mysqlEngineMajor — the MySQL engine major the chart is about to run,
+for the mysql-format-guard init container (backend#723). Resolution mirrors
+tracebloc.image's digest-wins precedence:
+  digest set   -> the known 5.7-lineage pin maps to "5.7"; any other digest is
+                  "unknown" (custom pin — the guard stands down).
+  digest empty -> derive from the tag: ""/prod/5.7* -> 5.7, 8.4* -> 8.4,
+                  8.0* -> 8.0, anything else -> unknown.
+The sha256 literal below MUST equal the images.mysqlClient.digest default in
+values.yaml — mysql_test.yaml pins the default render to "5.7", so re-pinning
+the digest without updating this helper fails CI instead of silently
+disarming the guard.
+*/}}
+{{- define "tracebloc.mysqlEngineMajor" -}}
+{{- $digest := .Values.images.mysqlClient.digest | default "" -}}
+{{- $tag := .Values.images.mysqlClient.tag | default "prod" -}}
+{{- if $digest -}}
+{{- if eq $digest "sha256:f546e47fb339e0982c902cef063b081ccf2cbbaf35b475287d583b9bf3163354" -}}
+5.7
+{{- else -}}
+unknown
+{{- end -}}
+{{- else if or (eq $tag "prod") (hasPrefix "5.7" $tag) -}}
+5.7
+{{- else if hasPrefix "8.4" $tag -}}
+8.4
+{{- else if hasPrefix "8.0" $tag -}}
+8.0
+{{- else -}}
+unknown
+{{- end -}}
+{{- end -}}

client/templates/mysql-deployment.yaml

@@ -32,12 +32,12 @@ spec:
       securityContext:
         fsGroup: 999
         fsGroupChangePolicy: "OnRootMismatch"
+      initContainers:
       {{- if .Values.hostPath.enabled }}
       # kubelet does not apply fsGroup to hostPath volumes
       # (kubernetes/kubernetes#138411), so bare-metal installs need a
       # privileged bootstrap to chown /var/lib/mysql to 999:999. CSI-backed
       # deployments (EKS/AKS/OC) skip this and rely on fsGroup.
-      initContainers:
       - name: init-mysql-data
         image: {{ include "tracebloc.image" (dict "repository" "library/busybox" "tag" .Values.images.busybox.tag "digest" .Values.images.busybox.digest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }}
         # Must run as root to chown the hostPath mount; kubelet does not apply
@@ -61,6 +61,60 @@ spec:
         - name: mysql-persistent-storage
           mountPath: /var/lib/mysql/
       {{- end }}
+      # Engine/datadir format guard (backend#723). MySQL datadirs are
+      # version-specific: 8.4 refuses a 5.7-format datadir (staged 5.7→8.0→8.4
+      # upgrades only), and 5.7 cannot open an 8.x one. A mismatched engine
+      # would CrashLoop with a cryptic mysqld error — this fails FIRST with an
+      # actionable message instead. `mysql.ibd` (the 8.0+ data dictionary) is
+      # the format marker: present = 8.x datadir, absent (with ibdata1) = 5.7.
+      # The 8.0 transit hop and unknown/custom pins are deliberately allowed —
+      # this is a backstop against accidental engine flips, not a gate.
+      - name: mysql-format-guard
+        image: {{ include "tracebloc.image" (dict "repository" "library/busybox" "tag" .Values.images.busybox.tag "digest" .Values.images.busybox.digest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }}
+        env:
+        - name: TB_EXPECTED_ENGINE
+          value: {{ include "tracebloc.mysqlEngineMajor" . | quote }}
+        securityContext:
+          runAsUser: 999
+          runAsGroup: 999
+          runAsNonRoot: true
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          capabilities:
+            drop: ["ALL"]
+          seccompProfile:
+            type: RuntimeDefault
+        command:
+          - 'sh'
+          - '-c'
+          - |
+            d=/var/lib/mysql
+            if [ ! -e "$d/ibdata1" ] && [ ! -e "$d/mysql.ibd" ]; then
+              exit 0  # fresh/empty datadir: the entrypoint initializes it
+            fi
+            case "$TB_EXPECTED_ENGINE" in
+              8.4)
+                if [ ! -e "$d/mysql.ibd" ]; then
+                  echo "REFUSING TO START: MySQL 8.4 image over a 5.7-format datadir (no mysql.ibd)."
+                  echo "8.4 cannot open 5.7 data — MySQL supports staged upgrades only (5.7 -> 8.0 -> 8.4)."
+                  echo "Either keep this edge on the 5.7 engine (remove the 8.4 image override) or"
+                  echo "migrate the datadir per the staged runbook in tracebloc/backend#723."
+                  exit 1
+                fi ;;
+              5.7)
+                if [ -e "$d/mysql.ibd" ]; then
+                  echo "REFUSING TO START: MySQL 5.7 image over an 8.x-format datadir (mysql.ibd present)."
+                  echo "This datadir was already upgraded and cannot go back to 5.7. Point this edge"
+                  echo "back at its 8.4 engine values (tracebloc/backend#723), or restore the"
+                  echo "pre-migration backup if a rollback is intended."
+                  exit 1
+                fi ;;
+              *) : ;;
+            esac
+        volumeMounts:
+        - name: mysql-persistent-storage
+          mountPath: /var/lib/mysql/
+          readOnly: true
       containers:
       - image: {{ include "tracebloc.image" (dict "repository" "tracebloc/mysql-client" "tag" (.Values.images.mysqlClient.tag | default "prod") "digest" .Values.images.mysqlClient.digest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }}
         imagePullPolicy: IfNotPresent

client/tests/mysql_test.yaml

@@ -185,3 +185,89 @@ tests:
       - matchRegex:
           path: spec.template.spec.containers[0].image
           pattern: ^docker\.io/tracebloc/mysql-client@sha256:[a-f0-9]{64}$
+
+  # ── Engine/datadir format guard (backend#723, decision A2) ──
+  # The first assert doubles as the digest↔helper sync pin: re-pinning
+  # images.mysqlClient.digest without updating tracebloc.mysqlEngineMajor
+  # makes the default render report "unknown" and fails here.
+  - it: renders the format guard armed for 5.7 by default (digest pin recognized)
+    template: templates/mysql-deployment.yaml
+    asserts:
+      - equal:
+          path: spec.template.spec.initContainers[0].name
+          value: mysql-format-guard
+      - contains:
+          path: spec.template.spec.initContainers[0].env
+          content:
+            name: TB_EXPECTED_ENGINE
+            value: "5.7"
+      - equal:
+          path: spec.template.spec.initContainers[0].volumeMounts[0].readOnly
+          value: true
+
+  - it: 8.4 opt-in (tag 8.4, empty digest) runs the 8.4 image and arms the guard for 8.4
+    template: templates/mysql-deployment.yaml
+    set:
+      images:
+        mysqlClient:
+          tag: "8.4"
+          digest: ""
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].image
+          value: docker.io/tracebloc/mysql-client:8.4
+      - contains:
+          path: spec.template.spec.initContainers[0].env
+          content:
+            name: TB_EXPECTED_ENGINE
+            value: "8.4"
+
+  - it: the 8.0 transit hop is recognized (guard stands down for the staged upgrade)
+    template: templates/mysql-deployment.yaml
+    set:
+      images:
+        mysqlClient:
+          tag: "8.0"
+          digest: ""
+    asserts:
+      - contains:
+          path: spec.template.spec.initContainers[0].env
+          content:
+            name: TB_EXPECTED_ENGINE
+            value: "8.0"
+
+  - it: a custom digest pin disarms the guard (engine unknown)
+    template: templates/mysql-deployment.yaml
+    set:
+      images:
+        mysqlClient:
+          digest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+    asserts:
+      - contains:
+          path: spec.template.spec.initContainers[0].env
+          content:
+            name: TB_EXPECTED_ENGINE
+            value: "unknown"
+
+  - it: hostPath keeps the chown bootstrap first and the guard second
+    template: templates/mysql-deployment.yaml
+    set:
+      hostPath:
+        enabled: true
+    asserts:
+      - equal:
+          path: spec.template.spec.initContainers[0].name
+          value: init-mysql-data
+      - equal:
+          path: spec.template.spec.initContainers[1].name
+          value: mysql-format-guard
+
+  - it: the format guard re-homes onto a private mirror like every other image (#585)
+    template: templates/mysql-deployment.yaml
+    set:
+      global:
+        imageRegistry: mirror.corp.example
+    asserts:
+      - equal:
+          path: spec.template.spec.initContainers[0].image
+          value: mirror.corp.example/library/busybox:1.35

client/values.yaml

@@ -507,6 +507,18 @@ images:
     # unchanged (arm64 hosts keep running it emulated). A future re-pin
     # (the 8.4 multi-arch image, backend#723) should pin the manifest-INDEX
     # digest, like images.ingestor.prodDigest does.
+    #
+    # 8.4 OPT-IN (backend#723, decision A2 2026-08-05) — FRESH datadirs only:
+    #   tag: "8.4"
+    #   digest: ""
+    # The bash installer sets exactly this on fresh arm64 installs (native
+    # multi-arch engine instead of amd64 emulation) and keeps it sticky across
+    # re-runs; TB_MYSQL_ENGINE=8.4|5.7 overrides. NEVER set it on an edge with
+    # an existing 5.7 datadir: 8.4 refuses to open 5.7 data (staged
+    # 5.7 -> 8.0 -> 8.4 only) — the mysql-format-guard init container blocks
+    # the mismatch with an actionable message instead of a mysqld CrashLoop.
+    # If you re-pin this digest, update tracebloc.mysqlEngineMajor in
+    # _helpers.tpl in the same commit (mysql_test.yaml enforces the pair).
     digest: "sha256:f546e47fb339e0982c902cef063b081ccf2cbbaf35b475287d583b9bf3163354"
   busybox:
     tag: "1.35"

scripts/lib/install-client-helm.sh

@@ -668,6 +668,72 @@ _download_services_progress() {
   return 0
 }
 
+# ── MySQL engine channel (backend#723, decision A2 2026-08-05) ─────────────
+# The chart's frozen 5.7 digest pin stays the default for every existing
+# install; FRESH installs may opt into the multi-arch 8.4 engine. `auto`
+# picks 8.4 only on a fresh arm64 install — the one cohort 5.7 actually hurts
+# (amd64-only image under emulation). Anything that smells like existing
+# state stays 5.7: an existing release, real mysql datadir content on the
+# host (legacy or per-release layout), or nothing at all to suggest 8.4.
+# A previous opt-in stays sticky across re-runs (the values file is
+# regenerated every run, so it is re-derived from the old file first). The
+# chart's mysql-format-guard init container backstops whatever this
+# heuristic misses — a wrong pick fails loudly before mysqld starts, it
+# never opens a datadir with the wrong engine.
+#   TB_MYSQL_ENGINE=auto|5.7|8.4    explicit value always wins (default auto)
+# Reads (bash dynamic scope, set by install_client_helm before the call):
+# values_file, existing_id, HOST_DATA_DIR, TB_NAMESPACE, ARCH.
+# Sets: TB_MYSQL_ENGINE_RESOLVED.
+# Content test for a host mysql datadir, FAIL-CLOSED on unlistable dirs
+# (mirrors _leftover_data_dirs, and the same Bugbot ownership case): a
+# uid-999/root-owned dir the host user can't read/enter cannot be proven
+# empty — treat it as content, so `auto` keeps 5.7 rather than opting a
+# reused datadir into 8.4 that the format guard would then refuse to boot.
+# Symlinks are never trusted as data (same stance as the leftover guard).
+_mysql_dir_has_content() {
+  local d="$1"
+  [[ -d "$d" && ! -L "$d" ]] || return 1   # absent -> no content
+  [[ -r "$d" && -x "$d" ]] || return 0     # unlistable -> fail closed
+  [[ -n "$(ls -A "$d" 2>/dev/null)" ]]
+}
+
+_resolve_mysql_engine() {
+  local requested="${TB_MYSQL_ENGINE:-auto}"
+  case "$requested" in
+    5.7|8.4)
+      TB_MYSQL_ENGINE_RESOLVED="$requested"
+      log "MySQL engine: ${requested} (explicit TB_MYSQL_ENGINE)"
+      return 0 ;;
+    auto) ;;
+    *)
+      error "TB_MYSQL_ENGINE must be 'auto', '5.7' or '8.4' (got '${requested}')" ;;
+  esac
+  # Sticky: an edge that opted into 8.4 stays there on every later re-run.
+  if [[ -f "${values_file:-}" ]] \
+    && grep -A 3 'mysqlClient:' "${values_file}" 2>/dev/null | grep -q 'tag: "8.4"'; then
+    TB_MYSQL_ENGINE_RESOLVED="8.4"
+    log "MySQL engine: 8.4 (kept from this machine's existing values.yaml)"
+    return 0
+  fi
+  # Never auto-flip existing state: a found release or real datadir content
+  # means a 5.7-format datadir may exist, and 8.4 refuses to open it. The
+  # empty dirs _ensure_tracebloc_dirs just created don't count — only files —
+  # but an UNLISTABLE dir counts as content (fail closed; see the helper).
+  if [[ -n "${existing_id:-}" ]] \
+    || _mysql_dir_has_content "${HOST_DATA_DIR:-/nonexistent}/mysql" \
+    || _mysql_dir_has_content "${HOST_DATA_DIR:-/nonexistent}/${TB_NAMESPACE:-}/mysql"; then
+    TB_MYSQL_ENGINE_RESOLVED="5.7"
+    return 0
+  fi
+  case "${ARCH:-$(uname -m)}" in
+    x86_64|amd64)
+      TB_MYSQL_ENGINE_RESOLVED="5.7" ;;
+    *)
+      TB_MYSQL_ENGINE_RESOLVED="8.4"
+      log "MySQL engine: 8.4 (fresh install on ${ARCH:-$(uname -m)} — native multi-arch engine, backend#723)" ;;
+  esac
+}
+
 install_client_helm() {
   # Step e (Install tracebloc) — main() prints the "e) Installing tracebloc"
   # header. The credential + namespace were provisioned in step d
@@ -903,6 +969,10 @@ install_client_helm() {
     log "No NVIDIA GPU — GPU_LIMITS and GPU_REQUESTS left empty"
   fi
 
+  # backend#723 A2: pick the MySQL engine for this install (before the heredoc
+  # below is rendered; see _resolve_mysql_engine for the full decision rules).
+  _resolve_mysql_engine
+
   # ── Write generated values.yaml ─────────────────────────────────────────
   log "Writing values to $values_file"
 
@@ -976,6 +1046,21 @@ hostPath:
 STORAGE
 [ -n "${HOST_DATASET_DIR:-}" ] && printf '  datasetPath: /tracebloc-data\n'
 fi)
+$(if [[ "${TB_MYSQL_ENGINE_RESOLVED:-5.7}" == "8.4" ]]; then
+cat <<'MYSQL84'
+
+# MySQL engine opt-in (backend#723, decision A2): this install runs the
+# multi-arch 8.4 engine natively — fresh datadirs only; the chart's
+# mysql-format-guard init container refuses a mismatched datadir. Explicit
+# tag + empty digest: the chart's 5.7 reproducibility pin stays for installs
+# on the default engine. Sticky across installer re-runs; override with
+# TB_MYSQL_ENGINE=5.7|8.4.
+images:
+  mysqlClient:
+    tag: "8.4"
+    digest: ""
+MYSQL84
+fi)
 pvc:
   mysql: 2Gi
   logs: 10Gi

scripts/manifest.sha256

@@ -8,7 +8,7 @@ fc3dacf419b66373a7e1b7c3ce0f44ec47fdfa3f7039cb2697ac2dd065344596  scripts/lib/se
 2a58a1f90c90d6759d7a8116331d7bc52b71ab80a74f37e8fbf15507ba98ebe0  scripts/lib/setup-linux.sh
 90750df5be3bb0266b06404df7dc38c0a7919e1487a34e1a02005de71736ba0e  scripts/lib/cluster.sh
 3c539322b19b31f21ff7c20594b52fcb825547c84046a45c2c382da99712a4ec  scripts/lib/gpu-plugins.sh
-e2c87c056afa99e8d98cdeb8f04e78a7eed54f1f7321538844506e7b5b62943e  scripts/lib/install-client-helm.sh
+935f1e09877da6ee56b3b5feb800f06141c79d521b0c5502231a5b9d1af9617f  scripts/lib/install-client-helm.sh
 61c1c887d158af52d4da4734b3bfa83205b2600ae7a291bfb3074daf3d9ffb55  scripts/lib/install-cli.sh
 725a85e4927761d8362221012ad1b69b380e36ec7fcfcf4c04b801f0994bce5c  scripts/lib/provision.sh
 e373403d7bb5ce3728b8d21af89e6bf672cc35bbf8938541eb527ae19cb9473b  scripts/lib/assess.sh