feat(chart): provision per-experiment DB credentials (RFC-0003 D10, backend#1181)

client/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: client
 description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift
 type: application
-version: 1.9.8
-appVersion: "1.9.8"
+version: 1.9.9
+appVersion: "1.9.9"
 keywords:
   - tracebloc
   - kubernetes

client/templates/jobs-manager-deployment.yaml

@@ -107,6 +107,24 @@ spec:
         - name: PER_INGESTION_TABLES
           value: "1"
         {{- end }}
+        {{- if .Values.perExperimentDbCreds }}
+        # RFC-0003 D10 (backend#1181): per-experiment MySQL credentials. When
+        # on, jobs-manager mints a short-lived MySQL user per experiment scoped
+        # to only its own table(s) and injects it into the training pod via a
+        # per-job Secret — instead of every pod sharing the root-equivalent
+        # edgeuser. TB_CREDMGR_* is the dedicated minting identity (provisioned
+        # by jobs-manager on startup from this generated Secret). Rendered only
+        # when enabled so default installs stay byte-identical.
+        - name: PER_EXPERIMENT_DB_CREDS
+          value: "1"
+        - name: TB_CREDMGR_USER
+          value: "tb_credmgr"
+        - name: TB_CREDMGR_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: {{ include "tracebloc.secretName" . }}
+              key: TB_CREDMGR_PASSWORD
+        {{- end }}
         # Ingestor image wiring for the POST /internal/submit-ingestion-run
         # endpoint. jobs-manager spawns each ingestion Job from these values
         # (see client-runtime submit_ingestion_run._build_image_reference):

client/templates/rbac.yaml

@@ -50,6 +50,15 @@ rules:
     # content matches before reusing it. Without `get`, those reads return
     # Forbidden and the endpoint 500s instead of the intended 409/replay.
     verbs: ["create", "get"]
+{{- if .Values.perExperimentDbCreds }}
+  # Per-experiment DB creds only: jobs-manager's revoke/sweep path deletes the
+  # per-job cred Secret. Scoped to `secrets` alone (never configmaps) and
+  # flag-gated so a default install grants no extra delete — least privilege,
+  # and byte-for-byte unchanged when off (Saqlain review).
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["delete"]
+{{- end }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -106,6 +115,15 @@ rules:
     # content matches before reusing it. Without `get`, those reads return
     # Forbidden and the endpoint 500s instead of the intended 409/replay.
     verbs: ["create", "get"]
+{{- if .Values.perExperimentDbCreds }}
+  # Per-experiment DB creds only: jobs-manager's revoke/sweep path deletes the
+  # per-job cred Secret. Scoped to `secrets` alone (never configmaps) and
+  # flag-gated so a default install grants no extra delete — least privilege,
+  # and byte-for-byte unchanged when off (Saqlain review).
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["delete"]
+{{- end }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding

client/templates/secrets.yaml

@@ -23,6 +23,35 @@
 {{- else -}}
 {{-   $podTokenSecret = randAlphaNum 48 -}}
 {{- end -}}
+{{- /*
+  Credential-manager password (RFC-0003 D10, backend#1181). Same generate-once
+  stability + 3-tier resolution as the signing secret above:
+    1. explicit .Values.credmgrPassword (operator pin — DR, a pre-created
+       MySQL account, or a forced rotation), else
+    2. the value already stored in the live Secret (preserve across upgrades), else
+    3. a freshly generated random secret (first install).
+  jobs-manager syncs the tb_credmgr account to this value on every startup, so
+  it must be stable across upgrades (regenerating it would churn the account).
+  Emitted only when perExperimentDbCreds is on.
+  NOTE (edge): because the key is emitted only while the flag is on, an
+  off→on→off→on toggle drops then regenerates it via tier 3, rotating the
+  account until jobs-manager re-syncs. Pin .Values.credmgrPassword to keep it
+  stable across toggles — acceptable for the dev-first rollout; the pin is the
+  fix (Saqlain review).
+*/ -}}
+{{- $credmgrPassword := "" -}}
+{{- if .Values.perExperimentDbCreds -}}
+{{-   if .Values.credmgrPassword -}}
+{{-     if not (regexMatch "^[A-Za-z0-9]+$" .Values.credmgrPassword) -}}
+{{-       fail "credmgrPassword must be alphanumeric ([A-Za-z0-9]+) — jobs-manager rejects other characters in the tb_credmgr password" -}}
+{{-     end -}}
+{{-     $credmgrPassword = .Values.credmgrPassword -}}
+{{-   else if and $existingSecret $existingSecret.data (hasKey $existingSecret.data "TB_CREDMGR_PASSWORD") -}}
+{{-     $credmgrPassword = (index $existingSecret.data "TB_CREDMGR_PASSWORD" | b64dec) -}}
+{{-   else -}}
+{{-     $credmgrPassword = randAlphaNum 48 -}}
+{{-   end -}}
+{{- end -}}
 apiVersion: v1
 kind: Secret
 metadata:
@@ -35,6 +64,9 @@ data:
   CLIENT_ID: {{ $clientId | b64enc | quote }}
   CLIENT_PASSWORD: {{ $clientPassword | b64enc | quote }}
   POD_TOKEN_SIGNING_SECRET: {{ $podTokenSecret | b64enc | quote }}
+{{- if .Values.perExperimentDbCreds }}
+  TB_CREDMGR_PASSWORD: {{ $credmgrPassword | b64enc | quote }}
+{{- end }}
 {{- if and (ne .Values.resourceMonitor false) (ne .Values.nodeAgents.namespace.name .Release.Namespace) }}
 ---
 # Mirrored into the node-agents namespace so the resource-monitor DaemonSet

client/tests/jobs_manager_test.yaml

@@ -343,3 +343,39 @@ tests:
           content:
             name: PER_INGESTION_TABLES
             value: "1"
+
+  - it: does not render per-experiment DB-cred env by default (RFC-0003 D10 knob off)
+    asserts:
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: PER_EXPERIMENT_DB_CREDS
+            value: "1"
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: TB_CREDMGR_USER
+            value: "tb_credmgr"
+
+  - it: renders per-experiment DB-cred env (flag + credmgr identity + secret ref) when enabled
+    set:
+      perExperimentDbCreds: true
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: PER_EXPERIMENT_DB_CREDS
+            value: "1"
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: TB_CREDMGR_USER
+            value: "tb_credmgr"
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: TB_CREDMGR_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: RELEASE-NAME-secrets
+                key: TB_CREDMGR_PASSWORD

client/tests/rbac_test.yaml

@@ -76,3 +76,55 @@ tests:
             apiGroups: [""]
             resources: ["pods", "pods/log", "events"]
             verbs: ["get", "list", "watch"]
+
+  # RFC-0003 D10 (backend#1181): the Secret-delete verb the jobs-manager
+  # revoke/sweep path needs must be flag-gated + secrets-only (Saqlain review).
+  - it: default-off grants no delete on configmaps/secrets (byte-for-byte)
+    set:
+      clusterScope: true
+    documentIndex: 1
+    asserts:
+      - contains:
+          path: rules
+          content:
+            apiGroups: [""]
+            resources: ["configmaps", "secrets"]
+            verbs: ["create", "get"]
+      - notContains:
+          path: rules
+          content:
+            apiGroups: [""]
+            resources: ["secrets"]
+            verbs: ["delete"]
+
+  - it: perExperimentDbCreds adds delete on secrets only (ClusterRole)
+    set:
+      clusterScope: true
+      perExperimentDbCreds: true
+    documentIndex: 1
+    asserts:
+      - contains:
+          path: rules
+          content:
+            apiGroups: [""]
+            resources: ["secrets"]
+            verbs: ["delete"]
+      - contains:
+          path: rules
+          content:
+            apiGroups: [""]
+            resources: ["configmaps", "secrets"]
+            verbs: ["create", "get"]
+
+  - it: perExperimentDbCreds adds delete on secrets only (Role)
+    set:
+      clusterScope: false
+      perExperimentDbCreds: true
+    documentIndex: 1
+    asserts:
+      - contains:
+          path: rules
+          content:
+            apiGroups: [""]
+            resources: ["secrets"]
+            verbs: ["delete"]

client/tests/secrets_test.yaml

@@ -95,3 +95,48 @@ tests:
     asserts:
       - failedTemplate: {}
 
+
+  # RFC-0003 D10 (backend#1181): TB_CREDMGR_PASSWORD is flag-gated (Saqlain #D).
+  - it: TB_CREDMGR_PASSWORD absent by default (byte-for-byte off)
+    template: templates/secrets.yaml
+    set:
+      clientId: "my-client-id"
+      clientPassword: "my-secret-pass"
+    asserts:
+      - notExists:
+          path: data.TB_CREDMGR_PASSWORD
+        documentIndex: 0
+
+  - it: TB_CREDMGR_PASSWORD rendered when perExperimentDbCreds is on
+    template: templates/secrets.yaml
+    set:
+      clientId: "my-client-id"
+      clientPassword: "my-secret-pass"
+      perExperimentDbCreds: true
+    asserts:
+      - isNotEmpty:
+          path: data.TB_CREDMGR_PASSWORD
+        documentIndex: 0
+
+  - it: credmgrPassword operator pin flows into the Secret
+    template: templates/secrets.yaml
+    set:
+      clientId: "my-client-id"
+      clientPassword: "my-secret-pass"
+      perExperimentDbCreds: true
+      credmgrPassword: "PinnedAlnum123"
+    asserts:
+      - equal:
+          path: data.TB_CREDMGR_PASSWORD
+          value: "UGlubmVkQWxudW0xMjM="
+        documentIndex: 0
+
+  - it: rejects a non-alphanumeric credmgrPassword pin
+    template: templates/secrets.yaml
+    set:
+      clientId: "my-client-id"
+      clientPassword: "my-secret-pass"
+      perExperimentDbCreds: true
+      credmgrPassword: "bad-pass!"
+    asserts:
+      - failedTemplate: {}

client/values.schema.json

@@ -287,6 +287,14 @@
         }
       }
     },
+    "perExperimentDbCreds": {
+      "type": "boolean",
+      "description": "RFC-0003 D10 (backend#1181): mint a per-experiment MySQL user scoped to the experiment's own table(s), injected via a per-job Secret, instead of the shared edgeuser. Flip per environment, dev first."
+    },
+    "credmgrPassword": {
+      "type": "string",
+      "description": "RFC-0003 D10 (backend#1181): optional operator pin for the tb_credmgr account password (3-tier resolution, mirrors podTokenSigningSecret). Empty = generate-and-persist. Set for DR / a pre-created MySQL account / forced rotation, or to keep it stable across a perExperimentDbCreds toggle. Must be alphanumeric."
+    },
     "perIngestionTables": {
       "type": "boolean",
       "description": "RFC-0003 D16 (backend#1204/#1205): stamp PER_INGESTION_TABLES=1 into every ingestion Job. Flip per environment, dev first; file-bearing categories are refused under the flag until client-runtime#203 phase 2."

client/values.yaml

@@ -765,6 +765,30 @@ imageRefresh:
 # Default false: legacy shared label-named tables, byte-for-byte.
 perIngestionTables: false
 
+# ============================================================
+# Per-experiment DB credentials (RFC-0003 D10 — backend#1181)
+# ============================================================
+# When true, jobs-manager mints a short-lived MySQL user for each experiment
+# scoped to ONLY that experiment's physical table(s), and injects it into the
+# training pod via a per-job Secret — instead of every pod sharing the
+# root-equivalent edgeuser (which today can read every dataset + the metadata
+# DB's Service Bus strings). jobs-manager provisions the dedicated tb_credmgr
+# minting identity on startup and revokes each user when its Job ends.
+#
+# Flip per environment, dev first. Default false: today's shared-credential
+# behavior, byte-for-byte. Retiring edgeuser is a later step once this is
+# universally on. Needs no per-edge action — the tb_credmgr password is a
+# generated, upgrade-stable Secret value.
+perExperimentDbCreds: false
+
+# Optional operator pin for the tb_credmgr account password (mirrors
+# podTokenSigningSecret's 3-tier resolution). Leave "" to generate-and-persist
+# on first install. Set it for DR, to match a pre-created MySQL account, or to
+# force a rotation — and to keep the password stable across a
+# perExperimentDbCreds off→on toggle. Must be alphanumeric ([A-Za-z0-9]+);
+# jobs-manager rejects other characters.
+credmgrPassword: ""
+
 # ============================================================
 # Ingestion endpoint authorization (client-runtime#21)
 # ============================================================