From 7522412e9cf959846c11ac22156becfff4632746 Mon Sep 17 00:00:00 2001 From: Lukas Wuttke Date: Thu, 30 Jul 2026 17:28:55 +0200 Subject: [PATCH 1/2] feat(chart): provision per-experiment DB credentials (RFC-0003 D10, backend#1181) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The chart half that flips client-runtime#235 from inert to live: - perExperimentDbCreds value (default false, schema-typed) → renders PER_EXPERIMENT_DB_CREDS + TB_CREDMGR_USER + TB_CREDMGR_PASSWORD onto jobs-manager (password via secretKeyRef, never plain). - secrets.yaml: generate-once TB_CREDMGR_PASSWORD (upgrade-stable, same lookup pattern as POD_TOKEN_SIGNING_SECRET), emitted only when enabled. - rbac: secrets verbs gain 'delete' — jobs-manager deletes the per-job cred Secret in the revoke path. - Chart.yaml 1.9.8 -> 1.9.9 (publishes on version change). Default installs render byte-identically (all flag-gated; 300 helm tests, both sides pinned). jobs-manager self-provisions the tb_credmgr account from this Secret on startup. Co-Authored-By: Claude Fable 5 --- client/Chart.yaml | 2 +- client/templates/jobs-manager-deployment.yaml | 18 ++++++++++ client/templates/rbac.yaml | 4 +-- client/templates/secrets.yaml | 18 ++++++++++ client/tests/jobs_manager_test.yaml | 36 +++++++++++++++++++ client/values.schema.json | 4 +++ client/values.yaml | 16 +++++++++ 7 files changed, 95 insertions(+), 3 deletions(-) diff --git a/client/Chart.yaml b/client/Chart.yaml index 5d3d77d6..b6bb3a4d 100644 --- a/client/Chart.yaml +++ b/client/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: client description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift type: application -version: 1.9.8 +version: 1.9.9 appVersion: "1.9.8" keywords: - tracebloc diff --git a/client/templates/jobs-manager-deployment.yaml b/client/templates/jobs-manager-deployment.yaml index d43674cb..67bc739d 100644 --- a/client/templates/jobs-manager-deployment.yaml +++ b/client/templates/jobs-manager-deployment.yaml @@ -107,6 +107,24 @@ spec: - name: PER_INGESTION_TABLES value: "1" {{- end }} + {{- if .Values.perExperimentDbCreds }} + # RFC-0003 D10 (backend#1181): per-experiment MySQL credentials. When + # on, jobs-manager mints a short-lived MySQL user per experiment scoped + # to only its own table(s) and injects it into the training pod via a + # per-job Secret — instead of every pod sharing the root-equivalent + # edgeuser. TB_CREDMGR_* is the dedicated minting identity (provisioned + # by jobs-manager on startup from this generated Secret). Rendered only + # when enabled so default installs stay byte-identical. + - name: PER_EXPERIMENT_DB_CREDS + value: "1" + - name: TB_CREDMGR_USER + value: "tb_credmgr" + - name: TB_CREDMGR_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "tracebloc.secretName" . }} + key: TB_CREDMGR_PASSWORD + {{- end }} # Ingestor image wiring for the POST /internal/submit-ingestion-run # endpoint. jobs-manager spawns each ingestion Job from these values # (see client-runtime submit_ingestion_run._build_image_reference): diff --git a/client/templates/rbac.yaml b/client/templates/rbac.yaml index 9f38644a..7eb45355 100644 --- a/client/templates/rbac.yaml +++ b/client/templates/rbac.yaml @@ -49,7 +49,7 @@ rules: # on a create-409 it reads the existing ConfigMap/Secret to confirm the # content matches before reusing it. Without `get`, those reads return # Forbidden and the endpoint 500s instead of the intended 409/replay. - verbs: ["create", "get"] + verbs: ["create", "get", "delete"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -105,7 +105,7 @@ rules: # on a create-409 it reads the existing ConfigMap/Secret to confirm the # content matches before reusing it. Without `get`, those reads return # Forbidden and the endpoint 500s instead of the intended 409/replay. - verbs: ["create", "get"] + verbs: ["create", "get", "delete"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding diff --git a/client/templates/secrets.yaml b/client/templates/secrets.yaml index 1cd17130..05657820 100644 --- a/client/templates/secrets.yaml +++ b/client/templates/secrets.yaml @@ -23,6 +23,21 @@ {{- else -}} {{- $podTokenSecret = randAlphaNum 48 -}} {{- end -}} +{{- /* + Credential-manager password (RFC-0003 D10, backend#1181). Same generate-once + stability as the signing secret above: jobs-manager syncs the tb_credmgr + account password to this value on every startup, so it must be stable across + upgrades (regenerating it would needlessly churn the account). Only emitted + when perExperimentDbCreds is on. +*/ -}} +{{- $credmgrPassword := "" -}} +{{- if .Values.perExperimentDbCreds -}} +{{- if and $existingSecret $existingSecret.data (hasKey $existingSecret.data "TB_CREDMGR_PASSWORD") -}} +{{- $credmgrPassword = (index $existingSecret.data "TB_CREDMGR_PASSWORD" | b64dec) -}} +{{- else -}} +{{- $credmgrPassword = randAlphaNum 48 -}} +{{- end -}} +{{- end -}} apiVersion: v1 kind: Secret metadata: @@ -35,6 +50,9 @@ data: CLIENT_ID: {{ $clientId | b64enc | quote }} CLIENT_PASSWORD: {{ $clientPassword | b64enc | quote }} POD_TOKEN_SIGNING_SECRET: {{ $podTokenSecret | b64enc | quote }} +{{- if .Values.perExperimentDbCreds }} + TB_CREDMGR_PASSWORD: {{ $credmgrPassword | b64enc | quote }} +{{- end }} {{- if and (ne .Values.resourceMonitor false) (ne .Values.nodeAgents.namespace.name .Release.Namespace) }} --- # Mirrored into the node-agents namespace so the resource-monitor DaemonSet diff --git a/client/tests/jobs_manager_test.yaml b/client/tests/jobs_manager_test.yaml index e5390544..2939ce3d 100644 --- a/client/tests/jobs_manager_test.yaml +++ b/client/tests/jobs_manager_test.yaml @@ -343,3 +343,39 @@ tests: content: name: PER_INGESTION_TABLES value: "1" + + - it: does not render per-experiment DB-cred env by default (RFC-0003 D10 knob off) + asserts: + - notContains: + path: spec.template.spec.containers[0].env + content: + name: PER_EXPERIMENT_DB_CREDS + value: "1" + - notContains: + path: spec.template.spec.containers[0].env + content: + name: TB_CREDMGR_USER + value: "tb_credmgr" + + - it: renders per-experiment DB-cred env (flag + credmgr identity + secret ref) when enabled + set: + perExperimentDbCreds: true + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: PER_EXPERIMENT_DB_CREDS + value: "1" + - contains: + path: spec.template.spec.containers[0].env + content: + name: TB_CREDMGR_USER + value: "tb_credmgr" + - contains: + path: spec.template.spec.containers[0].env + content: + name: TB_CREDMGR_PASSWORD + valueFrom: + secretKeyRef: + name: RELEASE-NAME-secrets + key: TB_CREDMGR_PASSWORD diff --git a/client/values.schema.json b/client/values.schema.json index c6fb0314..22981f59 100644 --- a/client/values.schema.json +++ b/client/values.schema.json @@ -287,6 +287,10 @@ } } }, + "perExperimentDbCreds": { + "type": "boolean", + "description": "RFC-0003 D10 (backend#1181): mint a per-experiment MySQL user scoped to the experiment's own table(s), injected via a per-job Secret, instead of the shared edgeuser. Flip per environment, dev first." + }, "perIngestionTables": { "type": "boolean", "description": "RFC-0003 D16 (backend#1204/#1205): stamp PER_INGESTION_TABLES=1 into every ingestion Job. Flip per environment, dev first; file-bearing categories are refused under the flag until client-runtime#203 phase 2." diff --git a/client/values.yaml b/client/values.yaml index 631a11fa..c3c810fe 100644 --- a/client/values.yaml +++ b/client/values.yaml @@ -765,6 +765,22 @@ imageRefresh: # Default false: legacy shared label-named tables, byte-for-byte. perIngestionTables: false +# ============================================================ +# Per-experiment DB credentials (RFC-0003 D10 — backend#1181) +# ============================================================ +# When true, jobs-manager mints a short-lived MySQL user for each experiment +# scoped to ONLY that experiment's physical table(s), and injects it into the +# training pod via a per-job Secret — instead of every pod sharing the +# root-equivalent edgeuser (which today can read every dataset + the metadata +# DB's Service Bus strings). jobs-manager provisions the dedicated tb_credmgr +# minting identity on startup and revokes each user when its Job ends. +# +# Flip per environment, dev first. Default false: today's shared-credential +# behavior, byte-for-byte. Retiring edgeuser is a later step once this is +# universally on. Needs no per-edge action — the tb_credmgr password is a +# generated, upgrade-stable Secret value. +perExperimentDbCreds: false + # ============================================================ # Ingestion endpoint authorization (client-runtime#21) # ============================================================ From 3afec19fce5168e34c5ee7feffe20acf8b1619df Mon Sep 17 00:00:00 2001 From: Lukas Wuttke Date: Fri, 31 Jul 2026 08:39:23 +0200 Subject: [PATCH 2/2] fix(chart): gate the RBAC delete verb + credmgr pin/tests (Saqlain + Bugbot #503) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - rbac: the Secret `delete` verb is now flag-gated on perExperimentDbCreds and scoped to `secrets` alone (was on configmaps+secrets, every install). Default-off is byte-for-byte `["create", "get"]` again — a cluster-wide delete-on-all-Secrets ClusterRole no longer ships to every install; the verb the revoke path needs only appears when the flag is on (Saqlain #A). - secrets: TB_CREDMGR_PASSWORD gains the 3rd resolution tier — an explicit `.Values.credmgrPassword` operator pin (DR / pre-created MySQL account / forced rotation), mirroring podTokenSigningSecret; validated alphanumeric (jobs-manager's constraint) with a fail-fast. Also documents the off->on->off->on regen edge and that the pin is its fix (Saqlain #B/#C). - Chart appVersion 1.9.8 -> 1.9.9, back in lockstep with version so the `app.kubernetes.io/version` labels report the shipped chart (Bugbot). - tests: secrets.yaml + rbac.yaml helm-unittests for both flag states (credmgr key absent/present, pin flows, non-alnum rejected, delete verb gated + secrets-only). 307 helm tests pass; helm lint clean (Saqlain #D). Co-Authored-By: Claude Fable 5 --- client/Chart.yaml | 2 +- client/templates/rbac.yaml | 22 ++++++++++++-- client/templates/secrets.yaml | 24 ++++++++++++---- client/tests/rbac_test.yaml | 52 ++++++++++++++++++++++++++++++++++ client/tests/secrets_test.yaml | 45 +++++++++++++++++++++++++++++ client/values.schema.json | 4 +++ client/values.yaml | 8 ++++++ 7 files changed, 149 insertions(+), 8 deletions(-) diff --git a/client/Chart.yaml b/client/Chart.yaml index b6bb3a4d..af0b4c3e 100644 --- a/client/Chart.yaml +++ b/client/Chart.yaml @@ -3,7 +3,7 @@ name: client description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift type: application version: 1.9.9 -appVersion: "1.9.8" +appVersion: "1.9.9" keywords: - tracebloc - kubernetes diff --git a/client/templates/rbac.yaml b/client/templates/rbac.yaml index 7eb45355..e100f221 100644 --- a/client/templates/rbac.yaml +++ b/client/templates/rbac.yaml @@ -49,7 +49,16 @@ rules: # on a create-409 it reads the existing ConfigMap/Secret to confirm the # content matches before reusing it. Without `get`, those reads return # Forbidden and the endpoint 500s instead of the intended 409/replay. - verbs: ["create", "get", "delete"] + verbs: ["create", "get"] +{{- if .Values.perExperimentDbCreds }} + # Per-experiment DB creds only: jobs-manager's revoke/sweep path deletes the + # per-job cred Secret. Scoped to `secrets` alone (never configmaps) and + # flag-gated so a default install grants no extra delete — least privilege, + # and byte-for-byte unchanged when off (Saqlain review). + - apiGroups: [""] + resources: ["secrets"] + verbs: ["delete"] +{{- end }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding @@ -105,7 +114,16 @@ rules: # on a create-409 it reads the existing ConfigMap/Secret to confirm the # content matches before reusing it. Without `get`, those reads return # Forbidden and the endpoint 500s instead of the intended 409/replay. - verbs: ["create", "get", "delete"] + verbs: ["create", "get"] +{{- if .Values.perExperimentDbCreds }} + # Per-experiment DB creds only: jobs-manager's revoke/sweep path deletes the + # per-job cred Secret. Scoped to `secrets` alone (never configmaps) and + # flag-gated so a default install grants no extra delete — least privilege, + # and byte-for-byte unchanged when off (Saqlain review). + - apiGroups: [""] + resources: ["secrets"] + verbs: ["delete"] +{{- end }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding diff --git a/client/templates/secrets.yaml b/client/templates/secrets.yaml index 05657820..90f2547a 100644 --- a/client/templates/secrets.yaml +++ b/client/templates/secrets.yaml @@ -25,14 +25,28 @@ {{- end -}} {{- /* Credential-manager password (RFC-0003 D10, backend#1181). Same generate-once - stability as the signing secret above: jobs-manager syncs the tb_credmgr - account password to this value on every startup, so it must be stable across - upgrades (regenerating it would needlessly churn the account). Only emitted - when perExperimentDbCreds is on. + stability + 3-tier resolution as the signing secret above: + 1. explicit .Values.credmgrPassword (operator pin — DR, a pre-created + MySQL account, or a forced rotation), else + 2. the value already stored in the live Secret (preserve across upgrades), else + 3. a freshly generated random secret (first install). + jobs-manager syncs the tb_credmgr account to this value on every startup, so + it must be stable across upgrades (regenerating it would churn the account). + Emitted only when perExperimentDbCreds is on. + NOTE (edge): because the key is emitted only while the flag is on, an + off→on→off→on toggle drops then regenerates it via tier 3, rotating the + account until jobs-manager re-syncs. Pin .Values.credmgrPassword to keep it + stable across toggles — acceptable for the dev-first rollout; the pin is the + fix (Saqlain review). */ -}} {{- $credmgrPassword := "" -}} {{- if .Values.perExperimentDbCreds -}} -{{- if and $existingSecret $existingSecret.data (hasKey $existingSecret.data "TB_CREDMGR_PASSWORD") -}} +{{- if .Values.credmgrPassword -}} +{{- if not (regexMatch "^[A-Za-z0-9]+$" .Values.credmgrPassword) -}} +{{- fail "credmgrPassword must be alphanumeric ([A-Za-z0-9]+) — jobs-manager rejects other characters in the tb_credmgr password" -}} +{{- end -}} +{{- $credmgrPassword = .Values.credmgrPassword -}} +{{- else if and $existingSecret $existingSecret.data (hasKey $existingSecret.data "TB_CREDMGR_PASSWORD") -}} {{- $credmgrPassword = (index $existingSecret.data "TB_CREDMGR_PASSWORD" | b64dec) -}} {{- else -}} {{- $credmgrPassword = randAlphaNum 48 -}} diff --git a/client/tests/rbac_test.yaml b/client/tests/rbac_test.yaml index 8421fbf8..ebd6feba 100644 --- a/client/tests/rbac_test.yaml +++ b/client/tests/rbac_test.yaml @@ -76,3 +76,55 @@ tests: apiGroups: [""] resources: ["pods", "pods/log", "events"] verbs: ["get", "list", "watch"] + + # RFC-0003 D10 (backend#1181): the Secret-delete verb the jobs-manager + # revoke/sweep path needs must be flag-gated + secrets-only (Saqlain review). + - it: default-off grants no delete on configmaps/secrets (byte-for-byte) + set: + clusterScope: true + documentIndex: 1 + asserts: + - contains: + path: rules + content: + apiGroups: [""] + resources: ["configmaps", "secrets"] + verbs: ["create", "get"] + - notContains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + verbs: ["delete"] + + - it: perExperimentDbCreds adds delete on secrets only (ClusterRole) + set: + clusterScope: true + perExperimentDbCreds: true + documentIndex: 1 + asserts: + - contains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + verbs: ["delete"] + - contains: + path: rules + content: + apiGroups: [""] + resources: ["configmaps", "secrets"] + verbs: ["create", "get"] + + - it: perExperimentDbCreds adds delete on secrets only (Role) + set: + clusterScope: false + perExperimentDbCreds: true + documentIndex: 1 + asserts: + - contains: + path: rules + content: + apiGroups: [""] + resources: ["secrets"] + verbs: ["delete"] diff --git a/client/tests/secrets_test.yaml b/client/tests/secrets_test.yaml index 0113e46c..9f0373da 100644 --- a/client/tests/secrets_test.yaml +++ b/client/tests/secrets_test.yaml @@ -95,3 +95,48 @@ tests: asserts: - failedTemplate: {} + + # RFC-0003 D10 (backend#1181): TB_CREDMGR_PASSWORD is flag-gated (Saqlain #D). + - it: TB_CREDMGR_PASSWORD absent by default (byte-for-byte off) + template: templates/secrets.yaml + set: + clientId: "my-client-id" + clientPassword: "my-secret-pass" + asserts: + - notExists: + path: data.TB_CREDMGR_PASSWORD + documentIndex: 0 + + - it: TB_CREDMGR_PASSWORD rendered when perExperimentDbCreds is on + template: templates/secrets.yaml + set: + clientId: "my-client-id" + clientPassword: "my-secret-pass" + perExperimentDbCreds: true + asserts: + - isNotEmpty: + path: data.TB_CREDMGR_PASSWORD + documentIndex: 0 + + - it: credmgrPassword operator pin flows into the Secret + template: templates/secrets.yaml + set: + clientId: "my-client-id" + clientPassword: "my-secret-pass" + perExperimentDbCreds: true + credmgrPassword: "PinnedAlnum123" + asserts: + - equal: + path: data.TB_CREDMGR_PASSWORD + value: "UGlubmVkQWxudW0xMjM=" + documentIndex: 0 + + - it: rejects a non-alphanumeric credmgrPassword pin + template: templates/secrets.yaml + set: + clientId: "my-client-id" + clientPassword: "my-secret-pass" + perExperimentDbCreds: true + credmgrPassword: "bad-pass!" + asserts: + - failedTemplate: {} diff --git a/client/values.schema.json b/client/values.schema.json index 22981f59..080fb9d5 100644 --- a/client/values.schema.json +++ b/client/values.schema.json @@ -291,6 +291,10 @@ "type": "boolean", "description": "RFC-0003 D10 (backend#1181): mint a per-experiment MySQL user scoped to the experiment's own table(s), injected via a per-job Secret, instead of the shared edgeuser. Flip per environment, dev first." }, + "credmgrPassword": { + "type": "string", + "description": "RFC-0003 D10 (backend#1181): optional operator pin for the tb_credmgr account password (3-tier resolution, mirrors podTokenSigningSecret). Empty = generate-and-persist. Set for DR / a pre-created MySQL account / forced rotation, or to keep it stable across a perExperimentDbCreds toggle. Must be alphanumeric." + }, "perIngestionTables": { "type": "boolean", "description": "RFC-0003 D16 (backend#1204/#1205): stamp PER_INGESTION_TABLES=1 into every ingestion Job. Flip per environment, dev first; file-bearing categories are refused under the flag until client-runtime#203 phase 2." diff --git a/client/values.yaml b/client/values.yaml index c3c810fe..9b719434 100644 --- a/client/values.yaml +++ b/client/values.yaml @@ -781,6 +781,14 @@ perIngestionTables: false # generated, upgrade-stable Secret value. perExperimentDbCreds: false +# Optional operator pin for the tb_credmgr account password (mirrors +# podTokenSigningSecret's 3-tier resolution). Leave "" to generate-and-persist +# on first install. Set it for DR, to match a pre-created MySQL account, or to +# force a rotation — and to keep the password stable across a +# perExperimentDbCreds off→on toggle. Must be alphanumeric ([A-Za-z0-9]+); +# jobs-manager rejects other characters. +credmgrPassword: "" + # ============================================================ # Ingestion endpoint authorization (client-runtime#21) # ============================================================