fix(installer): harden the Windows install→first-experiment path — checksum-driven downloads, reliable cluster-create, writable ingest & training volumes, local-chart support (#611)

client/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: client
 description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift
 type: application
-version: 1.9.15
-appVersion: "1.9.15"
+version: 1.9.20
+appVersion: "1.9.20"
 keywords:
   - tracebloc
   - kubernetes

client/templates/jobs-manager-deployment.yaml

@@ -35,8 +35,56 @@ spec:
       serviceAccountName: {{ include "tracebloc.serviceAccountName" . }}
       securityContext:
         runAsNonRoot: true
+        # No fsGroup here (deliberately). On hostPath kubelet ignores it
+        # (kubernetes/kubernetes#138411) — the init below does the work. On CSI an fsGroup
+        # relabel would only add GID 1000 to jobs-manager's OWN processes, yet it would flip
+        # the shared/logs volumes' group to 1000 (OnRootMismatch, first restart) — stripping
+        # the group-0 access the spawned training pods (UID 1001 / OpenShift arbitrary-UID,
+        # GID 0) and the host-UID ingestion pods actually rely on (see docs/SECURITY.md §5.3).
+        # It never reaches those spawned writers, so it's all risk and no gain.
         seccompProfile:
           type: RuntimeDefault
+      {{- if .Values.hostPath.enabled }}
+      # kubelet does NOT apply fsGroup to hostPath volumes (kubernetes/kubernetes#138411),
+      # so /data/shared AND /data/logs are created root-owned and non-root pods can't write
+      # to them. Two failures this causes: (1) dataset ingest — `tb data ingest` streams files
+      # into a staging pod that does `mkdir /data/shared/.tracebloc-staging/` (#611); and
+      # (2) training — a spawned training pod does `os.makedirs('/data/logs/<run>')` and hits
+      # "Permission denied". Both are the SAME hostPath dirs those spawned pods mount
+      # (client-pvc / client-logs-pvc), so fixing them once here reaches those pods too.
+      # Unlike mysql-data (one writer, UID 999), these have MULTIPLE non-root writers —
+      # jobs-manager, the training/ingestor pods it spawns, and the CLI's ingest-staging pod,
+      # whose UID this chart does not control — so they must be world-writable, not chowned to
+      # a single UID. Mode 3777 = setgid (2) so new files inherit GID 1000 + sticky (1) so one
+      # writer can't unlink or rename another writer's files (/tmp semantics); runs as root only
+      # long enough to fix the mounts. Caps: CHOWN for the chown; FOWNER so the chmod is idempotent
+      # on re-install; FSETID so the setgid bit survives the chmod — after the chown to GID 1000 the
+      # dir's group no longer matches the process (fsgid 0), and a root process without FSETID has
+      # the kernel silently strip S_ISGID, landing the mount at 1777 (setgid lost). Each dir is
+      # fixed INDEPENDENTLY and best-effort: a chown/chmod that can't complete (e.g. /data/shared on
+      # an NFS root_squash export via HOST_DATASET_DIR) is logged and skipped so the OTHER dir is
+      # still repaired and jobs-manager still starts — a genuinely unwritable mount then surfaces as
+      # a clear error at the writer pod rather than wedging the whole edge in Init. CSI skips this.
+      initContainers:
+      - name: init-writable-data
+        image: {{ include "tracebloc.image" (dict "repository" "library/busybox" "tag" .Values.images.busybox.tag "digest" .Values.images.busybox.digest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }}
+        securityContext:
+          runAsUser: 0
+          runAsNonRoot: false
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          capabilities:
+            drop: ["ALL"]
+            add: ["CHOWN", "FOWNER", "FSETID"]
+          seccompProfile:
+            type: RuntimeDefault
+        command: ['sh', '-c', 'for d in /data/shared /data/logs; do chown 1000:1000 "$d" && chmod 3777 "$d" || echo "init-writable-data: could not adjust $d (pre-provisioned or root_squash mount?); leaving as-is"; done']
+        volumeMounts:
+        - name: shared-volume
+          mountPath: /data/shared
+        - name: logs-volume
+          mountPath: /data/logs
+      {{- end }}
       containers:
       - name: api
         image: {{ include "tracebloc.image" (dict "repository" "tracebloc/jobs-manager" "tag" .Values.env.CLIENT_ENV "digest" .Values.images.jobsManager.digest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }}

client/tests/jobs_manager_test.yaml

@@ -414,3 +414,64 @@ tests:
               secretKeyRef:
                 name: RELEASE-NAME-secrets
                 key: TB_CREDMGR_PASSWORD
+
+  # #611: /data/shared (client-pvc) AND /data/logs (client-logs-pvc) must be writable by
+  # the non-root pods that mount them — the ingest-staging pod writes to /data/shared, the
+  # spawned training pods write to /data/logs. hostPath ignores fsGroup (k8s#138411), so a
+  # privileged init makes both volumes world-writable. No fsGroup is set: it's a no-op on
+  # hostPath and on CSI would strip the spawned pods' group-0 access; CSI skips the init.
+  - it: "hostPath install adds init-writable-data to make /data/shared and /data/logs writable (#611)"
+    set:
+      hostPath:
+        enabled: true
+    asserts:
+      # no fsGroup — no-op on hostPath, and a regression risk on CSI (see template comment)
+      - notExists:
+          path: spec.template.spec.securityContext.fsGroup
+      - equal:
+          path: spec.template.spec.initContainers[0].name
+          value: init-writable-data
+      - equal:
+          path: spec.template.spec.initContainers[0].securityContext.runAsUser
+          value: 0
+      - contains:
+          path: spec.template.spec.initContainers[0].securityContext.capabilities.add
+          content: CHOWN
+      - contains:
+          path: spec.template.spec.initContainers[0].securityContext.capabilities.add
+          content: FOWNER
+      # FSETID is required or the kernel strips the setgid bit when chmod runs after
+      # the chown to GID 1000 (root without FSETID can't set setgid on a non-owned-group dir)
+      - contains:
+          path: spec.template.spec.initContainers[0].securityContext.capabilities.add
+          content: FSETID
+      # each dir is fixed INDEPENDENTLY (a for-loop): chown 1000:1000 then chmod 3777
+      - matchRegex:
+          path: spec.template.spec.initContainers[0].command[2]
+          pattern: "for d in /data/shared /data/logs.*chown 1000:1000.*chmod 3777"
+      # best-effort: a failing chown/chmod is logged, not fatal, so the other dir is still fixed
+      # and jobs-manager still starts (e.g. /data/shared on an NFS root_squash export)
+      - matchRegex:
+          path: spec.template.spec.initContainers[0].command[2]
+          pattern: "\\|\\| echo"
+      # the init must mount BOTH hostPath volumes it chowns, or the chown is a no-op
+      - contains:
+          path: spec.template.spec.initContainers[0].volumeMounts
+          content:
+            name: shared-volume
+            mountPath: /data/shared
+      - contains:
+          path: spec.template.spec.initContainers[0].volumeMounts
+          content:
+            name: logs-volume
+            mountPath: /data/logs
+
+  - it: "CSI install (hostPath disabled) skips the privileged init and sets no fsGroup (#611)"
+    set:
+      hostPath:
+        enabled: false
+    asserts:
+      - notExists:
+          path: spec.template.spec.securityContext.fsGroup
+      - notExists:
+          path: spec.template.spec.initContainers