From 436ae6eb469f6c52e85e01beef8fa8a6e1871986 Mon Sep 17 00:00:00 2001
From: shujaat hasan <shujaat@tracebloc.io>
Date: Wed, 5 Aug 2026 16:19:15 +0200
Subject: [PATCH 01/10] fix(installer): make the checksum drive the download
 retry so a truncated tool binary self-heals (#611)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Field follow-up to #607/#608. On a Windows machine behind a filtering proxy, the
k3d download kept failing at "System tool checksum verification failed" even with
#608's multi-transport download — because #608 validated a download by SIZE FLOOR
+ magic bytes only. A binary truncated mid-transfer to somewhere between the 10 MB
floor and the real 25.8 MB still passed (it's >10 MB and starts with 'MZ'), so the
fallbacks never fired and it dead-ended at the separate, no-retry checksum step.
(Proven on the box: a manual download produced the correct hash at 25,805,312
bytes, while the installer's copy failed the checksum in ~2s with no fallback.)

Fix — the checksum is the authoritative completeness test:
- Get-VerifiedDownload gains -Sha256: after a transport lands a size/magic-valid
  file, its SHA-256 must equal the expected hash or the transport is treated as
  failed and the NEXT one (curl.exe -> BITS) is tried. A truncated/altered copy
  now self-heals instead of dead-ending.
- Get-VerifiedDownload gains -MustContain for the small checksum-list files, so a
  proxy error page lacking the expected asset line is retried too.
- k3d / kubectl / helm now fetch their checksum FIRST (resiliently) and pass the
  extracted, 64-hex-validated hash as the download gate. helm gains checksum
  verification on the PS path for the first time (parity with the bash path).

Tests: Pester source-guards for the -Sha256/-MustContain gates, the mismatch->retry
path, and per-tool checksum-first wiring; full suite green (444). Manifest
regenerated.

Contributes to #578. Closes #611.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 scripts/install-k8s.ps1             | 118 +++++++++++++++++++---------
 scripts/manifest.sha256             |   2 +-
 scripts/tests/install-k8s.Tests.ps1 |  35 ++++++++-
 3 files changed, 117 insertions(+), 38 deletions(-)

diff --git a/scripts/install-k8s.ps1 b/scripts/install-k8s.ps1
index 2d58bb8a..79e09c43 100644
--- a/scripts/install-k8s.ps1
+++ b/scripts/install-k8s.ps1
@@ -464,6 +464,19 @@ function Get-VerifiedDownload {
     [Parameter(Mandatory)][string]$Dest,
     [int]$MinBytes = 1MB,
     [string]$Magic = '',
+    # When set, the CHECKSUM is the authoritative completeness test (#609): after a
+    # transport lands a size/magic-valid file, its SHA-256 must equal $Sha256 or the
+    # transport is treated as failed and the NEXT one is tried. A size floor alone
+    # lets a mid-transfer truncation (>MinBytes, still starts with the magic bytes)
+    # slip through and dead-end at a downstream checksum with no retry -- the real
+    # field failure. With $Sha256, a truncated/corrupt copy just triggers curl.exe/
+    # BITS until a byte-correct copy lands.
+    [string]$Sha256 = '',
+    # When set, the downloaded TEXT must contain this substring or the transport is
+    # treated as failed (#609). Used for the small checksum-list files (checksums.txt
+    # / *.sha256), which have no fixed hash but must carry the expected asset line --
+    # a proxy error page that lacks it is caught and the next transport is tried.
+    [string]$MustContain = '',
     [string]$Label = 'download',
     [string]$Message = 'Downloading'
   )
@@ -491,10 +504,22 @@ function Get-VerifiedDownload {
     # NEXT transport should be tried, not the whole download aborted (Bugbot).
     try {
       $bad = Test-DownloadComplete -Path $Dest -MinBytes $MinBytes -Magic $Magic
+      if (-not $bad -and $Sha256) {
+        $got = (Get-FileHash -LiteralPath $Dest -Algorithm SHA256).Hash.ToLower()
+        if ($got -ne $Sha256.ToLower()) {
+          $bad = "checksum mismatch (got $got) -- the download is truncated or altered"
+        }
+      }
+      if (-not $bad -and $MustContain) {
+        $text = Get-Content -LiteralPath $Dest -Raw -ErrorAction Stop
+        if ($text -notmatch [regex]::Escape($MustContain)) {
+          $bad = "the file did not contain the expected entry '$MustContain' -- likely an error page"
+        }
+      }
     } catch {
       $bad = "could not read the downloaded file ($($_.Exception.Message)) -- it may be locked or quarantined by antivirus"
     }
-    if (-not $bad) { return }        # complete + valid -- done
+    if (-not $bad) { return }        # complete + (checksum/content) valid -- done
     $problems += "${name}: $bad"
     Warn "$Label via $name looked incomplete ($bad); trying another method..."
   }
@@ -1593,18 +1618,25 @@ function Install-Kubectl {
   $kubectlDest = "$TOOL_DIR\kubectl.exe"
   $kUrl = "https://dl.k8s.io/release/$kVer/bin/windows/$arch/kubectl.exe"
   $t0 = Get-Date
-  # Heartbeat during the otherwise-silent transfer (#422); retry wraps it.
-  Get-VerifiedDownload -Url $kUrl -Dest $kubectlDest -MinBytes 20MB -Magic 'MZ' `
-    -Label "kubectl download" -Message "Downloading kubectl $kVer (~60 MB)"
-  $expectedHash = Invoke-WithRetry -Label "checksum" -ScriptBlock {
-    (Invoke-WebRequest "https://dl.k8s.io/release/$kVer/bin/windows/$arch/kubectl.exe.sha256" `
-      -UseBasicParsing).Content.Trim()
+  # Fetch the .sha256 FIRST, then make it the download gate (#609): with -Sha256 the
+  # binary download retries transports (Invoke-WebRequest -> curl.exe -> BITS) until a
+  # byte-correct copy lands, so a mid-transfer truncation self-heals instead of
+  # dead-ending at a post-hoc checksum. dl.k8s.io publishes the bare 64-hex hash.
+  $kSums = "$env:TEMP\kubectl-sha-$([System.IO.Path]::GetRandomFileName()).txt"
+  try {
+    Get-VerifiedDownload -Url "https://dl.k8s.io/release/$kVer/bin/windows/$arch/kubectl.exe.sha256" `
+      -Dest $kSums -MinBytes 1 -Label "kubectl checksum" -Message "Fetching kubectl checksum"
+  } catch {
+    Remove-Item $kSums -Force -ErrorAction SilentlyContinue
+    Err "Couldn't fetch the kubectl checksum ($_). Check egress to dl.k8s.io and re-run."
   }
-  $actualHash = (Get-FileHash $kubectlDest -Algorithm SHA256).Hash.ToLower()
-  if ($actualHash -ne $expectedHash.ToLower()) {
-    Remove-Item $kubectlDest -Force
-    Err "System tool checksum verification failed."
+  $expectedHash = ((Get-Content $kSums -Raw).Trim())
+  Remove-Item $kSums -Force -ErrorAction SilentlyContinue
+  if ($expectedHash -notmatch '^[0-9a-fA-F]{64}$') {
+    Err "Couldn't read a valid kubectl checksum (got an error page?). Check egress to dl.k8s.io and re-run."
   }
+  Get-VerifiedDownload -Url $kUrl -Dest $kubectlDest -MinBytes 20MB -Magic 'MZ' -Sha256 $expectedHash `
+    -Label "kubectl download" -Message "Downloading kubectl $kVer (~60 MB)"
   RefreshPath
   Log "kubectl $kVer installed."
   Assert-ToolRuns -Name "kubectl" -VersionArgs @("version","--client") -BinPath $kubectlDest
@@ -1701,36 +1733,31 @@ function Install-K3dAndHelm {
         }
       $k3dDest = "$TOOL_DIR\k3d.exe"
       $k3dUrl = "https://github.com/k3d-io/k3d/releases/download/$k3dVer/k3d-windows-$arch.exe"
-      Get-VerifiedDownload -Url $k3dUrl -Dest $k3dDest -MinBytes 10MB -Magic 'MZ' `
-        -Label "k3d download" -Message "Downloading k3d $k3dVer (~25 MB)"
-      # Fail-closed verification, matching the Linux path and the kubectl
-      # precedent: an unfetchable checksums.txt, a missing asset line, or a
-      # mismatch all abort and remove the download — never install unverified
-      # bytes on a privileged path (Bugbot r3). The release's checksum asset is
-      # named checksums.txt ("<sha256>  _dist/<asset>" lines); the previous
-      # sha256sum.txt URL never existed, so the old fail-open verification
-      # silently never ran (#382).
+      # Fetch the checksum list FIRST, then make the SHA the download gate (#609).
+      # The release's checksum asset is checksums.txt ("<sha256>  _dist/<asset>"
+      # lines). Fetching it resiliently (multi-transport + must contain the asset
+      # line) means a proxy error page is retried, and passing the extracted hash to
+      # Get-VerifiedDownload makes the binary download retry transports until a
+      # byte-correct copy lands -- the fix for a mid-transfer truncation that used to
+      # slip past the size floor and dead-end at the checksum (the #607 field case).
+      $k3dSums = "$env:TEMP\k3d-checksums-$([System.IO.Path]::GetRandomFileName()).txt"
       try {
-        $checksums = Invoke-WithRetry -Label "k3d checksums" -ScriptBlock {
-          (Invoke-WebRequest "https://github.com/k3d-io/k3d/releases/download/$k3dVer/checksums.txt" `
-            -UseBasicParsing).Content
-        }
+        Get-VerifiedDownload -Url "https://github.com/k3d-io/k3d/releases/download/$k3dVer/checksums.txt" `
+          -Dest $k3dSums -MinBytes 1 -MustContain "k3d-windows-$arch.exe" `
+          -Label "k3d checksums" -Message "Fetching k3d checksums"
       } catch {
-        Remove-Item $k3dDest -Force -ErrorAction SilentlyContinue
+        Remove-Item $k3dSums -Force -ErrorAction SilentlyContinue
         Err "Couldn't fetch the k3d checksums ($_). Check egress to github.com and re-run."
       }
-      $expectedHash = (($checksums -split "`n" |
-        Where-Object { $_ -match "k3d-windows-$arch\.exe" }) -replace '\s+.*','' |
+      $expectedHash = (((Get-Content $k3dSums) |
+        Where-Object { $_ -match "k3d-windows-$arch\.exe" }) -replace '\s+.*', '' |
         Select-Object -First 1)
-      if (-not $expectedHash) {
-        Remove-Item $k3dDest -Force -ErrorAction SilentlyContinue
-        Err "System tool checksum verification failed."
-      }
-      $actualHash = (Get-FileHash $k3dDest -Algorithm SHA256).Hash.ToLower()
-      if ($actualHash -ne $expectedHash.Trim().ToLower()) {
-        Remove-Item $k3dDest -Force
-        Err "System tool checksum verification failed."
+      Remove-Item $k3dSums -Force -ErrorAction SilentlyContinue
+      if ($expectedHash -notmatch '^[0-9a-fA-F]{64}$') {
+        Err "Couldn't read a valid k3d checksum from checksums.txt. Check egress to github.com and re-run."
       }
+      Get-VerifiedDownload -Url $k3dUrl -Dest $k3dDest -MinBytes 10MB -Magic 'MZ' -Sha256 $expectedHash.Trim() `
+        -Label "k3d download" -Message "Downloading k3d $k3dVer (~25 MB)"
       Log "k3d checksum verified."
       RefreshPath
       # Compute the summary now (correct elapsed) but print it only AFTER the
@@ -1767,7 +1794,26 @@ function Install-K3dAndHelm {
       $t0helm = Get-Date
       $helmZip = "$env:TEMP\helm-$helmVer-windows-$arch.zip"
       $helmUrl = "https://get.helm.sh/helm-$helmVer-windows-$arch.zip"
-      Get-VerifiedDownload -Url $helmUrl -Dest $helmZip -MinBytes 5MB -Magic 'PK' `
+      # Checksum-gated like k3d/kubectl (#609): get.helm.sh publishes
+      # <zip>.sha256sum ("<hash>  <zip-name>"). Fetch it first so the zip download
+      # retries transports until byte-correct -- a truncated zip that used to pass
+      # size+magic and then fail at Expand-Archive now self-heals. (The PS path had
+      # no helm checksum at all before; this also brings it to parity with the
+      # bash path, which already verifies helm.)
+      $helmSums = "$env:TEMP\helm-sha-$([System.IO.Path]::GetRandomFileName()).txt"
+      try {
+        Get-VerifiedDownload -Url "$helmUrl.sha256sum" -Dest $helmSums -MinBytes 1 `
+          -MustContain "helm-$helmVer-windows-$arch.zip" -Label "helm checksum" -Message "Fetching Helm checksum"
+      } catch {
+        Remove-Item $helmSums -Force -ErrorAction SilentlyContinue
+        Err "Couldn't fetch the Helm checksum ($_). Check egress to get.helm.sh and re-run."
+      }
+      $helmHash = (((Get-Content $helmSums) -split '\s+' | Select-Object -First 1))
+      Remove-Item $helmSums -Force -ErrorAction SilentlyContinue
+      if ($helmHash -notmatch '^[0-9a-fA-F]{64}$') {
+        Err "Couldn't read a valid Helm checksum from get.helm.sh. Check egress and re-run."
+      }
+      Get-VerifiedDownload -Url $helmUrl -Dest $helmZip -MinBytes 5MB -Magic 'PK' -Sha256 $helmHash `
         -Label "helm download" -Message "Downloading Helm $helmVer (~20 MB)"
       $helmExtract = "$env:TEMP\helm-extract"
       if (Test-Path $helmExtract) { Remove-Item $helmExtract -Recurse -Force }
diff --git a/scripts/manifest.sha256 b/scripts/manifest.sha256
index 161eff88..9028436c 100644
--- a/scripts/manifest.sha256
+++ b/scripts/manifest.sha256
@@ -15,4 +15,4 @@ e373403d7bb5ce3728b8d21af89e6bf672cc35bbf8938541eb527ae19cb9473b  scripts/lib/as
 911fd0714b17357bb205fc8a8fa8e13eedc1a9632a2f63d4ead9f8d8c7ee546f  scripts/lib/probe.sh
 38761a6c56dc85b3f5742df036e6a2ec2baa0adb0c90b3753b6706779528b7be  scripts/lib/summary.sh
 77e03332ebfab1ef759c6148a57afcf479c02c5dc6cc7b0e0e680f58e20cd364  scripts/lib/diagnose.sh
-f3c3591c466a3959e06e26657d6e4403b97bebafeb9c0ee0375256687bb03e6d  scripts/install-k8s.ps1
+5f74c985064ddd52bfd1aa508182c8336acf72667c0ed05770ba9920c8ea14ce  scripts/install-k8s.ps1
diff --git a/scripts/tests/install-k8s.Tests.ps1 b/scripts/tests/install-k8s.Tests.ps1
index 3b14263a..5b5ae5fc 100644
--- a/scripts/tests/install-k8s.Tests.ps1
+++ b/scripts/tests/install-k8s.Tests.ps1
@@ -3416,6 +3416,39 @@ Describe "Get-VerifiedDownload resilience guards (#607, Bugbot)" {
   It "wraps the post-download validation so an I/O error tries the next transport, not aborts" {
     # Bugbot: Get-Item/OpenRead can throw if AV locks the just-written file; that
     # must fall through to curl.exe/BITS, not escape Get-VerifiedDownload.
-    $script:GVD | Should -Match 'try \{\s*\$bad = Test-DownloadComplete[\s\S]{0,220}catch \{\s*\$bad ='
+    $script:GVD | Should -Match 'try \{\s*\$bad = Test-DownloadComplete[\s\S]{0,700}catch \{\s*\$bad ='
+  }
+}
+
+Describe "Checksum-driven tool download (#609)" {
+  BeforeAll { $script:CDD = Get-Content "$PSScriptRoot/../install-k8s.ps1" -Raw }
+
+  It "Get-VerifiedDownload exposes the -Sha256 and -MustContain gates" {
+    $script:CDD | Should -Match '\[string\]\$Sha256'
+    $script:CDD | Should -Match '\[string\]\$MustContain'
+  }
+
+  It "a checksum mismatch is treated as a bad transport (retries the next one), not a dead end" {
+    # The whole point: -Sha256 makes the checksum the completeness test, so a
+    # truncated/altered copy triggers curl.exe/BITS instead of failing the install.
+    $script:CDD | Should -Match "if \(-not \`$bad -and \`$Sha256\)[\s\S]{0,200}checksum mismatch"
+  }
+
+  It "k3d gates the binary download on the checksum fetched first" {
+    $script:CDD | Should -Match 'checksums\.txt[\s\S]{0,160}-MustContain'
+    $script:CDD | Should -Match '\$k3dUrl[\s\S]{0,160}-Sha256'
+  }
+
+  It "kubectl gates the binary download on the .sha256 fetched first" {
+    $script:CDD | Should -Match 'Get-VerifiedDownload[\s\S]{0,80}kubectl\.exe\.sha256'
+    $script:CDD | Should -Match '\$kUrl[\s\S]{0,160}-Sha256'
+  }
+
+  It "helm gates the zip download on its published sha256sum (PS parity with bash)" {
+    $script:CDD | Should -Match '\$helmUrl[\s\S]{0,160}-Sha256'
+  }
+
+  It "each extracted checksum is validated as 64 hex before it gates a download" {
+    ([regex]::Matches($script:CDD, "notmatch '\^\[0-9a-fA-F\]\{64\}\`$'")).Count | Should -BeGreaterOrEqual 2
   }
 }

From 517242ab32d86b0d4a39f83141ad38f33af41817 Mon Sep 17 00:00:00 2001
From: shujaat hasan <shujaat@tracebloc.io>
Date: Wed, 5 Aug 2026 16:33:40 +0200
Subject: [PATCH 02/10] fix(#611): content-gate the kubectl .sha256 fetch so a
 proxy page retries transports (Bugbot)

The kubectl .sha256 is a bare 64-hex hash with no fixed substring, so it used
-MinBytes 1 with no content gate -- a proxy error page satisfied the floor, the
first transport 'succeeded', curl.exe/BITS never ran, and the later hex check
aborted. Add -MatchPattern (a regex content gate) to Get-VerifiedDownload and use
'[0-9a-fA-F]{64}' for the kubectl checksum fetch, matching how k3d/helm use
-MustContain. Pester guards updated; manifest regenerated.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 scripts/install-k8s.ps1             | 21 +++++++++++++++++----
 scripts/manifest.sha256             |  2 +-
 scripts/tests/install-k8s.Tests.ps1 | 14 ++++++++++++--
 3 files changed, 30 insertions(+), 7 deletions(-)

diff --git a/scripts/install-k8s.ps1 b/scripts/install-k8s.ps1
index 79e09c43..274564cf 100644
--- a/scripts/install-k8s.ps1
+++ b/scripts/install-k8s.ps1
@@ -473,10 +473,16 @@ function Get-VerifiedDownload {
     # BITS until a byte-correct copy lands.
     [string]$Sha256 = '',
     # When set, the downloaded TEXT must contain this substring or the transport is
-    # treated as failed (#609). Used for the small checksum-list files (checksums.txt
-    # / *.sha256), which have no fixed hash but must carry the expected asset line --
-    # a proxy error page that lacks it is caught and the next transport is tried.
+    # treated as failed (#609). Used for checksum-list files that carry the expected
+    # asset line (k3d checksums.txt, helm *.sha256sum) -- a proxy error page that
+    # lacks it is caught and the next transport is tried.
     [string]$MustContain = '',
+    # Like $MustContain but a REGEX (#611): the downloaded text must match it, else
+    # the transport is treated as failed. Used for checksum files with no fixed
+    # substring -- e.g. kubectl's .sha256 is a bare 64-hex hash, so '[0-9a-fA-F]{64}'
+    # makes a proxy HTML page (which has no such run) fall through to curl.exe/BITS
+    # instead of "succeeding" and dying at the later hex check (Bugbot).
+    [string]$MatchPattern = '',
     [string]$Label = 'download',
     [string]$Message = 'Downloading'
   )
@@ -516,6 +522,12 @@ function Get-VerifiedDownload {
           $bad = "the file did not contain the expected entry '$MustContain' -- likely an error page"
         }
       }
+      if (-not $bad -and $MatchPattern) {
+        $text = Get-Content -LiteralPath $Dest -Raw -ErrorAction Stop
+        if ($text -notmatch $MatchPattern) {
+          $bad = "the file did not match the expected pattern -- likely an error page"
+        }
+      }
     } catch {
       $bad = "could not read the downloaded file ($($_.Exception.Message)) -- it may be locked or quarantined by antivirus"
     }
@@ -1625,7 +1637,8 @@ function Install-Kubectl {
   $kSums = "$env:TEMP\kubectl-sha-$([System.IO.Path]::GetRandomFileName()).txt"
   try {
     Get-VerifiedDownload -Url "https://dl.k8s.io/release/$kVer/bin/windows/$arch/kubectl.exe.sha256" `
-      -Dest $kSums -MinBytes 1 -Label "kubectl checksum" -Message "Fetching kubectl checksum"
+      -Dest $kSums -MinBytes 1 -MatchPattern '[0-9a-fA-F]{64}' `
+      -Label "kubectl checksum" -Message "Fetching kubectl checksum"
   } catch {
     Remove-Item $kSums -Force -ErrorAction SilentlyContinue
     Err "Couldn't fetch the kubectl checksum ($_). Check egress to dl.k8s.io and re-run."
diff --git a/scripts/manifest.sha256 b/scripts/manifest.sha256
index 9028436c..a4135d75 100644
--- a/scripts/manifest.sha256
+++ b/scripts/manifest.sha256
@@ -15,4 +15,4 @@ e373403d7bb5ce3728b8d21af89e6bf672cc35bbf8938541eb527ae19cb9473b  scripts/lib/as
 911fd0714b17357bb205fc8a8fa8e13eedc1a9632a2f63d4ead9f8d8c7ee546f  scripts/lib/probe.sh
 38761a6c56dc85b3f5742df036e6a2ec2baa0adb0c90b3753b6706779528b7be  scripts/lib/summary.sh
 77e03332ebfab1ef759c6148a57afcf479c02c5dc6cc7b0e0e680f58e20cd364  scripts/lib/diagnose.sh
-5f74c985064ddd52bfd1aa508182c8336acf72667c0ed05770ba9920c8ea14ce  scripts/install-k8s.ps1
+ac8c577837c2364fe964b11dcd67c5a959254720ad12a1b0c84f1be3612a9259  scripts/install-k8s.ps1
diff --git a/scripts/tests/install-k8s.Tests.ps1 b/scripts/tests/install-k8s.Tests.ps1
index 5b5ae5fc..76171f28 100644
--- a/scripts/tests/install-k8s.Tests.ps1
+++ b/scripts/tests/install-k8s.Tests.ps1
@@ -3416,16 +3416,26 @@ Describe "Get-VerifiedDownload resilience guards (#607, Bugbot)" {
   It "wraps the post-download validation so an I/O error tries the next transport, not aborts" {
     # Bugbot: Get-Item/OpenRead can throw if AV locks the just-written file; that
     # must fall through to curl.exe/BITS, not escape Get-VerifiedDownload.
-    $script:GVD | Should -Match 'try \{\s*\$bad = Test-DownloadComplete[\s\S]{0,700}catch \{\s*\$bad ='
+    # Distance-independent: the try wraps the validation, and a catch turns an I/O
+    # error into a recorded problem (so the loop tries the next transport).
+    $script:GVD | Should -Match 'try \{\s*\$bad = Test-DownloadComplete'
+    $script:GVD | Should -Match 'catch \{\s*\$bad = "could not read the downloaded file'
   }
 }
 
 Describe "Checksum-driven tool download (#609)" {
   BeforeAll { $script:CDD = Get-Content "$PSScriptRoot/../install-k8s.ps1" -Raw }
 
-  It "Get-VerifiedDownload exposes the -Sha256 and -MustContain gates" {
+  It "Get-VerifiedDownload exposes the -Sha256, -MustContain and -MatchPattern gates" {
     $script:CDD | Should -Match '\[string\]\$Sha256'
     $script:CDD | Should -Match '\[string\]\$MustContain'
+    $script:CDD | Should -Match '\[string\]\$MatchPattern'
+  }
+
+  It "the kubectl .sha256 fetch has a content gate so a proxy page retries transports (Bugbot #611)" {
+    # kubectl's .sha256 is a bare hash (no fixed substring), so it must gate on a
+    # regex; -MinBytes 1 alone let a proxy error page 'succeed' and skip the fallbacks.
+    $script:CDD | Should -Match "kubectl\.exe\.sha256[\s\S]{0,120}-MatchPattern"
   }
 
   It "a checksum mismatch is treated as a bad transport (retries the next one), not a dead end" {

From 9c621f615ce9549e0e5b43038511358027e930de Mon Sep 17 00:00:00 2001
From: shujaat hasan <shujaat@tracebloc.io>
Date: Wed, 5 Aug 2026 16:42:43 +0200
Subject: [PATCH 03/10] fix(installer): don't misread a successful k3d
 cluster-create as failed (#611)

Field report (same Windows box, past the k3d download fix): Step 3 aborted with
"Failed to create compute environment" even though k3d printed
"Cluster 'tracebloc' created successfully!" with EMPTY stderr and the cluster was
actually up. Cause: Wait-ProcessWithDeadline polled HasExited but never called
WaitForExit(), and Start-Process -RedirectStandardOutput can leave $proc.ExitCode
$null in that window -- so `$null -ne 0` misread an exit-0 success as a failure.
Not machine-specific; a latent race any Windows user can hit.

- Wait-ProcessWithDeadline now calls $Process.WaitForExit() before returning
  success, so the redirected streams drain and ExitCode is reliable for EVERY
  caller (cluster create, partial delete, tracked installs).
- Cluster-create adds defense-in-depth: a still-null exit code falls back to
  k3d's own "created successfully" marker rather than failing a cluster that is up.
- Pester source-guards for both. Windows-only change (install-k8s.ps1); Linux/mac
  paths untouched and their suites remain green.

Contributes to #578.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 scripts/install-k8s.ps1             | 16 +++++++++++++++-
 scripts/manifest.sha256             |  2 +-
 scripts/tests/install-k8s.Tests.ps1 | 15 +++++++++++++++
 3 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/scripts/install-k8s.ps1 b/scripts/install-k8s.ps1
index 274564cf..176ed779 100644
--- a/scripts/install-k8s.ps1
+++ b/scripts/install-k8s.ps1
@@ -225,6 +225,14 @@ function Wait-ProcessWithDeadline {
     Start-Sleep -Seconds 2
   }
   Write-Host "`r                                                   `r" -NoNewline
+  # HasExited can flip true before the process's redirected stdout/stderr streams
+  # are fully drained, and in that window Start-Process -RedirectStandardOutput
+  # leaves $Process.ExitCode $null. Callers then read a null code and `$null -ne 0`
+  # misreads a SUCCESSFUL run as a failure -- the #611 field case: k3d printed
+  # "Cluster created successfully!" with empty stderr, yet the install aborted with
+  # the cluster actually up. WaitForExit() (bounded: the process has already exited)
+  # flushes the streams and guarantees ExitCode is populated for every caller.
+  try { $Process.WaitForExit() } catch {}
   return $true
 }
 
@@ -2671,9 +2679,15 @@ function New-K3dCluster {
       Err "Compute environment creation timed out after $timeoutMin minutes. Check that Docker is healthy and this network can pull images, then re-run. (TB_CREATE_TIMEOUT_MIN overrides the bound.)"
     }
 
-    $k3dExitCode = $k3dProc.ExitCode
     $k3dStdout = if (Test-Path $k3dOutLog) { Get-Content $k3dOutLog -Raw -ErrorAction SilentlyContinue } else { "" }
     $k3dStderr = if (Test-Path $k3dErrLog) { Get-Content $k3dErrLog -Raw -ErrorAction SilentlyContinue } else { "" }
+    $k3dExitCode = $k3dProc.ExitCode
+    # Defense-in-depth (#611): if the exit code is STILL unreadable after
+    # WaitForExit (Wait-ProcessWithDeadline), do not fail a cluster that k3d itself
+    # reported up -- trust its authoritative success marker over a null code.
+    if ($null -eq $k3dExitCode) {
+      $k3dExitCode = if ($k3dStdout -match 'created successfully') { 0 } else { 1 }
+    }
     Remove-Item $k3dOutLog, $k3dErrLog -Force -ErrorAction SilentlyContinue
     if ($proxyCfg) { Remove-Item (Split-Path $proxyCfg -Parent) -Recurse -Force -ErrorAction SilentlyContinue }
     if ($registriesCfg) { Remove-Item (Split-Path $registriesCfg -Parent) -Recurse -Force -ErrorAction SilentlyContinue }
diff --git a/scripts/manifest.sha256 b/scripts/manifest.sha256
index a4135d75..a699593f 100644
--- a/scripts/manifest.sha256
+++ b/scripts/manifest.sha256
@@ -15,4 +15,4 @@ e373403d7bb5ce3728b8d21af89e6bf672cc35bbf8938541eb527ae19cb9473b  scripts/lib/as
 911fd0714b17357bb205fc8a8fa8e13eedc1a9632a2f63d4ead9f8d8c7ee546f  scripts/lib/probe.sh
 38761a6c56dc85b3f5742df036e6a2ec2baa0adb0c90b3753b6706779528b7be  scripts/lib/summary.sh
 77e03332ebfab1ef759c6148a57afcf479c02c5dc6cc7b0e0e680f58e20cd364  scripts/lib/diagnose.sh
-ac8c577837c2364fe964b11dcd67c5a959254720ad12a1b0c84f1be3612a9259  scripts/install-k8s.ps1
+3d0e3d8a3771783db7501e54ce6d909d4ccf7bdc9dd02dc79b91b2dac5030a11  scripts/install-k8s.ps1
diff --git a/scripts/tests/install-k8s.Tests.ps1 b/scripts/tests/install-k8s.Tests.ps1
index 76171f28..da35675a 100644
--- a/scripts/tests/install-k8s.Tests.ps1
+++ b/scripts/tests/install-k8s.Tests.ps1
@@ -3462,3 +3462,18 @@ Describe "Checksum-driven tool download (#609)" {
     ([regex]::Matches($script:CDD, "notmatch '\^\[0-9a-fA-F\]\{64\}\`$'")).Count | Should -BeGreaterOrEqual 2
   }
 }
+
+Describe "Cluster-create exit-code reliability (#611)" {
+  BeforeAll { $script:CEC = Get-Content "$PSScriptRoot/../install-k8s.ps1" -Raw }
+
+  It "Wait-ProcessWithDeadline calls WaitForExit before returning success" {
+    # HasExited can flip true before redirected stdout/stderr drain, leaving
+    # $proc.ExitCode null; WaitForExit flushes them so every caller reads a real code.
+    $script:CEC | Should -Match 'function Wait-ProcessWithDeadline[\s\S]{0,1600}\$Process\.WaitForExit\(\)[\s\S]{0,80}return \$true'
+  }
+
+  It "cluster-create does not fail a cluster k3d reported up when the exit code is unreadable" {
+    # Defense-in-depth: a null exit code falls back to k3d's 'created successfully' marker.
+    $script:CEC | Should -Match "if \(\`$null -eq \`$k3dExitCode\)[\s\S]{0,160}created successfully"
+  }
+}

From 9b33a3a9cf48c59c4d0c1014f14e2308b04df225 Mon Sep 17 00:00:00 2001
From: shujaat hasan <shujaat@tracebloc.io>
Date: Thu, 6 Aug 2026 09:09:24 +0200
Subject: [PATCH 04/10] fix(#611): hash-anchor the checksum-list gates + check
 both k3d streams in the exit fallback (Bugbot)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Two Bugbot findings on the checksum-driven download work:

1. (High) The checksum-LIST fetch gates were fail-open, so a proxy error page
   "succeeded" on the first transport and skipped the curl.exe/BITS retry — exactly
   the case #611 exists to survive. Helm's -MustContain substring
   (helm-<ver>-windows-<arch>.zip) also appears in the request URL a proxy page can
   echo; kubectl's -MatchPattern was unanchored so any page with a 64-hex run passed;
   k3d gated on the bare asset name. Fix: drop the weak -MustContain entirely and gate
   every checksum-list fetch on the hash STRUCTURE — k3d/helm require a 64-hex hash
   adjacent to the asset, kubectl requires the hash at the start of the body. A
   proxy/HTML error page can't satisfy that, so it retries transports as intended.

2. (Medium) The null-exit-code cluster-create fallback only scanned $k3dStdout, but
   k3d logs its "Cluster created successfully!" line via logrus to STDERR — so a real
   success could be misread as failure. Fix: check both $k3dStdout and $k3dStderr.

Pester source-guards updated: kubectl gate is start-anchored, k3d/helm gates are
hash-anchored, no -MustContain remains, and the fallback inspects both streams. Full
suite green (447). Manifest regenerated.

Contributes to #578.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 scripts/install-k8s.ps1             | 40 +++++++++++++----------------
 scripts/manifest.sha256             |  2 +-
 scripts/tests/install-k8s.Tests.ps1 | 31 +++++++++++++---------
 3 files changed, 38 insertions(+), 35 deletions(-)

diff --git a/scripts/install-k8s.ps1 b/scripts/install-k8s.ps1
index 176ed779..cbe2538e 100644
--- a/scripts/install-k8s.ps1
+++ b/scripts/install-k8s.ps1
@@ -480,16 +480,15 @@ function Get-VerifiedDownload {
     # field failure. With $Sha256, a truncated/corrupt copy just triggers curl.exe/
     # BITS until a byte-correct copy lands.
     [string]$Sha256 = '',
-    # When set, the downloaded TEXT must contain this substring or the transport is
-    # treated as failed (#609). Used for checksum-list files that carry the expected
-    # asset line (k3d checksums.txt, helm *.sha256sum) -- a proxy error page that
-    # lacks it is caught and the next transport is tried.
-    [string]$MustContain = '',
-    # Like $MustContain but a REGEX (#611): the downloaded text must match it, else
-    # the transport is treated as failed. Used for checksum files with no fixed
-    # substring -- e.g. kubectl's .sha256 is a bare 64-hex hash, so '[0-9a-fA-F]{64}'
-    # makes a proxy HTML page (which has no such run) fall through to curl.exe/BITS
-    # instead of "succeeding" and dying at the later hex check (Bugbot).
+    # When set, the downloaded TEXT must MATCH this regex or the transport is treated
+    # as failed and the next one (curl.exe/BITS) is tried (#611). Used to gate the
+    # checksum-LIST files on the actual hash STRUCTURE, not a weak substring: an
+    # asset-name substring also appears in the request URL, so a proxy error page
+    # echoing the URL would satisfy a substring gate, "succeed" on the first
+    # transport, skip the retry, and then abort at the later hex parse (Bugbot). The
+    # call sites therefore require a 64-hex hash adjacent to the asset (k3d/helm) or
+    # anchored at the start of the body (kubectl's bare-hash .sha256) -- structure a
+    # proxy/HTML error page can't accidentally satisfy.
     [string]$MatchPattern = '',
     [string]$Label = 'download',
     [string]$Message = 'Downloading'
@@ -524,16 +523,10 @@ function Get-VerifiedDownload {
           $bad = "checksum mismatch (got $got) -- the download is truncated or altered"
         }
       }
-      if (-not $bad -and $MustContain) {
-        $text = Get-Content -LiteralPath $Dest -Raw -ErrorAction Stop
-        if ($text -notmatch [regex]::Escape($MustContain)) {
-          $bad = "the file did not contain the expected entry '$MustContain' -- likely an error page"
-        }
-      }
       if (-not $bad -and $MatchPattern) {
         $text = Get-Content -LiteralPath $Dest -Raw -ErrorAction Stop
         if ($text -notmatch $MatchPattern) {
-          $bad = "the file did not match the expected pattern -- likely an error page"
+          $bad = "the file did not match the expected checksum pattern -- likely a proxy error page; trying another method"
         }
       }
     } catch {
@@ -1645,7 +1638,7 @@ function Install-Kubectl {
   $kSums = "$env:TEMP\kubectl-sha-$([System.IO.Path]::GetRandomFileName()).txt"
   try {
     Get-VerifiedDownload -Url "https://dl.k8s.io/release/$kVer/bin/windows/$arch/kubectl.exe.sha256" `
-      -Dest $kSums -MinBytes 1 -MatchPattern '[0-9a-fA-F]{64}' `
+      -Dest $kSums -MinBytes 1 -MatchPattern '^\s*[0-9a-fA-F]{64}' `
       -Label "kubectl checksum" -Message "Fetching kubectl checksum"
   } catch {
     Remove-Item $kSums -Force -ErrorAction SilentlyContinue
@@ -1764,7 +1757,7 @@ function Install-K3dAndHelm {
       $k3dSums = "$env:TEMP\k3d-checksums-$([System.IO.Path]::GetRandomFileName()).txt"
       try {
         Get-VerifiedDownload -Url "https://github.com/k3d-io/k3d/releases/download/$k3dVer/checksums.txt" `
-          -Dest $k3dSums -MinBytes 1 -MustContain "k3d-windows-$arch.exe" `
+          -Dest $k3dSums -MinBytes 1 -MatchPattern "[0-9a-fA-F]{64}\s+\S*k3d-windows-$arch\.exe" `
           -Label "k3d checksums" -Message "Fetching k3d checksums"
       } catch {
         Remove-Item $k3dSums -Force -ErrorAction SilentlyContinue
@@ -1824,7 +1817,7 @@ function Install-K3dAndHelm {
       $helmSums = "$env:TEMP\helm-sha-$([System.IO.Path]::GetRandomFileName()).txt"
       try {
         Get-VerifiedDownload -Url "$helmUrl.sha256sum" -Dest $helmSums -MinBytes 1 `
-          -MustContain "helm-$helmVer-windows-$arch.zip" -Label "helm checksum" -Message "Fetching Helm checksum"
+          -MatchPattern "[0-9a-fA-F]{64}\s+\S*helm-\S*windows-$arch\.zip" -Label "helm checksum" -Message "Fetching Helm checksum"
       } catch {
         Remove-Item $helmSums -Force -ErrorAction SilentlyContinue
         Err "Couldn't fetch the Helm checksum ($_). Check egress to get.helm.sh and re-run."
@@ -2684,9 +2677,12 @@ function New-K3dCluster {
     $k3dExitCode = $k3dProc.ExitCode
     # Defense-in-depth (#611): if the exit code is STILL unreadable after
     # WaitForExit (Wait-ProcessWithDeadline), do not fail a cluster that k3d itself
-    # reported up -- trust its authoritative success marker over a null code.
+    # reported up -- trust its authoritative success marker over a null code. k3d
+    # logs via logrus to STDERR, so its "Cluster created successfully!" line lands in
+    # $k3dStderr, not $k3dStdout -- check BOTH streams or a real success is misread
+    # as failure (Bugbot).
     if ($null -eq $k3dExitCode) {
-      $k3dExitCode = if ($k3dStdout -match 'created successfully') { 0 } else { 1 }
+      $k3dExitCode = if ("$k3dStdout`n$k3dStderr" -match 'created successfully') { 0 } else { 1 }
     }
     Remove-Item $k3dOutLog, $k3dErrLog -Force -ErrorAction SilentlyContinue
     if ($proxyCfg) { Remove-Item (Split-Path $proxyCfg -Parent) -Recurse -Force -ErrorAction SilentlyContinue }
diff --git a/scripts/manifest.sha256 b/scripts/manifest.sha256
index a699593f..9cb888ca 100644
--- a/scripts/manifest.sha256
+++ b/scripts/manifest.sha256
@@ -15,4 +15,4 @@ e373403d7bb5ce3728b8d21af89e6bf672cc35bbf8938541eb527ae19cb9473b  scripts/lib/as
 911fd0714b17357bb205fc8a8fa8e13eedc1a9632a2f63d4ead9f8d8c7ee546f  scripts/lib/probe.sh
 38761a6c56dc85b3f5742df036e6a2ec2baa0adb0c90b3753b6706779528b7be  scripts/lib/summary.sh
 77e03332ebfab1ef759c6148a57afcf479c02c5dc6cc7b0e0e680f58e20cd364  scripts/lib/diagnose.sh
-3d0e3d8a3771783db7501e54ce6d909d4ccf7bdc9dd02dc79b91b2dac5030a11  scripts/install-k8s.ps1
+c758a024dbfabcdfd0b45959d1a219ca978b18e06be58f5261d11a26ea426585  scripts/install-k8s.ps1
diff --git a/scripts/tests/install-k8s.Tests.ps1 b/scripts/tests/install-k8s.Tests.ps1
index da35675a..a1a7f788 100644
--- a/scripts/tests/install-k8s.Tests.ps1
+++ b/scripts/tests/install-k8s.Tests.ps1
@@ -3426,16 +3426,19 @@ Describe "Get-VerifiedDownload resilience guards (#607, Bugbot)" {
 Describe "Checksum-driven tool download (#609)" {
   BeforeAll { $script:CDD = Get-Content "$PSScriptRoot/../install-k8s.ps1" -Raw }
 
-  It "Get-VerifiedDownload exposes the -Sha256, -MustContain and -MatchPattern gates" {
+  It "Get-VerifiedDownload exposes -Sha256 and -MatchPattern, and no fail-open substring gate" {
     $script:CDD | Should -Match '\[string\]\$Sha256'
-    $script:CDD | Should -Match '\[string\]\$MustContain'
     $script:CDD | Should -Match '\[string\]\$MatchPattern'
+    # -MustContain removed (Bugbot #611): a substring gate is fail-open because the
+    # asset name also appears in the request URL that a proxy error page can echo.
+    $script:CDD | Should -Not -Match 'MustContain'
   }
 
-  It "the kubectl .sha256 fetch has a content gate so a proxy page retries transports (Bugbot #611)" {
-    # kubectl's .sha256 is a bare hash (no fixed substring), so it must gate on a
-    # regex; -MinBytes 1 alone let a proxy error page 'succeed' and skip the fallbacks.
-    $script:CDD | Should -Match "kubectl\.exe\.sha256[\s\S]{0,120}-MatchPattern"
+  It "the kubectl .sha256 gate is START-anchored so a proxy page retries transports (Bugbot #611)" {
+    # kubectl's .sha256 is a bare hash; the gate must be anchored ('^...64hex') so an
+    # HTML error page (which starts with '<') fails it and falls through to curl.exe/
+    # BITS. An unanchored [0-9a-fA-F]{64} would pass on any page with a 64-hex run.
+    $script:CDD | Should -Match "kubectl\.exe\.sha256[\s\S]{0,140}-MatchPattern '\^"
   }
 
   It "a checksum mismatch is treated as a bad transport (retries the next one), not a dead end" {
@@ -3444,8 +3447,10 @@ Describe "Checksum-driven tool download (#609)" {
     $script:CDD | Should -Match "if \(-not \`$bad -and \`$Sha256\)[\s\S]{0,200}checksum mismatch"
   }
 
-  It "k3d gates the binary download on the checksum fetched first" {
-    $script:CDD | Should -Match 'checksums\.txt[\s\S]{0,160}-MustContain'
+  It "k3d gates the binary on the checksum fetched first, via a hash-anchored gate" {
+    # The checksum-list gate requires a 64-hex hash adjacent to the asset, not a bare
+    # asset-name substring (which also appears in the URL and would fail open) (Bugbot).
+    $script:CDD | Should -Match 'checksums\.txt[\s\S]{0,160}-MatchPattern "\[0-9a-fA-F\]\{64\}'
     $script:CDD | Should -Match '\$k3dUrl[\s\S]{0,160}-Sha256'
   }
 
@@ -3454,8 +3459,9 @@ Describe "Checksum-driven tool download (#609)" {
     $script:CDD | Should -Match '\$kUrl[\s\S]{0,160}-Sha256'
   }
 
-  It "helm gates the zip download on its published sha256sum (PS parity with bash)" {
+  It "helm gates the zip on its sha256sum with a hash-anchored gate (PS parity with bash)" {
     $script:CDD | Should -Match '\$helmUrl[\s\S]{0,160}-Sha256'
+    $script:CDD | Should -Match 'sha256sum[\s\S]{0,160}-MatchPattern "\[0-9a-fA-F\]\{64\}'
   }
 
   It "each extracted checksum is validated as 64 hex before it gates a download" {
@@ -3472,8 +3478,9 @@ Describe "Cluster-create exit-code reliability (#611)" {
     $script:CEC | Should -Match 'function Wait-ProcessWithDeadline[\s\S]{0,1600}\$Process\.WaitForExit\(\)[\s\S]{0,80}return \$true'
   }
 
-  It "cluster-create does not fail a cluster k3d reported up when the exit code is unreadable" {
-    # Defense-in-depth: a null exit code falls back to k3d's 'created successfully' marker.
-    $script:CEC | Should -Match "if \(\`$null -eq \`$k3dExitCode\)[\s\S]{0,160}created successfully"
+  It "the null-exit fallback checks BOTH k3d streams (logrus success goes to stderr) (Bugbot)" {
+    # k3d's 'Cluster created successfully!' is a logrus line on STDERR, so the null-
+    # exit fallback must inspect $k3dStderr too, not only $k3dStdout.
+    $script:CEC | Should -Match 'if \(\$null -eq \$k3dExitCode\)[\s\S]{0,120}k3dStdout[\s\S]{0,20}k3dStderr[\s\S]{0,40}created successfully'
   }
 }

From 7526cb00bb23a5177626793802ac45491f640031 Mon Sep 17 00:00:00 2001
From: shujaat hasan <shujaat@tracebloc.io>
Date: Thu, 6 Aug 2026 09:43:21 +0200
Subject: [PATCH 05/10] fix(#611): make /data/shared writable so dataset ingest
 works on hostPath installs

`tb data ingest` failed at the copy step with `mkdir: can't create directory
'/data/shared/.tracebloc-staging/': Permission denied`. On hostPath installs (the
Windows/WSL2 + bare-metal default) kubelet does not apply fsGroup to hostPath
volumes (kubernetes/kubernetes#138411), so /data/shared (client-pvc) is created
root-owned and the non-root ingest-staging pod can't write to it. mysql-data has a
privileged init-chown for exactly this reason; the shared data volume had none.

- jobs-manager gains fsGroup: 1000 (CSI clusters apply it to the shared volume).
- On hostPath, a privileged init-shared-data container (root, CHOWN+FOWNER only)
  chowns /data/shared to 1000:1000 and chmod 2777. World-writable, unlike
  mysql-data's single-UID chown, because the shared volume has multiple non-root
  writers whose UIDs this chart doesn't control -- jobs-manager, the training/
  ingestor pods it spawns, and the CLI's ingest-staging pod. setgid keeps new files
  in GID 1000; the init is gated on hostPath (CSI relies on fsGroup).
- helm-unittest: init present + world-writable on hostPath; absent (fsGroup kept)
  on CSI. Chart bumped 1.9.15 -> 1.9.16.

Client-side companion to the installer fixes on this PR (requested to land here).
Contributes to #578.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 client/Chart.yaml                             |  4 +-
 client/templates/jobs-manager-deployment.yaml | 34 +++++++++++++++++
 client/tests/jobs_manager_test.yaml           | 38 +++++++++++++++++++
 3 files changed, 74 insertions(+), 2 deletions(-)

diff --git a/client/Chart.yaml b/client/Chart.yaml
index 672adbd6..9c879548 100644
--- a/client/Chart.yaml
+++ b/client/Chart.yaml
@@ -2,8 +2,8 @@ apiVersion: v2
 name: client
 description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift
 type: application
-version: 1.9.15
-appVersion: "1.9.15"
+version: 1.9.16
+appVersion: "1.9.16"
 keywords:
   - tracebloc
   - kubernetes
diff --git a/client/templates/jobs-manager-deployment.yaml b/client/templates/jobs-manager-deployment.yaml
index 71ca1db3..de1fe0d0 100644
--- a/client/templates/jobs-manager-deployment.yaml
+++ b/client/templates/jobs-manager-deployment.yaml
@@ -35,8 +35,42 @@ spec:
       serviceAccountName: {{ include "tracebloc.serviceAccountName" . }}
       securityContext:
         runAsNonRoot: true
+        # CSI-backed clusters (EKS/AKS/OC) apply this to the shared data volume so
+        # every non-root writer shares GID 1000; hostPath ignores it (see the init
+        # below). fsGroupChangePolicy limits the relabel to a root-owned mount.
+        fsGroup: 1000
+        fsGroupChangePolicy: "OnRootMismatch"
         seccompProfile:
           type: RuntimeDefault
+      {{- if .Values.hostPath.enabled }}
+      # kubelet does NOT apply fsGroup to hostPath volumes (kubernetes/kubernetes#138411),
+      # so /data/shared is created root-owned and non-root pods can't write to it. That
+      # breaks dataset ingest: `tb data ingest` streams files into a staging pod that does
+      # `mkdir /data/shared/.tracebloc-staging/` and hits "Permission denied" (#611). Unlike
+      # mysql-data (one writer, UID 999), the shared volume has MULTIPLE non-root writers —
+      # jobs-manager, the training/ingestor pods it spawns, and the CLI's ingest-staging pod,
+      # whose UID this chart does not control — so it must be world-writable, not chowned to
+      # a single UID. setgid (2) makes new files inherit GID 1000; runs as root only long
+      # enough to fix the mount, with FOWNER so the chmod is idempotent on re-install. CSI
+      # clusters rely on fsGroup above and skip this.
+      initContainers:
+      - name: init-shared-data
+        image: {{ include "tracebloc.image" (dict "repository" "library/busybox" "tag" .Values.images.busybox.tag "digest" .Values.images.busybox.digest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }}
+        securityContext:
+          runAsUser: 0
+          runAsNonRoot: false
+          allowPrivilegeEscalation: false
+          readOnlyRootFilesystem: true
+          capabilities:
+            drop: ["ALL"]
+            add: ["CHOWN", "FOWNER"]
+          seccompProfile:
+            type: RuntimeDefault
+        command: ['sh', '-c', 'chown 1000:1000 /data/shared && chmod 2777 /data/shared']
+        volumeMounts:
+        - name: shared-volume
+          mountPath: /data/shared
+      {{- end }}
       containers:
       - name: api
         image: {{ include "tracebloc.image" (dict "repository" "tracebloc/jobs-manager" "tag" .Values.env.CLIENT_ENV "digest" .Values.images.jobsManager.digest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }}
diff --git a/client/tests/jobs_manager_test.yaml b/client/tests/jobs_manager_test.yaml
index fe0991a9..c0312a84 100644
--- a/client/tests/jobs_manager_test.yaml
+++ b/client/tests/jobs_manager_test.yaml
@@ -414,3 +414,41 @@ tests:
               secretKeyRef:
                 name: RELEASE-NAME-secrets
                 key: TB_CREDMGR_PASSWORD
+
+  # #611: /data/shared (client-pvc) must be writable by the non-root ingest-staging
+  # pod. hostPath ignores fsGroup (k8s#138411), so a privileged init makes the shared
+  # volume world-writable; CSI relies on fsGroup and skips the init.
+  - it: "hostPath install adds init-shared-data to make /data/shared writable for ingest (#611)"
+    set:
+      hostPath:
+        enabled: true
+    asserts:
+      - equal:
+          path: spec.template.spec.securityContext.fsGroup
+          value: 1000
+      - equal:
+          path: spec.template.spec.initContainers[0].name
+          value: init-shared-data
+      - equal:
+          path: spec.template.spec.initContainers[0].securityContext.runAsUser
+          value: 0
+      - contains:
+          path: spec.template.spec.initContainers[0].securityContext.capabilities.add
+          content: CHOWN
+      - contains:
+          path: spec.template.spec.initContainers[0].securityContext.capabilities.add
+          content: FOWNER
+      - matchRegex:
+          path: spec.template.spec.initContainers[0].command[2]
+          pattern: "chown 1000:1000 /data/shared && chmod 2777 /data/shared"
+
+  - it: "CSI install (hostPath disabled) keeps fsGroup but skips the privileged init (#611)"
+    set:
+      hostPath:
+        enabled: false
+    asserts:
+      - equal:
+          path: spec.template.spec.securityContext.fsGroup
+          value: 1000
+      - notExists:
+          path: spec.template.spec.initContainers

From ab698b8c7c2c5b56d105bc09b793105423cb4641 Mon Sep 17 00:00:00 2001
From: shujaat hasan <shujaat@tracebloc.io>
Date: Thu, 6 Aug 2026 09:50:24 +0200
Subject: [PATCH 06/10] feat(installer): support TRACEBLOC_CHART_PATH on
 Windows (local-chart parity with bash)

The Windows installer could only ever install the PUBLISHED chart (helm repo), so a
branch-only chart change (e.g. the #611 /data/shared fix, or #585's global.imageRegistry)
was impossible to test from a Windows install. The bash installer already supports a
local chart via TRACEBLOC_CHART_PATH (_resolve_chart_ref); this brings Windows to parity.

- When $env:TRACEBLOC_CHART_PATH is set, install-k8s.ps1 installs from that local chart
  directory (validated) and skips `helm repo add`; otherwise it uses the published repo
  as before. Applied to both the fresh-install and adopt/reconcile helm upgrades.
- Pester source-guards for the local-chart ref, the repo-add skip, and the not-a-directory
  error. Manifest regenerated. Full suite green (451).

Enables a from-scratch Windows test of the branch chart. Contributes to #578.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 scripts/install-k8s.ps1             | 29 ++++++++++++++++++++++-------
 scripts/manifest.sha256             |  2 +-
 scripts/tests/install-k8s.Tests.ps1 | 19 +++++++++++++++++++
 3 files changed, 42 insertions(+), 8 deletions(-)

diff --git a/scripts/install-k8s.ps1 b/scripts/install-k8s.ps1
index cbe2538e..0ff300ed 100644
--- a/scripts/install-k8s.ps1
+++ b/scripts/install-k8s.ps1
@@ -3639,17 +3639,32 @@ $envBlock
   # with this script's own ...\tracebloc-installer-<n>\install-k8s.ps1 temp path --
   # which contains "tracebloc" -- so the guard skipped the add on every fresh
   # install and Step 4 died later with "Error: repo tracebloc not found". #385)
-  Log "Adding Helm repo: $TRACEBLOC_HELM_REPO_URL"
-  $addOutput = (helm repo add $TRACEBLOC_HELM_REPO_NAME $TRACEBLOC_HELM_REPO_URL --force-update 2>&1) | Out-String
-  Log "helm repo add: $addOutput"
-  if ($LASTEXITCODE -ne 0) { Err "Couldn't add the tracebloc chart repo ($TRACEBLOC_HELM_REPO_URL)." $addOutput }
+  # Chart source: $env:TRACEBLOC_CHART_PATH points at a LOCAL chart directory for
+  # dev/testing an unreleased chart (parity with the bash installer's
+  # _resolve_chart_ref, lib/install-client-helm.sh) -- without it, Windows could only
+  # ever install the published chart, so branch-only chart fixes were untestable here.
+  # A local path skips `helm repo add` entirely; otherwise use the published repo.
+  if ($env:TRACEBLOC_CHART_PATH) {
+    if (-not (Test-Path -LiteralPath $env:TRACEBLOC_CHART_PATH -PathType Container)) {
+      Err "TRACEBLOC_CHART_PATH is set but is not a directory: $($env:TRACEBLOC_CHART_PATH)"
+    }
+    $chartRef = $env:TRACEBLOC_CHART_PATH
+    Info "Dev mode: installing the chart from local path $chartRef (skipping the Helm repo)."
+    Log "Using local chart: $chartRef"
+  } else {
+    $chartRef = "$TRACEBLOC_HELM_REPO_NAME/$TRACEBLOC_CHART_NAME"
+    Log "Adding Helm repo: $TRACEBLOC_HELM_REPO_URL"
+    $addOutput = (helm repo add $TRACEBLOC_HELM_REPO_NAME $TRACEBLOC_HELM_REPO_URL --force-update 2>&1) | Out-String
+    Log "helm repo add: $addOutput"
+    if ($LASTEXITCODE -ne 0) { Err "Couldn't add the tracebloc chart repo ($TRACEBLOC_HELM_REPO_URL)." $addOutput }
+  }
 
   Write-Host ""
   if ($adoptedReuse) {
     # Surgical reconcile of the LIVE release: --reuse-values preserves the
     # deployed configuration + secret; only clientId is healed (#397 r2).
     Log "Reconciling release '$existingName' in namespace '$existingNs' (adopted; --reuse-values; healing clientId)..."
-    $helmOutput = (helm upgrade $existingName "$TRACEBLOC_HELM_REPO_NAME/$TRACEBLOC_CHART_NAME" `
+    $helmOutput = (helm upgrade $existingName $chartRef `
       --namespace $existingNs `
       --reuse-values `
       --set-string "clientId=$TB_CLIENT_ID" 2>&1) | Out-String
@@ -3663,8 +3678,8 @@ $envBlock
       Set-Content -Path $valuesFile -Value $vals -Encoding UTF8
     }
   } else {
-    Log "Installing $TB_NAMESPACE from $TRACEBLOC_HELM_REPO_NAME/$TRACEBLOC_CHART_NAME in namespace '$TB_NAMESPACE'..."
-    $helmOutput = (helm upgrade --install $TB_NAMESPACE "$TRACEBLOC_HELM_REPO_NAME/$TRACEBLOC_CHART_NAME" `
+    Log "Installing $TB_NAMESPACE from $chartRef in namespace '$TB_NAMESPACE'..."
+    $helmOutput = (helm upgrade --install $TB_NAMESPACE $chartRef `
       --namespace $TB_NAMESPACE `
       --create-namespace `
       --values $valuesFile 2>&1) | Out-String
diff --git a/scripts/manifest.sha256 b/scripts/manifest.sha256
index 9cb888ca..97d8f4b5 100644
--- a/scripts/manifest.sha256
+++ b/scripts/manifest.sha256
@@ -15,4 +15,4 @@ e373403d7bb5ce3728b8d21af89e6bf672cc35bbf8938541eb527ae19cb9473b  scripts/lib/as
 911fd0714b17357bb205fc8a8fa8e13eedc1a9632a2f63d4ead9f8d8c7ee546f  scripts/lib/probe.sh
 38761a6c56dc85b3f5742df036e6a2ec2baa0adb0c90b3753b6706779528b7be  scripts/lib/summary.sh
 77e03332ebfab1ef759c6148a57afcf479c02c5dc6cc7b0e0e680f58e20cd364  scripts/lib/diagnose.sh
-c758a024dbfabcdfd0b45959d1a219ca978b18e06be58f5261d11a26ea426585  scripts/install-k8s.ps1
+eba65b40cd98ec67c0bdd8a34ac48d50c5845be0e4ffa08549fd8fbb9d8cd205  scripts/install-k8s.ps1
diff --git a/scripts/tests/install-k8s.Tests.ps1 b/scripts/tests/install-k8s.Tests.ps1
index a1a7f788..12a3288a 100644
--- a/scripts/tests/install-k8s.Tests.ps1
+++ b/scripts/tests/install-k8s.Tests.ps1
@@ -3484,3 +3484,22 @@ Describe "Cluster-create exit-code reliability (#611)" {
     $script:CEC | Should -Match 'if \(\$null -eq \$k3dExitCode\)[\s\S]{0,120}k3dStdout[\s\S]{0,20}k3dStderr[\s\S]{0,40}created successfully'
   }
 }
+
+Describe "Local chart path support (#611 — Windows/bash parity)" {
+  BeforeAll { $script:LCP = Get-Content "$PSScriptRoot/../install-k8s.ps1" -Raw }
+
+  It "uses TRACEBLOC_CHART_PATH as the chart ref when set (test an unreleased chart)" {
+    $script:LCP | Should -Match 'if \(\$env:TRACEBLOC_CHART_PATH\)'
+    $script:LCP | Should -Match '\$chartRef = \$env:TRACEBLOC_CHART_PATH'
+  }
+  It "installs from `$chartRef, not a hardcoded repo path" {
+    $script:LCP | Should -Match 'helm upgrade --install \$TB_NAMESPACE \$chartRef'
+    $script:LCP | Should -Match 'helm upgrade \$existingName \$chartRef'
+  }
+  It "skips 'helm repo add' when a local chart path is given (it's in the else branch)" {
+    $script:LCP | Should -Match '\$chartRef = "\$TRACEBLOC_HELM_REPO_NAME/\$TRACEBLOC_CHART_NAME"[\s\S]{0,140}helm repo add'
+  }
+  It "errors if TRACEBLOC_CHART_PATH is set but is not a directory" {
+    $script:LCP | Should -Match 'TRACEBLOC_CHART_PATH is set but is not a directory'
+  }
+}

From f46ce62c22e7d799f3c5f43f4bdcf466dd455a99 Mon Sep 17 00:00:00 2001
From: shujaat hasan <shujaat@tracebloc.io>
Date: Thu, 6 Aug 2026 10:49:57 +0200
Subject: [PATCH 07/10] fix(installer): make /data/logs writable for training &
 inference pods (#611)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

hostPath ignores fsGroup (kubernetes/kubernetes#138411), so /data/logs was
created root-owned and non-root training/inference pods hit
`PermissionError [Errno 13]` creating their per-run log dir
(`os.makedirs('/data/logs/<run>')`). The #611 init-container chowned
/data/shared but not /data/logs — the same class of bug on the logs volume.

Extend the init (renamed init-shared-data -> init-writable-data) to
chown+chmod BOTH hostPath volumes and mount both. Training and inference
pods share one spec (job.yaml), so this covers both; ingestion already
covered by the /data/shared chmod. Bump chart 1.9.16 -> 1.9.17.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 client/Chart.yaml                             |  4 ++--
 client/templates/jobs-manager-deployment.yaml | 21 +++++++++-------
 client/tests/jobs_manager_test.yaml           | 24 ++++++++++++++-----
 3 files changed, 33 insertions(+), 16 deletions(-)

diff --git a/client/Chart.yaml b/client/Chart.yaml
index 9c879548..f18dff2f 100644
--- a/client/Chart.yaml
+++ b/client/Chart.yaml
@@ -2,8 +2,8 @@ apiVersion: v2
 name: client
 description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift
 type: application
-version: 1.9.16
-appVersion: "1.9.16"
+version: 1.9.17
+appVersion: "1.9.17"
 keywords:
   - tracebloc
   - kubernetes
diff --git a/client/templates/jobs-manager-deployment.yaml b/client/templates/jobs-manager-deployment.yaml
index de1fe0d0..fb39c69b 100644
--- a/client/templates/jobs-manager-deployment.yaml
+++ b/client/templates/jobs-manager-deployment.yaml
@@ -44,17 +44,20 @@ spec:
           type: RuntimeDefault
       {{- if .Values.hostPath.enabled }}
       # kubelet does NOT apply fsGroup to hostPath volumes (kubernetes/kubernetes#138411),
-      # so /data/shared is created root-owned and non-root pods can't write to it. That
-      # breaks dataset ingest: `tb data ingest` streams files into a staging pod that does
-      # `mkdir /data/shared/.tracebloc-staging/` and hits "Permission denied" (#611). Unlike
-      # mysql-data (one writer, UID 999), the shared volume has MULTIPLE non-root writers —
+      # so /data/shared AND /data/logs are created root-owned and non-root pods can't write
+      # to them. Two failures this causes: (1) dataset ingest — `tb data ingest` streams files
+      # into a staging pod that does `mkdir /data/shared/.tracebloc-staging/` (#611); and
+      # (2) training — a spawned training pod does `os.makedirs('/data/logs/<run>')` and hits
+      # "Permission denied". Both are the SAME hostPath dirs those spawned pods mount
+      # (client-pvc / client-logs-pvc), so fixing them once here reaches those pods too.
+      # Unlike mysql-data (one writer, UID 999), these have MULTIPLE non-root writers —
       # jobs-manager, the training/ingestor pods it spawns, and the CLI's ingest-staging pod,
-      # whose UID this chart does not control — so it must be world-writable, not chowned to
+      # whose UID this chart does not control — so they must be world-writable, not chowned to
       # a single UID. setgid (2) makes new files inherit GID 1000; runs as root only long
-      # enough to fix the mount, with FOWNER so the chmod is idempotent on re-install. CSI
+      # enough to fix the mounts, with FOWNER so the chmod is idempotent on re-install. CSI
       # clusters rely on fsGroup above and skip this.
       initContainers:
-      - name: init-shared-data
+      - name: init-writable-data
         image: {{ include "tracebloc.image" (dict "repository" "library/busybox" "tag" .Values.images.busybox.tag "digest" .Values.images.busybox.digest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }}
         securityContext:
           runAsUser: 0
@@ -66,10 +69,12 @@ spec:
             add: ["CHOWN", "FOWNER"]
           seccompProfile:
             type: RuntimeDefault
-        command: ['sh', '-c', 'chown 1000:1000 /data/shared && chmod 2777 /data/shared']
+        command: ['sh', '-c', 'chown 1000:1000 /data/shared /data/logs && chmod 2777 /data/shared /data/logs']
         volumeMounts:
         - name: shared-volume
           mountPath: /data/shared
+        - name: logs-volume
+          mountPath: /data/logs
       {{- end }}
       containers:
       - name: api
diff --git a/client/tests/jobs_manager_test.yaml b/client/tests/jobs_manager_test.yaml
index c0312a84..d794aebf 100644
--- a/client/tests/jobs_manager_test.yaml
+++ b/client/tests/jobs_manager_test.yaml
@@ -415,10 +415,11 @@ tests:
                 name: RELEASE-NAME-secrets
                 key: TB_CREDMGR_PASSWORD
 
-  # #611: /data/shared (client-pvc) must be writable by the non-root ingest-staging
-  # pod. hostPath ignores fsGroup (k8s#138411), so a privileged init makes the shared
-  # volume world-writable; CSI relies on fsGroup and skips the init.
-  - it: "hostPath install adds init-shared-data to make /data/shared writable for ingest (#611)"
+  # #611: /data/shared (client-pvc) AND /data/logs (client-logs-pvc) must be writable by
+  # the non-root pods that mount them — the ingest-staging pod writes to /data/shared, the
+  # spawned training pods write to /data/logs. hostPath ignores fsGroup (k8s#138411), so a
+  # privileged init makes both volumes world-writable; CSI relies on fsGroup and skips it.
+  - it: "hostPath install adds init-writable-data to make /data/shared and /data/logs writable (#611)"
     set:
       hostPath:
         enabled: true
@@ -428,7 +429,7 @@ tests:
           value: 1000
       - equal:
           path: spec.template.spec.initContainers[0].name
-          value: init-shared-data
+          value: init-writable-data
       - equal:
           path: spec.template.spec.initContainers[0].securityContext.runAsUser
           value: 0
@@ -440,7 +441,18 @@ tests:
           content: FOWNER
       - matchRegex:
           path: spec.template.spec.initContainers[0].command[2]
-          pattern: "chown 1000:1000 /data/shared && chmod 2777 /data/shared"
+          pattern: "chown 1000:1000 /data/shared /data/logs && chmod 2777 /data/shared /data/logs"
+      # the init must mount BOTH hostPath volumes it chowns, or the chown is a no-op
+      - contains:
+          path: spec.template.spec.initContainers[0].volumeMounts
+          content:
+            name: shared-volume
+            mountPath: /data/shared
+      - contains:
+          path: spec.template.spec.initContainers[0].volumeMounts
+          content:
+            name: logs-volume
+            mountPath: /data/logs
 
   - it: "CSI install (hostPath disabled) keeps fsGroup but skips the privileged init (#611)"
     set:

From 9fe357eaa8cbfff1af2ec07e782db4e6b932c15e Mon Sep 17 00:00:00 2001
From: shujaat hasan <shujaat@tracebloc.io>
Date: Thu, 6 Aug 2026 11:03:27 +0200
Subject: [PATCH 08/10] =?UTF-8?q?fix(installer):=20address=20review=20?=
 =?UTF-8?q?=E2=80=94=20sticky=20bit=20on=20shared=20dirs=20+=20reset-then-?=
 =?UTF-8?q?reuse=20parity?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Two reviewer follow-ups on #612:

- chart: chmod the writable hostPath dirs 3777 (was 2777) — add the sticky
  bit so one writer can't unlink/rename another writer's files in
  /data/shared // /data/logs (/tmp semantics). setgid is retained. Safe given
  the uid topology (dir owned by 1000; training pods run as 1000; the ingestor
  writes its own subtrees as a stable uid) and no cross-uid filesystem deletes
  exist in client-runtime. Chart 1.9.17 -> 1.9.18.

- install-k8s.ps1: the adopt/reconcile helm upgrade now prefers
  --reset-then-reuse-values when `helm upgrade --help` advertises it (Helm
  >= 3.14), falling back to --reuse-values otherwise — so NEW chart defaults
  reach adopted Windows edges on auto-upgrade (bash parity with
  install-client-helm.sh). manifest.sha256 regenerated.

Tests: helm-unittest updated (3777); new Pester test asserts the
reset-then-reuse preference; full Pester suite 452 pass.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 client/Chart.yaml                             |  4 ++--
 client/templates/jobs-manager-deployment.yaml |  7 ++++---
 client/tests/jobs_manager_test.yaml           |  2 +-
 scripts/install-k8s.ps1                       | 16 ++++++++++++----
 scripts/manifest.sha256                       |  2 +-
 scripts/tests/install-k8s.Tests.ps1           | 19 +++++++++++++++++++
 6 files changed, 39 insertions(+), 11 deletions(-)

diff --git a/client/Chart.yaml b/client/Chart.yaml
index f18dff2f..2ef999ba 100644
--- a/client/Chart.yaml
+++ b/client/Chart.yaml
@@ -2,8 +2,8 @@ apiVersion: v2
 name: client
 description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift
 type: application
-version: 1.9.17
-appVersion: "1.9.17"
+version: 1.9.18
+appVersion: "1.9.18"
 keywords:
   - tracebloc
   - kubernetes
diff --git a/client/templates/jobs-manager-deployment.yaml b/client/templates/jobs-manager-deployment.yaml
index fb39c69b..0c8422eb 100644
--- a/client/templates/jobs-manager-deployment.yaml
+++ b/client/templates/jobs-manager-deployment.yaml
@@ -53,8 +53,9 @@ spec:
       # Unlike mysql-data (one writer, UID 999), these have MULTIPLE non-root writers —
       # jobs-manager, the training/ingestor pods it spawns, and the CLI's ingest-staging pod,
       # whose UID this chart does not control — so they must be world-writable, not chowned to
-      # a single UID. setgid (2) makes new files inherit GID 1000; runs as root only long
-      # enough to fix the mounts, with FOWNER so the chmod is idempotent on re-install. CSI
+      # a single UID. Mode 3777 = setgid (2) so new files inherit GID 1000 + sticky (1) so one
+      # writer can't unlink or rename another writer's files (/tmp semantics); runs as root only
+      # long enough to fix the mounts, with FOWNER so the chmod is idempotent on re-install. CSI
       # clusters rely on fsGroup above and skip this.
       initContainers:
       - name: init-writable-data
@@ -69,7 +70,7 @@ spec:
             add: ["CHOWN", "FOWNER"]
           seccompProfile:
             type: RuntimeDefault
-        command: ['sh', '-c', 'chown 1000:1000 /data/shared /data/logs && chmod 2777 /data/shared /data/logs']
+        command: ['sh', '-c', 'chown 1000:1000 /data/shared /data/logs && chmod 3777 /data/shared /data/logs']
         volumeMounts:
         - name: shared-volume
           mountPath: /data/shared
diff --git a/client/tests/jobs_manager_test.yaml b/client/tests/jobs_manager_test.yaml
index d794aebf..87784117 100644
--- a/client/tests/jobs_manager_test.yaml
+++ b/client/tests/jobs_manager_test.yaml
@@ -441,7 +441,7 @@ tests:
           content: FOWNER
       - matchRegex:
           path: spec.template.spec.initContainers[0].command[2]
-          pattern: "chown 1000:1000 /data/shared /data/logs && chmod 2777 /data/shared /data/logs"
+          pattern: "chown 1000:1000 /data/shared /data/logs && chmod 3777 /data/shared /data/logs"
       # the init must mount BOTH hostPath volumes it chowns, or the chown is a no-op
       - contains:
           path: spec.template.spec.initContainers[0].volumeMounts
diff --git a/scripts/install-k8s.ps1 b/scripts/install-k8s.ps1
index 0ff300ed..7d72eaf2 100644
--- a/scripts/install-k8s.ps1
+++ b/scripts/install-k8s.ps1
@@ -3661,12 +3661,20 @@ $envBlock
 
   Write-Host ""
   if ($adoptedReuse) {
-    # Surgical reconcile of the LIVE release: --reuse-values preserves the
-    # deployed configuration + secret; only clientId is healed (#397 r2).
-    Log "Reconciling release '$existingName' in namespace '$existingNs' (adopted; --reuse-values; healing clientId)..."
+    # Surgical reconcile of the LIVE release: preserve the deployed configuration +
+    # secret; only clientId is healed (#397 r2). Prefer --reset-then-reuse-values
+    # (Helm >= 3.14: reset to chart defaults, then re-apply the user's overrides, so
+    # NEW chart defaults reach adopted edges on auto-upgrade) over --reuse-values
+    # (keeps only stored values, so new chart defaults never land); feature-detect via
+    # --help and fall back on older Helm (bash parity: install-client-helm.sh).
+    $reuseFlag = "--reuse-values"
+    if ((helm upgrade --help 2>$null | Out-String) -match '--reset-then-reuse-values') {
+      $reuseFlag = "--reset-then-reuse-values"
+    }
+    Log "Reconciling release '$existingName' in namespace '$existingNs' (adopted; $reuseFlag; healing clientId)..."
     $helmOutput = (helm upgrade $existingName $chartRef `
       --namespace $existingNs `
-      --reuse-values `
+      $reuseFlag `
       --set-string "clientId=$TB_CLIENT_ID" 2>&1) | Out-String
     Log "Helm Output: $helmOutput"
     if ($LASTEXITCODE -ne 0) { Err "Client reconcile failed." $helmOutput }
diff --git a/scripts/manifest.sha256 b/scripts/manifest.sha256
index 97d8f4b5..e855e585 100644
--- a/scripts/manifest.sha256
+++ b/scripts/manifest.sha256
@@ -15,4 +15,4 @@ e373403d7bb5ce3728b8d21af89e6bf672cc35bbf8938541eb527ae19cb9473b  scripts/lib/as
 911fd0714b17357bb205fc8a8fa8e13eedc1a9632a2f63d4ead9f8d8c7ee546f  scripts/lib/probe.sh
 38761a6c56dc85b3f5742df036e6a2ec2baa0adb0c90b3753b6706779528b7be  scripts/lib/summary.sh
 77e03332ebfab1ef759c6148a57afcf479c02c5dc6cc7b0e0e680f58e20cd364  scripts/lib/diagnose.sh
-eba65b40cd98ec67c0bdd8a34ac48d50c5845be0e4ffa08549fd8fbb9d8cd205  scripts/install-k8s.ps1
+35745834c814b03950f6e6a36d3df40fadec51f8f9fe743b6d647a8eedd3679c  scripts/install-k8s.ps1
diff --git a/scripts/tests/install-k8s.Tests.ps1 b/scripts/tests/install-k8s.Tests.ps1
index 12a3288a..b88cf79d 100644
--- a/scripts/tests/install-k8s.Tests.ps1
+++ b/scripts/tests/install-k8s.Tests.ps1
@@ -988,6 +988,25 @@ Describe "Install-ClientHelm" {
     Install-ClientHelm
     Should -Invoke helm -ParameterFilter { ($args -contains "upgrade") -and ($args -contains "--reuse-values") }
   }
+  It "adopted mode prefers --reset-then-reuse-values when Helm >= 3.14 exposes it (bash parity: new chart defaults reach adopted edges)" {
+    # When `helm upgrade --help` advertises --reset-then-reuse-values (Helm >= 3.14),
+    # the reconcile must use it so NEW chart defaults land on adopted Windows edges on
+    # auto-upgrade — not stay pinned to stored values as plain --reuse-values would.
+    $HOST_DATA_DIR = "$TestDrive/d-adopt-reset"
+    $script:TB_PROV_MODE = "adopted"; $script:TB_PROV_ID = "uuid-9"; $script:TB_PROV_NS = "lukas-01"
+    Mock helm {
+      if (($args -contains "upgrade") -and ($args -contains "--help")) { "      --reset-then-reuse-values   reset then reuse"; $global:LASTEXITCODE = 0; return }
+      if ($args -contains "list") { '[{"name":"oldrel","namespace":"lukas-01","chart":"client-1.4.3"}]'; $global:LASTEXITCODE = 0; return }
+      if ($args -contains "get") {
+        if ($args -contains "json") { '{"clientId":"uuid-9"}' } else { 'clientId: uuid-9' }
+        $global:LASTEXITCODE = 0; return
+      }
+      $global:LASTEXITCODE = 0
+    }
+    Install-ClientHelm
+    Should -Invoke helm -ParameterFilter { ($args -contains "upgrade") -and ($args -contains "--reset-then-reuse-values") }
+    Should -Not -Invoke helm -ParameterFilter { ($args -contains "upgrade") -and ($args -contains "--reuse-values") }
+  }
   It "a DIFFERENT existing client still refuses outside adopted mode (guard intact)" {
     $HOST_DATA_DIR = "$TestDrive/d-guard-minted"
     $script:TB_PROV_MODE = "minted"; $script:TB_PROV_ID = "uuid-new"

From 15e0f37872819c57f76729fdfb4a11acda04f3d9 Mon Sep 17 00:00:00 2001
From: shujaat hasan <shujaat@tracebloc.io>
Date: Thu, 6 Aug 2026 11:07:59 +0200
Subject: [PATCH 09/10] fix(installer): add CAP_FSETID so the init's setgid bit
 survives (Bugbot)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

init-writable-data chowns the shared/logs hostPath dirs to GID 1000 then
chmods 3777. With caps dropped to CHOWN+FOWNER only, the kernel silently
strips S_ISGID on the chmod — after the chown the dir's group no longer
matches the process (fsgid 0), and a root process without CAP_FSETID can't
set setgid on it — so the mount landed at 1777 and new files did NOT inherit
GID 1000 as documented. Add FSETID to the cap set; setgid now sticks.
helm-unittest asserts FSETID present. Chart 1.9.18 -> 1.9.19.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 client/Chart.yaml                             | 4 ++--
 client/templates/jobs-manager-deployment.yaml | 9 ++++++---
 client/tests/jobs_manager_test.yaml           | 5 +++++
 3 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/client/Chart.yaml b/client/Chart.yaml
index 2ef999ba..0e262262 100644
--- a/client/Chart.yaml
+++ b/client/Chart.yaml
@@ -2,8 +2,8 @@ apiVersion: v2
 name: client
 description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift
 type: application
-version: 1.9.18
-appVersion: "1.9.18"
+version: 1.9.19
+appVersion: "1.9.19"
 keywords:
   - tracebloc
   - kubernetes
diff --git a/client/templates/jobs-manager-deployment.yaml b/client/templates/jobs-manager-deployment.yaml
index 0c8422eb..f8584dfc 100644
--- a/client/templates/jobs-manager-deployment.yaml
+++ b/client/templates/jobs-manager-deployment.yaml
@@ -55,8 +55,11 @@ spec:
       # whose UID this chart does not control — so they must be world-writable, not chowned to
       # a single UID. Mode 3777 = setgid (2) so new files inherit GID 1000 + sticky (1) so one
       # writer can't unlink or rename another writer's files (/tmp semantics); runs as root only
-      # long enough to fix the mounts, with FOWNER so the chmod is idempotent on re-install. CSI
-      # clusters rely on fsGroup above and skip this.
+      # long enough to fix the mounts. Caps: CHOWN for the chown; FOWNER so the chmod is idempotent
+      # on re-install; FSETID so the setgid bit survives the chmod — after the chown to GID 1000 the
+      # dir's group no longer matches the process (fsgid 0), and a root process without FSETID has
+      # the kernel silently strip S_ISGID, landing the mount at 1777 (setgid lost). CSI clusters
+      # rely on fsGroup above and skip this.
       initContainers:
       - name: init-writable-data
         image: {{ include "tracebloc.image" (dict "repository" "library/busybox" "tag" .Values.images.busybox.tag "digest" .Values.images.busybox.digest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }}
@@ -67,7 +70,7 @@ spec:
           readOnlyRootFilesystem: true
           capabilities:
             drop: ["ALL"]
-            add: ["CHOWN", "FOWNER"]
+            add: ["CHOWN", "FOWNER", "FSETID"]
           seccompProfile:
             type: RuntimeDefault
         command: ['sh', '-c', 'chown 1000:1000 /data/shared /data/logs && chmod 3777 /data/shared /data/logs']
diff --git a/client/tests/jobs_manager_test.yaml b/client/tests/jobs_manager_test.yaml
index 87784117..51e57d48 100644
--- a/client/tests/jobs_manager_test.yaml
+++ b/client/tests/jobs_manager_test.yaml
@@ -439,6 +439,11 @@ tests:
       - contains:
           path: spec.template.spec.initContainers[0].securityContext.capabilities.add
           content: FOWNER
+      # FSETID is required or the kernel strips the setgid bit when chmod runs after
+      # the chown to GID 1000 (root without FSETID can't set setgid on a non-owned-group dir)
+      - contains:
+          path: spec.template.spec.initContainers[0].securityContext.capabilities.add
+          content: FSETID
       - matchRegex:
           path: spec.template.spec.initContainers[0].command[2]
           pattern: "chown 1000:1000 /data/shared /data/logs && chmod 3777 /data/shared /data/logs"

From 3a7f940121b2dcf38e70a4eef55ccede5e423592 Mon Sep 17 00:00:00 2001
From: shujaat hasan <shujaat@tracebloc.io>
Date: Thu, 6 Aug 2026 11:25:40 +0200
Subject: [PATCH 10/10] fix(installer): drop risky fsGroup + make init
 per-volume best-effort (Bugbot)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Two Bugbot findings on the writable-volume fix:

- HIGH — remove `fsGroup: 1000` / `fsGroupChangePolicy` from jobs-manager. It is
  a no-op on hostPath (kubelet ignores fsGroup — the init does the work) and on
  CSI it only grants jobs-manager's OWN processes GID 1000 while its
  OnRootMismatch relabel flips the shared/logs volumes to group 1000 — stripping
  the group-0 access the spawned training pods (UID 1001 / OpenShift arbitrary
  UID, GID 0) and the host-UID ingestion pods rely on (docs/SECURITY.md §5.3). It
  never reaches those spawned writers, so it was all regression risk and no gain.
  Those pods keep their own documented posture; CSI is untouched (matches develop).

- MEDIUM — the init now fixes each dir INDEPENDENTLY and best-effort:
  `for d in /data/shared /data/logs; do chown && chmod || echo <warn>; done`. A
  chown that can't complete (e.g. /data/shared on an NFS root_squash export) no
  longer aborts the chain and skips /data/logs — the other dir is still repaired
  and jobs-manager still starts; a truly unwritable mount surfaces as a clear
  error at the writer pod instead of wedging the edge in Init.

helm-unittest updated (no fsGroup on either path; per-dir loop; CSI skips init).
Chart 1.9.19 -> 1.9.20.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 client/Chart.yaml                             |  4 ++--
 client/templates/jobs-manager-deployment.yaml | 21 ++++++++++++-------
 client/tests/jobs_manager_test.yaml           | 20 +++++++++++-------
 3 files changed, 28 insertions(+), 17 deletions(-)

diff --git a/client/Chart.yaml b/client/Chart.yaml
index 0e262262..b4107817 100644
--- a/client/Chart.yaml
+++ b/client/Chart.yaml
@@ -2,8 +2,8 @@ apiVersion: v2
 name: client
 description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift
 type: application
-version: 1.9.19
-appVersion: "1.9.19"
+version: 1.9.20
+appVersion: "1.9.20"
 keywords:
   - tracebloc
   - kubernetes
diff --git a/client/templates/jobs-manager-deployment.yaml b/client/templates/jobs-manager-deployment.yaml
index f8584dfc..c5ef49d0 100644
--- a/client/templates/jobs-manager-deployment.yaml
+++ b/client/templates/jobs-manager-deployment.yaml
@@ -35,11 +35,13 @@ spec:
       serviceAccountName: {{ include "tracebloc.serviceAccountName" . }}
       securityContext:
         runAsNonRoot: true
-        # CSI-backed clusters (EKS/AKS/OC) apply this to the shared data volume so
-        # every non-root writer shares GID 1000; hostPath ignores it (see the init
-        # below). fsGroupChangePolicy limits the relabel to a root-owned mount.
-        fsGroup: 1000
-        fsGroupChangePolicy: "OnRootMismatch"
+        # No fsGroup here (deliberately). On hostPath kubelet ignores it
+        # (kubernetes/kubernetes#138411) — the init below does the work. On CSI an fsGroup
+        # relabel would only add GID 1000 to jobs-manager's OWN processes, yet it would flip
+        # the shared/logs volumes' group to 1000 (OnRootMismatch, first restart) — stripping
+        # the group-0 access the spawned training pods (UID 1001 / OpenShift arbitrary-UID,
+        # GID 0) and the host-UID ingestion pods actually rely on (see docs/SECURITY.md §5.3).
+        # It never reaches those spawned writers, so it's all risk and no gain.
         seccompProfile:
           type: RuntimeDefault
       {{- if .Values.hostPath.enabled }}
@@ -58,8 +60,11 @@ spec:
       # long enough to fix the mounts. Caps: CHOWN for the chown; FOWNER so the chmod is idempotent
       # on re-install; FSETID so the setgid bit survives the chmod — after the chown to GID 1000 the
       # dir's group no longer matches the process (fsgid 0), and a root process without FSETID has
-      # the kernel silently strip S_ISGID, landing the mount at 1777 (setgid lost). CSI clusters
-      # rely on fsGroup above and skip this.
+      # the kernel silently strip S_ISGID, landing the mount at 1777 (setgid lost). Each dir is
+      # fixed INDEPENDENTLY and best-effort: a chown/chmod that can't complete (e.g. /data/shared on
+      # an NFS root_squash export via HOST_DATASET_DIR) is logged and skipped so the OTHER dir is
+      # still repaired and jobs-manager still starts — a genuinely unwritable mount then surfaces as
+      # a clear error at the writer pod rather than wedging the whole edge in Init. CSI skips this.
       initContainers:
       - name: init-writable-data
         image: {{ include "tracebloc.image" (dict "repository" "library/busybox" "tag" .Values.images.busybox.tag "digest" .Values.images.busybox.digest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }}
@@ -73,7 +78,7 @@ spec:
             add: ["CHOWN", "FOWNER", "FSETID"]
           seccompProfile:
             type: RuntimeDefault
-        command: ['sh', '-c', 'chown 1000:1000 /data/shared /data/logs && chmod 3777 /data/shared /data/logs']
+        command: ['sh', '-c', 'for d in /data/shared /data/logs; do chown 1000:1000 "$d" && chmod 3777 "$d" || echo "init-writable-data: could not adjust $d (pre-provisioned or root_squash mount?); leaving as-is"; done']
         volumeMounts:
         - name: shared-volume
           mountPath: /data/shared
diff --git a/client/tests/jobs_manager_test.yaml b/client/tests/jobs_manager_test.yaml
index 51e57d48..09dc4b30 100644
--- a/client/tests/jobs_manager_test.yaml
+++ b/client/tests/jobs_manager_test.yaml
@@ -418,15 +418,16 @@ tests:
   # #611: /data/shared (client-pvc) AND /data/logs (client-logs-pvc) must be writable by
   # the non-root pods that mount them — the ingest-staging pod writes to /data/shared, the
   # spawned training pods write to /data/logs. hostPath ignores fsGroup (k8s#138411), so a
-  # privileged init makes both volumes world-writable; CSI relies on fsGroup and skips it.
+  # privileged init makes both volumes world-writable. No fsGroup is set: it's a no-op on
+  # hostPath and on CSI would strip the spawned pods' group-0 access; CSI skips the init.
   - it: "hostPath install adds init-writable-data to make /data/shared and /data/logs writable (#611)"
     set:
       hostPath:
         enabled: true
     asserts:
-      - equal:
+      # no fsGroup — no-op on hostPath, and a regression risk on CSI (see template comment)
+      - notExists:
           path: spec.template.spec.securityContext.fsGroup
-          value: 1000
       - equal:
           path: spec.template.spec.initContainers[0].name
           value: init-writable-data
@@ -444,9 +445,15 @@ tests:
       - contains:
           path: spec.template.spec.initContainers[0].securityContext.capabilities.add
           content: FSETID
+      # each dir is fixed INDEPENDENTLY (a for-loop): chown 1000:1000 then chmod 3777
+      - matchRegex:
+          path: spec.template.spec.initContainers[0].command[2]
+          pattern: "for d in /data/shared /data/logs.*chown 1000:1000.*chmod 3777"
+      # best-effort: a failing chown/chmod is logged, not fatal, so the other dir is still fixed
+      # and jobs-manager still starts (e.g. /data/shared on an NFS root_squash export)
       - matchRegex:
           path: spec.template.spec.initContainers[0].command[2]
-          pattern: "chown 1000:1000 /data/shared /data/logs && chmod 3777 /data/shared /data/logs"
+          pattern: "\\|\\| echo"
       # the init must mount BOTH hostPath volumes it chowns, or the chown is a no-op
       - contains:
           path: spec.template.spec.initContainers[0].volumeMounts
@@ -459,13 +466,12 @@ tests:
             name: logs-volume
             mountPath: /data/logs
 
-  - it: "CSI install (hostPath disabled) keeps fsGroup but skips the privileged init (#611)"
+  - it: "CSI install (hostPath disabled) skips the privileged init and sets no fsGroup (#611)"
     set:
       hostPath:
         enabled: false
     asserts:
-      - equal:
+      - notExists:
           path: spec.template.spec.securityContext.fsGroup
-          value: 1000
       - notExists:
           path: spec.template.spec.initContainers