diff --git a/client/Chart.yaml b/client/Chart.yaml index 672adbd6..b4107817 100644 --- a/client/Chart.yaml +++ b/client/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: client description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift type: application -version: 1.9.15 -appVersion: "1.9.15" +version: 1.9.20 +appVersion: "1.9.20" keywords: - tracebloc - kubernetes diff --git a/client/templates/jobs-manager-deployment.yaml b/client/templates/jobs-manager-deployment.yaml index 71ca1db3..c5ef49d0 100644 --- a/client/templates/jobs-manager-deployment.yaml +++ b/client/templates/jobs-manager-deployment.yaml @@ -35,8 +35,56 @@ spec: serviceAccountName: {{ include "tracebloc.serviceAccountName" . }} securityContext: runAsNonRoot: true + # No fsGroup here (deliberately). On hostPath kubelet ignores it + # (kubernetes/kubernetes#138411) — the init below does the work. On CSI an fsGroup + # relabel would only add GID 1000 to jobs-manager's OWN processes, yet it would flip + # the shared/logs volumes' group to 1000 (OnRootMismatch, first restart) — stripping + # the group-0 access the spawned training pods (UID 1001 / OpenShift arbitrary-UID, + # GID 0) and the host-UID ingestion pods actually rely on (see docs/SECURITY.md §5.3). + # It never reaches those spawned writers, so it's all risk and no gain. seccompProfile: type: RuntimeDefault + {{- if .Values.hostPath.enabled }} + # kubelet does NOT apply fsGroup to hostPath volumes (kubernetes/kubernetes#138411), + # so /data/shared AND /data/logs are created root-owned and non-root pods can't write + # to them. Two failures this causes: (1) dataset ingest — `tb data ingest` streams files + # into a staging pod that does `mkdir /data/shared/.tracebloc-staging/` (#611); and + # (2) training — a spawned training pod does `os.makedirs('/data/logs/')` and hits + # "Permission denied". Both are the SAME hostPath dirs those spawned pods mount + # (client-pvc / client-logs-pvc), so fixing them once here reaches those pods too. + # Unlike mysql-data (one writer, UID 999), these have MULTIPLE non-root writers — + # jobs-manager, the training/ingestor pods it spawns, and the CLI's ingest-staging pod, + # whose UID this chart does not control — so they must be world-writable, not chowned to + # a single UID. Mode 3777 = setgid (2) so new files inherit GID 1000 + sticky (1) so one + # writer can't unlink or rename another writer's files (/tmp semantics); runs as root only + # long enough to fix the mounts. Caps: CHOWN for the chown; FOWNER so the chmod is idempotent + # on re-install; FSETID so the setgid bit survives the chmod — after the chown to GID 1000 the + # dir's group no longer matches the process (fsgid 0), and a root process without FSETID has + # the kernel silently strip S_ISGID, landing the mount at 1777 (setgid lost). Each dir is + # fixed INDEPENDENTLY and best-effort: a chown/chmod that can't complete (e.g. /data/shared on + # an NFS root_squash export via HOST_DATASET_DIR) is logged and skipped so the OTHER dir is + # still repaired and jobs-manager still starts — a genuinely unwritable mount then surfaces as + # a clear error at the writer pod rather than wedging the whole edge in Init. CSI skips this. + initContainers: + - name: init-writable-data + image: {{ include "tracebloc.image" (dict "repository" "library/busybox" "tag" .Values.images.busybox.tag "digest" .Values.images.busybox.digest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }} + securityContext: + runAsUser: 0 + runAsNonRoot: false + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + add: ["CHOWN", "FOWNER", "FSETID"] + seccompProfile: + type: RuntimeDefault + command: ['sh', '-c', 'for d in /data/shared /data/logs; do chown 1000:1000 "$d" && chmod 3777 "$d" || echo "init-writable-data: could not adjust $d (pre-provisioned or root_squash mount?); leaving as-is"; done'] + volumeMounts: + - name: shared-volume + mountPath: /data/shared + - name: logs-volume + mountPath: /data/logs + {{- end }} containers: - name: api image: {{ include "tracebloc.image" (dict "repository" "tracebloc/jobs-manager" "tag" .Values.env.CLIENT_ENV "digest" .Values.images.jobsManager.digest "registry" (dig "imageRegistry" "docker.io" (.Values.global | default dict))) | quote }} diff --git a/client/tests/jobs_manager_test.yaml b/client/tests/jobs_manager_test.yaml index fe0991a9..09dc4b30 100644 --- a/client/tests/jobs_manager_test.yaml +++ b/client/tests/jobs_manager_test.yaml @@ -414,3 +414,64 @@ tests: secretKeyRef: name: RELEASE-NAME-secrets key: TB_CREDMGR_PASSWORD + + # #611: /data/shared (client-pvc) AND /data/logs (client-logs-pvc) must be writable by + # the non-root pods that mount them — the ingest-staging pod writes to /data/shared, the + # spawned training pods write to /data/logs. hostPath ignores fsGroup (k8s#138411), so a + # privileged init makes both volumes world-writable. No fsGroup is set: it's a no-op on + # hostPath and on CSI would strip the spawned pods' group-0 access; CSI skips the init. + - it: "hostPath install adds init-writable-data to make /data/shared and /data/logs writable (#611)" + set: + hostPath: + enabled: true + asserts: + # no fsGroup — no-op on hostPath, and a regression risk on CSI (see template comment) + - notExists: + path: spec.template.spec.securityContext.fsGroup + - equal: + path: spec.template.spec.initContainers[0].name + value: init-writable-data + - equal: + path: spec.template.spec.initContainers[0].securityContext.runAsUser + value: 0 + - contains: + path: spec.template.spec.initContainers[0].securityContext.capabilities.add + content: CHOWN + - contains: + path: spec.template.spec.initContainers[0].securityContext.capabilities.add + content: FOWNER + # FSETID is required or the kernel strips the setgid bit when chmod runs after + # the chown to GID 1000 (root without FSETID can't set setgid on a non-owned-group dir) + - contains: + path: spec.template.spec.initContainers[0].securityContext.capabilities.add + content: FSETID + # each dir is fixed INDEPENDENTLY (a for-loop): chown 1000:1000 then chmod 3777 + - matchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: "for d in /data/shared /data/logs.*chown 1000:1000.*chmod 3777" + # best-effort: a failing chown/chmod is logged, not fatal, so the other dir is still fixed + # and jobs-manager still starts (e.g. /data/shared on an NFS root_squash export) + - matchRegex: + path: spec.template.spec.initContainers[0].command[2] + pattern: "\\|\\| echo" + # the init must mount BOTH hostPath volumes it chowns, or the chown is a no-op + - contains: + path: spec.template.spec.initContainers[0].volumeMounts + content: + name: shared-volume + mountPath: /data/shared + - contains: + path: spec.template.spec.initContainers[0].volumeMounts + content: + name: logs-volume + mountPath: /data/logs + + - it: "CSI install (hostPath disabled) skips the privileged init and sets no fsGroup (#611)" + set: + hostPath: + enabled: false + asserts: + - notExists: + path: spec.template.spec.securityContext.fsGroup + - notExists: + path: spec.template.spec.initContainers diff --git a/scripts/install-k8s.ps1 b/scripts/install-k8s.ps1 index 2d58bb8a..7d72eaf2 100644 --- a/scripts/install-k8s.ps1 +++ b/scripts/install-k8s.ps1 @@ -225,6 +225,14 @@ function Wait-ProcessWithDeadline { Start-Sleep -Seconds 2 } Write-Host "`r `r" -NoNewline + # HasExited can flip true before the process's redirected stdout/stderr streams + # are fully drained, and in that window Start-Process -RedirectStandardOutput + # leaves $Process.ExitCode $null. Callers then read a null code and `$null -ne 0` + # misreads a SUCCESSFUL run as a failure -- the #611 field case: k3d printed + # "Cluster created successfully!" with empty stderr, yet the install aborted with + # the cluster actually up. WaitForExit() (bounded: the process has already exited) + # flushes the streams and guarantees ExitCode is populated for every caller. + try { $Process.WaitForExit() } catch {} return $true } @@ -464,6 +472,24 @@ function Get-VerifiedDownload { [Parameter(Mandatory)][string]$Dest, [int]$MinBytes = 1MB, [string]$Magic = '', + # When set, the CHECKSUM is the authoritative completeness test (#609): after a + # transport lands a size/magic-valid file, its SHA-256 must equal $Sha256 or the + # transport is treated as failed and the NEXT one is tried. A size floor alone + # lets a mid-transfer truncation (>MinBytes, still starts with the magic bytes) + # slip through and dead-end at a downstream checksum with no retry -- the real + # field failure. With $Sha256, a truncated/corrupt copy just triggers curl.exe/ + # BITS until a byte-correct copy lands. + [string]$Sha256 = '', + # When set, the downloaded TEXT must MATCH this regex or the transport is treated + # as failed and the next one (curl.exe/BITS) is tried (#611). Used to gate the + # checksum-LIST files on the actual hash STRUCTURE, not a weak substring: an + # asset-name substring also appears in the request URL, so a proxy error page + # echoing the URL would satisfy a substring gate, "succeed" on the first + # transport, skip the retry, and then abort at the later hex parse (Bugbot). The + # call sites therefore require a 64-hex hash adjacent to the asset (k3d/helm) or + # anchored at the start of the body (kubectl's bare-hash .sha256) -- structure a + # proxy/HTML error page can't accidentally satisfy. + [string]$MatchPattern = '', [string]$Label = 'download', [string]$Message = 'Downloading' ) @@ -491,10 +517,22 @@ function Get-VerifiedDownload { # NEXT transport should be tried, not the whole download aborted (Bugbot). try { $bad = Test-DownloadComplete -Path $Dest -MinBytes $MinBytes -Magic $Magic + if (-not $bad -and $Sha256) { + $got = (Get-FileHash -LiteralPath $Dest -Algorithm SHA256).Hash.ToLower() + if ($got -ne $Sha256.ToLower()) { + $bad = "checksum mismatch (got $got) -- the download is truncated or altered" + } + } + if (-not $bad -and $MatchPattern) { + $text = Get-Content -LiteralPath $Dest -Raw -ErrorAction Stop + if ($text -notmatch $MatchPattern) { + $bad = "the file did not match the expected checksum pattern -- likely a proxy error page; trying another method" + } + } } catch { $bad = "could not read the downloaded file ($($_.Exception.Message)) -- it may be locked or quarantined by antivirus" } - if (-not $bad) { return } # complete + valid -- done + if (-not $bad) { return } # complete + (checksum/content) valid -- done $problems += "${name}: $bad" Warn "$Label via $name looked incomplete ($bad); trying another method..." } @@ -1593,18 +1631,26 @@ function Install-Kubectl { $kubectlDest = "$TOOL_DIR\kubectl.exe" $kUrl = "https://dl.k8s.io/release/$kVer/bin/windows/$arch/kubectl.exe" $t0 = Get-Date - # Heartbeat during the otherwise-silent transfer (#422); retry wraps it. - Get-VerifiedDownload -Url $kUrl -Dest $kubectlDest -MinBytes 20MB -Magic 'MZ' ` - -Label "kubectl download" -Message "Downloading kubectl $kVer (~60 MB)" - $expectedHash = Invoke-WithRetry -Label "checksum" -ScriptBlock { - (Invoke-WebRequest "https://dl.k8s.io/release/$kVer/bin/windows/$arch/kubectl.exe.sha256" ` - -UseBasicParsing).Content.Trim() + # Fetch the .sha256 FIRST, then make it the download gate (#609): with -Sha256 the + # binary download retries transports (Invoke-WebRequest -> curl.exe -> BITS) until a + # byte-correct copy lands, so a mid-transfer truncation self-heals instead of + # dead-ending at a post-hoc checksum. dl.k8s.io publishes the bare 64-hex hash. + $kSums = "$env:TEMP\kubectl-sha-$([System.IO.Path]::GetRandomFileName()).txt" + try { + Get-VerifiedDownload -Url "https://dl.k8s.io/release/$kVer/bin/windows/$arch/kubectl.exe.sha256" ` + -Dest $kSums -MinBytes 1 -MatchPattern '^\s*[0-9a-fA-F]{64}' ` + -Label "kubectl checksum" -Message "Fetching kubectl checksum" + } catch { + Remove-Item $kSums -Force -ErrorAction SilentlyContinue + Err "Couldn't fetch the kubectl checksum ($_). Check egress to dl.k8s.io and re-run." } - $actualHash = (Get-FileHash $kubectlDest -Algorithm SHA256).Hash.ToLower() - if ($actualHash -ne $expectedHash.ToLower()) { - Remove-Item $kubectlDest -Force - Err "System tool checksum verification failed." + $expectedHash = ((Get-Content $kSums -Raw).Trim()) + Remove-Item $kSums -Force -ErrorAction SilentlyContinue + if ($expectedHash -notmatch '^[0-9a-fA-F]{64}$') { + Err "Couldn't read a valid kubectl checksum (got an error page?). Check egress to dl.k8s.io and re-run." } + Get-VerifiedDownload -Url $kUrl -Dest $kubectlDest -MinBytes 20MB -Magic 'MZ' -Sha256 $expectedHash ` + -Label "kubectl download" -Message "Downloading kubectl $kVer (~60 MB)" RefreshPath Log "kubectl $kVer installed." Assert-ToolRuns -Name "kubectl" -VersionArgs @("version","--client") -BinPath $kubectlDest @@ -1701,36 +1747,31 @@ function Install-K3dAndHelm { } $k3dDest = "$TOOL_DIR\k3d.exe" $k3dUrl = "https://github.com/k3d-io/k3d/releases/download/$k3dVer/k3d-windows-$arch.exe" - Get-VerifiedDownload -Url $k3dUrl -Dest $k3dDest -MinBytes 10MB -Magic 'MZ' ` - -Label "k3d download" -Message "Downloading k3d $k3dVer (~25 MB)" - # Fail-closed verification, matching the Linux path and the kubectl - # precedent: an unfetchable checksums.txt, a missing asset line, or a - # mismatch all abort and remove the download — never install unverified - # bytes on a privileged path (Bugbot r3). The release's checksum asset is - # named checksums.txt (" _dist/" lines); the previous - # sha256sum.txt URL never existed, so the old fail-open verification - # silently never ran (#382). + # Fetch the checksum list FIRST, then make the SHA the download gate (#609). + # The release's checksum asset is checksums.txt (" _dist/" + # lines). Fetching it resiliently (multi-transport + must contain the asset + # line) means a proxy error page is retried, and passing the extracted hash to + # Get-VerifiedDownload makes the binary download retry transports until a + # byte-correct copy lands -- the fix for a mid-transfer truncation that used to + # slip past the size floor and dead-end at the checksum (the #607 field case). + $k3dSums = "$env:TEMP\k3d-checksums-$([System.IO.Path]::GetRandomFileName()).txt" try { - $checksums = Invoke-WithRetry -Label "k3d checksums" -ScriptBlock { - (Invoke-WebRequest "https://github.com/k3d-io/k3d/releases/download/$k3dVer/checksums.txt" ` - -UseBasicParsing).Content - } + Get-VerifiedDownload -Url "https://github.com/k3d-io/k3d/releases/download/$k3dVer/checksums.txt" ` + -Dest $k3dSums -MinBytes 1 -MatchPattern "[0-9a-fA-F]{64}\s+\S*k3d-windows-$arch\.exe" ` + -Label "k3d checksums" -Message "Fetching k3d checksums" } catch { - Remove-Item $k3dDest -Force -ErrorAction SilentlyContinue + Remove-Item $k3dSums -Force -ErrorAction SilentlyContinue Err "Couldn't fetch the k3d checksums ($_). Check egress to github.com and re-run." } - $expectedHash = (($checksums -split "`n" | - Where-Object { $_ -match "k3d-windows-$arch\.exe" }) -replace '\s+.*','' | + $expectedHash = (((Get-Content $k3dSums) | + Where-Object { $_ -match "k3d-windows-$arch\.exe" }) -replace '\s+.*', '' | Select-Object -First 1) - if (-not $expectedHash) { - Remove-Item $k3dDest -Force -ErrorAction SilentlyContinue - Err "System tool checksum verification failed." - } - $actualHash = (Get-FileHash $k3dDest -Algorithm SHA256).Hash.ToLower() - if ($actualHash -ne $expectedHash.Trim().ToLower()) { - Remove-Item $k3dDest -Force - Err "System tool checksum verification failed." + Remove-Item $k3dSums -Force -ErrorAction SilentlyContinue + if ($expectedHash -notmatch '^[0-9a-fA-F]{64}$') { + Err "Couldn't read a valid k3d checksum from checksums.txt. Check egress to github.com and re-run." } + Get-VerifiedDownload -Url $k3dUrl -Dest $k3dDest -MinBytes 10MB -Magic 'MZ' -Sha256 $expectedHash.Trim() ` + -Label "k3d download" -Message "Downloading k3d $k3dVer (~25 MB)" Log "k3d checksum verified." RefreshPath # Compute the summary now (correct elapsed) but print it only AFTER the @@ -1767,7 +1808,26 @@ function Install-K3dAndHelm { $t0helm = Get-Date $helmZip = "$env:TEMP\helm-$helmVer-windows-$arch.zip" $helmUrl = "https://get.helm.sh/helm-$helmVer-windows-$arch.zip" - Get-VerifiedDownload -Url $helmUrl -Dest $helmZip -MinBytes 5MB -Magic 'PK' ` + # Checksum-gated like k3d/kubectl (#609): get.helm.sh publishes + # .sha256sum (" "). Fetch it first so the zip download + # retries transports until byte-correct -- a truncated zip that used to pass + # size+magic and then fail at Expand-Archive now self-heals. (The PS path had + # no helm checksum at all before; this also brings it to parity with the + # bash path, which already verifies helm.) + $helmSums = "$env:TEMP\helm-sha-$([System.IO.Path]::GetRandomFileName()).txt" + try { + Get-VerifiedDownload -Url "$helmUrl.sha256sum" -Dest $helmSums -MinBytes 1 ` + -MatchPattern "[0-9a-fA-F]{64}\s+\S*helm-\S*windows-$arch\.zip" -Label "helm checksum" -Message "Fetching Helm checksum" + } catch { + Remove-Item $helmSums -Force -ErrorAction SilentlyContinue + Err "Couldn't fetch the Helm checksum ($_). Check egress to get.helm.sh and re-run." + } + $helmHash = (((Get-Content $helmSums) -split '\s+' | Select-Object -First 1)) + Remove-Item $helmSums -Force -ErrorAction SilentlyContinue + if ($helmHash -notmatch '^[0-9a-fA-F]{64}$') { + Err "Couldn't read a valid Helm checksum from get.helm.sh. Check egress and re-run." + } + Get-VerifiedDownload -Url $helmUrl -Dest $helmZip -MinBytes 5MB -Magic 'PK' -Sha256 $helmHash ` -Label "helm download" -Message "Downloading Helm $helmVer (~20 MB)" $helmExtract = "$env:TEMP\helm-extract" if (Test-Path $helmExtract) { Remove-Item $helmExtract -Recurse -Force } @@ -2612,9 +2672,18 @@ function New-K3dCluster { Err "Compute environment creation timed out after $timeoutMin minutes. Check that Docker is healthy and this network can pull images, then re-run. (TB_CREATE_TIMEOUT_MIN overrides the bound.)" } - $k3dExitCode = $k3dProc.ExitCode $k3dStdout = if (Test-Path $k3dOutLog) { Get-Content $k3dOutLog -Raw -ErrorAction SilentlyContinue } else { "" } $k3dStderr = if (Test-Path $k3dErrLog) { Get-Content $k3dErrLog -Raw -ErrorAction SilentlyContinue } else { "" } + $k3dExitCode = $k3dProc.ExitCode + # Defense-in-depth (#611): if the exit code is STILL unreadable after + # WaitForExit (Wait-ProcessWithDeadline), do not fail a cluster that k3d itself + # reported up -- trust its authoritative success marker over a null code. k3d + # logs via logrus to STDERR, so its "Cluster created successfully!" line lands in + # $k3dStderr, not $k3dStdout -- check BOTH streams or a real success is misread + # as failure (Bugbot). + if ($null -eq $k3dExitCode) { + $k3dExitCode = if ("$k3dStdout`n$k3dStderr" -match 'created successfully') { 0 } else { 1 } + } Remove-Item $k3dOutLog, $k3dErrLog -Force -ErrorAction SilentlyContinue if ($proxyCfg) { Remove-Item (Split-Path $proxyCfg -Parent) -Recurse -Force -ErrorAction SilentlyContinue } if ($registriesCfg) { Remove-Item (Split-Path $registriesCfg -Parent) -Recurse -Force -ErrorAction SilentlyContinue } @@ -3570,19 +3639,42 @@ $envBlock # with this script's own ...\tracebloc-installer-\install-k8s.ps1 temp path -- # which contains "tracebloc" -- so the guard skipped the add on every fresh # install and Step 4 died later with "Error: repo tracebloc not found". #385) - Log "Adding Helm repo: $TRACEBLOC_HELM_REPO_URL" - $addOutput = (helm repo add $TRACEBLOC_HELM_REPO_NAME $TRACEBLOC_HELM_REPO_URL --force-update 2>&1) | Out-String - Log "helm repo add: $addOutput" - if ($LASTEXITCODE -ne 0) { Err "Couldn't add the tracebloc chart repo ($TRACEBLOC_HELM_REPO_URL)." $addOutput } + # Chart source: $env:TRACEBLOC_CHART_PATH points at a LOCAL chart directory for + # dev/testing an unreleased chart (parity with the bash installer's + # _resolve_chart_ref, lib/install-client-helm.sh) -- without it, Windows could only + # ever install the published chart, so branch-only chart fixes were untestable here. + # A local path skips `helm repo add` entirely; otherwise use the published repo. + if ($env:TRACEBLOC_CHART_PATH) { + if (-not (Test-Path -LiteralPath $env:TRACEBLOC_CHART_PATH -PathType Container)) { + Err "TRACEBLOC_CHART_PATH is set but is not a directory: $($env:TRACEBLOC_CHART_PATH)" + } + $chartRef = $env:TRACEBLOC_CHART_PATH + Info "Dev mode: installing the chart from local path $chartRef (skipping the Helm repo)." + Log "Using local chart: $chartRef" + } else { + $chartRef = "$TRACEBLOC_HELM_REPO_NAME/$TRACEBLOC_CHART_NAME" + Log "Adding Helm repo: $TRACEBLOC_HELM_REPO_URL" + $addOutput = (helm repo add $TRACEBLOC_HELM_REPO_NAME $TRACEBLOC_HELM_REPO_URL --force-update 2>&1) | Out-String + Log "helm repo add: $addOutput" + if ($LASTEXITCODE -ne 0) { Err "Couldn't add the tracebloc chart repo ($TRACEBLOC_HELM_REPO_URL)." $addOutput } + } Write-Host "" if ($adoptedReuse) { - # Surgical reconcile of the LIVE release: --reuse-values preserves the - # deployed configuration + secret; only clientId is healed (#397 r2). - Log "Reconciling release '$existingName' in namespace '$existingNs' (adopted; --reuse-values; healing clientId)..." - $helmOutput = (helm upgrade $existingName "$TRACEBLOC_HELM_REPO_NAME/$TRACEBLOC_CHART_NAME" ` + # Surgical reconcile of the LIVE release: preserve the deployed configuration + + # secret; only clientId is healed (#397 r2). Prefer --reset-then-reuse-values + # (Helm >= 3.14: reset to chart defaults, then re-apply the user's overrides, so + # NEW chart defaults reach adopted edges on auto-upgrade) over --reuse-values + # (keeps only stored values, so new chart defaults never land); feature-detect via + # --help and fall back on older Helm (bash parity: install-client-helm.sh). + $reuseFlag = "--reuse-values" + if ((helm upgrade --help 2>$null | Out-String) -match '--reset-then-reuse-values') { + $reuseFlag = "--reset-then-reuse-values" + } + Log "Reconciling release '$existingName' in namespace '$existingNs' (adopted; $reuseFlag; healing clientId)..." + $helmOutput = (helm upgrade $existingName $chartRef ` --namespace $existingNs ` - --reuse-values ` + $reuseFlag ` --set-string "clientId=$TB_CLIENT_ID" 2>&1) | Out-String Log "Helm Output: $helmOutput" if ($LASTEXITCODE -ne 0) { Err "Client reconcile failed." $helmOutput } @@ -3594,8 +3686,8 @@ $envBlock Set-Content -Path $valuesFile -Value $vals -Encoding UTF8 } } else { - Log "Installing $TB_NAMESPACE from $TRACEBLOC_HELM_REPO_NAME/$TRACEBLOC_CHART_NAME in namespace '$TB_NAMESPACE'..." - $helmOutput = (helm upgrade --install $TB_NAMESPACE "$TRACEBLOC_HELM_REPO_NAME/$TRACEBLOC_CHART_NAME" ` + Log "Installing $TB_NAMESPACE from $chartRef in namespace '$TB_NAMESPACE'..." + $helmOutput = (helm upgrade --install $TB_NAMESPACE $chartRef ` --namespace $TB_NAMESPACE ` --create-namespace ` --values $valuesFile 2>&1) | Out-String diff --git a/scripts/manifest.sha256 b/scripts/manifest.sha256 index 161eff88..e855e585 100644 --- a/scripts/manifest.sha256 +++ b/scripts/manifest.sha256 @@ -15,4 +15,4 @@ e373403d7bb5ce3728b8d21af89e6bf672cc35bbf8938541eb527ae19cb9473b scripts/lib/as 911fd0714b17357bb205fc8a8fa8e13eedc1a9632a2f63d4ead9f8d8c7ee546f scripts/lib/probe.sh 38761a6c56dc85b3f5742df036e6a2ec2baa0adb0c90b3753b6706779528b7be scripts/lib/summary.sh 77e03332ebfab1ef759c6148a57afcf479c02c5dc6cc7b0e0e680f58e20cd364 scripts/lib/diagnose.sh -f3c3591c466a3959e06e26657d6e4403b97bebafeb9c0ee0375256687bb03e6d scripts/install-k8s.ps1 +35745834c814b03950f6e6a36d3df40fadec51f8f9fe743b6d647a8eedd3679c scripts/install-k8s.ps1 diff --git a/scripts/tests/install-k8s.Tests.ps1 b/scripts/tests/install-k8s.Tests.ps1 index 3b14263a..b88cf79d 100644 --- a/scripts/tests/install-k8s.Tests.ps1 +++ b/scripts/tests/install-k8s.Tests.ps1 @@ -988,6 +988,25 @@ Describe "Install-ClientHelm" { Install-ClientHelm Should -Invoke helm -ParameterFilter { ($args -contains "upgrade") -and ($args -contains "--reuse-values") } } + It "adopted mode prefers --reset-then-reuse-values when Helm >= 3.14 exposes it (bash parity: new chart defaults reach adopted edges)" { + # When `helm upgrade --help` advertises --reset-then-reuse-values (Helm >= 3.14), + # the reconcile must use it so NEW chart defaults land on adopted Windows edges on + # auto-upgrade — not stay pinned to stored values as plain --reuse-values would. + $HOST_DATA_DIR = "$TestDrive/d-adopt-reset" + $script:TB_PROV_MODE = "adopted"; $script:TB_PROV_ID = "uuid-9"; $script:TB_PROV_NS = "lukas-01" + Mock helm { + if (($args -contains "upgrade") -and ($args -contains "--help")) { " --reset-then-reuse-values reset then reuse"; $global:LASTEXITCODE = 0; return } + if ($args -contains "list") { '[{"name":"oldrel","namespace":"lukas-01","chart":"client-1.4.3"}]'; $global:LASTEXITCODE = 0; return } + if ($args -contains "get") { + if ($args -contains "json") { '{"clientId":"uuid-9"}' } else { 'clientId: uuid-9' } + $global:LASTEXITCODE = 0; return + } + $global:LASTEXITCODE = 0 + } + Install-ClientHelm + Should -Invoke helm -ParameterFilter { ($args -contains "upgrade") -and ($args -contains "--reset-then-reuse-values") } + Should -Not -Invoke helm -ParameterFilter { ($args -contains "upgrade") -and ($args -contains "--reuse-values") } + } It "a DIFFERENT existing client still refuses outside adopted mode (guard intact)" { $HOST_DATA_DIR = "$TestDrive/d-guard-minted" $script:TB_PROV_MODE = "minted"; $script:TB_PROV_ID = "uuid-new" @@ -3416,6 +3435,90 @@ Describe "Get-VerifiedDownload resilience guards (#607, Bugbot)" { It "wraps the post-download validation so an I/O error tries the next transport, not aborts" { # Bugbot: Get-Item/OpenRead can throw if AV locks the just-written file; that # must fall through to curl.exe/BITS, not escape Get-VerifiedDownload. - $script:GVD | Should -Match 'try \{\s*\$bad = Test-DownloadComplete[\s\S]{0,220}catch \{\s*\$bad =' + # Distance-independent: the try wraps the validation, and a catch turns an I/O + # error into a recorded problem (so the loop tries the next transport). + $script:GVD | Should -Match 'try \{\s*\$bad = Test-DownloadComplete' + $script:GVD | Should -Match 'catch \{\s*\$bad = "could not read the downloaded file' + } +} + +Describe "Checksum-driven tool download (#609)" { + BeforeAll { $script:CDD = Get-Content "$PSScriptRoot/../install-k8s.ps1" -Raw } + + It "Get-VerifiedDownload exposes -Sha256 and -MatchPattern, and no fail-open substring gate" { + $script:CDD | Should -Match '\[string\]\$Sha256' + $script:CDD | Should -Match '\[string\]\$MatchPattern' + # -MustContain removed (Bugbot #611): a substring gate is fail-open because the + # asset name also appears in the request URL that a proxy error page can echo. + $script:CDD | Should -Not -Match 'MustContain' + } + + It "the kubectl .sha256 gate is START-anchored so a proxy page retries transports (Bugbot #611)" { + # kubectl's .sha256 is a bare hash; the gate must be anchored ('^...64hex') so an + # HTML error page (which starts with '<') fails it and falls through to curl.exe/ + # BITS. An unanchored [0-9a-fA-F]{64} would pass on any page with a 64-hex run. + $script:CDD | Should -Match "kubectl\.exe\.sha256[\s\S]{0,140}-MatchPattern '\^" + } + + It "a checksum mismatch is treated as a bad transport (retries the next one), not a dead end" { + # The whole point: -Sha256 makes the checksum the completeness test, so a + # truncated/altered copy triggers curl.exe/BITS instead of failing the install. + $script:CDD | Should -Match "if \(-not \`$bad -and \`$Sha256\)[\s\S]{0,200}checksum mismatch" + } + + It "k3d gates the binary on the checksum fetched first, via a hash-anchored gate" { + # The checksum-list gate requires a 64-hex hash adjacent to the asset, not a bare + # asset-name substring (which also appears in the URL and would fail open) (Bugbot). + $script:CDD | Should -Match 'checksums\.txt[\s\S]{0,160}-MatchPattern "\[0-9a-fA-F\]\{64\}' + $script:CDD | Should -Match '\$k3dUrl[\s\S]{0,160}-Sha256' + } + + It "kubectl gates the binary download on the .sha256 fetched first" { + $script:CDD | Should -Match 'Get-VerifiedDownload[\s\S]{0,80}kubectl\.exe\.sha256' + $script:CDD | Should -Match '\$kUrl[\s\S]{0,160}-Sha256' + } + + It "helm gates the zip on its sha256sum with a hash-anchored gate (PS parity with bash)" { + $script:CDD | Should -Match '\$helmUrl[\s\S]{0,160}-Sha256' + $script:CDD | Should -Match 'sha256sum[\s\S]{0,160}-MatchPattern "\[0-9a-fA-F\]\{64\}' + } + + It "each extracted checksum is validated as 64 hex before it gates a download" { + ([regex]::Matches($script:CDD, "notmatch '\^\[0-9a-fA-F\]\{64\}\`$'")).Count | Should -BeGreaterOrEqual 2 + } +} + +Describe "Cluster-create exit-code reliability (#611)" { + BeforeAll { $script:CEC = Get-Content "$PSScriptRoot/../install-k8s.ps1" -Raw } + + It "Wait-ProcessWithDeadline calls WaitForExit before returning success" { + # HasExited can flip true before redirected stdout/stderr drain, leaving + # $proc.ExitCode null; WaitForExit flushes them so every caller reads a real code. + $script:CEC | Should -Match 'function Wait-ProcessWithDeadline[\s\S]{0,1600}\$Process\.WaitForExit\(\)[\s\S]{0,80}return \$true' + } + + It "the null-exit fallback checks BOTH k3d streams (logrus success goes to stderr) (Bugbot)" { + # k3d's 'Cluster created successfully!' is a logrus line on STDERR, so the null- + # exit fallback must inspect $k3dStderr too, not only $k3dStdout. + $script:CEC | Should -Match 'if \(\$null -eq \$k3dExitCode\)[\s\S]{0,120}k3dStdout[\s\S]{0,20}k3dStderr[\s\S]{0,40}created successfully' + } +} + +Describe "Local chart path support (#611 — Windows/bash parity)" { + BeforeAll { $script:LCP = Get-Content "$PSScriptRoot/../install-k8s.ps1" -Raw } + + It "uses TRACEBLOC_CHART_PATH as the chart ref when set (test an unreleased chart)" { + $script:LCP | Should -Match 'if \(\$env:TRACEBLOC_CHART_PATH\)' + $script:LCP | Should -Match '\$chartRef = \$env:TRACEBLOC_CHART_PATH' + } + It "installs from `$chartRef, not a hardcoded repo path" { + $script:LCP | Should -Match 'helm upgrade --install \$TB_NAMESPACE \$chartRef' + $script:LCP | Should -Match 'helm upgrade \$existingName \$chartRef' + } + It "skips 'helm repo add' when a local chart path is given (it's in the else branch)" { + $script:LCP | Should -Match '\$chartRef = "\$TRACEBLOC_HELM_REPO_NAME/\$TRACEBLOC_CHART_NAME"[\s\S]{0,140}helm repo add' + } + It "errors if TRACEBLOC_CHART_PATH is set but is not a directory" { + $script:LCP | Should -Match 'TRACEBLOC_CHART_PATH is set but is not a directory' } }