diff --git a/.github/workflows/helm-ci.yaml b/.github/workflows/helm-ci.yaml index 98b92ceb..2313270c 100644 --- a/.github/workflows/helm-ci.yaml +++ b/.github/workflows/helm-ci.yaml @@ -143,11 +143,27 @@ jobs: echo "OK — $label is multi-arch (amd64 + arm64)." } - # The floating tag is the default spawn target — always validate it. - if [ -z "$tag" ] || [ "$tag" = "null" ]; then - echo "::error::images.ingestor.tag is empty — the chart must define a floating tag to spawn by."; exit 1 + # Every tag this chart can spawn by must be multi-arch. Since + # backend#1360 that is `tag` (an explicit override, empty by default) + # PLUS each per-environment entry in `channelTags` — an edge resolves + # exactly one of them, so a single-arch value in any of them breaks + # ingestion on arm64 for whichever environment lands on it. + checked=0 + if [ -n "$tag" ] && [ "$tag" != "null" ]; then + assert_multiarch "${repo}:${tag}" "explicit tag override (images.ingestor.tag)" + checked=$((checked + 1)) + fi + for env_key in dev stg prod; do + channel=$(yq ".images.ingestor.channelTags.${env_key}" client/values.yaml) + if [ -z "$channel" ] || [ "$channel" = "null" ]; then + continue + fi + assert_multiarch "${repo}:${channel}" "channelTags.${env_key}" + checked=$((checked + 1)) + done + if [ "$checked" -eq 0 ]; then + echo "::error::no spawnable ingestor tag is defined — images.ingestor.tag is empty and channelTags has no entries, so an edge with no digest has nothing to spawn."; exit 1 fi - assert_multiarch "${repo}:${tag}" "floating tag" # The prod pin is a chart DEFAULT that reaches every prod edge through # the fleet auto-upgrade, so a single-arch value here would break diff --git a/client/Chart.yaml b/client/Chart.yaml index 5d3d77d6..af0b4c3e 100644 --- a/client/Chart.yaml +++ b/client/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: client description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift type: application -version: 1.9.8 -appVersion: "1.9.8" +version: 1.9.9 +appVersion: "1.9.9" keywords: - tracebloc - kubernetes diff --git a/client/templates/_helpers.tpl b/client/templates/_helpers.tpl index 67fda447..d2020364 100644 --- a/client/templates/_helpers.tpl +++ b/client/templates/_helpers.tpl @@ -274,6 +274,62 @@ defeating the pin. Every read is nil-guarded for the same reason. Usage: {{ include "tracebloc.ingestorDigest" . }} */}} +{{/* + Resolved CLIENT_ENV, with the documented aliases normalized to the + canonical dev|stg|prod keys. + + ONE definition on purpose. Bugbot caught the first cut normalizing inside + tracebloc.ingestorTag only, so CLIENT_ENV=production selected the prod + float tag while tracebloc.ingestorDigest still compared the RAW value to + "prod" and returned nothing -- silently dropping the reproducibility pin + (backend#1028/#1245) on an edge that looked correctly configured. Any future + consumer of CLIENT_ENV must go through here rather than re-deriving it, the + same reason ENV_ALIASES lives once in client-runtime proxy_config. +*/}} +{{- define "tracebloc.clientEnv" -}} +{{- $raw := (default dict .Values.env).CLIENT_ENV | default "prod" -}} +{{- $aliases := dict "development" "dev" "staging" "stg" "production" "prod" -}} +{{- if hasKey $aliases $raw -}} +{{- get $aliases $raw -}} +{{- else -}} +{{- $raw -}} +{{- end -}} +{{- end }} + +{{/* + Effective floating tag for spawned ingestion Jobs (backend#1360). + + Precedence, mirroring tracebloc.ingestorDigest: + 1. `images.ingestor.tag` explicit override, any environment + 2. `images.ingestor.channelTags[CLIENT_ENV]` per-environment channel + 3. "0.7" last-resort literal, so a release that + predates these keys still renders under + `--reuse-values` + + Only consulted when no digest applies: jobs-manager builds `repo@digest` + when tracebloc.ingestorDigest is non-empty, and `repo:tag` otherwise + (client-runtime submit_ingestion_run._build_image_reference). + + dev/stg resolve to the UNSIGNED internal channels. Prod is a semver float, + not a `:prod` tag — none is published. +*/}} +{{- define "tracebloc.ingestorTag" -}} +{{- $ing := default dict .Values.images.ingestor -}} +{{- $explicit := $ing.tag | default "" -}} +{{- if $explicit -}} +{{- $explicit -}} +{{- else -}} +{{- $clientEnv := include "tracebloc.clientEnv" . -}} +{{- $channels := default dict $ing.channelTags -}} +{{- $channel := get $channels $clientEnv | default "" -}} +{{- if $channel -}} +{{- $channel -}} +{{- else -}} +{{- "0.7" -}} +{{- end -}} +{{- end -}} +{{- end }} + {{- define "tracebloc.ingestorDigest" -}} {{- $ing := default dict .Values.images.ingestor -}} {{- $explicit := $ing.digest | default "" -}} @@ -284,7 +340,7 @@ Usage: {{ include "tracebloc.ingestorDigest" . }} {{- if hasKey $ing "prodPin" -}} {{- $prodPin = $ing.prodPin -}} {{- end -}} -{{- $clientEnv := (default dict .Values.env).CLIENT_ENV | default "prod" -}} +{{- $clientEnv := include "tracebloc.clientEnv" . -}} {{- if and $prodPin (eq $clientEnv "prod") -}} {{- $ing.prodDigest | default "" -}} {{- end -}} diff --git a/client/templates/jobs-manager-deployment.yaml b/client/templates/jobs-manager-deployment.yaml index d43674cb..fbdd12e2 100644 --- a/client/templates/jobs-manager-deployment.yaml +++ b/client/templates/jobs-manager-deployment.yaml @@ -134,7 +134,7 @@ spec: - name: INGESTOR_IMAGE_REPOSITORY value: {{ (default dict .Values.images.ingestor).repository | default "ghcr.io/tracebloc/ingestor" | quote }} - name: INGESTOR_IMAGE_TAG - value: {{ (default dict .Values.images.ingestor).tag | default "0.7" | quote }} + value: {{ include "tracebloc.ingestorTag" . | quote }} - name: INGESTOR_IMAGE_DIGEST value: {{ include "tracebloc.ingestorDigest" . | quote }} - name: REQUESTS_PROXY_URL diff --git a/client/tests/ingestor_channel_tag_test.yaml b/client/tests/ingestor_channel_tag_test.yaml new file mode 100644 index 00000000..bf26c94d --- /dev/null +++ b/client/tests/ingestor_channel_tag_test.yaml @@ -0,0 +1,149 @@ +suite: ingestor per-environment channel tag (backend#1360) +templates: + - templates/jobs-manager-deployment.yaml +release: + name: stg + namespace: tracebloc +set: + clientId: "test-id" + clientPassword: "test" +tests: + - it: dev edges spawn the :dev internal channel + set: + env.CLIENT_ENV: dev + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: INGESTOR_IMAGE_TAG + value: "dev" + + - it: staging edges spawn the :stg internal channel + set: + env.CLIENT_ENV: stg + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: INGESTOR_IMAGE_TAG + value: "stg" + + - it: prod stays on the semver float, never a :prod tag (none is published) + set: + env.CLIENT_ENV: prod + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: INGESTOR_IMAGE_TAG + value: "0.7" + + - it: defaults to the prod float when CLIENT_ENV is unset + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: INGESTOR_IMAGE_TAG + value: "0.7" + + - it: an explicit tag override wins over the channel for every environment + set: + env.CLIENT_ENV: dev + images.ingestor.tag: "0.8.0" + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: INGESTOR_IMAGE_TAG + value: "0.8.0" + + - it: an unknown CLIENT_ENV falls back to the literal rather than rendering empty + set: + env.CLIENT_ENV: produktion + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: INGESTOR_IMAGE_TAG + value: "0.7" + + - it: renders for a release predating channelTags (--reuse-values replay) + set: + env.CLIENT_ENV: dev + images.ingestor.channelTags: null + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: INGESTOR_IMAGE_TAG + value: "0.7" + + - it: prod still pins by digest, so the tag is not the spawn target there + set: + env.CLIENT_ENV: prod + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: INGESTOR_IMAGE_DIGEST + value: "sha256:9098b3c9b83951f825e262d6ff1e9c47c46c5b5cafbfc8fef1ed82ccbadebb24" + + - it: dev gets no digest, so it spawns by the floating channel tag + set: + env.CLIENT_ENV: dev + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: INGESTOR_IMAGE_DIGEST + value: "" + + - it: the documented alias "staging" resolves to the stg channel, not the prod float + # Review catch (#494): the schema documents dev|staging|prod while these + # keys are dev|stg|prod, so CLIENT_ENV=staging previously missed + # channelTags and silently fell back to 0.7 while client-runtime + # normalized it to stg at runtime — the split-brain of client-runtime#227. + set: + env.CLIENT_ENV: staging + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: INGESTOR_IMAGE_TAG + value: "stg" + + - it: the documented alias "development" resolves to the dev channel + set: + env.CLIENT_ENV: development + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: INGESTOR_IMAGE_TAG + value: "dev" + + - it: the documented alias "production" resolves to the prod float + set: + env.CLIENT_ENV: production + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: INGESTOR_IMAGE_TAG + value: "0.7" + + - it: the "production" alias keeps the prod reproducibility pin + # Bugbot catch (#494): the first alias fix normalized inside + # tracebloc.ingestorTag only, so CLIENT_ENV=production got the prod float + # tag while tracebloc.ingestorDigest still compared the RAW value to + # "prod" and returned nothing — silently dropping the pin + # (backend#1028/#1245) on an edge that looked correctly configured. Both + # helpers now share tracebloc.clientEnv. + set: + env.CLIENT_ENV: production + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: INGESTOR_IMAGE_DIGEST + value: "sha256:9098b3c9b83951f825e262d6ff1e9c47c46c5b5cafbfc8fef1ed82ccbadebb24" diff --git a/client/tests/jobs_manager_test.yaml b/client/tests/jobs_manager_test.yaml index e5390544..1a0009c1 100644 --- a/client/tests/jobs_manager_test.yaml +++ b/client/tests/jobs_manager_test.yaml @@ -123,7 +123,11 @@ tests: name: INGESTOR_IMAGE_DIGEST value: "sha256:9098b3c9b83951f825e262d6ff1e9c47c46c5b5cafbfc8fef1ed82ccbadebb24" - - it: keeps the ingestor floating on dev (CLIENT_ENV=dev => no digest) + - it: keeps the ingestor floating on dev (CLIENT_ENV=dev => no digest, :dev channel) + # Since backend#1360 the dev float is the per-environment INTERNAL + # channel (:dev) rather than the 0.7 release float — the point of the + # change: a dev edge tracks develop, so an ingestor change can be + # validated on a real edge without a production release. # dev/staging carry env.CLIENT_ENV in their user-supplied values, so the prod # pin does not apply: DIGEST empty => client-runtime spawns repo:tag with # imagePullPolicy=Always and the edge is never frozen on a stale build. @@ -140,7 +144,7 @@ tests: path: spec.template.spec.containers[0].env content: name: INGESTOR_IMAGE_TAG - value: "0.7" + value: "dev" - it: keeps the ingestor floating on staging (CLIENT_ENV=stg => no digest) set: diff --git a/client/values.schema.json b/client/values.schema.json index c6fb0314..c734975a 100644 --- a/client/values.schema.json +++ b/client/values.schema.json @@ -11,7 +11,7 @@ "CLIENT_ENV": { "type": "string", "default": "prod", - "description": "Environment identifier and Docker image tag (dev, staging, prod). Defaults to 'prod' if not set or empty." + "description": "Environment identifier and Docker image tag. Canonical values are `dev`, `stg`, `prod`; the documented aliases `development`/`staging`/`production` are accepted and normalized (matching client-runtime proxy_config.ENV_ALIASES). Load-bearing: it selects images.ingestor.channelTags and the prod digest pin. Defaults to 'prod' if not set or empty." }, "HTTP_PROXY_HOST": { "type": "string" }, "HTTP_PROXY_PORT": { "type": "string" }, @@ -468,9 +468,32 @@ }, "tag": { "type": "string", - "minLength": 1, "not": { "const": "latest" }, - "description": "Floating ghcr.io/tracebloc/ingestor tag spawned when no digest applies (imagePullPolicy=Always). `latest` is rejected — spawn behaviour must not drift. Surfaces as INGESTOR_IMAGE_TAG." + "description": "Explicit floating-tag override for the spawned ingestor. EMPTY by default: the effective tag then comes from channelTags[CLIENT_ENV] (backend#1360). `latest` is rejected — spawn behaviour must not drift. To pin an exact image prefer `digest`, which wins over any tag. Surfaces as INGESTOR_IMAGE_TAG." + }, + "channelTags": { + "type": "object", + "description": "Per-environment floating tags, keyed on the resolved CLIENT_ENV, used when `tag` is empty. `dev`/`stg` track the UNSIGNED internal channels published from the matching data-ingestors branch; `prod` stays a semver float because no :prod tag is published. backend#1360.", + "properties": { + "dev": { + "type": "string", + "minLength": 1, + "not": { "const": "latest" }, + "description": "Tag spawned when CLIENT_ENV resolves to dev." + }, + "stg": { + "type": "string", + "minLength": 1, + "not": { "const": "latest" }, + "description": "Tag spawned when CLIENT_ENV resolves to stg." + }, + "prod": { + "type": "string", + "minLength": 1, + "not": { "const": "latest" }, + "description": "Tag spawned when CLIENT_ENV resolves to prod and no digest applies." + } + } }, "prodDigest": { "type": "string", diff --git a/client/values.yaml b/client/values.yaml index 631a11fa..26e47c3c 100644 --- a/client/values.yaml +++ b/client/values.yaml @@ -389,7 +389,31 @@ images: # newer categories re-opens the submit-vs-run drift bug (categories # accepted at submit that the spawned image can't process). See # client-runtime#162 discussion. - tag: "0.7" + # Empty by DEFAULT: the effective tag now comes from `channelTags` below, + # keyed on the resolved CLIENT_ENV. Set this to force one tag on an edge + # regardless of environment (to pin an exact immutable image, prefer + # `digest` — it wins over any tag). backend#1360. + tag: "" + + # -- PER-ENVIRONMENT CHANNELS (backend#1360) + # + # dev and staging edges track the internal channels published from the + # matching data-ingestors branch (`publish-images.yml`: develop -> :dev, + # staging -> :stg), so an ingestor change can be validated on a real edge + # WITHOUT a production release. Before this, the ingestor image existed + # only as a byproduct of a prod release, so testing one change cost a prod + # PyPI publish plus an FR-gate override (2026-07-30). + # + # `prod` deliberately stays a semver float, NOT a `:prod` channel — no + # such tag is published; prod normally runs `prodDigest` anyway and this is + # only the fallback when pinning is disabled. Keep the prod entry tracking + # the current release line for the same submit-vs-run reasons noted above. + # + # These are unsigned internal images. Do not point a prod edge at them. + channelTags: + dev: "dev" + stg: "stg" + prod: "0.7" # -- PROD REPRODUCIBILITY PIN — the digest prod edges run. # diff --git a/scripts/resolve-ingestor-digest.sh b/scripts/resolve-ingestor-digest.sh index ee416b2e..fa275f22 100755 --- a/scripts/resolve-ingestor-digest.sh +++ b/scripts/resolve-ingestor-digest.sh @@ -72,6 +72,39 @@ read_ingestor_tag() { ' "$file" } +# Portable, yq-free reader for images.ingestor.channelTags.prod. Same scoping +# discipline as read_ingestor_tag: only the 6-space `prod:` leaf inside +# images: -> ingestor: -> channelTags: can match, so no sibling key can be +# picked up by mistake. bash-3.2 / macOS-safe. +read_ingestor_prod_channel() { + local file="$1" + [[ -f "$file" ]] || return 1 + awk ' + /^images:[[:space:]]*$/ { in_images = 1; next } + /^[^[:space:]#]/ { in_images = 0; in_ingestor = 0; in_channels = 0 } + in_images { + if ($0 ~ /^ [A-Za-z_][A-Za-z0-9_]*:[[:space:]]*$/) { + in_ingestor = ($0 ~ /^ ingestor:[[:space:]]*$/) ? 1 : 0 + in_channels = 0 + next + } + if (in_ingestor && $0 ~ /^ [A-Za-z_][A-Za-z0-9_]*:[[:space:]]*$/) { + in_channels = ($0 ~ /^ channelTags:[[:space:]]*$/) ? 1 : 0 + next + } + if (in_channels && $0 ~ /^ prod:[[:space:]]*/) { + line = $0 + sub(/^ prod:[[:space:]]*/, "", line) # drop the key + sub(/[[:space:]]+#.*$/, "", line) # drop a trailing comment + gsub(/^[[:space:]]+|[[:space:]]+$/, "", line) # trim + gsub(/^"|"$/, "", line) # unwrap double quotes + gsub(/^'"'"'|'"'"'$/, "", line) # unwrap single quotes + if (line != "") { print line; exit } + } + } + ' "$file" +} + if [[ -z "$tag" ]]; then # No explicit TAG arg → default to the chart's images.ingestor.tag so this # helper always resolves the SAME line the chart ships. NEVER hardcode a @@ -79,15 +112,30 @@ if [[ -z "$tag" ]]; then # after the chart tag moves (e.g. 0.7 -> 0.8) while appearing to follow the # chart. Prefer yq; fall back to the portable yq-free parse above; if # neither can determine it, fail loudly rather than guess. + # Since backend#1360 `images.ingestor.tag` is an OVERRIDE that is empty by + # default, and the prod float lives in `images.ingestor.channelTags.prod`. + # This script resolves the PROD pin, so prefer the explicit override when an + # operator set one and otherwise read the prod channel. Without this the + # documented no-arg / --write path exits on an empty tag -- which is the very + # command the chart comments and the ingestor-multiarch CI error tell people + # to run (Bugbot, #494). if command -v yq >/dev/null 2>&1 && [[ -f "$chart_values" ]]; then tag="$(yq -r '.images.ingestor.tag' "$chart_values")" - [[ "$tag" == "null" ]] && tag="" # yq prints literal "null" for a missing key + [[ "$tag" == "null" ]] && tag="" + if [[ -z "$tag" ]]; then + tag="$(yq -r '.images.ingestor.channelTags.prod' "$chart_values")" + [[ "$tag" == "null" ]] && tag="" + fi else tag="$(read_ingestor_tag "$chart_values" || true)" + if [[ -z "$tag" ]]; then + tag="$(read_ingestor_prod_channel "$chart_values" || true)" + fi fi if [[ -z "$tag" ]]; then echo "ERROR: could not determine the default ingestor tag from ${chart_values#$here/../}." >&2 - echo " (images.ingestor.tag was unreadable: file missing, key absent, or yq not" >&2 + echo " (neither images.ingestor.tag nor images.ingestor.channelTags.prod was" >&2 + echo " readable: file missing, keys absent, or yq not" >&2 echo " installed and the yq-free parse found nothing.)" >&2 echo " Fix: pass TAG explicitly — scripts/resolve-ingestor-digest.sh [--write] —" >&2 echo " or install yq. Refusing to fall back to a hardcoded tag." >&2