feat(ingestor): resolve the spawned tag per environment (:dev / :stg)

.github/workflows/helm-ci.yaml

@@ -143,11 +143,27 @@ jobs:
             echo "OK — $label is multi-arch (amd64 + arm64)."
           }
 
-          # The floating tag is the default spawn target — always validate it.
-          if [ -z "$tag" ] || [ "$tag" = "null" ]; then
-            echo "::error::images.ingestor.tag is empty — the chart must define a floating tag to spawn by."; exit 1
+          # Every tag this chart can spawn by must be multi-arch. Since
+          # backend#1360 that is `tag` (an explicit override, empty by default)
+          # PLUS each per-environment entry in `channelTags` — an edge resolves
+          # exactly one of them, so a single-arch value in any of them breaks
+          # ingestion on arm64 for whichever environment lands on it.
+          checked=0
+          if [ -n "$tag" ] && [ "$tag" != "null" ]; then
+            assert_multiarch "${repo}:${tag}" "explicit tag override (images.ingestor.tag)"
+            checked=$((checked + 1))
+          fi
+          for env_key in dev stg prod; do
+            channel=$(yq ".images.ingestor.channelTags.${env_key}" client/values.yaml)
+            if [ -z "$channel" ] || [ "$channel" = "null" ]; then
+              continue
+            fi
+            assert_multiarch "${repo}:${channel}" "channelTags.${env_key}"
+            checked=$((checked + 1))
+          done
+          if [ "$checked" -eq 0 ]; then
+            echo "::error::no spawnable ingestor tag is defined — images.ingestor.tag is empty and channelTags has no entries, so an edge with no digest has nothing to spawn."; exit 1
           fi
-          assert_multiarch "${repo}:${tag}" "floating tag"
 
           # The prod pin is a chart DEFAULT that reaches every prod edge through
           # the fleet auto-upgrade, so a single-arch value here would break

client/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: client
 description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift
 type: application
-version: 1.9.8
-appVersion: "1.9.8"
+version: 1.9.9
+appVersion: "1.9.9"
 keywords:
   - tracebloc
   - kubernetes

client/templates/_helpers.tpl

@@ -274,6 +274,62 @@ defeating the pin. Every read is nil-guarded for the same reason.
 
 Usage: {{ include "tracebloc.ingestorDigest" . }}
 */}}
+{{/*
+  Resolved CLIENT_ENV, with the documented aliases normalized to the
+  canonical dev|stg|prod keys.
+
+  ONE definition on purpose. Bugbot caught the first cut normalizing inside
+  tracebloc.ingestorTag only, so CLIENT_ENV=production selected the prod
+  float tag while tracebloc.ingestorDigest still compared the RAW value to
+  "prod" and returned nothing -- silently dropping the reproducibility pin
+  (backend#1028/#1245) on an edge that looked correctly configured. Any future
+  consumer of CLIENT_ENV must go through here rather than re-deriving it, the
+  same reason ENV_ALIASES lives once in client-runtime proxy_config.
+*/}}
+{{- define "tracebloc.clientEnv" -}}
+{{- $raw := (default dict .Values.env).CLIENT_ENV | default "prod" -}}
+{{- $aliases := dict "development" "dev" "staging" "stg" "production" "prod" -}}
+{{- if hasKey $aliases $raw -}}
+{{- get $aliases $raw -}}
+{{- else -}}
+{{- $raw -}}
+{{- end -}}
+{{- end }}
+
+{{/*
+  Effective floating tag for spawned ingestion Jobs (backend#1360).
+
+  Precedence, mirroring tracebloc.ingestorDigest:
+    1. `images.ingestor.tag`          explicit override, any environment
+    2. `images.ingestor.channelTags[CLIENT_ENV]`   per-environment channel
+    3. "0.7"                          last-resort literal, so a release that
+                                      predates these keys still renders under
+                                      `--reuse-values`
+
+  Only consulted when no digest applies: jobs-manager builds `repo@digest`
+  when tracebloc.ingestorDigest is non-empty, and `repo:tag` otherwise
+  (client-runtime submit_ingestion_run._build_image_reference).
+
+  dev/stg resolve to the UNSIGNED internal channels. Prod is a semver float,
+  not a `:prod` tag — none is published.
+*/}}
+{{- define "tracebloc.ingestorTag" -}}
+{{- $ing := default dict .Values.images.ingestor -}}
+{{- $explicit := $ing.tag | default "" -}}
+{{- if $explicit -}}
+{{- $explicit -}}
+{{- else -}}
+{{- $clientEnv := include "tracebloc.clientEnv" . -}}
+{{- $channels := default dict $ing.channelTags -}}
+{{- $channel := get $channels $clientEnv | default "" -}}
+{{- if $channel -}}
+{{- $channel -}}
+{{- else -}}
+{{- "0.7" -}}
+{{- end -}}
+{{- end -}}
+{{- end }}
+
 {{- define "tracebloc.ingestorDigest" -}}
 {{- $ing := default dict .Values.images.ingestor -}}
 {{- $explicit := $ing.digest | default "" -}}
@@ -284,7 +340,7 @@ Usage: {{ include "tracebloc.ingestorDigest" . }}
 {{- if hasKey $ing "prodPin" -}}
 {{- $prodPin = $ing.prodPin -}}
 {{- end -}}
-{{- $clientEnv := (default dict .Values.env).CLIENT_ENV | default "prod" -}}
+{{- $clientEnv := include "tracebloc.clientEnv" . -}}
 {{- if and $prodPin (eq $clientEnv "prod") -}}
 {{- $ing.prodDigest | default "" -}}
 {{- end -}}

client/templates/jobs-manager-deployment.yaml

@@ -134,7 +134,7 @@ spec:
         - name: INGESTOR_IMAGE_REPOSITORY
           value: {{ (default dict .Values.images.ingestor).repository | default "ghcr.io/tracebloc/ingestor" | quote }}
         - name: INGESTOR_IMAGE_TAG
-          value: {{ (default dict .Values.images.ingestor).tag | default "0.7" | quote }}
+          value: {{ include "tracebloc.ingestorTag" . | quote }}
         - name: INGESTOR_IMAGE_DIGEST
           value: {{ include "tracebloc.ingestorDigest" . | quote }}
         - name: REQUESTS_PROXY_URL

client/tests/ingestor_channel_tag_test.yaml

@@ -0,0 +1,149 @@
+suite: ingestor per-environment channel tag (backend#1360)
+templates:
+  - templates/jobs-manager-deployment.yaml
+release:
+  name: stg
+  namespace: tracebloc
+set:
+  clientId: "test-id"
+  clientPassword: "test"
+tests:
+  - it: dev edges spawn the :dev internal channel
+    set:
+      env.CLIENT_ENV: dev
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: INGESTOR_IMAGE_TAG
+            value: "dev"
+
+  - it: staging edges spawn the :stg internal channel
+    set:
+      env.CLIENT_ENV: stg
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: INGESTOR_IMAGE_TAG
+            value: "stg"
+
+  - it: prod stays on the semver float, never a :prod tag (none is published)
+    set:
+      env.CLIENT_ENV: prod
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: INGESTOR_IMAGE_TAG
+            value: "0.7"
+
+  - it: defaults to the prod float when CLIENT_ENV is unset
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: INGESTOR_IMAGE_TAG
+            value: "0.7"
+
+  - it: an explicit tag override wins over the channel for every environment
+    set:
+      env.CLIENT_ENV: dev
+      images.ingestor.tag: "0.8.0"
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: INGESTOR_IMAGE_TAG
+            value: "0.8.0"
+
+  - it: an unknown CLIENT_ENV falls back to the literal rather than rendering empty
+    set:
+      env.CLIENT_ENV: produktion
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: INGESTOR_IMAGE_TAG
+            value: "0.7"
+
+  - it: renders for a release predating channelTags (--reuse-values replay)
+    set:
+      env.CLIENT_ENV: dev
+      images.ingestor.channelTags: null
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: INGESTOR_IMAGE_TAG
+            value: "0.7"
+
+  - it: prod still pins by digest, so the tag is not the spawn target there
+    set:
+      env.CLIENT_ENV: prod
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: INGESTOR_IMAGE_DIGEST
+            value: "sha256:9098b3c9b83951f825e262d6ff1e9c47c46c5b5cafbfc8fef1ed82ccbadebb24"
+
+  - it: dev gets no digest, so it spawns by the floating channel tag
+    set:
+      env.CLIENT_ENV: dev
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: INGESTOR_IMAGE_DIGEST
+            value: ""
+
+  - it: the documented alias "staging" resolves to the stg channel, not the prod float
+    # Review catch (#494): the schema documents dev|staging|prod while these
+    # keys are dev|stg|prod, so CLIENT_ENV=staging previously missed
+    # channelTags and silently fell back to 0.7 while client-runtime
+    # normalized it to stg at runtime — the split-brain of client-runtime#227.
+    set:
+      env.CLIENT_ENV: staging
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: INGESTOR_IMAGE_TAG
+            value: "stg"
+
+  - it: the documented alias "development" resolves to the dev channel
+    set:
+      env.CLIENT_ENV: development
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: INGESTOR_IMAGE_TAG
+            value: "dev"
+
+  - it: the documented alias "production" resolves to the prod float
+    set:
+      env.CLIENT_ENV: production
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: INGESTOR_IMAGE_TAG
+            value: "0.7"
+
+  - it: the "production" alias keeps the prod reproducibility pin
+    # Bugbot catch (#494): the first alias fix normalized inside
+    # tracebloc.ingestorTag only, so CLIENT_ENV=production got the prod float
+    # tag while tracebloc.ingestorDigest still compared the RAW value to
+    # "prod" and returned nothing — silently dropping the pin
+    # (backend#1028/#1245) on an edge that looked correctly configured. Both
+    # helpers now share tracebloc.clientEnv.
+    set:
+      env.CLIENT_ENV: production
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: INGESTOR_IMAGE_DIGEST
+            value: "sha256:9098b3c9b83951f825e262d6ff1e9c47c46c5b5cafbfc8fef1ed82ccbadebb24"

client/tests/jobs_manager_test.yaml

@@ -123,7 +123,11 @@ tests:
             name: INGESTOR_IMAGE_DIGEST
             value: "sha256:9098b3c9b83951f825e262d6ff1e9c47c46c5b5cafbfc8fef1ed82ccbadebb24"
 
-  - it: keeps the ingestor floating on dev (CLIENT_ENV=dev => no digest)
+  - it: keeps the ingestor floating on dev (CLIENT_ENV=dev => no digest, :dev channel)
+    # Since backend#1360 the dev float is the per-environment INTERNAL
+    # channel (:dev) rather than the 0.7 release float — the point of the
+    # change: a dev edge tracks develop, so an ingestor change can be
+    # validated on a real edge without a production release.
     # dev/staging carry env.CLIENT_ENV in their user-supplied values, so the prod
     # pin does not apply: DIGEST empty => client-runtime spawns repo:tag with
     # imagePullPolicy=Always and the edge is never frozen on a stale build.
@@ -140,7 +144,7 @@ tests:
           path: spec.template.spec.containers[0].env
           content:
             name: INGESTOR_IMAGE_TAG
-            value: "0.7"
+            value: "dev"
 
   - it: keeps the ingestor floating on staging (CLIENT_ENV=stg => no digest)
     set:

client/values.schema.json

@@ -11,7 +11,7 @@
         "CLIENT_ENV": {
           "type": "string",
           "default": "prod",
-          "description": "Environment identifier and Docker image tag (dev, staging, prod). Defaults to 'prod' if not set or empty."
+          "description": "Environment identifier and Docker image tag. Canonical values are `dev`, `stg`, `prod`; the documented aliases `development`/`staging`/`production` are accepted and normalized (matching client-runtime proxy_config.ENV_ALIASES). Load-bearing: it selects images.ingestor.channelTags and the prod digest pin. Defaults to 'prod' if not set or empty."
         },
         "HTTP_PROXY_HOST": { "type": "string" },
         "HTTP_PROXY_PORT": { "type": "string" },
@@ -468,9 +468,32 @@
             },
             "tag": {
               "type": "string",
-              "minLength": 1,
               "not": { "const": "latest" },
-              "description": "Floating ghcr.io/tracebloc/ingestor tag spawned when no digest applies (imagePullPolicy=Always). `latest` is rejected — spawn behaviour must not drift. Surfaces as INGESTOR_IMAGE_TAG."
+              "description": "Explicit floating-tag override for the spawned ingestor. EMPTY by default: the effective tag then comes from channelTags[CLIENT_ENV] (backend#1360). `latest` is rejected — spawn behaviour must not drift. To pin an exact image prefer `digest`, which wins over any tag. Surfaces as INGESTOR_IMAGE_TAG."
+            },
+            "channelTags": {
+              "type": "object",
+              "description": "Per-environment floating tags, keyed on the resolved CLIENT_ENV, used when `tag` is empty. `dev`/`stg` track the UNSIGNED internal channels published from the matching data-ingestors branch; `prod` stays a semver float because no :prod tag is published. backend#1360.",
+              "properties": {
+                "dev": {
+                  "type": "string",
+                  "minLength": 1,
+                  "not": { "const": "latest" },
+                  "description": "Tag spawned when CLIENT_ENV resolves to dev."
+                },
+                "stg": {
+                  "type": "string",
+                  "minLength": 1,
+                  "not": { "const": "latest" },
+                  "description": "Tag spawned when CLIENT_ENV resolves to stg."
+                },
+                "prod": {
+                  "type": "string",
+                  "minLength": 1,
+                  "not": { "const": "latest" },
+                  "description": "Tag spawned when CLIENT_ENV resolves to prod and no digest applies."
+                }
+              }
             },
             "prodDigest": {
               "type": "string",

client/values.yaml

@@ -389,7 +389,31 @@ images:
     # newer categories re-opens the submit-vs-run drift bug (categories
     # accepted at submit that the spawned image can't process). See
     # client-runtime#162 discussion.
-    tag: "0.7"
+    # Empty by DEFAULT: the effective tag now comes from `channelTags` below,
+    # keyed on the resolved CLIENT_ENV. Set this to force one tag on an edge
+    # regardless of environment (to pin an exact immutable image, prefer
+    # `digest` — it wins over any tag). backend#1360.
+    tag: ""
+
+    # -- PER-ENVIRONMENT CHANNELS (backend#1360)
+    #
+    # dev and staging edges track the internal channels published from the
+    # matching data-ingestors branch (`publish-images.yml`: develop -> :dev,
+    # staging -> :stg), so an ingestor change can be validated on a real edge
+    # WITHOUT a production release. Before this, the ingestor image existed
+    # only as a byproduct of a prod release, so testing one change cost a prod
+    # PyPI publish plus an FR-gate override (2026-07-30).
+    #
+    # `prod` deliberately stays a semver float, NOT a `:prod` channel — no
+    # such tag is published; prod normally runs `prodDigest` anyway and this is
+    # only the fallback when pinning is disabled. Keep the prod entry tracking
+    # the current release line for the same submit-vs-run reasons noted above.
+    #
+    # These are unsigned internal images. Do not point a prod edge at them.
+    channelTags:
+      dev: "dev"
+      stg: "stg"
+      prod: "0.7"
 
     # -- PROD REPRODUCIBILITY PIN — the digest prod edges run.
     #