feat(self-managed): support a pre-created LLM request-router TLS Secret

deploy/helm/llm-request-router/README.md

@@ -75,7 +75,8 @@ Important settings to review before deployment:
 - `llmRequestRouter.metrics.enabled` to expose the metrics port on the Service (default: `false`)
 - `llmRequestRouter.metrics.serviceMonitor.enabled` to create a Prometheus `ServiceMonitor` (requires `metrics.enabled`)
 - `llmRequestRouter.certificate.*` to let cert-manager issue the Stargate QUIC server certificate
-- `llmRequestRouter.tls.*` to mount the issued TLS Secret and pass cert/key paths to Stargate
+- `llmRequestRouter.tls.*` to mount the TLS Secret and pass cert/key paths to Stargate
+- `llmRequestRouter.tls.mode` to choose the source of the QUIC server identity. `certManager` (default) mounts the Secret cert-manager writes for `certificate.*`. `existingSecret` mounts a pre-created Secret instead: the chart renders no `Certificate` and adds no issuer dependency, `certificate.enabled` must stay `false`, `tls.secretName`, `tls.certPath`, and `tls.keyPath` are required, and the operator owns issuance, renewal, rotation, and recovery. The Secret must provide the `tls.crt` and `tls.key` entries. The chart cannot read a pre-created Secret, so it does not validate its SANs or expiry.
 - `llmRequestRouter.pki.*` to provision the OpenBao service-issuing PKI hierarchy that cert-manager mints the Certificate from. Opt-in via `pki.enabled=true`. Mirrors the SIS chart's `hook-lls-migrations.yaml` pattern: a Helm pre-install/pre-upgrade Job runs the `nvcf-openbao-migrations` image with `CORE_MIGRATIONS_ENABLED=false` + `ADDONS_LLM_ENABLED=true` so only the LLM addon executes. `pki.allowedDomains` (comma-separated DNS suffixes) is required when enabled and is the OpenBao PKI role's `allowed_domains` security constraint. Typically this is `<customer-domain>,cluster.local`. Job-level fail-hard is handled by `restartPolicy: OnFailure` + `pki.backoffLimit` combined with the migrations image's `FAILED_MIGRATIONS` accumulator (image `>= 0.12.1`).
 - `llmRequestRouter.vault.audience` for the projected ServiceAccount token audience used to authenticate to OpenBao
 - `llmRequestRouter.vault.noVaultAnnotations` to disable Vault Agent injection (useful for local testing without OpenBao)

deploy/helm/llm-request-router/llm-request-router/templates/_helpers.tpl

@@ -180,6 +180,43 @@ comparing suffixes.
 {{- end -}}
 {{- end }}
 
+{{/*
+Validate the QUIC server identity source. certManager keeps cert-manager as the
+owner of issuance and renewal. existingSecret mounts a pre-created TLS Secret,
+renders no Certificate, and makes the operator the owner. The two are mutually
+exclusive, and existingSecret needs the Secret name plus both file paths because
+the mount and the Stargate arguments are all conditional on them.
+*/}}
+{{- define "llm-request-router.validateTlsIdentity" -}}
+{{- $tls := .Values.llmRequestRouter.tls | default dict -}}
+{{- $certificate := .Values.llmRequestRouter.certificate | default dict -}}
+{{- $mode := $tls.mode | default "certManager" -}}
+{{- if not (has $mode (list "certManager" "existingSecret")) -}}
+{{- fail (printf "llmRequestRouter.tls.mode must be certManager or existingSecret, got %q" (toString $mode)) -}}
+{{- end -}}
+{{- if eq $mode "existingSecret" -}}
+{{- if $certificate.enabled -}}
+{{- fail "llmRequestRouter.certificate.enabled must be false when llmRequestRouter.tls.mode is existingSecret; cert-manager and the operator cannot both own the request-router certificate" -}}
+{{- end -}}
+{{- if not $tls.secretName -}}
+{{- fail "llmRequestRouter.tls.secretName is required when llmRequestRouter.tls.mode is existingSecret" -}}
+{{- end -}}
+{{- if not $tls.certPath -}}
+{{- fail "llmRequestRouter.tls.certPath is required when llmRequestRouter.tls.mode is existingSecret" -}}
+{{- end -}}
+{{- if not $tls.keyPath -}}
+{{- fail "llmRequestRouter.tls.keyPath is required when llmRequestRouter.tls.mode is existingSecret" -}}
+{{- end -}}
+{{- $tlsMountPath := include "llm-request-router.tlsMountPath" . | trim -}}
+{{- if not (hasPrefix "/" $tlsMountPath) -}}
+{{- fail "llmRequestRouter.tls.mountPath must be an absolute path when llmRequestRouter.tls.mode is existingSecret" -}}
+{{- end -}}
+{{- if or (ne $tlsMountPath (dir $tls.certPath)) (ne $tlsMountPath (dir $tls.keyPath)) -}}
+{{- fail "llmRequestRouter.tls.mountPath must match the directory containing llmRequestRouter.tls.certPath and llmRequestRouter.tls.keyPath when llmRequestRouter.tls.mode is existingSecret" -}}
+{{- end -}}
+{{- end -}}
+{{- end }}
+
 {{- define "llm-request-router.tlsSecretName" -}}
 {{- $tls := .Values.llmRequestRouter.tls | default dict -}}
 {{- $certificate := .Values.llmRequestRouter.certificate | default dict -}}

deploy/helm/llm-request-router/llm-request-router/templates/deployment.yaml

@@ -31,6 +31,11 @@ tunnel target.
 {{- if and $disableDnsDiscovery (gt $replicaCount 1) }}
 {{- fail "llmRequestRouter.discovery.disableDnsDiscovery cannot be true when llmRequestRouter.replicaCount is greater than 1; multi-replica routers require DNS discovery" }}
 {{- end }}
+{{- /*
+The identity guard lives here, not in certificate.yaml: existingSecret mode
+renders no Certificate, so a guard in that template would never run.
+*/}}
+{{- include "llm-request-router.validateTlsIdentity" . }}
 {{- $advertisedHostnameTemplate := include "llm-request-router.advertisedHostnameTemplate" . }}
 spec:
   serviceName: {{ .Values.llmRequestRouter.service.headlessName }}

deploy/helm/llm-request-router/llm-request-router/values.yaml

@@ -175,6 +175,17 @@ llmRequestRouter:
     audience: ""
 
   tls:
+    # Source of the QUIC server identity.
+    #   certManager    - certificate.enabled controls cert-manager issuance and
+    #                    the chart mounts the Secret cert-manager writes.
+    #   existingSecret - mount a pre-created TLS Secret. The chart renders no
+    #                    Certificate and adds no issuer dependency. secretName,
+    #                    certPath, and keyPath are required, certificate.enabled
+    #                    must stay false, and the operator owns issuance,
+    #                    renewal, rotation, and recovery.
+    mode: certManager
+    # Secret holding the server certificate. It must provide the tls.crt and
+    # tls.key entries, as a kubernetes.io/tls Secret does.
     # The mount directory is polled every 30 seconds for certificate and key
     # updates, which apply to new connections without a pod restart. Keep both
     # files in one Secret so Kubernetes projects one atomic generation.

deploy/helm/llm-request-router/scripts/check-pki-render.sh

@@ -290,4 +290,144 @@ if render_certificate_case \
   fail "advertised hostname containing non-DNS braces unexpectedly rendered"
 fi
 
+# Pass 3: existing-Secret identity mode. The operator owns issuance, so the
+# chart must mount the pre-created Secret without rendering a Certificate or
+# the OpenBao provisioning hook.
+existing_secret_manifest="${tmp_dir}/existing-secret.yaml"
+render_existing_secret_case() {
+  output="$1"
+  shift
+
+  helm template llm-request-router ./llm-request-router \
+    --namespace nvcf \
+    --values ./llm-request-router/values.yaml \
+    --set llmRequestRouter.image.repository=stargate \
+    --set-string llmRequestRouter.tls.mode=existingSecret \
+    --set llmRequestRouter.tls.quicInsecure=false \
+    "$@" \
+    > "${output}"
+}
+
+render_existing_secret_case \
+  "${existing_secret_manifest}" \
+  --set-string llmRequestRouter.tls.secretName=operator-quic-tls \
+  --set-string llmRequestRouter.tls.certPath=/etc/stargate/tls/tls.crt \
+  --set-string llmRequestRouter.tls.keyPath=/etc/stargate/tls/tls.key
+
+existing_secret_cert="$(yq -rN 'select(.kind == "Certificate") | .metadata.name' "${existing_secret_manifest}" | head -n1)"
+[ -z "${existing_secret_cert}" ] || fail "existing-Secret mode rendered a Certificate: ${existing_secret_cert}"
+
+existing_secret_job="$(yq -rN 'select(.kind == "Job" and .metadata.name == "addons-llm-migrations") | .metadata.name' "${existing_secret_manifest}" | head -n1)"
+[ -z "${existing_secret_job}" ] || fail "existing-Secret mode rendered the OpenBao provisioning hook"
+
+existing_secret_volume="$(yq -rN 'select(.kind == "StatefulSet" and .metadata.name == "llm-request-router") | .spec.template.spec.volumes[] | select(.name == "stargate-tls" and .secret.secretName == "operator-quic-tls") | .name' "${existing_secret_manifest}")"
+[ "${existing_secret_volume}" = "stargate-tls" ] || fail "existing-Secret mode did not mount the pre-created Secret"
+
+existing_secret_mount="$(yq -rN 'select(.kind == "StatefulSet" and .metadata.name == "llm-request-router") | .spec.template.spec.containers[0].volumeMounts[] | select(.name == "stargate-tls" and .mountPath == "/etc/stargate/tls" and .readOnly == true) | .name' "${existing_secret_manifest}")"
+[ "${existing_secret_mount}" = "stargate-tls" ] || fail "existing-Secret mode did not mount stargate-tls read-only"
+
+existing_secret_args="$(yq -rN 'select(.kind == "StatefulSet" and .metadata.name == "llm-request-router") | .spec.template.spec.containers[0].args[]' "${existing_secret_manifest}")"
+printf '%s\n' "${existing_secret_args}" | grep -qx -- "--tls-cert-path=/etc/stargate/tls/tls.crt" || fail "existing-Secret mode did not pass the certificate path"
+printf '%s\n' "${existing_secret_args}" | grep -qx -- "--tls-key-path=/etc/stargate/tls/tls.key" || fail "existing-Secret mode did not pass the private key path"
+if printf '%s\n' "${existing_secret_args}" | grep -qx -- "--quic-insecure"; then
+  fail "existing-Secret mode enabled insecure request-router transport"
+fi
+
+# Mixed ownership: cert-manager and the operator cannot both own the identity.
+mixed_ownership_error="${tmp_dir}/mixed-ownership.err"
+if render_existing_secret_case \
+  /dev/null \
+  --set llmRequestRouter.certificate.enabled=true \
+  --set-string llmRequestRouter.certificate.issuerRef.name=nvcf-openbao-pki \
+  --set-string 'llmRequestRouter.certificate.dnsNames[0]=*.llm-request-router-headless.nvcf.svc.cluster.local' \
+  --set-string llmRequestRouter.tls.secretName=operator-quic-tls \
+  --set-string llmRequestRouter.tls.certPath=/etc/stargate/tls/tls.crt \
+  --set-string llmRequestRouter.tls.keyPath=/etc/stargate/tls/tls.key \
+  2> "${mixed_ownership_error}"; then
+  fail "mixed certificate ownership unexpectedly rendered"
+fi
+grep -Fq \
+  "llmRequestRouter.certificate.enabled must be false when llmRequestRouter.tls.mode is existingSecret" \
+  "${mixed_ownership_error}" || fail "mixed ownership render did not return the expected guard message"
+
+# Incomplete configuration: every required value reports itself by name. The
+# mount and the Stargate arguments are conditional, so a missing value would
+# otherwise leave the router on plaintext QUIC without any diagnostic.
+check_required_existing_secret_value() {
+  case_name="$1"
+  expected_message="$2"
+  shift 2
+
+  error_file="${tmp_dir}/${case_name}.err"
+  if render_existing_secret_case /dev/null "$@" 2> "${error_file}"; then
+    fail "existing-Secret render without ${case_name} unexpectedly succeeded"
+  fi
+  grep -Fq "${expected_message}" "${error_file}" ||
+    fail "${case_name} render did not return the expected guard message"
+}
+
+check_required_existing_secret_value \
+  secret-name \
+  "llmRequestRouter.tls.secretName is required when llmRequestRouter.tls.mode is existingSecret" \
+  --set-string llmRequestRouter.tls.certPath=/etc/stargate/tls/tls.crt \
+  --set-string llmRequestRouter.tls.keyPath=/etc/stargate/tls/tls.key
+
+check_required_existing_secret_value \
+  cert-path \
+  "llmRequestRouter.tls.certPath is required when llmRequestRouter.tls.mode is existingSecret" \
+  --set-string llmRequestRouter.tls.secretName=operator-quic-tls \
+  --set-string llmRequestRouter.tls.keyPath=/etc/stargate/tls/tls.key
+
+check_required_existing_secret_value \
+  key-path \
+  "llmRequestRouter.tls.keyPath is required when llmRequestRouter.tls.mode is existingSecret" \
+  --set-string llmRequestRouter.tls.secretName=operator-quic-tls \
+  --set-string llmRequestRouter.tls.certPath=/etc/stargate/tls/tls.crt
+
+# The Secret is mounted as a directory, so changing its mount location without
+# changing the certificate paths would leave the router unable to read them.
+mismatched_mount_path_error="${tmp_dir}/mismatched-mount-path.err"
+if render_existing_secret_case \
+  /dev/null \
+  --set-string llmRequestRouter.tls.secretName=operator-quic-tls \
+  --set-string llmRequestRouter.tls.mountPath=/var/run/router-tls \
+  --set-string llmRequestRouter.tls.certPath=/etc/stargate/tls/tls.crt \
+  --set-string llmRequestRouter.tls.keyPath=/etc/stargate/tls/tls.key \
+  2> "${mismatched_mount_path_error}"; then
+  fail "existing-Secret mode accepted a mount path that does not contain the TLS files"
+fi
+grep -Fq \
+  "llmRequestRouter.tls.mountPath must match the directory containing llmRequestRouter.tls.certPath and llmRequestRouter.tls.keyPath when llmRequestRouter.tls.mode is existingSecret" \
+  "${mismatched_mount_path_error}" || fail "mismatched mount path did not return the expected guard message"
+
+# Kubernetes volume mounts must be absolute paths. Matching relative paths
+# would otherwise pass the directory consistency check but fail at deployment.
+relative_mount_path_error="${tmp_dir}/relative-mount-path.err"
+if render_existing_secret_case \
+  /dev/null \
+  --set-string llmRequestRouter.tls.secretName=operator-quic-tls \
+  --set-string llmRequestRouter.tls.mountPath=tls \
+  --set-string llmRequestRouter.tls.certPath=tls/tls.crt \
+  --set-string llmRequestRouter.tls.keyPath=tls/tls.key \
+  2> "${relative_mount_path_error}"; then
+  fail "existing-Secret mode accepted a relative TLS mount path"
+fi
+grep -Fq \
+  "llmRequestRouter.tls.mountPath must be an absolute path when llmRequestRouter.tls.mode is existingSecret" \
+  "${relative_mount_path_error}" || fail "relative mount path did not return the expected guard message"
+
+# An unknown mode must fail rather than silently fall back to cert-manager.
+invalid_mode_error="${tmp_dir}/invalid-mode.err"
+if helm template llm-request-router ./llm-request-router \
+  --namespace nvcf \
+  --values ./llm-request-router/values.yaml \
+  --set llmRequestRouter.image.repository=stargate \
+  --set-string llmRequestRouter.tls.mode=externalSecret \
+  > /dev/null 2> "${invalid_mode_error}"; then
+  fail "unknown llmRequestRouter.tls.mode unexpectedly rendered"
+fi
+grep -Fq \
+  'llmRequestRouter.tls.mode must be certManager or existingSecret, got "externalSecret"' \
+  "${invalid_mode_error}" || fail "unknown mode render did not return the expected guard message"
+
 echo "PKI render checks passed"

deploy/stacks/self-managed/environments/base.yaml

@@ -257,21 +257,35 @@ addons:
   # LLM addon: gateway + request router (stargate) for LLM function invocation
   llm:
     enabled: false
-    # QUIC TLS certificate for the request router (Stargate). When enabled,
-    # the chart requests a Certificate from the configured issuer and mounts
-    # the resulting Secret. Managed mode also provisions the OpenBao
-    # service-issuing hierarchy and defaults to
-    # ClusterIssuer/nvcf-openbao-pki. Disabled by default; opt in per env.
+    # QUIC TLS certificate for the request router (Stargate). Disabled by
+    # default; opt in per env.
+    #
+    # mode selects who owns the server identity:
+    #   certManager    - the chart requests a Certificate from the configured
+    #                    issuer and mounts the resulting Secret. Managed mode
+    #                    also provisions the OpenBao service-issuing hierarchy
+    #                    and defaults to ClusterIssuer/nvcf-openbao-pki.
+    #   existingSecret - the chart mounts a Secret you created and manages no
+    #                    issuance. Requires secretName. clusterIssuer.enabled,
+    #                    dnsNames, and allowedDomains must be unset because
+    #                    they only steer stack-managed issuance. You own
+    #                    issuance, renewal, rotation, and recovery, and the
+    #                    stack validates neither the SANs nor the expiry.
     pki:
       enabled: false
+      mode: certManager
       # REQUIRED for a managed issuer. Comma-separated DNS suffixes the
       # OpenBao PKI role accepts. Typically the customer domain plus
       # cluster.local for in-cluster service identity.
       allowedDomains: ""
-      # REQUIRED when enabled. SANs requested on the issued certificate.
+      # REQUIRED for mode certManager. SANs requested on the issued
+      # certificate. Must be empty for mode existingSecret.
       dnsNames: []
-      # Optional overrides; defaults are usually correct.
+      # REQUIRED for mode existingSecret; the kubernetes.io/tls Secret in the
+      # nvcf namespace holding the tls.crt and tls.key entries. Optional for
+      # mode certManager, where it names the Secret cert-manager writes.
       # secretName: stargate-quic-tls
+      # Optional overrides; defaults are usually correct.
       # issuerKind: ClusterIssuer
       # issuerName: nvcf-openbao-pki
       # Stack management defaults to true only for the default