From 9cad89b7a7be5290d8b165a6f18dc25619d0e236 Mon Sep 17 00:00:00 2001 From: Mike Camp Date: Fri, 21 Aug 2026 12:08:29 -0400 Subject: [PATCH 1/4] feat(llm-request-router): route LLM workers through backend router Route worker gRPC registration by HTTP/2 authority and reverse QUIC by SNI through a highly available backend router. Wire it into the LLM addon, expose the required Gateway routes, harden PKI and render validation, and make local self-managed testing use the same path. --- .github/workflows/bazel.yml | 1 + .github/workflows/image-push-manual.yml | 16 + deploy/helm/gateway-routes/Makefile | 7 +- deploy/helm/gateway-routes/README.md | 29 +- .../chart/templates/_helpers.tpl | 4 + .../templates/referencegrant-llm-worker.yaml | 25 ++ .../chart/templates/tcproute-llm-worker.yaml | 28 ++ .../chart/templates/udproute-llm-worker.yaml | 28 ++ deploy/helm/gateway-routes/chart/values.yaml | 26 ++ .../scripts/check-llm-worker-routes.sh | 86 ++++ .../scripts/test-render-routes.sh | 31 ++ deploy/helm/llm-request-router/Makefile | 7 +- deploy/helm/llm-request-router/README.md | 58 ++- .../llm-request-router/templates/_helpers.tpl | 188 ++++++++- .../templates/backend-router-rbac.yaml | 32 ++ .../backend-router-serviceaccount.yaml | 16 + .../backend-router-servicemonitor.yaml | 25 ++ .../templates/backend-router.yaml | 164 ++++++++ .../templates/certificate.yaml | 7 +- .../templates/deployment.yaml | 25 +- .../llm-request-router/values.yaml | 62 ++- .../scripts/check-backend-router-render.sh | 386 ++++++++++++++++++ .../scripts/check-pki-render.sh | 84 ++++ deploy/stacks/self-managed/Makefile | 6 + .../self-managed/environments/base.yaml | 55 +++ deploy/stacks/self-managed/global.yaml.gotmpl | 71 +++- .../helmfile.d/01-dependencies.yaml.gotmpl | 6 + .../helmfile.d/02-core.yaml.gotmpl | 12 +- .../openbao-migrations-llm-env.yaml.gotmpl | 13 + .../tests/api-keys-startup-probe.sh | 44 ++ .../tests/gateway-routes-local-chart.sh | 26 ++ .../tests/grpc-proxy-nats-endpoint.sh | 44 ++ .../tests/llm-pki-openbao-migration.sh | 71 ++++ .../tests/llm-router-local-chart.sh | 21 + .../tests/llm-router-split-cluster.sh | 82 ++++ src/libraries/rust/stargate/Dockerfile | 1 + src/libraries/rust/stargate/README.md | 4 + .../rust/stargate/crates/pylon/BUILD.bazel | 6 +- .../crates/stargate-k8s-router/BUILD.bazel | 6 +- .../crates/stargate-k8s-router/src/metrics.rs | 51 +++ .../rust/stargate/crates/stargate/BUILD.bazel | 13 + .../rust/stargate/tools/ci/BUILD.bazel | 1 + .../tools/ci/oci_image_contains_path_test.sh | 35 ++ tools/ci/test-image-push-manual | 97 +++++ tools/ncp-local-cluster/Makefile | 4 + .../apps/envoy-gateway/gateway-grpc.yaml | 12 + .../static/fake-gpu-operator.yaml | 14 +- .../apps/fake-gpu-operator/values.yaml | 5 +- .../k3d-config-control-plane.yaml | 6 + .../tests/test-multicluster-make.sh | 30 ++ 50 files changed, 2022 insertions(+), 49 deletions(-) create mode 100644 deploy/helm/gateway-routes/chart/templates/referencegrant-llm-worker.yaml create mode 100644 deploy/helm/gateway-routes/chart/templates/tcproute-llm-worker.yaml create mode 100644 deploy/helm/gateway-routes/chart/templates/udproute-llm-worker.yaml create mode 100755 deploy/helm/gateway-routes/scripts/check-llm-worker-routes.sh create mode 100644 deploy/helm/llm-request-router/llm-request-router/templates/backend-router-rbac.yaml create mode 100644 deploy/helm/llm-request-router/llm-request-router/templates/backend-router-serviceaccount.yaml create mode 100644 deploy/helm/llm-request-router/llm-request-router/templates/backend-router-servicemonitor.yaml create mode 100644 deploy/helm/llm-request-router/llm-request-router/templates/backend-router.yaml create mode 100755 deploy/helm/llm-request-router/scripts/check-backend-router-render.sh create mode 100644 deploy/stacks/self-managed/openbao-migrations-llm-env.yaml.gotmpl create mode 100755 deploy/stacks/self-managed/tests/api-keys-startup-probe.sh create mode 100755 deploy/stacks/self-managed/tests/gateway-routes-local-chart.sh create mode 100755 deploy/stacks/self-managed/tests/grpc-proxy-nats-endpoint.sh create mode 100755 deploy/stacks/self-managed/tests/llm-pki-openbao-migration.sh create mode 100755 deploy/stacks/self-managed/tests/llm-router-local-chart.sh create mode 100755 deploy/stacks/self-managed/tests/llm-router-split-cluster.sh create mode 100755 src/libraries/rust/stargate/tools/ci/oci_image_contains_path_test.sh create mode 100755 tools/ci/test-image-push-manual diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 840114d0c..fee22ffff 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -92,6 +92,7 @@ jobs: bash tools/ci/test-bazel-remote-probe bash tools/ci/test-bazel-lock-touches-java python3 tools/ci/test-bazel-lock-touches-java-properties + bash tools/ci/test-image-push-manual - name: Compute changed subtrees id: detect diff --git a/.github/workflows/image-push-manual.yml b/.github/workflows/image-push-manual.yml index 4b5424951..d37ee8f60 100644 --- a/.github/workflows/image-push-manual.yml +++ b/.github/workflows/image-push-manual.yml @@ -305,6 +305,8 @@ jobs: exit 1 fi echo "discovered: ${indexes[*]}" + declare -A seen_repos=() + repos=() for tgt in "${indexes[@]}"; do name="${tgt##*:}"; name="${name%_index}" # Two naming conventions exist in the tree and they mean different @@ -324,6 +326,20 @@ jobs: *-image) repo="${name%-image}" ;; *) sub="$(printf '%s' "$name" | tr '_' '-')"; repo="${svc}-${sub}" ;; esac + if [ -n "${seen_repos[$repo]:-}" ]; then + echo "ERROR: ${tgt} and ${seen_repos[$repo]} both map to image repository ${repo}" >&2 + echo "Give each oci_image_index target a distinct image name before publishing." >&2 + exit 1 + fi + seen_repos["$repo"]="$tgt" + repos+=("$repo") + done + # Validate every target-to-repository mapping before starting any + # push. A later collision must not leave earlier repositories with + # a new snapshot or latest-dispatch tag. + for i in "${!indexes[@]}"; do + tgt="${indexes[$i]}" + repo="${repos[$i]}" dest="${REGISTRY}/${repo}" echo "[push] ${tgt} -> ${dest}:${TAG} (+ latest-dispatch)" mkdir -p ci-ghcr diff --git a/deploy/helm/gateway-routes/Makefile b/deploy/helm/gateway-routes/Makefile index e3d0f06f7..ce1d41401 100644 --- a/deploy/helm/gateway-routes/Makefile +++ b/deploy/helm/gateway-routes/Makefile @@ -14,12 +14,15 @@ # limitations under the License. -.PHONY: test test-render-routes test-vanity-gateway-route +.PHONY: test test-render-routes test-vanity-gateway-route check-llm-worker-routes -test: test-render-routes test-vanity-gateway-route +test: test-render-routes test-vanity-gateway-route check-llm-worker-routes test-render-routes: @sh ./scripts/test-render-routes.sh test-vanity-gateway-route: @sh ./scripts/test-vanity-gateway-route.sh + +check-llm-worker-routes: + @sh ./scripts/check-llm-worker-routes.sh diff --git a/deploy/helm/gateway-routes/README.md b/deploy/helm/gateway-routes/README.md index 306823874..41d6189c2 100644 --- a/deploy/helm/gateway-routes/README.md +++ b/deploy/helm/gateway-routes/README.md @@ -4,9 +4,17 @@ This repository contains the Helm chart for deploying NVCF ingress routes via th ## Overview -The chart deploys `HTTPRoute`, `TCPRoute`, and `ReferenceGrant` resources that attach to an existing Gateway provisioned separately by the cluster operator (e.g. Envoy Gateway, Istio, Traefik, Kong). It also includes optional `PodMonitor` resources for scraping Envoy Gateway proxy metrics with Prometheus. - -The chart deploys routing configuration only. It does not include any container images. Backend services referenced by the routes (`api`, `nvct-api`, `api-keys`, `invocation`, `llm-api-gateway`, `vanity-gateway`, `reval`, `sis`, `grpc`, `nats`) must already be deployed separately. +The chart deploys `HTTPRoute`, `GRPCRoute`, `TCPRoute`, `UDPRoute`, and +`ReferenceGrant` resources that attach to an existing Gateway provisioned +separately by the cluster operator, such as Envoy Gateway, Istio, Traefik, or +Kong. It also includes optional `PodMonitor` resources for scraping Envoy +Gateway proxy metrics with Prometheus. + +The chart deploys routing configuration only. It does not include any +container images. Backend services referenced by the routes (`api`, +`nvct-api`, `api-keys`, `invocation`, `llm-api-gateway`, +`llm-request-router-backend-router`, `vanity-gateway`, `reval`, `sis`, `grpc`, +`nats`) must already be deployed separately. ## Prerequisites @@ -14,7 +22,8 @@ The chart deploys routing configuration only. It does not include any container - Helm 3.x - `kubectl` - A Gateway API compatible controller installed in the cluster -- An existing `Gateway` resource with an HTTP listener (and TCP listeners if the gRPC or NATS routes are enabled) +- Existing `Gateway` resources with the listeners required by each enabled route +- A Gateway controller with `UDPRoute` support when LLM worker routing is enabled - The backend services that the routes target, deployed in their respective namespaces ## Getting Started @@ -57,6 +66,8 @@ Important settings to review before deployment: - `nvcfGatewayRoutes.gateways.shared.*` for the HTTP Gateway name, namespace, and listener - `nvcfGatewayRoutes.gateways.grpc.*` for the TCP Gateway name, namespace, and listener - `nvcfGatewayRoutes.gateways.nats.*` for the NATS TCP Gateway name, namespace, and listener +- `nvcfGatewayRoutes.gateways.llmGrpc.*` for the LLM worker gRPC TCP listener +- `nvcfGatewayRoutes.gateways.llmQuic.*` for the LLM reverse-tunnel UDP listener - `nvcfGatewayRoutes.routes..enabled` to toggle individual routes - `nvcfGatewayRoutes.routes.nvcfApi.grpc.enabled` and `nvcfGatewayRoutes.routes.nvctApi.grpc.enabled` to expose API gRPC routes @@ -87,6 +98,7 @@ Enabled `HTTPRoute` entries must not share a resolved hostname because each `HTT | `grpc` | TCPRoute | Not rendered | `grpc.nvcf:10081` | | `grpcWorker` | TCPRoute (disabled by default) | Not rendered | `grpc.nvcf:10086` | | `nats` | TCPRoute (disabled by default) | Not rendered | `nats.nats-system:4222` | +| `llmWorker` | TCPRoute and UDPRoute (disabled by default) | Not rendered | `llm-request-router-backend-router.:50071/TCP,50072/UDP` | Cross-namespace routing is supported via `ReferenceGrant` resources rendered into each backend namespace. @@ -97,3 +109,12 @@ Cross-namespace routing is supported via `ReferenceGrant` resources rendered int - The `grpc` TCPRoute does not enforce HTTP hostname matching at the Gateway layer. Configure DNS or TCP load balancer routing outside this chart. - The `grpcWorker` TCPRoute is beta support for split or multi-cluster gRPC worker callbacks. It carries HTTP/1 CONNECT callback traffic only. Enable it only when the control-plane grpc-proxy runs one replica with HPA disabled. Multi-replica grpc-proxy requires pod-specific callback routing and is not supported by this shared TCPRoute. - Enabling the `nats` route requires a reachable TCP listener for NATS on the referenced Gateway. The HTTP Gateway address does not imply NATS reachability unless that same Gateway also has the NATS TCP listener configured. +- The `llmWorker` routes target Stargate's authority/SNI-aware backend router. + Set `nvcfGatewayRoutes.routes.llmWorker.backend.namespace` to the effective + namespace of the `llm-request-router` release. The gateway chart cannot + derive the namespace of a separate Helm release. + Use `nvcfGatewayRoutes.routes.llmWorker.backend.grpcPort` for registration + traffic and `nvcfGatewayRoutes.routes.llmWorker.backend.quicPort` for reverse + tunnels; this route does not use the generic `backend.port` setting. + Keep the TCP and UDP Gateways separate when the infrastructure requires + separate load balancers for each protocol. diff --git a/deploy/helm/gateway-routes/chart/templates/_helpers.tpl b/deploy/helm/gateway-routes/chart/templates/_helpers.tpl index e22ae145d..124712752 100644 --- a/deploy/helm/gateway-routes/chart/templates/_helpers.tpl +++ b/deploy/helm/gateway-routes/chart/templates/_helpers.tpl @@ -53,6 +53,10 @@ app.kubernetes.io/name: {{ include "nvcf-gateway.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} +{{- define "nvcf-gateway.llmWorkerBackendNamespace" -}} +{{- required "nvcfGatewayRoutes.routes.llmWorker.backend.namespace is required when llmWorker.enabled is true" .Values.nvcfGatewayRoutes.routes.llmWorker.backend.namespace -}} +{{- end }} + {{/* Validate that enabled HTTPRoutes do not compete for the same hostname and root PathPrefix match on the shared Gateway. All HTTPRoute templates in this diff --git a/deploy/helm/gateway-routes/chart/templates/referencegrant-llm-worker.yaml b/deploy/helm/gateway-routes/chart/templates/referencegrant-llm-worker.yaml new file mode 100644 index 000000000..fe7f592ad --- /dev/null +++ b/deploy/helm/gateway-routes/chart/templates/referencegrant-llm-worker.yaml @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{{- if and .Values.nvcfGatewayRoutes.enabled .Values.nvcfGatewayRoutes.routes.llmWorker.enabled }} +{{- $backendNamespace := include "nvcf-gateway.llmWorkerBackendNamespace" . }} +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: ReferenceGrant +metadata: + name: allow-llm-worker-routes + namespace: {{ $backendNamespace }} + labels: + {{- include "nvcf-gateway.labels" . | nindent 4 }} +spec: + from: + - group: gateway.networking.k8s.io + kind: TCPRoute + namespace: {{ .Values.nvcfGatewayRoutes.gateways.llmGrpc.namespace }} + - group: gateway.networking.k8s.io + kind: UDPRoute + namespace: {{ .Values.nvcfGatewayRoutes.gateways.llmQuic.namespace }} + to: + - group: "" + kind: Service + name: {{ .Values.nvcfGatewayRoutes.routes.llmWorker.backend.name }} +{{- end }} diff --git a/deploy/helm/gateway-routes/chart/templates/tcproute-llm-worker.yaml b/deploy/helm/gateway-routes/chart/templates/tcproute-llm-worker.yaml new file mode 100644 index 000000000..892ec696e --- /dev/null +++ b/deploy/helm/gateway-routes/chart/templates/tcproute-llm-worker.yaml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{{- if and .Values.nvcfGatewayRoutes.enabled .Values.nvcfGatewayRoutes.routes.llmWorker.enabled }} +{{- $backendNamespace := include "nvcf-gateway.llmWorkerBackendNamespace" . }} +apiVersion: gateway.networking.k8s.io/v1alpha2 +kind: TCPRoute +metadata: + name: {{ .Values.nvcfGatewayRoutes.routes.llmWorker.name }}-grpc + namespace: {{ .Values.nvcfGatewayRoutes.gateways.llmGrpc.namespace }} + labels: + {{- include "nvcf-gateway.labels" . | nindent 4 }} + app.kubernetes.io/component: llm-worker-grpc-route + {{- with .Values.nvcfGatewayRoutes.routes.llmWorker.routeAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + parentRefs: + - name: {{ .Values.nvcfGatewayRoutes.gateways.llmGrpc.name }} + namespace: {{ .Values.nvcfGatewayRoutes.gateways.llmGrpc.namespace }} + sectionName: {{ .Values.nvcfGatewayRoutes.gateways.llmGrpc.listenerName }} + rules: + - backendRefs: + - name: {{ .Values.nvcfGatewayRoutes.routes.llmWorker.backend.name }} + namespace: {{ $backendNamespace }} + port: {{ .Values.nvcfGatewayRoutes.routes.llmWorker.backend.grpcPort }} +{{- end }} diff --git a/deploy/helm/gateway-routes/chart/templates/udproute-llm-worker.yaml b/deploy/helm/gateway-routes/chart/templates/udproute-llm-worker.yaml new file mode 100644 index 000000000..cba18b322 --- /dev/null +++ b/deploy/helm/gateway-routes/chart/templates/udproute-llm-worker.yaml @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{{- if and .Values.nvcfGatewayRoutes.enabled .Values.nvcfGatewayRoutes.routes.llmWorker.enabled }} +{{- $backendNamespace := include "nvcf-gateway.llmWorkerBackendNamespace" . }} +apiVersion: gateway.networking.k8s.io/v1alpha2 +kind: UDPRoute +metadata: + name: {{ .Values.nvcfGatewayRoutes.routes.llmWorker.name }}-quic + namespace: {{ .Values.nvcfGatewayRoutes.gateways.llmQuic.namespace }} + labels: + {{- include "nvcf-gateway.labels" . | nindent 4 }} + app.kubernetes.io/component: llm-worker-quic-route + {{- with .Values.nvcfGatewayRoutes.routes.llmWorker.routeAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + parentRefs: + - name: {{ .Values.nvcfGatewayRoutes.gateways.llmQuic.name }} + namespace: {{ .Values.nvcfGatewayRoutes.gateways.llmQuic.namespace }} + sectionName: {{ .Values.nvcfGatewayRoutes.gateways.llmQuic.listenerName }} + rules: + - backendRefs: + - name: {{ .Values.nvcfGatewayRoutes.routes.llmWorker.backend.name }} + namespace: {{ $backendNamespace }} + port: {{ .Values.nvcfGatewayRoutes.routes.llmWorker.backend.quicPort }} +{{- end }} diff --git a/deploy/helm/gateway-routes/chart/values.yaml b/deploy/helm/gateway-routes/chart/values.yaml index 462b53485..63c982fd1 100644 --- a/deploy/helm/gateway-routes/chart/values.yaml +++ b/deploy/helm/gateway-routes/chart/values.yaml @@ -51,6 +51,17 @@ nvcfGatewayRoutes: namespace: gateway # Listener (sectionName) on the Gateway to attach the route to listenerName: nats + # TCP Gateway for LLM worker registration and Stargate watches. + llmGrpc: + name: llm-grpc-gateway + namespace: gateway + listenerName: llm-grpc + # UDP Gateway for LLM reverse tunnels. This may be a separate load + # balancer from the TCP Gateway. + llmQuic: + name: llm-quic-gateway + namespace: gateway + listenerName: llm-quic # HTTPRoute configurations routes: @@ -239,6 +250,21 @@ nvcfGatewayRoutes: namespace: nats-system port: 4222 routeAnnotations: {} + + # Backend-facing LLM registration/watch and reverse-tunnel routes. + # The backend router selects the correct Stargate pod by gRPC authority + # and QUIC SNI, so this route supports multi-replica request routers. + llmWorker: + enabled: false + name: llm-worker + backend: + name: llm-request-router-backend-router + # Required when llmWorker is enabled. Set this to the effective + # llm-request-router namespace. + namespace: "" + grpcPort: 50071 + quicPort: 50072 + routeAnnotations: {} # NVCF Worker Container needs to be able to fetch secrets for function pods # In split/multi cluster planes, ess resides in control-plane and requires diff --git a/deploy/helm/gateway-routes/scripts/check-llm-worker-routes.sh b/deploy/helm/gateway-routes/scripts/check-llm-worker-routes.sh new file mode 100755 index 000000000..229bf124f --- /dev/null +++ b/deploy/helm/gateway-routes/scripts/check-llm-worker-routes.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +chart_dir="${script_dir}/../chart" +rendered="$(mktemp)" +disabled="$(mktemp)" +invalid_backend_namespace_error="$(mktemp)" +trap 'rm -f "$rendered" "$disabled" "$invalid_backend_namespace_error"' EXIT + +helm template nvcf-gateway-routes "$chart_dir" \ + --namespace gateway \ + --set nvcfGatewayRoutes.routes.llmWorker.enabled=true \ + --set nvcfGatewayRoutes.gateways.llmGrpc.name=llm-grpc-gateway \ + --set nvcfGatewayRoutes.gateways.llmGrpc.namespace=gateway \ + --set nvcfGatewayRoutes.gateways.llmQuic.name=llm-quic-gateway \ + --set nvcfGatewayRoutes.gateways.llmQuic.namespace=gateway \ + --set nvcfGatewayRoutes.routes.llmWorker.backend.namespace=router-system \ + >"$rendered" + +assert_contains() { + local pattern="$1" + local message="$2" + if ! grep -Fq -- "$pattern" "$rendered"; then + echo "FAIL: ${message}" >&2 + exit 1 + fi +} + +assert_contains "kind: TCPRoute" \ + "LLM worker routing must expose gRPC registration over TCP" +assert_contains "kind: UDPRoute" \ + "LLM worker routing must expose reverse tunnels over UDP" +assert_contains "name: llm-request-router-backend-router" \ + "LLM worker routes must target the authority/SNI-aware backend router" +assert_contains "name: allow-llm-worker-routes" \ + "ReferenceGrant must permit cross-namespace LLM worker routes" +assert_contains "sectionName: llm-grpc" \ + "TCPRoute must attach to the configured LLM gRPC listener" +assert_contains "sectionName: llm-quic" \ + "UDPRoute must attach to the configured LLM QUIC listener" + +reference_grant_service_name="$(awk ' + $0 == "kind: ReferenceGrant" { in_grant = 1; target_grant = 0; in_to = 0 } + in_grant && !target_grant && $1 == "name:" && $2 == "allow-llm-worker-routes" { target_grant = 1 } + target_grant && $0 == " to:" { in_to = 1 } + target_grant && in_to && $1 == "name:" { print $2; exit } +' "$rendered")" +if [[ "$reference_grant_service_name" != "llm-request-router-backend-router" ]]; then + echo "FAIL: LLM worker ReferenceGrant must stay scoped to the configured backend Service" >&2 + exit 1 +fi + +backend_namespace_references="$(grep -Fc -- "namespace: router-system" "$rendered" || true)" +if [[ "$backend_namespace_references" != "3" ]]; then + echo "FAIL: LLM worker routes and ReferenceGrant must use the configured backend namespace" >&2 + exit 1 +fi + +if helm template nvcf-gateway-routes "$chart_dir" \ + --namespace gateway \ + --set nvcfGatewayRoutes.routes.llmWorker.enabled=true \ + --set-string nvcfGatewayRoutes.routes.llmWorker.backend.namespace= \ + >/dev/null 2>"$invalid_backend_namespace_error"; then + echo "FAIL: enabled LLM worker routing must require an explicit backend namespace" >&2 + exit 1 +fi +if ! grep -Fq -- "nvcfGatewayRoutes.routes.llmWorker.backend.namespace is required when llmWorker.enabled is true" "$invalid_backend_namespace_error"; then + echo "FAIL: missing LLM worker backend namespace must return the expected validation error" >&2 + exit 1 +fi + +helm template nvcf-gateway-routes "$chart_dir" \ + --namespace gateway \ + --set nvcfGatewayRoutes.routes.llmWorker.enabled=false \ + >"$disabled" + +if grep -Eq '^ name: (llm-worker-(grpc|quic)|allow-llm-worker-routes)$' "$disabled"; then + echo "FAIL: disabled LLM worker routing must not render route resources" >&2 + exit 1 +fi + +echo "PASS: LLM worker Gateway routes render correctly" diff --git a/deploy/helm/gateway-routes/scripts/test-render-routes.sh b/deploy/helm/gateway-routes/scripts/test-render-routes.sh index 464bc2c29..11dc6075c 100755 --- a/deploy/helm/gateway-routes/scripts/test-render-routes.sh +++ b/deploy/helm/gateway-routes/scripts/test-render-routes.sh @@ -164,6 +164,9 @@ assert_resource_count "$default_render" GRPCRoute nvcf-api-grpc gateway 0 assert_resource_count "$default_render" GRPCRoute nvct-api-grpc gateway 0 assert_resource_count "$default_render" TCPRoute grpc-worker gateway 0 assert_resource_count "$default_render" TCPRoute nats gateway 0 +assert_resource_count "$default_render" TCPRoute llm-worker-grpc gateway 0 +assert_resource_count "$default_render" UDPRoute llm-worker-quic gateway 0 +assert_resource_count "$default_render" ReferenceGrant allow-llm-worker-routes nvcf 0 assert_resource_count "$default_render" ReferenceGrant allow-tcproute-to-nats nats-system 0 helm template nvcf-gateway-routes "$repo_root/chart" \ @@ -172,6 +175,8 @@ helm template nvcf-gateway-routes "$repo_root/chart" \ --set nvcfGatewayRoutes.routes.nvctApi.grpc.enabled=true \ --set nvcfGatewayRoutes.routes.grpcWorker.enabled=true \ --set nvcfGatewayRoutes.routes.nats.enabled=true \ + --set nvcfGatewayRoutes.routes.llmWorker.enabled=true \ + --set nvcfGatewayRoutes.routes.llmWorker.backend.namespace=nvcf \ --set nvcfGatewayRoutes.gateways.nats.name=nats-gateway \ --set nvcfGatewayRoutes.gateways.nats.namespace=gateway \ --set nvcfGatewayRoutes.gateways.nats.listenerName=nats \ @@ -225,6 +230,32 @@ assert_resource_field "$enabled_render" TCPRoute nats gateway '.spec.rules[0].ba assert_resource_field "$enabled_render" TCPRoute nats gateway '.spec.hostnames' null assert_resource_field "$enabled_render" TCPRoute nats gateway '.metadata.annotations' null +assert_resource_count "$enabled_render" TCPRoute llm-worker-grpc gateway 1 +assert_resource_field "$enabled_render" TCPRoute llm-worker-grpc gateway '.metadata.labels."app.kubernetes.io/component"' llm-worker-grpc-route +assert_resource_field "$enabled_render" TCPRoute llm-worker-grpc gateway '.spec.parentRefs[0].name' llm-grpc-gateway +assert_resource_field "$enabled_render" TCPRoute llm-worker-grpc gateway '.spec.parentRefs[0].namespace' gateway +assert_resource_field "$enabled_render" TCPRoute llm-worker-grpc gateway '.spec.parentRefs[0].sectionName' llm-grpc +assert_resource_field "$enabled_render" TCPRoute llm-worker-grpc gateway '.spec.rules[0].backendRefs[0].name' llm-request-router-backend-router +assert_resource_field "$enabled_render" TCPRoute llm-worker-grpc gateway '.spec.rules[0].backendRefs[0].namespace' nvcf +assert_resource_field "$enabled_render" TCPRoute llm-worker-grpc gateway '.spec.rules[0].backendRefs[0].port' 50071 + +assert_resource_count "$enabled_render" UDPRoute llm-worker-quic gateway 1 +assert_resource_field "$enabled_render" UDPRoute llm-worker-quic gateway '.metadata.labels."app.kubernetes.io/component"' llm-worker-quic-route +assert_resource_field "$enabled_render" UDPRoute llm-worker-quic gateway '.spec.parentRefs[0].name' llm-quic-gateway +assert_resource_field "$enabled_render" UDPRoute llm-worker-quic gateway '.spec.parentRefs[0].namespace' gateway +assert_resource_field "$enabled_render" UDPRoute llm-worker-quic gateway '.spec.parentRefs[0].sectionName' llm-quic +assert_resource_field "$enabled_render" UDPRoute llm-worker-quic gateway '.spec.rules[0].backendRefs[0].name' llm-request-router-backend-router +assert_resource_field "$enabled_render" UDPRoute llm-worker-quic gateway '.spec.rules[0].backendRefs[0].namespace' nvcf +assert_resource_field "$enabled_render" UDPRoute llm-worker-quic gateway '.spec.rules[0].backendRefs[0].port' 50072 + +assert_resource_count "$enabled_render" ReferenceGrant allow-llm-worker-routes nvcf 1 +assert_resource_field "$enabled_render" ReferenceGrant allow-llm-worker-routes nvcf '.spec.from[0].kind' TCPRoute +assert_resource_field "$enabled_render" ReferenceGrant allow-llm-worker-routes nvcf '.spec.from[0].namespace' gateway +assert_resource_field "$enabled_render" ReferenceGrant allow-llm-worker-routes nvcf '.spec.from[1].kind' UDPRoute +assert_resource_field "$enabled_render" ReferenceGrant allow-llm-worker-routes nvcf '.spec.from[1].namespace' gateway +assert_resource_field "$enabled_render" ReferenceGrant allow-llm-worker-routes nvcf '.spec.to[0].kind' Service +assert_resource_field "$enabled_render" ReferenceGrant allow-llm-worker-routes nvcf '.spec.to[0].name' llm-request-router-backend-router + assert_resource_count "$enabled_render" ReferenceGrant allow-tcproute-to-nats nats-system 1 assert_resource_field "$enabled_render" ReferenceGrant allow-tcproute-to-nats nats-system '.spec.from[0].kind' TCPRoute assert_resource_field "$enabled_render" ReferenceGrant allow-tcproute-to-nats nats-system '.spec.from[0].namespace' gateway diff --git a/deploy/helm/llm-request-router/Makefile b/deploy/helm/llm-request-router/Makefile index 5e14e15d9..605e20cb5 100644 --- a/deploy/helm/llm-request-router/Makefile +++ b/deploy/helm/llm-request-router/Makefile @@ -27,7 +27,7 @@ OCI_REGISTRY_NAMESPACE ?= CHART_NAME := $(shell yq -r .name $(helm_dir)/Chart.yaml) CHART_VERSION := $(shell yq -r .version $(helm_dir)/Chart.yaml) -.PHONY: install uninstall status lint template validate clean package push-oci test check-pki-render check-multi-replica-render +.PHONY: install uninstall status lint template validate clean package push-oci test check-pki-render check-multi-replica-render check-backend-router-render install: ifndef values @@ -65,7 +65,7 @@ template: validate: template kubeconform -strict -summary -output pretty -kubernetes-version 1.31.5 bin/manifest.yaml -test: check-pki-render check-multi-replica-render +test: check-pki-render check-multi-replica-render check-backend-router-render check-pki-render: sh ./scripts/check-pki-render.sh @@ -73,6 +73,9 @@ check-pki-render: check-multi-replica-render: sh ./scripts/check-multi-replica-render.sh +check-backend-router-render: + sh ./scripts/check-backend-router-render.sh + clean: rm -rf ./packaged-charts bin diff --git a/deploy/helm/llm-request-router/README.md b/deploy/helm/llm-request-router/README.md index d1683c608..eea2553f0 100644 --- a/deploy/helm/llm-request-router/README.md +++ b/deploy/helm/llm-request-router/README.md @@ -4,7 +4,17 @@ This repository contains the Helm chart for deploying the NVCF LLM Request Route ## Overview -The chart packages the LLM Request Router StatefulSet with HTTP and gRPC services, a metrics endpoint, and a headless service for multi-instance DNS discovery. A Vault Agent sidecar is configured to fetch a service token from a Vault or OpenBao backend; the application reads `nvcfApiToken` from `/vault/secrets/secrets.json` and attaches it as a Bearer token to outgoing worker authentication gRPC calls. +The chart packages the LLM Request Router StatefulSet with HTTP and gRPC +services, a metrics endpoint, and a headless service for multi-instance DNS +discovery. It can also deploy the Stargate Kubernetes backend router for +worker gRPC registration and reverse QUIC tunnels through a shared Gateway or +load balancer. The backend router selects the correct Stargate pod from gRPC +authority and QUIC SNI. + +A Vault Agent sidecar is configured to fetch a service token from a Vault or +OpenBao backend. The application reads `nvcfApiToken` from +`/vault/secrets/secrets.json` and attaches it as a Bearer token to outgoing +worker authentication gRPC calls. The default chart values do not set the required image registry and repository. They must be supplied through an additional values file at install time, and access to those images must be arranged separately. @@ -72,6 +82,7 @@ Important settings to review before deployment: - `llmRequestRouter.imagePullSecrets` for private registry access - `llmRequestRouter.replicaCount`, resource requests, and limits for your environment - `llmRequestRouter.service.*` for HTTP, gRPC, metrics, and headless service ports +- `llmRequestRouter.backendRouter.*` for multi-replica worker gRPC and reverse-tunnel routing - `llmRequestRouter.metrics.enabled` to expose the metrics port on the Service (default: `false`) - `llmRequestRouter.metrics.serviceMonitor.enabled` to create a Prometheus `ServiceMonitor` (requires `metrics.enabled`) - `llmRequestRouter.certificate.*` to let cert-manager issue the Stargate QUIC server certificate @@ -83,6 +94,51 @@ Important settings to review before deployment: The default values include development-oriented placeholders. Override them before using the chart in any shared or production environment. +## Backend Worker Routing + +Enable `llmRequestRouter.backendRouter.enabled` when workers reach a +multi-replica request router through a shared endpoint. Set both pylon dial +addresses to the external endpoints that workers can resolve: + +```yaml +llmRequestRouter: + backendRouter: + enabled: true + image: + tag: + pylonGrpcDialAddress: llm-router.example.com:443 + pylonReverseTunnelDialAddress: llm-router.example.com:8080 +``` + +The chart uses the main Stargate image for both workloads. The image must +contain `/usr/local/bin/stargate-k8s-router`. Set +`llmRequestRouter.backendRouter.image.tag` to an image version that contains +that binary. The chart requires this explicit pin when backend routing is +enabled. + +The backend router watches EndpointSlices. The chart creates a dedicated +ServiceAccount by default and binds a namespaced Role to it when +`llmRequestRouter.rbac.create=true`. When +`llmRequestRouter.backendRouter.serviceAccount.create=false`, set +`llmRequestRouter.backendRouter.serviceAccount.name` to an existing account. +When `rbac.create=false`, grant `get`, `list`, and `watch` on +`discovery.k8s.io/endpointslices` to that account outside this chart. + +Route TCP port `50071` and UDP port `50072` to the +`llm-request-router-backend-router` Service. The NVCF gateway-routes chart can +create the matching `TCPRoute`, `UDPRoute`, and `ReferenceGrant` resources. +The Gateway implementation must support Gateway API `UDPRoute`. + +When QUIC verification is enabled, the mounted certificate must cover the +worker-facing reverse-tunnel hostname and the per-pod hostname template. The +default template is +`{pod_name}.llm-request-router-headless..svc.cluster.local`. + +Stargate and the backend router read the TLS certificate and key only during +process startup. After cert-manager or another issuer renews the Secret, +restart both workloads or configure a Secret reloader that triggers their +rollouts. + ## Load Balancer Configuration The chart can pass a Stargate load-balancer config in either of two ways: diff --git a/deploy/helm/llm-request-router/llm-request-router/templates/_helpers.tpl b/deploy/helm/llm-request-router/llm-request-router/templates/_helpers.tpl index 512a69cea..8bc30c0cf 100644 --- a/deploy/helm/llm-request-router/llm-request-router/templates/_helpers.tpl +++ b/deploy/helm/llm-request-router/llm-request-router/templates/_helpers.tpl @@ -49,21 +49,29 @@ app.kubernetes.io/instance: {{ .Release.Name }} {{- printf "app.kubernetes.io/name=%s,app.kubernetes.io/instance=%s" (include "llm-request-router.name" .) .Release.Name -}} {{- end }} +{{- define "llm-request-router.backendRouterName" -}} +{{- printf "%s-backend-router" (include "llm-request-router.fullname" .) | trunc 63 | trimSuffix "-" -}} +{{- end }} + +{{- define "llm-request-router.backendRouterSelectorLabels" -}} +app.kubernetes.io/name: {{ include "llm-request-router.backendRouterName" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{- define "llm-request-router.backendRouterLabels" -}} +helm.sh/chart: {{ include "llm-request-router.chart" . }} +{{ include "llm-request-router.backendRouterSelectorLabels" . }} +app.kubernetes.io/component: backend-router +{{- with .Values.llmRequestRouter.backendRouter.image.tag }} +app.kubernetes.io/version: {{ . | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + {{- define "llm-request-router.namespace" -}} {{- default .Release.Namespace .Values.llmRequestRouter.namespace -}} {{- end -}} -{{- define "llm-request-router.advertisedHostnameTemplate" -}} -{{- $configuredTemplate := .Values.llmRequestRouter.kubernetes.advertisedHostnameTemplate -}} -{{- if $configuredTemplate -}} -{{- $configuredTemplate -}} -{{- else if eq (.Values.llmRequestRouter.replicaCount | int) 1 -}} -{{- printf "%s.%s.svc.cluster.local" (include "llm-request-router.fullname" .) (include "llm-request-router.namespace" .) -}} -{{- else -}} -{{- printf "{pod_name}.%s.%s.svc.cluster.local" .Values.llmRequestRouter.service.headlessName (include "llm-request-router.namespace" .) -}} -{{- end -}} -{{- end -}} - {{- define "llm-request-router.isValidDnsName" -}} {{- $name := .name | toString | lower -}} {{- $labels := splitList "." $name -}} @@ -73,7 +81,7 @@ app.kubernetes.io/instance: {{ .Release.Name }} (not (hasPrefix "." $name)) (not (hasSuffix "." $name)) -}} {{- range $label := $labels -}} -{{- if not (regexMatch "^[a-z0-9_]([a-z0-9_-]{0,61}[a-z0-9_])?$" $label) -}} +{{- if not (regexMatch "^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$" $label) -}} {{- $valid = false -}} {{- end -}} {{- end -}} @@ -83,6 +91,23 @@ app.kubernetes.io/instance: {{ .Release.Name }} {{- if $valid -}}true{{- end -}} {{- end -}} +{{/* +The Certificate that gets issued is the configured dnsNames plus, when backend +routing is on, the wildcard form of the advertised pod hostname. Validation and +rendering must agree on that list, so both read it from here. +*/}} +{{- define "llm-request-router.effectiveCertificateDnsNames" -}} +{{- $certificate := .Values.llmRequestRouter.certificate | default dict -}} +{{- $dnsNames := dig "dnsNames" (list) $certificate -}} +{{- if dig "backendRouter" "enabled" false .Values.llmRequestRouter -}} +{{- $wildcard := replace "{pod_name}" "*" (include "llm-request-router.advertisedHostnameTemplate" .) -}} +{{- if not (has $wildcard $dnsNames) -}} +{{- $dnsNames = append $dnsNames $wildcard -}} +{{- end -}} +{{- end -}} +{{- toJson $dnsNames -}} +{{- end -}} + {{/* Certificate wildcards follow rustls-webpki rules: only a complete leftmost label may be a wildcard, and it matches exactly one hostname label. Replace @@ -92,7 +117,7 @@ comparing suffixes. {{- define "llm-request-router.validateCertificateDnsNames" -}} {{- $certificate := .Values.llmRequestRouter.certificate | default dict -}} {{- if $certificate.enabled -}} -{{- $dnsNames := $certificate.dnsNames | default (list) -}} +{{- $dnsNames := include "llm-request-router.effectiveCertificateDnsNames" . | fromJsonArray -}} {{- if eq (len $dnsNames) 0 -}} {{- fail "llmRequestRouter.certificate.dnsNames is required when certificate.enabled is true" -}} {{- end -}} @@ -149,6 +174,48 @@ comparing suffixes. {{- end -}} {{- end -}} +{{/* +The OpenBao signing role this chart provisions is created with +allow_subdomains=true, allow_bare_domains=false, and allow_wildcard_certificates=true. +A SAN outside allowed_domains renders cleanly and then fails at issuance, so +check it here instead. Only applies when this chart owns both the Certificate +and the role. + +Coverage, matching the role flags: + name.sub.domain covered when it is a strict subdomain of an allowed domain + *.sub.domain same, and additionally when the wildcard sits directly on an + allowed domain + domain never covered on its own, because bare issuance is refused +*/}} +{{- define "llm-request-router.validatePkiAllowedDomains" -}} +{{- $pki := .Values.llmRequestRouter.pki | default dict -}} +{{- $certificate := .Values.llmRequestRouter.certificate | default dict -}} +{{- if and $pki.enabled $certificate.enabled -}} +{{- $configured := $pki.allowedDomains | default "" | toString -}} +{{- $allowed := splitList "," $configured -}} +{{- $dnsNames := include "llm-request-router.effectiveCertificateDnsNames" . | fromJsonArray -}} +{{- range $dnsName := $dnsNames -}} +{{- $name := $dnsName | toString | lower | trim -}} +{{- $isWildcard := hasPrefix "*." $name -}} +{{- $base := trimPrefix "*." $name -}} +{{- $covered := false -}} +{{- range $allowedDomain := $allowed -}} +{{- $domain := $allowedDomain | toString | lower | trim -}} +{{- if $domain -}} +{{- if hasSuffix (printf ".%s" $domain) $base -}} +{{- $covered = true -}} +{{- else if and $isWildcard (eq $base $domain) -}} +{{- $covered = true -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- if not $covered -}} +{{- fail (printf "certificate DNS name %q is not covered by llmRequestRouter.pki.allowedDomains %q. The OpenBao signing role uses allow_subdomains=true and allow_bare_domains=false, so cert-manager issuance would be rejected after a successful render. Add a covering suffix, for example cluster.local for in-cluster names." $dnsName $configured) -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} + {{- define "llm-request-router.serviceAccountName" -}} {{- if .Values.llmRequestRouter.serviceAccount.create }} {{- default (include "llm-request-router.fullname" .) .Values.llmRequestRouter.serviceAccount.name }} @@ -157,6 +224,15 @@ comparing suffixes. {{- end }} {{- end }} +{{- define "llm-request-router.backendRouterServiceAccountName" -}} +{{- $serviceAccount := .Values.llmRequestRouter.backendRouter.serviceAccount | default dict -}} +{{- if $serviceAccount.create -}} +{{- default (include "llm-request-router.backendRouterName" .) $serviceAccount.name -}} +{{- else -}} +{{- required "llmRequestRouter.backendRouter.serviceAccount.name is required when backendRouter is enabled and backendRouter.serviceAccount.create is false" $serviceAccount.name -}} +{{- end -}} +{{- end }} + {{- define "llm-request-router.image" -}} {{- $registry := .Values.llmRequestRouter.image.registry -}} {{- $repository := .Values.llmRequestRouter.image.repository -}} @@ -168,6 +244,59 @@ comparing suffixes. {{- end -}} {{- end }} +{{- define "llm-request-router.advertisedHostnameTemplate" -}} +{{- $configured := .Values.llmRequestRouter.kubernetes.advertisedHostnameTemplate -}} +{{- $backendRouterEnabled := dig "backendRouter" "enabled" false .Values.llmRequestRouter -}} +{{- if and $backendRouterEnabled $configured (ne (len (splitList "{pod_name}" $configured)) 2) -}} +{{- fail "llmRequestRouter.kubernetes.advertisedHostnameTemplate must contain exactly one {pod_name} when backendRouter.enabled is true" -}} +{{- end -}} +{{- if $configured -}} +{{- $configured -}} +{{- else if or $backendRouterEnabled (gt (.Values.llmRequestRouter.replicaCount | int) 1) -}} +{{- printf "{pod_name}.%s.%s.svc.cluster.local" .Values.llmRequestRouter.service.headlessName (include "llm-request-router.namespace" .) -}} +{{- else -}} +{{- printf "%s.%s.svc.cluster.local" (include "llm-request-router.fullname" .) (include "llm-request-router.namespace" .) -}} +{{- end -}} +{{- end }} + +{{/* +Workers dial these to reach the router. In-cluster the backend-router Service is +the right answer and needs no configuration, which is what lets backend routing +be on by default. Split-cluster and multi-region operators override both with an +externally reachable address. +*/}} +{{- define "llm-request-router.backendRouterGrpcDialAddress" -}} +{{- $backendRouter := .Values.llmRequestRouter.backendRouter | default dict -}} +{{- $configured := dig "pylonGrpcDialAddress" "" $backendRouter | toString | trim -}} +{{- if $configured -}} +{{- $configured -}} +{{- else -}} +{{- printf "%s.%s.svc.cluster.local:%v" (include "llm-request-router.backendRouterName" .) (include "llm-request-router.namespace" .) (dig "service" "grpcPort" 50071 $backendRouter) -}} +{{- end -}} +{{- end -}} + +{{- define "llm-request-router.backendRouterReverseTunnelDialAddress" -}} +{{- $backendRouter := .Values.llmRequestRouter.backendRouter | default dict -}} +{{- $configured := dig "pylonReverseTunnelDialAddress" "" $backendRouter | toString | trim -}} +{{- if $configured -}} +{{- $configured -}} +{{- else -}} +{{- printf "%s.%s.svc.cluster.local:%v" (include "llm-request-router.backendRouterName" .) (include "llm-request-router.namespace" .) (dig "service" "reverseTunnelPort" 50072 $backendRouter) -}} +{{- end -}} +{{- end -}} + +{{- define "llm-request-router.backendRouterImage" -}} +{{- $image := .Values.llmRequestRouter.backendRouter.image -}} +{{- $registry := default .Values.llmRequestRouter.image.registry $image.registry -}} +{{- $repository := required "llmRequestRouter.backendRouter.image.repository or llmRequestRouter.image.repository is required when backend routing is enabled" (default .Values.llmRequestRouter.image.repository $image.repository) -}} +{{- $tag := default .Chart.AppVersion $image.tag -}} +{{- if $registry -}} +{{- printf "%s/%s:%s" $registry $repository $tag -}} +{{- else -}} +{{- printf "%s:%s" $repository $tag -}} +{{- end -}} +{{- end }} + {{- define "llm-request-router.pkiMigrationsImage" -}} {{- $img := .Values.llmRequestRouter.pki.image -}} {{- $registry := $img.registry -}} @@ -240,6 +369,39 @@ the mount and the Stargate arguments are all conditional on them. {{- end -}} {{- end }} +{{- define "llm-request-router.validateTlsCertKeyDir" -}} +{{- $tls := .Values.llmRequestRouter.tls | default dict -}} +{{- if and $tls.certPath $tls.keyPath (ne (dir $tls.certPath) (dir $tls.keyPath)) -}} +{{- fail "llmRequestRouter.tls.certPath and llmRequestRouter.tls.keyPath must use the same directory" -}} +{{- end -}} +{{- end }} + +{{- define "llm-request-router.validateBackendRouterTls" -}} +{{- $tls := .Values.llmRequestRouter.tls | default dict -}} +{{- $secretName := include "llm-request-router.tlsSecretName" . -}} +{{- $hasSecret := not (empty $secretName) -}} +{{- $hasCert := not (empty $tls.certPath) -}} +{{- $hasKey := not (empty $tls.keyPath) -}} +{{- $hasAny := or $hasSecret $hasCert $hasKey -}} +{{- $hasAll := and $hasSecret $hasCert $hasKey -}} +{{- if and $hasAny (not $hasAll) -}} +{{- fail "llmRequestRouter backend routing requires tls.secretName (or certificate secret), tls.certPath, and tls.keyPath together" -}} +{{- end -}} +{{- if and (not $tls.quicInsecure) (not $hasAll) -}} +{{- fail "llmRequestRouter backend routing requires a TLS Secret and cert/key paths when tls.quicInsecure is false" -}} +{{- end -}} +{{- if $hasAll -}} +{{- include "llm-request-router.validateTlsCertKeyDir" . -}} +{{- end -}} +{{- if and $hasAll (ne (clean (include "llm-request-router.tlsMountPath" .)) (clean (dir $tls.certPath))) -}} +{{- fail "llmRequestRouter.tls.mountPath must match the directory containing tls.certPath and tls.keyPath" -}} +{{- end -}} +{{- end }} + +{{- define "llm-request-router.validateBackendRouterServiceAccount" -}} +{{- $_ := include "llm-request-router.backendRouterServiceAccountName" . -}} +{{- end }} + {{/* Vault Annotations */}} diff --git a/deploy/helm/llm-request-router/llm-request-router/templates/backend-router-rbac.yaml b/deploy/helm/llm-request-router/llm-request-router/templates/backend-router-rbac.yaml new file mode 100644 index 000000000..6e8dae34a --- /dev/null +++ b/deploy/helm/llm-request-router/llm-request-router/templates/backend-router-rbac.yaml @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{{- if and .Values.llmRequestRouter.backendRouter.enabled .Values.llmRequestRouter.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "llm-request-router.backendRouterName" . }}-endpointslice-reader + namespace: {{ include "llm-request-router.namespace" . }} + labels: + {{- include "llm-request-router.backendRouterLabels" . | nindent 4 }} +rules: + - apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "llm-request-router.backendRouterName" . }}-endpointslice-reader + namespace: {{ include "llm-request-router.namespace" . }} + labels: + {{- include "llm-request-router.backendRouterLabels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "llm-request-router.backendRouterName" . }}-endpointslice-reader +subjects: + - kind: ServiceAccount + name: {{ include "llm-request-router.backendRouterServiceAccountName" . }} + namespace: {{ include "llm-request-router.namespace" . }} +{{- end }} diff --git a/deploy/helm/llm-request-router/llm-request-router/templates/backend-router-serviceaccount.yaml b/deploy/helm/llm-request-router/llm-request-router/templates/backend-router-serviceaccount.yaml new file mode 100644 index 000000000..e92bcdb85 --- /dev/null +++ b/deploy/helm/llm-request-router/llm-request-router/templates/backend-router-serviceaccount.yaml @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{{- if and .Values.llmRequestRouter.backendRouter.enabled .Values.llmRequestRouter.backendRouter.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "llm-request-router.backendRouterServiceAccountName" . }} + namespace: {{ include "llm-request-router.namespace" . }} + labels: + {{- include "llm-request-router.backendRouterLabels" . | nindent 4 }} + {{- with .Values.llmRequestRouter.backendRouter.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/deploy/helm/llm-request-router/llm-request-router/templates/backend-router-servicemonitor.yaml b/deploy/helm/llm-request-router/llm-request-router/templates/backend-router-servicemonitor.yaml new file mode 100644 index 000000000..b3b48854a --- /dev/null +++ b/deploy/helm/llm-request-router/llm-request-router/templates/backend-router-servicemonitor.yaml @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{{- $metricsEnabled := dig "metrics" "enabled" false .Values.llmRequestRouter }} +{{- $serviceMonitorEnabled := dig "metrics" "serviceMonitor" "enabled" false .Values.llmRequestRouter }} +{{- if and .Values.llmRequestRouter.backendRouter.enabled $metricsEnabled $serviceMonitorEnabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "llm-request-router.backendRouterName" . }}-metrics + namespace: {{ include "llm-request-router.namespace" . }} + labels: + {{- include "llm-request-router.backendRouterLabels" . | nindent 4 }} +spec: + endpoints: + - interval: {{ dig "metrics" "serviceMonitor" "interval" "30s" .Values.llmRequestRouter | quote }} + path: /metrics + port: health + namespaceSelector: + matchNames: + - {{ include "llm-request-router.namespace" . }} + selector: + matchLabels: + {{- include "llm-request-router.backendRouterSelectorLabels" . | nindent 6 }} +{{- end }} diff --git a/deploy/helm/llm-request-router/llm-request-router/templates/backend-router.yaml b/deploy/helm/llm-request-router/llm-request-router/templates/backend-router.yaml new file mode 100644 index 000000000..e9e5efed1 --- /dev/null +++ b/deploy/helm/llm-request-router/llm-request-router/templates/backend-router.yaml @@ -0,0 +1,164 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{{- if .Values.llmRequestRouter.backendRouter.enabled }} +{{- include "llm-request-router.validateBackendRouterTls" . }} +{{- include "llm-request-router.validateBackendRouterServiceAccount" . }} +{{- $advertisedHostnameTemplate := include "llm-request-router.advertisedHostnameTemplate" . }} +{{- $tlsSecretName := include "llm-request-router.tlsSecretName" . }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "llm-request-router.backendRouterName" . }} + namespace: {{ include "llm-request-router.namespace" . }} + labels: + {{- include "llm-request-router.backendRouterLabels" . | nindent 4 }} +spec: + {{- /* + Each router replica terminates QUIC itself and cannot resume a session that + began on another replica. Pin a client to one replica so a rehash mid-session + does not cost the worker a stateless reset and reconnect. + */}} + sessionAffinity: ClientIP + selector: + {{- include "llm-request-router.backendRouterSelectorLabels" . | nindent 4 }} + ports: + - name: grpc + port: {{ .Values.llmRequestRouter.backendRouter.service.grpcPort }} + targetPort: grpc + protocol: TCP + - name: quic + port: {{ .Values.llmRequestRouter.backendRouter.service.reverseTunnelPort }} + targetPort: quic + protocol: UDP + - name: health + port: {{ .Values.llmRequestRouter.backendRouter.service.healthPort }} + targetPort: health + protocol: TCP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "llm-request-router.backendRouterName" . }} + namespace: {{ include "llm-request-router.namespace" . }} + labels: + {{- include "llm-request-router.backendRouterLabels" . | nindent 4 }} +spec: + replicas: {{ .Values.llmRequestRouter.backendRouter.replicaCount }} + selector: + matchLabels: + {{- include "llm-request-router.backendRouterSelectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "llm-request-router.backendRouterSelectorLabels" . | nindent 8 }} + app.kubernetes.io/component: backend-router + {{- with .Values.llmRequestRouter.backendRouter.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.llmRequestRouter.backendRouter.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "llm-request-router.backendRouterServiceAccountName" . }} + {{- with .Values.llmRequestRouter.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + {{- toYaml .Values.llmRequestRouter.podSecurityContext | nindent 8 }} + containers: + - name: backend-router + image: {{ include "llm-request-router.backendRouterImage" . }} + imagePullPolicy: {{ default .Values.llmRequestRouter.image.pullPolicy .Values.llmRequestRouter.backendRouter.image.pullPolicy }} + command: + - /usr/local/bin/stargate-k8s-router + args: + - --listen-addr=0.0.0.0:{{ .Values.llmRequestRouter.backendRouter.service.grpcPort }} + - --reverse-tunnel-listen-addr=0.0.0.0:{{ .Values.llmRequestRouter.backendRouter.service.reverseTunnelPort }} + - --health-listen-addr=0.0.0.0:{{ .Values.llmRequestRouter.backendRouter.service.healthPort }} + - --target-namespace={{ include "llm-request-router.namespace" . }} + - --target-service-name={{ include "llm-request-router.fullname" . }} + - --advertised-hostname-template={{ $advertisedHostnameTemplate }} + - --grpc-port-name=grpc + - --quic-port-name=quic + - --tunnel-protocol=raw-quic + {{- with .Values.llmRequestRouter.tls.certPath }} + - --tls-cert-path={{ . }} + {{- end }} + {{- with .Values.llmRequestRouter.tls.keyPath }} + - --tls-key-path={{ . }} + {{- end }} + {{- if .Values.llmRequestRouter.tls.quicInsecure }} + - --quic-insecure + {{- end }} + ports: + - name: grpc + containerPort: {{ .Values.llmRequestRouter.backendRouter.service.grpcPort }} + protocol: TCP + - name: quic + containerPort: {{ .Values.llmRequestRouter.backendRouter.service.reverseTunnelPort }} + protocol: UDP + - name: health + containerPort: {{ .Values.llmRequestRouter.backendRouter.service.healthPort }} + protocol: TCP + readinessProbe: + httpGet: + path: /readyz + port: health + periodSeconds: 2 + livenessProbe: + httpGet: + path: /livez + port: health + periodSeconds: 5 + resources: + {{- toYaml .Values.llmRequestRouter.backendRouter.resources | nindent 12 }} + securityContext: + {{- toYaml .Values.llmRequestRouter.securityContext | nindent 12 }} + {{- if and $tlsSecretName .Values.llmRequestRouter.tls.certPath .Values.llmRequestRouter.tls.keyPath }} + volumeMounts: + - name: stargate-tls + mountPath: {{ include "llm-request-router.tlsMountPath" . | quote }} + readOnly: true + {{- end }} + {{- if and $tlsSecretName .Values.llmRequestRouter.tls.certPath .Values.llmRequestRouter.tls.keyPath }} + volumes: + - name: stargate-tls + secret: + secretName: {{ $tlsSecretName | quote }} + items: + - key: tls.crt + path: {{ base .Values.llmRequestRouter.tls.certPath | quote }} + - key: tls.key + path: {{ base .Values.llmRequestRouter.tls.keyPath | quote }} + {{- end }} + {{- with .Values.llmRequestRouter.backendRouter.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.llmRequestRouter.backendRouter.affinity }} + affinity: + {{- toYaml .Values.llmRequestRouter.backendRouter.affinity | nindent 8 }} + {{- else }} + {{- /* + Soft, not required: two replicas on one node still survive a pod crash, + and a hard constraint would leave the second replica unschedulable on a + single-node cluster such as k3d. + */}} + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + {{- include "llm-request-router.backendRouterSelectorLabels" . | nindent 20 }} + {{- end }} + {{- with .Values.llmRequestRouter.backendRouter.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/deploy/helm/llm-request-router/llm-request-router/templates/certificate.yaml b/deploy/helm/llm-request-router/llm-request-router/templates/certificate.yaml index f54f91c85..02a63c5c1 100644 --- a/deploy/helm/llm-request-router/llm-request-router/templates/certificate.yaml +++ b/deploy/helm/llm-request-router/llm-request-router/templates/certificate.yaml @@ -16,7 +16,12 @@ {{ $certificate := .Values.llmRequestRouter.certificate | default dict -}} {{- if $certificate.enabled }} {{- include "llm-request-router.validateCertificateDnsNames" . }} +{{- include "llm-request-router.validatePkiAllowedDomains" . }} {{- $issuerRef := $certificate.issuerRef | default dict -}} +{{- $dnsNames := include "llm-request-router.effectiveCertificateDnsNames" . | fromJsonArray }} +{{- if empty $dnsNames -}} +{{- fail "llmRequestRouter.certificate.dnsNames is required when certificate.enabled is true" -}} +{{- end }} apiVersion: cert-manager.io/v1 kind: Certificate metadata: @@ -32,7 +37,7 @@ spec: renewBefore: {{ . | quote }} {{- end }} dnsNames: - {{- required "llmRequestRouter.certificate.dnsNames is required when certificate.enabled is true" $certificate.dnsNames | toYaml | nindent 4 }} + {{- $dnsNames | toYaml | nindent 4 }} issuerRef: kind: {{ default "ClusterIssuer" $issuerRef.kind | quote }} name: {{ required "llmRequestRouter.certificate.issuerRef.name is required when certificate.enabled is true" $issuerRef.name | quote }} diff --git a/deploy/helm/llm-request-router/llm-request-router/templates/deployment.yaml b/deploy/helm/llm-request-router/llm-request-router/templates/deployment.yaml index 6d5059d5f..7cde249c2 100644 --- a/deploy/helm/llm-request-router/llm-request-router/templates/deployment.yaml +++ b/deploy/helm/llm-request-router/llm-request-router/templates/deployment.yaml @@ -37,6 +37,22 @@ renders no Certificate, so a guard in that template would never run. */}} {{- include "llm-request-router.validateTlsIdentity" . }} {{- $advertisedHostnameTemplate := include "llm-request-router.advertisedHostnameTemplate" . }} +{{- $backendRouterEnabled := dig "backendRouter" "enabled" false .Values.llmRequestRouter }} +{{- if and $backendRouterEnabled (not .Values.llmRequestRouter.transport.reverseTunnelListenAddr) }} +{{- fail "llmRequestRouter.backendRouter.enabled requires llmRequestRouter.transport.reverseTunnelListenAddr" }} +{{- end }} +{{- if $backendRouterEnabled }} +{{- $reverseTunnelListenAddr := .Values.llmRequestRouter.transport.reverseTunnelListenAddr | toString | trim }} +{{- $reverseTunnelListenPortMatch := regexFind ":[0-9]+$" $reverseTunnelListenAddr }} +{{- if not $reverseTunnelListenPortMatch }} +{{- fail "llmRequestRouter.transport.reverseTunnelListenAddr must end with a numeric port when backend routing is enabled" }} +{{- end }} +{{- $reverseTunnelListenPort := trimPrefix ":" $reverseTunnelListenPortMatch | int }} +{{- $reverseTunnelServicePort := .Values.llmRequestRouter.service.reverseTunnelPort | int }} +{{- if ne $reverseTunnelListenPort $reverseTunnelServicePort }} +{{- fail (printf "llmRequestRouter.transport.reverseTunnelListenAddr port %d must match llmRequestRouter.service.reverseTunnelPort %d when backend routing is enabled" $reverseTunnelListenPort $reverseTunnelServicePort) }} +{{- end }} +{{- end }} spec: serviceName: {{ .Values.llmRequestRouter.service.headlessName }} replicas: {{ $replicaCount }} @@ -95,7 +111,10 @@ spec: - --backend-connectivity=reverse - --reverse-tunnel-listen-addr={{ .Values.llmRequestRouter.transport.reverseTunnelListenAddr }} {{- end }} - {{- if and .Values.llmRequestRouter.transport.reverseTunnelListenAddr (gt $replicaCount 1) }} + {{- if $backendRouterEnabled }} + - --grpc-pylon-dial-addr={{ include "llm-request-router.backendRouterGrpcDialAddress" . }} + - --reverse-tunnel-pylon-dial-addr={{ include "llm-request-router.backendRouterReverseTunnelDialAddress" . }} + {{- else if and .Values.llmRequestRouter.transport.reverseTunnelListenAddr (gt $replicaCount 1) }} - --reverse-tunnel-pylon-dial-addr=$(POD_IP):{{ .Values.llmRequestRouter.service.reverseTunnelPort }} {{- end }} {{- if .Values.llmRequestRouter.transport.reverseTunnelConnectTimeoutMs }} @@ -178,9 +197,7 @@ spec: {{- end }} {{- $tlsSecretNameMount := include "llm-request-router.tlsSecretName" . }} {{- if and $tlsSecretNameMount .Values.llmRequestRouter.tls.certPath .Values.llmRequestRouter.tls.keyPath }} - {{- if ne (dir .Values.llmRequestRouter.tls.certPath) (dir .Values.llmRequestRouter.tls.keyPath) }} - {{- fail "llmRequestRouter.tls.certPath and llmRequestRouter.tls.keyPath must use the same directory" }} - {{- end }} + {{- include "llm-request-router.validateTlsCertKeyDir" . }} - name: stargate-tls mountPath: {{ include "llm-request-router.tlsMountPath" . | quote }} readOnly: true diff --git a/deploy/helm/llm-request-router/llm-request-router/values.yaml b/deploy/helm/llm-request-router/llm-request-router/values.yaml index a0a3429dc..8196f31ed 100644 --- a/deploy/helm/llm-request-router/llm-request-router/values.yaml +++ b/deploy/helm/llm-request-router/llm-request-router/values.yaml @@ -71,6 +71,64 @@ llmRequestRouter: reverseTunnelPort: 50072 headlessName: llm-request-router-headless + # Routes backend registration and reverse-tunnel traffic to the Stargate pod + # named by gRPC authority or QUIC SNI. Enable this when pylons connect through + # a shared Gateway or load balancer and request-router replicaCount is greater + # than one. + backendRouter: + enabled: false + # The router is the single front door for worker registration and reverse + # tunnels in its region, so one replica makes it a single point of failure + # for every worker in that region. The router keeps no state beyond the QUIC + # sessions it terminates and an EndpointSlice watch, so it scales + # horizontally. Two replicas are the default for that reason. + # + # Each replica terminates QUIC itself and cannot resume a session started on + # another replica, so the Service pins clients with sessionAffinity: + # ClientIP. Replicas also carry a soft anti-affinity so they prefer separate + # nodes; override backendRouter.affinity to change that. + replicaCount: 2 + # The router binary ships in the same versioned release as Stargate, so + # registry, repository, and tag all fall back to the main image settings and + # the chart appVersion when unset. + image: + registry: "" + repository: "" + tag: "" + pullPolicy: "" + # Addresses workers dial to reach the router. Both default to the + # backend-router Service inside the cluster, which is correct when workers + # run alongside the control plane. Override both with externally reachable + # addresses when workers run in a separate cluster or region. + pylonGrpcDialAddress: "" + pylonReverseTunnelDialAddress: "" + serviceAccount: + create: true + annotations: {} + name: "" + service: + grpcPort: 50071 + reverseTunnelPort: 50072 + healthPort: 8080 + podAnnotations: {} + podLabels: {} + # The router relays every byte of the reverse tunnel, and that tunnel + # carries inference traffic rather than only control messages. It is a data + # path component, so it is sized closer to the router it fronts than to a + # sidecar. CPU is the binding resource: each connection is terminated and + # re-originated, so TLS work is paid twice. These values have not been load + # validated; revisit them with real token throughput. + resources: + requests: + cpu: 250m + memory: 256Mi + limits: + cpu: "2" + memory: 1Gi + nodeSelector: {} + tolerations: [] + affinity: {} + resources: requests: cpu: 100m @@ -121,8 +179,8 @@ llmRequestRouter: # suffixes). REQUIRED when enabled. Typically: # ",cluster.local" # so the signing role accepts both the customer's external DNS and - # in-cluster service names. Missing value → script aborts non-zero → - # entrypoint accumulator → Job fails on backoff exhaustion. + # in-cluster service names. Missing value -> script aborts non-zero -> + # entrypoint accumulator -> Job fails on backoff exhaustion. allowedDomains: "" # nvcf-openbao-migrations image. The chart picks up the same image # that the k8s-openbao Helm hook uses; supply registry/repository/tag diff --git a/deploy/helm/llm-request-router/scripts/check-backend-router-render.sh b/deploy/helm/llm-request-router/scripts/check-backend-router-render.sh new file mode 100755 index 000000000..7e6a8c84f --- /dev/null +++ b/deploy/helm/llm-request-router/scripts/check-backend-router-render.sh @@ -0,0 +1,386 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +chart_dir="${script_dir}/../llm-request-router" +rendered="$(mktemp)" +disabled="$(mktemp)" +external_service_account="$(mktemp)" +wildcard_certificate="$(mktemp)" +zero_config="$(mktemp)" +service_monitor_namespace_file="$(mktemp)" +trap 'rm -f "$rendered" "$disabled" "$external_service_account" "$wildcard_certificate" "$zero_config" "$service_monitor_namespace_file"' EXIT + +helm template llm-request-router "$chart_dir" \ + --namespace nvcf \ + --set llmRequestRouter.image.registry=registry.example.invalid \ + --set llmRequestRouter.image.repository=nvcf/stargate \ + --set llmRequestRouter.backendRouter.enabled=true \ + --set llmRequestRouter.backendRouter.image.tag=next \ + --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=llm-router.example.invalid:443 \ + --set llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress=llm-router.example.invalid:8080 \ + --set llmRequestRouter.certificate.enabled=true \ + --set llmRequestRouter.certificate.issuerRef.name=test-issuer \ + --set 'llmRequestRouter.certificate.dnsNames[0]=llm-request-router.nvcf.svc.cluster.local' \ + --set llmRequestRouter.tls.secretName=stargate-quic-tls \ + --set llmRequestRouter.tls.certPath=/etc/stargate/tls/tls.crt \ + --set llmRequestRouter.tls.keyPath=/etc/stargate/tls/tls.key \ + --set llmRequestRouter.tls.quicInsecure=false \ + --set llmRequestRouter.metrics.enabled=true \ + --set llmRequestRouter.metrics.serviceMonitor.enabled=true \ + >"$rendered" + +assert_contains() { + local pattern="$1" + local message="$2" + if ! grep -Fq -- "$pattern" "$rendered"; then + echo "FAIL: ${message}" >&2 + exit 1 + fi +} + +assert_render_fails() { + local expected_error="$1" + local error_file + shift + error_file="$(mktemp)" + if helm template llm-request-router "$chart_dir" --namespace nvcf "$@" >/dev/null 2>"$error_file"; then + rm -f "$error_file" + echo "FAIL: expected Helm render to fail: ${expected_error}" >&2 + exit 1 + fi + if ! grep -Fq -- "$expected_error" "$error_file"; then + rm -f "$error_file" + echo "FAIL: render did not return the expected validation error: ${expected_error}" >&2 + exit 1 + fi + rm -f "$error_file" +} + +assert_backend_router_replicas() { + local expected="$1" + local actual + actual="$(awk ' + $0 == "---" { in_deployment = 0; backend_router = 0 } + $0 == "kind: Deployment" { in_deployment = 1; backend_router = 0 } + in_deployment && $1 == "name:" && $2 == "llm-request-router-backend-router" { backend_router = 1 } + backend_router && $1 == "replicas:" { print $2; exit } + ' "$rendered")" + if [[ "$actual" != "$expected" ]]; then + echo "FAIL: backend router must default to ${expected} replica; rendered ${actual:-none}" >&2 + exit 1 + fi +} + +assert_backend_router_role_binding_subject() { + local rendered_file="$1" + local expected="$2" + local actual + actual="$(awk ' + $0 == "---" { in_binding = 0; target_binding = 0; in_subjects = 0 } + $0 == "kind: RoleBinding" { in_binding = 1; target_binding = 0; in_subjects = 0 } + in_binding && !target_binding && $1 == "name:" && $2 == "llm-request-router-backend-router-endpointslice-reader" { target_binding = 1 } + target_binding && $0 == "subjects:" { in_subjects = 1 } + target_binding && in_subjects && $1 == "name:" { print $2; exit } + ' "$rendered_file")" + if [[ "$actual" != "$expected" ]]; then + echo "FAIL: backend router RoleBinding must target ${expected}; rendered ${actual:-none}" >&2 + exit 1 + fi +} + +assert_service_account_exists() { + local rendered_file="$1" + local expected="$2" + if ! awk -v expected="$expected" ' + $0 == "---" { in_service_account = 0 } + $0 == "kind: ServiceAccount" { in_service_account = 1 } + in_service_account && $1 == "name:" && $2 == expected { found = 1 } + END { exit found ? 0 : 1 } + ' "$rendered_file"; then + echo "FAIL: chart must render the dedicated ${expected} ServiceAccount" >&2 + exit 1 + fi +} + +assert_backend_router_service_monitor_namespace() { + local rendered_file="$1" + local expected="$2" + local actual + actual="$(awk ' + $0 == "---" { in_monitor = 0; target_monitor = 0 } + $0 == "kind: ServiceMonitor" { in_monitor = 1 } + in_monitor && $1 == "name:" && $2 == "llm-request-router-backend-router-metrics" { target_monitor = 1; next } + target_monitor && $1 == "namespace:" { print $2; exit } + ' "$rendered_file")" + if [[ "$actual" != "$expected" ]]; then + echo "FAIL: backend router ServiceMonitor must be created in ${expected}; rendered ${actual:-none}" >&2 + exit 1 + fi +} + +assert_contains "name: llm-request-router-backend-router" \ + "backend router workload and Service must use a stable name" +assert_contains "kind: Deployment" \ + "backend router must render as a Deployment" +assert_backend_router_replicas "2" +assert_contains "kind: Role" \ + "backend router must render namespaced RBAC" +assert_contains "resources: [\"endpointslices\"]" \ + "backend router must be allowed to watch EndpointSlices" + +# Backend routing follows the LLM addon, so it has to render with no +# operator-supplied dial addresses at all. This renders exactly that case. +helm template llm-request-router "$chart_dir" \ + --namespace nvcf \ + --set llmRequestRouter.image.registry=registry.example.invalid \ + --set llmRequestRouter.image.repository=nvcf/stargate \ + --set llmRequestRouter.backendRouter.enabled=true \ + --set llmRequestRouter.backendRouter.image.repository=nvcf/stargate-k8s-router \ + >"$zero_config" + +assert_zero_config_contains() { + local pattern="$1" + local message="$2" + if ! grep -Fq -- "$pattern" "$zero_config"; then + echo "FAIL: ${message}" >&2 + exit 1 + fi +} + +assert_zero_config_contains "--grpc-pylon-dial-addr=llm-request-router-backend-router.nvcf.svc.cluster.local:50071" \ + "gRPC dial address must default to the in-cluster backend-router Service" +assert_zero_config_contains "--reverse-tunnel-pylon-dial-addr=llm-request-router-backend-router.nvcf.svc.cluster.local:50072" \ + "reverse-tunnel dial address must default to the in-cluster backend-router Service" + +# An explicitly configured address must still win over the default. +assert_contains "--grpc-pylon-dial-addr=llm-router.example.invalid:443" \ + "configured gRPC dial address must override the in-cluster default" + +# Each replica terminates QUIC itself and cannot resume another replica's +# session, so clients must be pinned. +assert_contains "sessionAffinity: ClientIP" \ + "backend router Service must pin clients so QUIC sessions do not rehash" + +# Two replicas on one node would not survive node loss. +assert_contains "podAntiAffinity:" \ + "backend router must default to spreading replicas across nodes" +assert_contains "topologyKey: kubernetes.io/hostname" \ + "backend router anti-affinity must spread across nodes, not a narrower topology" +assert_contains "serviceAccountName: llm-request-router-backend-router" \ + "backend router must use its dedicated ServiceAccount" +assert_service_account_exists "$rendered" "llm-request-router-backend-router" +assert_backend_router_role_binding_subject "$rendered" "llm-request-router-backend-router" +assert_contains "command:" \ + "backend router must override the Stargate image entrypoint" +assert_contains "/usr/local/bin/stargate-k8s-router" \ + "Stargate image must include the Kubernetes router binary" +assert_contains "--target-service-name=llm-request-router" \ + "backend router must watch the readiness-respecting request-router Service" +assert_contains "--advertised-hostname-template={pod_name}.llm-request-router-headless.nvcf.svc.cluster.local" \ + "backend router authority and SNI template must match Stargate" +assert_contains "- '*.llm-request-router-headless.nvcf.svc.cluster.local'" \ + "request-router certificate must cover pod-specific backend routing hostnames" +assert_contains "image: registry.example.invalid/nvcf/stargate:next" \ + "backend router must use its explicitly pinned Stargate image" +assert_contains "app.kubernetes.io/version: \"next\"" \ + "backend router labels must identify the explicitly pinned image version" +assert_contains "--grpc-pylon-dial-addr=llm-router.example.invalid:443" \ + "Stargate must advertise the external gRPC endpoint to pylon" +assert_contains "--reverse-tunnel-pylon-dial-addr=llm-router.example.invalid:8080" \ + "Stargate must advertise the external reverse-tunnel endpoint to pylon" +assert_contains "--tls-cert-path=/etc/stargate/tls/tls.crt" \ + "backend router must use the Stargate TLS certificate" +assert_contains "secretName: \"stargate-quic-tls\"" \ + "backend router must mount the configured Stargate TLS Secret" +assert_contains "name: llm-request-router-backend-router-metrics" \ + "backend router metrics must be discoverable by the existing ServiceMonitor option" +assert_backend_router_service_monitor_namespace "$rendered" "nvcf" + +helm template llm-request-router "$chart_dir" \ + --namespace nvcf \ + --set llmRequestRouter.image.registry=registry.example.invalid \ + --set llmRequestRouter.image.repository=nvcf/stargate \ + --set llmRequestRouter.backendRouter.enabled=false \ + >"$disabled" + +if grep -Fq "llm-request-router-backend-router" "$disabled"; then + echo "FAIL: disabled backend router must not render router resources" >&2 + exit 1 +fi + +assert_render_fails "llmRequestRouter.kubernetes.advertisedHostnameTemplate must contain exactly one {pod_name} when backendRouter.enabled is true" \ + --set llmRequestRouter.image.registry=registry.example.invalid \ + --set llmRequestRouter.image.repository=nvcf/stargate \ + --set-string 'llmRequestRouter.kubernetes.advertisedHostnameTemplate=\{pod_name\}\{pod_name\}' \ + --set llmRequestRouter.backendRouter.enabled=true \ + --set llmRequestRouter.backendRouter.image.tag=next \ + --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=llm-router.example.invalid:443 \ + --set llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress=llm-router.example.invalid:8080 + +assert_render_fails "llmRequestRouter.transport.reverseTunnelListenAddr port 50073 must match llmRequestRouter.service.reverseTunnelPort 50072 when backend routing is enabled" \ + --set llmRequestRouter.image.registry=registry.example.invalid \ + --set llmRequestRouter.image.repository=nvcf/stargate \ + --set llmRequestRouter.backendRouter.enabled=true \ + --set llmRequestRouter.transport.reverseTunnelListenAddr=0.0.0.0:50073 + +assert_render_fails "llmRequestRouter.backendRouter.image.repository or llmRequestRouter.image.repository is required when backend routing is enabled" \ + --set llmRequestRouter.backendRouter.enabled=true \ + --set-string llmRequestRouter.image.repository= \ + --set-string llmRequestRouter.backendRouter.image.repository= + +service_monitor_namespace="$(helm template llm-request-router "$chart_dir" \ + --namespace release-namespace \ + --set llmRequestRouter.namespace=router-system \ + --set llmRequestRouter.image.registry=registry.example.invalid \ + --set llmRequestRouter.image.repository=nvcf/stargate \ + --set llmRequestRouter.backendRouter.enabled=true \ + --set llmRequestRouter.backendRouter.image.tag=next \ + --set llmRequestRouter.metrics.enabled=true \ + --set llmRequestRouter.metrics.serviceMonitor.enabled=true)" +printf '%s\n' "$service_monitor_namespace" >"$service_monitor_namespace_file" +assert_backend_router_service_monitor_namespace "$service_monitor_namespace_file" "router-system" + +assert_render_fails "llmRequestRouter.backendRouter.serviceAccount.name is required when backendRouter is enabled and backendRouter.serviceAccount.create is false" \ + --set llmRequestRouter.image.registry=registry.example.invalid \ + --set llmRequestRouter.image.repository=nvcf/stargate \ + --set llmRequestRouter.backendRouter.enabled=true \ + --set llmRequestRouter.backendRouter.image.tag=next \ + --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=llm-router.example.invalid:443 \ + --set llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress=llm-router.example.invalid:8080 \ + --set llmRequestRouter.backendRouter.serviceAccount.create=false + +assert_render_fails "llmRequestRouter.backendRouter.serviceAccount.name is required when backendRouter is enabled and backendRouter.serviceAccount.create is false" \ + --set llmRequestRouter.image.registry=registry.example.invalid \ + --set llmRequestRouter.image.repository=nvcf/stargate \ + --set llmRequestRouter.backendRouter.enabled=true \ + --set llmRequestRouter.backendRouter.image.tag=next \ + --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=llm-router.example.invalid:443 \ + --set llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress=llm-router.example.invalid:8080 \ + --set llmRequestRouter.backendRouter.serviceAccount.create=false \ + --set llmRequestRouter.rbac.create=false + +helm template llm-request-router "$chart_dir" \ + --namespace nvcf \ + --set llmRequestRouter.image.registry=registry.example.invalid \ + --set llmRequestRouter.image.repository=nvcf/stargate \ + --set llmRequestRouter.backendRouter.enabled=true \ + --set llmRequestRouter.backendRouter.image.tag=next \ + --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=llm-router.example.invalid:443 \ + --set llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress=llm-router.example.invalid:8080 \ + --set llmRequestRouter.backendRouter.serviceAccount.create=false \ + --set llmRequestRouter.backendRouter.serviceAccount.name=external-backend-router \ + >"$external_service_account" + +if ! grep -Fq -- "serviceAccountName: external-backend-router" "$external_service_account"; then + echo "FAIL: backend router must use the configured external ServiceAccount" >&2 + exit 1 +fi +assert_backend_router_role_binding_subject "$external_service_account" "external-backend-router" + +assert_zero_config_contains "image: registry.example.invalid/nvcf/stargate-k8s-router:0.9.0" \ + "backend router image must fall back to the main registry and the chart appVersion" + +helm template llm-request-router "$chart_dir" \ + --namespace nvcf \ + --set llmRequestRouter.image.registry=registry.example.invalid \ + --set llmRequestRouter.image.repository=nvcf/stargate \ + --set llmRequestRouter.backendRouter.enabled=true \ + --set llmRequestRouter.backendRouter.image.tag=next \ + --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=llm-router.example.invalid:443 \ + --set llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress=llm-router.example.invalid:8080 \ + --set llmRequestRouter.certificate.enabled=true \ + --set llmRequestRouter.certificate.issuerRef.name=test-issuer \ + --set llmRequestRouter.tls.secretName=stargate-quic-tls \ + --set llmRequestRouter.tls.certPath=/etc/stargate/tls/tls.crt \ + --set llmRequestRouter.tls.keyPath=/etc/stargate/tls/tls.key \ + >"$wildcard_certificate" + +if ! grep -Fq -- "- '*.llm-request-router-headless.nvcf.svc.cluster.local'" "$wildcard_certificate"; then + echo "FAIL: backend routing must add its wildcard before certificate DNS-name validation" >&2 + exit 1 +fi + +assert_render_fails "llmRequestRouter.certificate.dnsNames is required when certificate.enabled is true" \ + --set llmRequestRouter.image.registry=registry.example.invalid \ + --set llmRequestRouter.image.repository=nvcf/stargate \ + --set llmRequestRouter.backendRouter.enabled=false \ + --set llmRequestRouter.certificate.enabled=true \ + --set llmRequestRouter.certificate.issuerRef.name=test-issuer + +assert_render_fails "llmRequestRouter.kubernetes.advertisedHostnameTemplate must contain exactly one {pod_name} when backendRouter.enabled is true" \ + --set llmRequestRouter.image.registry=registry.example.invalid \ + --set llmRequestRouter.image.repository=nvcf/stargate \ + --set llmRequestRouter.kubernetes.advertisedHostnameTemplate=llm-request-router.nvcf.svc.cluster.local \ + --set llmRequestRouter.backendRouter.enabled=true \ + --set llmRequestRouter.backendRouter.image.tag=next \ + --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=llm-router.example.invalid:443 \ + --set llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress=llm-router.example.invalid:8080 + +assert_render_fails "llmRequestRouter backend routing requires a TLS Secret and cert/key paths when tls.quicInsecure is false" \ + --set llmRequestRouter.image.registry=registry.example.invalid \ + --set llmRequestRouter.image.repository=nvcf/stargate \ + --set llmRequestRouter.backendRouter.enabled=true \ + --set llmRequestRouter.backendRouter.image.tag=next \ + --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=llm-router.example.invalid:443 \ + --set llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress=llm-router.example.invalid:8080 \ + --set llmRequestRouter.tls.quicInsecure=false + +assert_render_fails "llmRequestRouter backend routing requires tls.secretName (or certificate secret), tls.certPath, and tls.keyPath together" \ + --set llmRequestRouter.image.registry=registry.example.invalid \ + --set llmRequestRouter.image.repository=nvcf/stargate \ + --set llmRequestRouter.backendRouter.enabled=true \ + --set llmRequestRouter.backendRouter.image.tag=next \ + --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=llm-router.example.invalid:443 \ + --set llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress=llm-router.example.invalid:8080 \ + --set llmRequestRouter.tls.certPath=/etc/stargate/tls/tls.crt + +assert_render_fails "llmRequestRouter.tls.certPath and llmRequestRouter.tls.keyPath must use the same directory" \ + --set llmRequestRouter.image.registry=registry.example.invalid \ + --set llmRequestRouter.image.repository=nvcf/stargate \ + --set llmRequestRouter.backendRouter.enabled=true \ + --set llmRequestRouter.backendRouter.image.tag=next \ + --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=llm-router.example.invalid:443 \ + --set llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress=llm-router.example.invalid:8080 \ + --set llmRequestRouter.tls.secretName=stargate-quic-tls \ + --set llmRequestRouter.tls.certPath=/etc/stargate/tls/tls.crt \ + --set llmRequestRouter.tls.keyPath=/var/run/stargate/tls.key + +assert_render_fails "llmRequestRouter.tls.certPath and llmRequestRouter.tls.keyPath must use the same directory" \ + --set llmRequestRouter.image.registry=registry.example.invalid \ + --set llmRequestRouter.image.repository=nvcf/stargate \ + --set llmRequestRouter.backendRouter.enabled=false \ + --set llmRequestRouter.tls.secretName=stargate-quic-tls \ + --set llmRequestRouter.tls.certPath=/etc/stargate/tls/tls.crt \ + --set llmRequestRouter.tls.keyPath=/var/run/stargate/tls.key + +assert_render_fails "llmRequestRouter.tls.mountPath must match the directory containing tls.certPath and tls.keyPath" \ + --set llmRequestRouter.image.registry=registry.example.invalid \ + --set llmRequestRouter.image.repository=nvcf/stargate \ + --set llmRequestRouter.backendRouter.enabled=true \ + --set llmRequestRouter.backendRouter.image.tag=next \ + --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=llm-router.example.invalid:443 \ + --set llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress=llm-router.example.invalid:8080 \ + --set llmRequestRouter.tls.secretName=stargate-quic-tls \ + --set llmRequestRouter.tls.mountPath=/var/run/stargate \ + --set llmRequestRouter.tls.certPath=/etc/stargate/tls/tls.crt \ + --set llmRequestRouter.tls.keyPath=/etc/stargate/tls/tls.key + +single_replica="$(helm template llm-request-router "$chart_dir" \ + --namespace nvcf \ + --set llmRequestRouter.image.registry=registry.example.invalid \ + --set llmRequestRouter.image.repository=nvcf/stargate \ + --set llmRequestRouter.replicaCount=1 \ + --set llmRequestRouter.backendRouter.enabled=true \ + --set llmRequestRouter.backendRouter.image.tag=next \ + --set llmRequestRouter.backendRouter.pylonGrpcDialAddress=llm-router.example.invalid:443 \ + --set llmRequestRouter.backendRouter.pylonReverseTunnelDialAddress=llm-router.example.invalid:8080)" +if ! grep -Fq -- "--advertised-hostname-template={pod_name}.llm-request-router-headless.nvcf.svc.cluster.local" <<<"$single_replica"; then + echo "FAIL: backend routing must retain per-pod authority and SNI for one replica" >&2 + exit 1 +fi + +echo "PASS: LLM request-router backend routing renders correctly" diff --git a/deploy/helm/llm-request-router/scripts/check-pki-render.sh b/deploy/helm/llm-request-router/scripts/check-pki-render.sh index 8dbfcc263..4147f7811 100644 --- a/deploy/helm/llm-request-router/scripts/check-pki-render.sh +++ b/deploy/helm/llm-request-router/scripts/check-pki-render.sh @@ -290,6 +290,15 @@ if render_certificate_case \ fail "advertised hostname containing non-DNS braces unexpectedly rendered" fi +invalid_underscore_error="${tmp_dir}/invalid-underscore.err" +if render_certificate_case \ + /dev/null \ + "router_name.example.internal" \ + "*.example.internal" \ + 2> "${invalid_underscore_error}"; then + fail "advertised hostname containing an underscore unexpectedly rendered" +fi + # Pass 3: existing-Secret identity mode. The operator owns issuance, so the # chart must mount the pre-created Secret without rendering a Certificate or # the OpenBao provisioning hook. @@ -430,4 +439,79 @@ grep -Fq \ 'llmRequestRouter.tls.mode must be certManager or existingSecret, got "externalSecret"' \ "${invalid_mode_error}" || fail "unknown mode render did not return the expected guard message" +# The OpenBao signing role is created with allow_subdomains=true and +# allow_bare_domains=false, so a SAN outside allowed_domains renders cleanly and +# then fails at cert-manager issuance. Catch it at render instead. These cases +# pin the coverage rules, including the ones that must NOT fail: a wrong guard +# here would block valid deployments, which is worse than the trap it replaces. +assert_allowed_domains_case() { + local description="$1" + local expectation="$2" + local allowed_domains="$3" + local dns_name="$4" + local advertised_hostname_template="${5:-}" + local case_values="${tmp_dir}/allowed-domains-values.yaml" + local case_error="${tmp_dir}/allowed-domains.err" + + cat > "${case_values}" <> "${case_values}" < /dev/null 2> "${case_error}"; then + [ "${expectation}" = "pass" ] || fail "${description} unexpectedly rendered" + else + [ "${expectation}" = "fail" ] || fail "${description} unexpectedly failed to render" + grep -Fq "is not covered by llmRequestRouter.pki.allowedDomains" "${case_error}" || + fail "${description} did not return the allowed-domains guard message" + fi +} + +# Must render: these are valid deployments. +assert_allowed_domains_case "documented customer-domain plus cluster.local" \ + pass "example.com,cluster.local" "llm-request-router.nvcf.svc.cluster.local" +assert_allowed_domains_case "whitespace around the comma separators" \ + pass " example.com , cluster.local " "llm-request-router.nvcf.svc.cluster.local" +assert_allowed_domains_case "allowed domain deeper in the suffix" \ + pass "svc.cluster.local" "llm-request-router.nvcf.svc.cluster.local" +assert_allowed_domains_case "direct wildcard covering the advertised pod hostname" \ + pass "example.com" "*.example.com" "{pod_name}.example.com" + +# Must fail: issuance would be rejected. +assert_allowed_domains_case "no overlap with the certificate names" \ + fail "example.com" "llm-request-router.nvcf.svc.cluster.local" +assert_allowed_domains_case "name that ends with the domain but is not a subdomain" \ + fail "cluster.local" "evilcluster.local" +assert_allowed_domains_case "bare domain, which the role refuses to issue" \ + fail "llm-request-router-headless.nvcf.svc.cluster.local" "llm-request-router-headless.nvcf.svc.cluster.local" + echo "PKI render checks passed" diff --git a/deploy/stacks/self-managed/Makefile b/deploy/stacks/self-managed/Makefile index 2dec7deb0..43483f73b 100644 --- a/deploy/stacks/self-managed/Makefile +++ b/deploy/stacks/self-managed/Makefile @@ -4,6 +4,12 @@ include Makefile.dist .PHONY: test test: @tests/llm-router-worker-address.sh + @tests/llm-router-split-cluster.sh + @tests/llm-router-local-chart.sh + @tests/gateway-routes-local-chart.sh + @tests/grpc-proxy-nats-endpoint.sh + @tests/llm-pki-openbao-migration.sh + @tests/api-keys-startup-probe.sh @tests/llm-pki-release.sh @tests/check-llm-pki-issuer.sh @tests/pdb-value-wiring.sh diff --git a/deploy/stacks/self-managed/environments/base.yaml b/deploy/stacks/self-managed/environments/base.yaml index 8d451b369..6120641e6 100644 --- a/deploy/stacks/self-managed/environments/base.yaml +++ b/deploy/stacks/self-managed/environments/base.yaml @@ -257,6 +257,34 @@ addons: # LLM addon: gateway + request router (stargate) for LLM function invocation llm: enabled: false + + requestRouter: + replicaCount: 3 + # Set only for source-tree validation. Empty uses the released OCI chart. + chartPath: "" + + # Authority and SNI aware backend router. A worker holds one registration + # stream and one reverse QUIC tunnel per router replica, and each must + # reach the specific replica named by the gRPC authority or QUIC SNI, so + # ordinary load balancing cannot carry this traffic. The router reads that + # identity and forwards to the named pod. + # + # Follows addons.llm.enabled by default. It is deliberately not a separate + # opt-in: keeping it on in single-cluster installs means they exercise the + # same connection path that split-cluster and multi-region installs + # depend on. + # + # Note that the router relays the reverse tunnel, and that tunnel carries + # inference traffic, so it sits in the data path and terminates QUIC TLS. + backendRouter: + # enabled: defaults to addons.llm.enabled + replicaCount: 2 + # Addresses workers dial to reach the router. Both default to the + # backend-router Service in-cluster, which is correct when workers run + # alongside the control plane. Set both to externally reachable + # addresses when workers run in a separate cluster or region. + pylonGrpcDialAddress: "" + pylonReverseTunnelDialAddress: "" # QUIC TLS certificate for the request router (Stargate). Disabled by # default; opt in per env. # @@ -416,6 +444,9 @@ ingress: gatewayApi: enabled: true controllerNamespace: "" # must be set by the environment + # Optional path to a local gateway-routes chart. This lets integration + # validation deploy route templates that have not been released yet. + chartPath: "" routes: nvcfApi: routeAnnotations: {} @@ -450,6 +481,18 @@ ingress: nats: enabled: false routeAnnotations: {} + # Backend-router routes for split-cluster LLM worker registration and + # reverse QUIC tunnels. The target router selects the named Stargate pod + # by HTTP/2 authority and QUIC SNI. + llmWorker: + enabled: false + backend: + name: llm-request-router-backend-router + # Required when llmWorker is enabled; normally the NVCF namespace. + namespace: "" + grpcPort: 50071 + quicPort: 50072 + routeAnnotations: {} # ESS route. NVCF worker containers fetch function secrets from ess-api. # In split/multi-cluster installs ess runs in the control plane, so # compute-plane workers reach it through this route. Opt-in per environment. @@ -467,6 +510,14 @@ ingress: name: "" # must be set by the environment when routes.nats.enabled=true namespace: "" # must be set by the environment when routes.nats.enabled=true listenerName: nats + llmGrpc: + name: "" # required when routes.llmWorker.enabled=true + namespace: "" # required when routes.llmWorker.enabled=true + listenerName: llm-grpc + llmQuic: + name: "" # required when routes.llmWorker.enabled=true + namespace: "" # required when routes.llmWorker.enabled=true + listenerName: llm-quic grpcproxy: # PDB: disabled by default. Enable when running multiple grpc-proxy replicas. @@ -479,4 +530,8 @@ grpcproxy: # need an external Gateway or load-balancer URL, but grpc-proxy should stay # on a control-plane-local API service address. nvcfGrpcServiceURL: "" + # Optional NATS endpoint used only by grpc-proxy. It defaults to the local + # NATS Service even when global.workerEndpoints.nvcfNatsServiceURL points at + # an external compute-worker endpoint. + natsServiceURL: "" workerConnectBaseURL: "" diff --git a/deploy/stacks/self-managed/global.yaml.gotmpl b/deploy/stacks/self-managed/global.yaml.gotmpl index 27b6023be..25c7ef470 100644 --- a/deploy/stacks/self-managed/global.yaml.gotmpl +++ b/deploy/stacks/self-managed/global.yaml.gotmpl @@ -115,11 +115,20 @@ openbao: tag: {{ dig "openbao" "migrations" "image" "tag" "" .Values }} {{- end }} + env: + {{- with dig "openbao" "migrations" "env" (list) .Values }} + {{- toYaml . | nindent 6 }} + {{- end }} + - name: ADDONS_LLM_ENABLED + value: {{ dig "addons" "llm" "enabled" false .Values | quote }} + - name: NVCF_SERVICE_PKI_ALLOWED_DOMAINS + value: {{ dig "addons" "llm" "pki" "allowedDomains" "" .Values | quote }} + # merge the rest of the user-supplied migrations block, but drop - # its image sub-block so we don't clobber the defaults above + # image and env sub-blocks so we don't clobber the defaults above. {{- with .Values.openbao.migrations }} {{- range $key, $value := . }} - {{- if ne $key "image" }} + {{- if and (ne $key "image") (ne $key "env") }} {{ $key }}: {{- toYaml $value | nindent 6 }} {{- end }} @@ -257,6 +266,10 @@ apikeys: podDisruptionBudget: {{- toYaml . | nindent 4 }} {{- end }} + {{- with dig "apikeys" "startupProbe" dict .Values }} + startupProbe: + {{- toYaml . | nindent 4 }} + {{- end }} natsAuthCalloutService: {{- if .Values.global.imagePullSecrets }} @@ -272,6 +285,7 @@ natsAuthCalloutService: {{- $nvcfWorkerGrpcServiceURL := dig "nvcfGrpcServiceURL" "" $workerEndpoints }} {{- $grpcProxyNVCFGrpcServiceURL := dig "grpcproxy" "nvcfGrpcServiceURL" "" .Values | default $nvcfWorkerGrpcServiceURL }} {{- $nvcfNatsWorkerServiceURL := dig "nvcfNatsServiceURL" "" $workerEndpoints }} +{{- $grpcProxyNatsServiceURL := dig "grpcproxy" "natsServiceURL" "" .Values | default "nats://nats.nats-system.svc.cluster.local:4222" }} {{- $essWorkerBaseURL := dig "essServiceURL" "" $workerEndpoints }} {{- $nvctWorkerServiceURL := dig "nvctServiceURL" "" $workerEndpoints }} {{- $nvctWorkerGrpcServiceURL := dig "nvctGrpcServiceURL" "" $workerEndpoints }} @@ -541,7 +555,7 @@ grpcproxy: {{- with $grpcProxyNVCFGrpcServiceURL }} NVCF_FQDN_GRPC: {{ . | quote }} {{- end }} - {{- with $nvcfNatsWorkerServiceURL }} + {{- with $grpcProxyNatsServiceURL }} NATS_FQDN: {{ . | quote }} {{- end }} {{- if .Values.rateLimiter.enabled }} @@ -842,6 +856,35 @@ llmRequestRouter: image: registry: {{ .Values.global.image.registry }} repository: {{ .Values.global.image.repository }}/stargate + {{- /* + Backend routing follows the LLM addon rather than being separately opted into. + A worker holds one registration stream and one reverse tunnel per replica, and + those must reach a named replica, so the router is the connection path in + every topology. Keeping it always on means single-cluster installs exercise + the same path that split-cluster and multi-region installs depend on, instead + of that path only appearing where it is hardest to test. + + The dial addresses default to the backend-router Service in-cluster, so no + configuration is required. Split-cluster and multi-region operators override + addons.llm.requestRouter.backendRouter.pylon*DialAddress with externally + reachable addresses. + */}} + backendRouter: + enabled: {{ dig "addons" "llm" "requestRouter" "backendRouter" "enabled" (dig "addons" "llm" "enabled" false .Values) .Values }} + replicaCount: {{ dig "addons" "llm" "requestRouter" "backendRouter" "replicaCount" 2 .Values }} + image: + registry: {{ .Values.global.image.registry }} + repository: {{ .Values.global.image.repository }}/stargate-k8s-router + {{- with dig "addons" "llm" "requestRouter" "backendRouter" "pylonGrpcDialAddress" "" .Values }} + pylonGrpcDialAddress: {{ . | quote }} + {{- end }} + {{- with dig "addons" "llm" "requestRouter" "backendRouter" "pylonReverseTunnelDialAddress" "" .Values }} + pylonReverseTunnelDialAddress: {{ . | quote }} + {{- end }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- toYaml .Values.global.imagePullSecrets | nindent 6 }} + {{- end }} {{- with include "nvcf.nodeSelector" (dict "type" "controlplane" "selectors" .Values.global.nodeSelectors) }} {{- . | nindent 2 }} {{- end }} @@ -1253,6 +1296,7 @@ nvcfUi: nvcfGatewayRoutes: {{- $natsRouteEnabled := dig "ingress" "gatewayApi" "routes" "nats" "enabled" false .Values }} {{- $essRouteEnabled := dig "ingress" "gatewayApi" "routes" "ess" "enabled" false .Values }} + {{- $llmWorkerRouteEnabled := dig "ingress" "gatewayApi" "routes" "llmWorker" "enabled" false .Values }} domain: "{{ .Values.global.domain }}" gateways: shared: @@ -1267,6 +1311,16 @@ nvcfGatewayRoutes: namespace: {{ required "ingress.gatewayApi.gateways.nats.namespace is required when ingress.gatewayApi.routes.nats.enabled is true" .Values.ingress.gatewayApi.gateways.nats.namespace }} listenerName: {{ dig "ingress" "gatewayApi" "gateways" "nats" "listenerName" "nats" .Values }} {{- end }} + {{- if $llmWorkerRouteEnabled }} + llmGrpc: + name: {{ required "ingress.gatewayApi.gateways.llmGrpc.name is required when ingress.gatewayApi.routes.llmWorker.enabled is true" .Values.ingress.gatewayApi.gateways.llmGrpc.name }} + namespace: {{ required "ingress.gatewayApi.gateways.llmGrpc.namespace is required when ingress.gatewayApi.routes.llmWorker.enabled is true" .Values.ingress.gatewayApi.gateways.llmGrpc.namespace }} + listenerName: {{ dig "ingress" "gatewayApi" "gateways" "llmGrpc" "listenerName" "llm-grpc" .Values }} + llmQuic: + name: {{ required "ingress.gatewayApi.gateways.llmQuic.name is required when ingress.gatewayApi.routes.llmWorker.enabled is true" .Values.ingress.gatewayApi.gateways.llmQuic.name }} + namespace: {{ required "ingress.gatewayApi.gateways.llmQuic.namespace is required when ingress.gatewayApi.routes.llmWorker.enabled is true" .Values.ingress.gatewayApi.gateways.llmQuic.namespace }} + listenerName: {{ dig "ingress" "gatewayApi" "gateways" "llmQuic" "listenerName" "llm-quic" .Values }} + {{- end }} routes: nvcfApi: routeAnnotations: @@ -1329,6 +1383,17 @@ nvcfGatewayRoutes: enabled: {{ $natsRouteEnabled }} routeAnnotations: {{ dig "ingress" "gatewayApi" "routes" "nats" "routeAnnotations" dict .Values | toYaml | nindent 8 | trim }} + llmWorker: + enabled: {{ $llmWorkerRouteEnabled }} + {{- if $llmWorkerRouteEnabled }} + backend: + name: {{ dig "ingress" "gatewayApi" "routes" "llmWorker" "backend" "name" "llm-request-router-backend-router" .Values }} + namespace: {{ required "ingress.gatewayApi.routes.llmWorker.backend.namespace is required when ingress.gatewayApi.routes.llmWorker.enabled is true" .Values.ingress.gatewayApi.routes.llmWorker.backend.namespace }} + grpcPort: {{ dig "ingress" "gatewayApi" "routes" "llmWorker" "backend" "grpcPort" 50071 .Values }} + quicPort: {{ dig "ingress" "gatewayApi" "routes" "llmWorker" "backend" "quicPort" 50072 .Values }} + {{- end }} + routeAnnotations: + {{ dig "ingress" "gatewayApi" "routes" "llmWorker" "routeAnnotations" dict .Values | toYaml | nindent 8 | trim }} # ESS route attaches to the shared Gateway (backend/hostname come from chart # defaults); only the enable flag and annotations are environment-driven. ess: diff --git a/deploy/stacks/self-managed/helmfile.d/01-dependencies.yaml.gotmpl b/deploy/stacks/self-managed/helmfile.d/01-dependencies.yaml.gotmpl index 2bbfd218b..e456e84dd 100644 --- a/deploy/stacks/self-managed/helmfile.d/01-dependencies.yaml.gotmpl +++ b/deploy/stacks/self-managed/helmfile.d/01-dependencies.yaml.gotmpl @@ -124,6 +124,12 @@ releases: condition: openbao.enabled # From defaults.yaml or env overrides namespace: vault-system <<: *dependency # Inherits base values from the dependency template + # Secret values own migrations.env. Apply the generated LLM addition last + # so it preserves that list while adding the addon migration switch. + values: + - ../global.yaml.gotmpl + - ../secrets/{{ requiredEnv "HELMFILE_ENV" }}-secrets.yaml + - ../openbao-migrations-llm-env.yaml.gotmpl needs: - nats-system/nats diff --git a/deploy/stacks/self-managed/helmfile.d/02-core.yaml.gotmpl b/deploy/stacks/self-managed/helmfile.d/02-core.yaml.gotmpl index 79160b37f..fc2c976e0 100644 --- a/deploy/stacks/self-managed/helmfile.d/02-core.yaml.gotmpl +++ b/deploy/stacks/self-managed/helmfile.d/02-core.yaml.gotmpl @@ -165,8 +165,11 @@ releases: release-group: services - name: llm-request-router - chart: nvcf/helm-nvcf-llm-request-router - version: 1.7.2 + {{- $llmRequestRouterChartPath := dig "addons" "llm" "requestRouter" "chartPath" "" .Values }} + chart: {{ $llmRequestRouterChartPath | default "nvcf/helm-nvcf-llm-request-router" | quote }} + {{- if not $llmRequestRouterChartPath }} + version: 1.9.0 + {{- end }} namespace: nvcf condition: addons.llm.enabled values: @@ -211,8 +214,11 @@ releases: # --- Gateway API Ingress --- - name: ingress - chart: nvcf/nvcf-gateway-routes + {{- $gatewayRoutesChartPath := dig "ingress" "gatewayApi" "chartPath" "" .Values }} + chart: {{ $gatewayRoutesChartPath | default "nvcf/nvcf-gateway-routes" | quote }} + {{- if not $gatewayRoutesChartPath }} version: 1.15.0 + {{- end }} needs: - nvcf/notary-service - api-keys/api-keys diff --git a/deploy/stacks/self-managed/openbao-migrations-llm-env.yaml.gotmpl b/deploy/stacks/self-managed/openbao-migrations-llm-env.yaml.gotmpl new file mode 100644 index 000000000..b28f80805 --- /dev/null +++ b/deploy/stacks/self-managed/openbao-migrations-llm-env.yaml.gotmpl @@ -0,0 +1,13 @@ +{{- $secretPath := printf "secrets/%s-secrets.yaml" (requiredEnv "HELMFILE_ENV") -}} +{{- $secretValues := readFile $secretPath | fromYaml -}} +{{- $migrationEnv := dig "openbao" "migrations" "env" (list) $secretValues -}} +openbao: + migrations: + env: +{{- with $migrationEnv }} +{{ toYaml . | nindent 6 }} +{{- end }} + - name: ADDONS_LLM_ENABLED + value: {{ dig "addons" "llm" "enabled" false .Values | quote }} + - name: NVCF_SERVICE_PKI_ALLOWED_DOMAINS + value: {{ dig "addons" "llm" "pki" "allowedDomains" "" .Values | quote }} diff --git a/deploy/stacks/self-managed/tests/api-keys-startup-probe.sh b/deploy/stacks/self-managed/tests/api-keys-startup-probe.sh new file mode 100755 index 000000000..6645bb880 --- /dev/null +++ b/deploy/stacks/self-managed/tests/api-keys-startup-probe.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +set -euo pipefail + +stack_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +work_dir="$(mktemp -d)" +test_stack_dir="$work_dir/self-managed" +environment_name="api-keys-startup-probe-test" +trap 'rm -rf "$work_dir"' EXIT + +fail() { + echo "api-keys-startup-probe: $*" >&2 + exit 1 +} + +mkdir -p "$test_stack_dir" +cp -R "$stack_dir"/. "$test_stack_dir" +: >"$test_stack_dir/secrets/$environment_name-secrets.yaml" +printf '%s\n' \ + 'apikeys:' \ + ' startupProbe:' \ + ' failureThreshold: 60' \ + >"$test_stack_dir/environments/$environment_name.yaml" + +values_file="$work_dir/api-keys-values.yaml" +HELMFILE_ENV="$environment_name" \ + HELMFILE_CACHE_HOME="$work_dir/helmfile-cache" \ + helmfile \ + --file "$test_stack_dir/helmfile.d/02-core.yaml.gotmpl" \ + --environment default \ + --state-values-set ingress.gatewayApi.controllerNamespace=envoy-gateway-system \ + --state-values-set ingress.gatewayApi.gateways.shared.name=shared-gw \ + --state-values-set ingress.gatewayApi.gateways.shared.namespace=envoy-gateway-system \ + --state-values-set ingress.gatewayApi.gateways.grpc.name=grpc-gw \ + --state-values-set ingress.gatewayApi.gateways.grpc.namespace=envoy-gateway-system \ + --state-values-set apikeys.startupProbe.failureThreshold=60 \ + --selector name=api-keys \ + write-values \ + --output-file-template "$values_file" + +actual="$(yq -r '.apikeys.startupProbe.failureThreshold // "missing"' "$values_file")" +test "$actual" = "60" || + fail "expected apikeys.startupProbe.failureThreshold=60, got $actual" + +echo "api-keys-startup-probe: all checks passed" diff --git a/deploy/stacks/self-managed/tests/gateway-routes-local-chart.sh b/deploy/stacks/self-managed/tests/gateway-routes-local-chart.sh new file mode 100755 index 000000000..6b5352d91 --- /dev/null +++ b/deploy/stacks/self-managed/tests/gateway-routes-local-chart.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail + +stack_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +chart_path="../../../helm/gateway-routes/chart" + +result="$(cd "$stack_dir" && HELMFILE_ENV=base helmfile \ + --file helmfile.d/02-core.yaml.gotmpl \ + --environment default \ + --state-values-set ingress.gatewayApi.enabled=true \ + --state-values-set ingress.gatewayApi.controllerNamespace=gateway \ + --state-values-set ingress.gatewayApi.gateways.shared.name=shared-gw \ + --state-values-set ingress.gatewayApi.gateways.shared.namespace=gateway \ + --state-values-set ingress.gatewayApi.gateways.grpc.name=grpc-gw \ + --state-values-set ingress.gatewayApi.gateways.grpc.namespace=gateway \ + --state-values-set-string "ingress.gatewayApi.chartPath=$chart_path" \ + --selector name=ingress \ + list --skip-charts --output json)" + +actual="$(jq -r '.[0].chart' <<<"$result")" +test "$actual" = "$chart_path" || { + echo "gateway-routes-local-chart: expected $chart_path, got ${actual:-missing}" >&2 + exit 1 +} + +echo "gateway-routes-local-chart: all checks passed" diff --git a/deploy/stacks/self-managed/tests/grpc-proxy-nats-endpoint.sh b/deploy/stacks/self-managed/tests/grpc-proxy-nats-endpoint.sh new file mode 100755 index 000000000..78f6d70f7 --- /dev/null +++ b/deploy/stacks/self-managed/tests/grpc-proxy-nats-endpoint.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +set -euo pipefail + +stack_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +work_dir="$(mktemp -d)" +test_stack_dir="$work_dir/self-managed" +environment_name="grpc-proxy-nats-endpoint-test" +trap 'rm -rf "$work_dir"' EXIT + +fail() { + echo "grpc-proxy-nats-endpoint: $*" >&2 + exit 1 +} + +mkdir -p "$test_stack_dir" +cp -R "$stack_dir"/. "$test_stack_dir" +: >"$test_stack_dir/secrets/$environment_name-secrets.yaml" +printf '%s\n' \ + 'global:' \ + ' workerEndpoints:' \ + ' nvcfNatsServiceURL: nats://control.example.test:4222' \ + >"$test_stack_dir/environments/$environment_name.yaml" + +values_file="$work_dir/grpc-proxy-values.yaml" +HELMFILE_ENV="$environment_name" \ + HELMFILE_CACHE_HOME="$work_dir/helmfile-cache" \ + helmfile \ + --file "$test_stack_dir/helmfile.d/02-core.yaml.gotmpl" \ + --environment default \ + --state-values-set ingress.gatewayApi.controllerNamespace=envoy-gateway-system \ + --state-values-set ingress.gatewayApi.gateways.shared.name=shared-gw \ + --state-values-set ingress.gatewayApi.gateways.shared.namespace=envoy-gateway-system \ + --state-values-set ingress.gatewayApi.gateways.grpc.name=grpc-gw \ + --state-values-set ingress.gatewayApi.gateways.grpc.namespace=envoy-gateway-system \ + --selector name=grpc-proxy \ + write-values \ + --output-file-template "$values_file" + +actual="$(yq -r '.grpcproxy.env.NATS_FQDN // "missing"' "$values_file")" +expected='nats://nats.nats-system.svc.cluster.local:4222' +test "$actual" = "$expected" || + fail "expected control-plane grpc-proxy NATS_FQDN=$expected, got $actual" + +echo "grpc-proxy-nats-endpoint: all checks passed" diff --git a/deploy/stacks/self-managed/tests/llm-pki-openbao-migration.sh b/deploy/stacks/self-managed/tests/llm-pki-openbao-migration.sh new file mode 100755 index 000000000..bbf83cf23 --- /dev/null +++ b/deploy/stacks/self-managed/tests/llm-pki-openbao-migration.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +set -euo pipefail + +stack_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +work_dir="$(mktemp -d)" +test_stack_dir="$work_dir/self-managed" +environment_name="llm-pki-openbao-migration-test" +environment_file="$test_stack_dir/environments/$environment_name.yaml" +secrets_file="$test_stack_dir/secrets/$environment_name-secrets.yaml" +trap 'rm -rf "$work_dir"' EXIT + +fail() { + echo "llm-pki-openbao-migration: $*" >&2 + exit 1 +} + +mkdir -p "$test_stack_dir" +cp -R "$stack_dir"/. "$test_stack_dir" +printf '%s\n' \ + 'openbao:' \ + ' migrations:' \ + ' env:' \ + ' - name: EXISTING_SECRET_ENV' \ + ' value: preserved' \ + >"$secrets_file" +printf '%s\n' \ + 'addons:' \ + ' llm:' \ + ' enabled: true' \ + ' pki:' \ + ' enabled: true' \ + ' allowedDomains: cluster.local' \ + ' dnsNames:' \ + ' - llm-request-router.nvcf.svc.cluster.local' \ + ' image:' \ + ' tag: test' \ + >"$environment_file" + +values_file="$work_dir/openbao-values.yaml" +HELMFILE_ENV="$environment_name" \ + HELMFILE_CACHE_HOME="$work_dir/helmfile-cache" \ + helmfile \ + --file "$test_stack_dir/helmfile.d/01-dependencies.yaml.gotmpl" \ + --environment default \ + --state-values-set ingress.gatewayApi.controllerNamespace=envoy-gateway-system \ + --state-values-set ingress.gatewayApi.gateways.shared.name=shared-gw \ + --state-values-set ingress.gatewayApi.gateways.shared.namespace=envoy-gateway-system \ + --state-values-set ingress.gatewayApi.gateways.grpc.name=grpc-gw \ + --state-values-set ingress.gatewayApi.gateways.grpc.namespace=envoy-gateway-system \ + --selector name=openbao-server \ + write-values \ + --output-file-template "$values_file" + +environment_value() { + local name="$1" + yq -r ".openbao.migrations.env[]? | select(.name == \"$name\") | .value" "$values_file" +} + +actual="$(environment_value ADDONS_LLM_ENABLED)" +test "$actual" = "true" || + fail "expected ADDONS_LLM_ENABLED=true in the OpenBao migration environment, got ${actual:-missing}" + +actual="$(environment_value NVCF_SERVICE_PKI_ALLOWED_DOMAINS)" +test "$actual" = "cluster.local" || + fail "expected NVCF_SERVICE_PKI_ALLOWED_DOMAINS=cluster.local in the OpenBao migration environment, got ${actual:-missing}" + +actual="$(environment_value EXISTING_SECRET_ENV)" +test "$actual" = "preserved" || + fail "expected existing migration environment entries to be preserved, got ${actual:-missing}" + +echo "llm-pki-openbao-migration: all checks passed" diff --git a/deploy/stacks/self-managed/tests/llm-router-local-chart.sh b/deploy/stacks/self-managed/tests/llm-router-local-chart.sh new file mode 100755 index 000000000..6829fefa9 --- /dev/null +++ b/deploy/stacks/self-managed/tests/llm-router-local-chart.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -euo pipefail + +stack_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +chart_path="../../../helm/llm-request-router/llm-request-router" + +result="$(cd "$stack_dir" && HELMFILE_ENV=base helmfile \ + --file helmfile.d/02-core.yaml.gotmpl \ + --environment default \ + --state-values-set addons.llm.enabled=true \ + --state-values-set-string "addons.llm.requestRouter.chartPath=$chart_path" \ + --selector name=llm-request-router \ + list --skip-charts --output json)" + +actual="$(jq -r '.[0].chart' <<<"$result")" +test "$actual" = "$chart_path" || { + echo "llm-router-local-chart: expected $chart_path, got ${actual:-missing}" >&2 + exit 1 +} + +echo "llm-router-local-chart: all checks passed" diff --git a/deploy/stacks/self-managed/tests/llm-router-split-cluster.sh b/deploy/stacks/self-managed/tests/llm-router-split-cluster.sh new file mode 100755 index 000000000..07c961035 --- /dev/null +++ b/deploy/stacks/self-managed/tests/llm-router-split-cluster.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +set -euo pipefail + +stack_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +work_dir="$(mktemp -d)" +test_stack_dir="$work_dir/self-managed" +environment_name="llm-router-split-cluster-test" +environment_file="$test_stack_dir/environments/$environment_name.yaml" +secrets_file="$test_stack_dir/secrets/$environment_name-secrets.yaml" +trap 'rm -rf "$work_dir"' EXIT + +fail() { + echo "llm-router-split-cluster: $*" >&2 + exit 1 +} + +mkdir -p "$test_stack_dir" +cp -R "$stack_dir"/. "$test_stack_dir" +printf '{}\n' >"$secrets_file" + +printf '%s\n' \ + 'addons:' \ + ' llm:' \ + ' enabled: true' \ + ' requestRouter:' \ + ' backendRouter:' \ + ' pylonGrpcDialAddress: control.example.test:50071' \ + ' pylonReverseTunnelDialAddress: control.example.test:50072' \ + 'ingress:' \ + ' gatewayApi:' \ + ' controllerNamespace: envoy-gateway-system' \ + ' routes:' \ + ' llmWorker:' \ + ' enabled: true' \ + ' backend:' \ + ' namespace: nvcf' \ + ' gateways:' \ + ' shared:' \ + ' name: shared-gw' \ + ' namespace: envoy-gateway-system' \ + ' grpc:' \ + ' name: grpc-gw' \ + ' namespace: envoy-gateway-system' \ + ' llmGrpc:' \ + ' name: llm-grpc-gw' \ + ' namespace: envoy-gateway-system' \ + ' listenerName: llm-grpc' \ + ' llmQuic:' \ + ' name: llm-quic-gw' \ + ' namespace: envoy-gateway-system' \ + ' listenerName: llm-quic' \ + >"$environment_file" + +HELMFILE_ENV="$environment_name" \ + HELMFILE_CACHE_HOME="$work_dir/helmfile-cache" \ + helmfile \ + --file "$test_stack_dir/helmfile.d/02-core.yaml.gotmpl" \ + --environment default \ + --selector name=ingress \ + write-values \ + --output-file-template "$work_dir/ingress-values.yaml" + +values_file="$work_dir/ingress-values.yaml" +test -f "$values_file" || fail "ingress values were not rendered" + +assert_value() { + local path="$1" + local expected="$2" + local actual + actual="$(yq -r "$path" "$values_file")" + test "$actual" = "$expected" || + fail "expected $path to be $expected, got $actual" +} + +assert_value '.nvcfGatewayRoutes.routes.llmWorker.enabled' 'true' +assert_value '.nvcfGatewayRoutes.routes.llmWorker.backend.namespace' 'nvcf' +assert_value '.nvcfGatewayRoutes.gateways.llmGrpc.name' 'llm-grpc-gw' +assert_value '.nvcfGatewayRoutes.gateways.llmGrpc.listenerName' 'llm-grpc' +assert_value '.nvcfGatewayRoutes.gateways.llmQuic.name' 'llm-quic-gw' +assert_value '.nvcfGatewayRoutes.gateways.llmQuic.listenerName' 'llm-quic' + +echo "llm-router-split-cluster: all checks passed" diff --git a/src/libraries/rust/stargate/Dockerfile b/src/libraries/rust/stargate/Dockerfile index 0a301e513..217a98932 100644 --- a/src/libraries/rust/stargate/Dockerfile +++ b/src/libraries/rust/stargate/Dockerfile @@ -129,6 +129,7 @@ ARG TARGETARCH # Health probe changes less often than the binary, so copy it first for layer caching. COPY --from=health-probe-downloader /usr/local/bin/grpc_health_probe /usr/local/bin/grpc_health_probe COPY --from=binary-builder /out/stargate /usr/local/bin/stargate +COPY --from=binary-builder /out/stargate-k8s-router /usr/local/bin/stargate-k8s-router ENTRYPOINT ["stargate"] CMD [] diff --git a/src/libraries/rust/stargate/README.md b/src/libraries/rust/stargate/README.md index 70cfe3b5d..6bad98426 100644 --- a/src/libraries/rust/stargate/README.md +++ b/src/libraries/rust/stargate/README.md @@ -91,6 +91,10 @@ load-balancer topology for production backend traffic. - `crates/mock-dynamo`: local OpenAI-style backend - `crates/stargate-bench`: benchmark runner +The versioned Stargate runtime image also includes +`/usr/local/bin/stargate-k8s-router`. Kubernetes deployments can run the main +Stargate process and the backend router from the same immutable image tag. + ## Benchmarks ```bash diff --git a/src/libraries/rust/stargate/crates/pylon/BUILD.bazel b/src/libraries/rust/stargate/crates/pylon/BUILD.bazel index cec1a0fa0..0f35b17ee 100644 --- a/src/libraries/rust/stargate/crates/pylon/BUILD.bazel +++ b/src/libraries/rust/stargate/crates/pylon/BUILD.bazel @@ -37,7 +37,7 @@ rust_test( # Multi-arch OCI image. distroless/cc base, binary at /usr/local/bin/pylon. rust_oci_image( - name = "image", + name = "pylon-image", base = "@distroless_cc", binary = ":pylon", binary_path = "/usr/local/bin/pylon", @@ -52,8 +52,8 @@ sh_test( name = "image_entrypoint_mode_test", srcs = ["//src/libraries/rust/stargate/tools/ci:image_entrypoint_mode_test.sh"], args = [ - "$(location :image_layer)", + "$(location :pylon-image_layer)", "/usr/local/bin/pylon", ], - data = [":image_layer"], + data = [":pylon-image_layer"], ) diff --git a/src/libraries/rust/stargate/crates/stargate-k8s-router/BUILD.bazel b/src/libraries/rust/stargate/crates/stargate-k8s-router/BUILD.bazel index bf65e0377..d34789325 100644 --- a/src/libraries/rust/stargate/crates/stargate-k8s-router/BUILD.bazel +++ b/src/libraries/rust/stargate/crates/stargate-k8s-router/BUILD.bazel @@ -48,7 +48,7 @@ rust_test( # Multi-arch OCI image. distroless/cc base, binary at # /usr/local/bin/stargate-k8s-router. rust_oci_image( - name = "image", + name = "stargate-k8s-router-image", base = "@distroless_cc", binary = ":stargate-k8s-router", binary_path = "/usr/local/bin/stargate-k8s-router", @@ -62,8 +62,8 @@ sh_test( name = "image_entrypoint_mode_test", srcs = ["//src/libraries/rust/stargate/tools/ci:image_entrypoint_mode_test.sh"], args = [ - "$(location :image_layer)", + "$(location :stargate-k8s-router-image_layer)", "/usr/local/bin/stargate-k8s-router", ], - data = [":image_layer"], + data = [":stargate-k8s-router-image_layer"], ) diff --git a/src/libraries/rust/stargate/crates/stargate-k8s-router/src/metrics.rs b/src/libraries/rust/stargate/crates/stargate-k8s-router/src/metrics.rs index 6c218c73a..83400b56a 100644 --- a/src/libraries/rust/stargate/crates/stargate-k8s-router/src/metrics.rs +++ b/src/libraries/rust/stargate/crates/stargate-k8s-router/src/metrics.rs @@ -19,6 +19,26 @@ use anyhow::Result; use prometheus::{Encoder, IntCounterVec, IntGaugeVec, Opts, Registry, TextEncoder}; use stargate_tls::{SERVER_IDENTITY_MATERIAL, TlsIdentityStatus, TlsReloadOutcome}; +const QUIC_CONNECTION_OUTCOMES: &[&str] = &[ + "accepted", + "completed", + "missing_sni", + "relay_error", + "target_unavailable", + "unknown_sni", +]; +const WEBTRANSPORT_SESSION_OUTCOMES: &[&str] = &[ + "accepted", + "completed", + "invalid_connect", + "missing_sni", + "relay_error", + "target_unavailable", + "unknown_sni", + "upstream_connect_error", + "upstream_rejected", +]; + #[derive(Clone)] pub struct RouterMetrics { registry: Registry, @@ -71,6 +91,13 @@ impl RouterMetrics { .inc_by(0); } + for outcome in QUIC_CONNECTION_OUTCOMES { + let _ = quic_connections_total.with_label_values(&[outcome]); + } + for outcome in WEBTRANSPORT_SESSION_OUTCOMES { + let _ = webtransport_sessions_total.with_label_values(&[outcome]); + } + Ok(Self { registry, quic_connections_total, @@ -134,6 +161,30 @@ impl RouterMetrics { mod tests { use super::*; + #[test] + fn metrics_export_known_outcomes_before_traffic() { + let metrics = RouterMetrics::new().expect("metrics should initialize"); + + let body = metrics.gather().expect("metrics should encode"); + + for &outcome in QUIC_CONNECTION_OUTCOMES { + assert!( + body.contains(&format!( + r#"stargate_k8s_router_quic_connections_total{{outcome="{outcome}"}} 0"# + )), + "missing zero-valued QUIC series for {outcome}" + ); + } + for &outcome in WEBTRANSPORT_SESSION_OUTCOMES { + assert!( + body.contains(&format!( + r#"stargate_k8s_router_webtransport_sessions_total{{outcome="{outcome}"}} 0"# + )), + "missing zero-valued WebTransport series for {outcome}" + ); + } + } + #[test] fn metrics_exports_quic_connection_outcomes() { let metrics = RouterMetrics::new().expect("metrics should initialize"); diff --git a/src/libraries/rust/stargate/crates/stargate/BUILD.bazel b/src/libraries/rust/stargate/crates/stargate/BUILD.bazel index a813c2f6a..8e8715930 100644 --- a/src/libraries/rust/stargate/crates/stargate/BUILD.bazel +++ b/src/libraries/rust/stargate/crates/stargate/BUILD.bazel @@ -116,6 +116,7 @@ rust_oci_image( base = "@distroless_cc", binary = ":stargate", binary_path = "/usr/local/bin/stargate", + extra_layers = ["//src/libraries/rust/stargate/crates/stargate-k8s-router:stargate-k8s-router-image_layer"], tags = ["stargate"], visibility = ["//visibility:public"], ) @@ -132,3 +133,15 @@ sh_test( ], data = [":image_layer"], ) + +# Composite-image guard. Inspect the assembled OCI layout so removing the +# router's extra layer fails this test. +sh_test( + name = "image_router_binary_test", + srcs = ["//src/libraries/rust/stargate/tools/ci:oci_image_contains_path_test.sh"], + args = [ + "$(location :image)", + "/usr/local/bin/stargate-k8s-router", + ], + data = [":image"], +) diff --git a/src/libraries/rust/stargate/tools/ci/BUILD.bazel b/src/libraries/rust/stargate/tools/ci/BUILD.bazel index 96e468f21..2f322349f 100644 --- a/src/libraries/rust/stargate/tools/ci/BUILD.bazel +++ b/src/libraries/rust/stargate/tools/ci/BUILD.bazel @@ -4,6 +4,7 @@ exports_files( [ "image_entrypoint_mode_test.sh", + "oci_image_contains_path_test.sh", ], visibility = ["//visibility:public"], ) diff --git a/src/libraries/rust/stargate/tools/ci/oci_image_contains_path_test.sh b/src/libraries/rust/stargate/tools/ci/oci_image_contains_path_test.sh new file mode 100755 index 000000000..c389a8607 --- /dev/null +++ b/src/libraries/rust/stargate/tools/ci/oci_image_contains_path_test.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +if [[ $# -ne 2 ]]; then + echo "usage: $0 " >&2 + exit 2 +fi + +image_layout="$1" +image_path="${2#/}" + +if [[ ! -f "${image_layout}/index.json" || ! -d "${image_layout}/blobs/sha256" ]]; then + echo "${image_layout} is not an OCI image layout" >&2 + exit 1 +fi + +while IFS= read -r -d '' blob; do + if entries="$(tar -tf "${blob}" 2>/dev/null)"; then + while IFS= read -r entry; do + entry="${entry#./}" + entry="${entry#/}" + if [[ "${entry}" == "${image_path}" ]]; then + exit 0 + fi + done <<< "${entries}" + fi +# rules_oci may symlink blob files to their source layers. Select those links +# directly without following symlinked directories outside the blob tree. +done < <(find "${image_layout}/blobs/sha256" \( -type f -o -type l \) -print0) + +echo "missing /${image_path} in ${image_layout}" >&2 +exit 1 diff --git a/tools/ci/test-image-push-manual b/tools/ci/test-image-push-manual new file mode 100755 index 000000000..1647c578b --- /dev/null +++ b/tools/ci/test-image-push-manual @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Behavioral regression test for the manual image-push workflow. A repository +# collision must fail before any image push begins, because every push also +# updates the shared latest-dispatch tag. +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +workflow="${repo_root}/.github/workflows/image-push-manual.yml" +work="$(mktemp -d)" +trap 'rm -rf "${work}"' EXIT + +awk ' + $0 == " - name: Build and push multi-arch image(s)" { + in_step = 1 + next + } + in_step && $0 == " run: |" { + in_script = 1 + next + } + in_script { + if ($0 ~ /^ - name:/) { + exit + } + sub(/^ /, "") + print + } +' "${workflow}" > "${work}/image-push-step" + +if [[ ! -s "${work}/image-push-step" ]]; then + echo "FAIL: could not extract the image-push shell block" >&2 + exit 1 +fi + +mkdir -p "${work}/bin" "${work}/home" "${work}/run" +cat > "${work}/bin/bazel" <<'STUB' +#!/usr/bin/env bash +set -euo pipefail + +printf '%s\n' "$*" >> "${BAZEL_CALL_LOG}" +case "${1:-}" in + query) + # Both targets map to example-service-worker under the workflow's two + # supported naming conventions. + printf '%s\n' \ + '//first:worker_image_index' \ + '//second:example-service-worker-image_index' + ;; + run) + touch "${PUSH_MARKER}" + ;; + *) + echo "unexpected bazel invocation: $*" >&2 + exit 1 + ;; +esac +STUB +chmod +x "${work}/bin/bazel" + +output="${work}/output" +calls="${work}/bazel-calls" +push_marker="${work}/push-started" + +if ( + cd "${work}/run" + HOME="${work}/home" \ + PATH="${work}/bin:${PATH}" \ + BAZEL_CALL_LOG="${calls}" \ + PUSH_MARKER="${push_marker}" \ + SVC_PATH="src/example-service" \ + SCOPE="//..." \ + TAG="gh.1-deadbeef" \ + REGISTRY="registry.example.com/nvcf" \ + bash "${work}/image-push-step" +) > "${output}" 2>&1; then + echo "FAIL: duplicate repository mapping unexpectedly succeeded" >&2 + exit 1 +fi + +if [[ -e "${push_marker}" ]] || grep -q '^run ' "${calls}"; then + echo "FAIL: bazel run executed before all repository mappings were validated" >&2 + sed 's/^/ /' "${calls}" >&2 + exit 1 +fi + +if ! grep -Fq \ + "//second:example-service-worker-image_index and //first:worker_image_index both map to image repository example-service-worker" \ + "${output}"; then + echo "FAIL: workflow did not report the conflicting target mappings" >&2 + sed 's/^/ /' "${output}" >&2 + exit 1 +fi + +echo "image-push-manual: duplicate mappings prevent every push" diff --git a/tools/ncp-local-cluster/Makefile b/tools/ncp-local-cluster/Makefile index 354da8727..a79a20bff 100644 --- a/tools/ncp-local-cluster/Makefile +++ b/tools/ncp-local-cluster/Makefile @@ -36,6 +36,8 @@ CONTROL_PLANE_HTTPS_PORT ?= 8443 CONTROL_PLANE_GRPC_PORT ?= 9090 CONTROL_PLANE_GRPC_PROXY_PORT ?= 10081 CONTROL_PLANE_GRPC_WORKER_PORT ?= 10086 +CONTROL_PLANE_LLM_GRPC_PORT ?= 50071 +CONTROL_PLANE_LLM_QUIC_PORT ?= 50072 CONTROL_PLANE_NATS_PORT ?= 4222 CONTROL_PLANE_DOMAIN ?= nvcf-control-plane.test COMPUTE_CLUSTER_PREFIX ?= ncp-local-compute @@ -100,6 +102,8 @@ ensure-cluster: ## Create cluster if it doesn't exist, otherwise skip CONTROL_PLANE_GRPC_PORT="$(CONTROL_PLANE_GRPC_PORT)" \ CONTROL_PLANE_GRPC_PROXY_PORT="$(CONTROL_PLANE_GRPC_PROXY_PORT)" \ CONTROL_PLANE_GRPC_WORKER_PORT="$(CONTROL_PLANE_GRPC_WORKER_PORT)" \ + CONTROL_PLANE_LLM_GRPC_PORT="$(CONTROL_PLANE_LLM_GRPC_PORT)" \ + CONTROL_PLANE_LLM_QUIC_PORT="$(CONTROL_PLANE_LLM_QUIC_PORT)" \ CONTROL_PLANE_NATS_PORT="$(CONTROL_PLANE_NATS_PORT)" \ k3d cluster create $(CLUSTER_NAME) --config $(K3D_CONFIG_FILE); \ echo "OK Cluster $(CLUSTER_NAME) created."; \ diff --git a/tools/ncp-local-cluster/apps/envoy-gateway/gateway-grpc.yaml b/tools/ncp-local-cluster/apps/envoy-gateway/gateway-grpc.yaml index 238386750..f19c39668 100644 --- a/tools/ncp-local-cluster/apps/envoy-gateway/gateway-grpc.yaml +++ b/tools/ncp-local-cluster/apps/envoy-gateway/gateway-grpc.yaml @@ -36,3 +36,15 @@ spec: allowedRoutes: namespaces: from: All + - name: llm-grpc + protocol: TCP + port: 50071 + allowedRoutes: + namespaces: + from: All + - name: llm-quic + protocol: UDP + port: 50072 + allowedRoutes: + namespaces: + from: All diff --git a/tools/ncp-local-cluster/apps/fake-gpu-operator/static/fake-gpu-operator.yaml b/tools/ncp-local-cluster/apps/fake-gpu-operator/static/fake-gpu-operator.yaml index 8d58046c4..dea0ba206 100644 --- a/tools/ncp-local-cluster/apps/fake-gpu-operator/static/fake-gpu-operator.yaml +++ b/tools/ncp-local-cluster/apps/fake-gpu-operator/static/fake-gpu-operator.yaml @@ -63,7 +63,7 @@ data: nodePoolLabelKey: run.ai/simulated-gpu-node-pool nodePools: default: - gpuCount: 128 + gpuCount: 2 gpuMemory: 81559 gpuProduct: NVIDIA-H100-SXM4-80GB kind: ConfigMap @@ -418,7 +418,7 @@ spec: metadata: annotations: - checksum/topology: 63a9de7b0d41b449b9a0ba036e23ea7e55eb174d6a822ff9dc2531fe61db971b + checksum/topology: d4293b3607dec297d688bfb0dfe5ffa78eb892015c926bf1d5b61500788468c4 labels: app: device-plugin component: device-plugin @@ -496,7 +496,7 @@ spec: template: metadata: annotations: - checksum/topology: 63a9de7b0d41b449b9a0ba036e23ea7e55eb174d6a822ff9dc2531fe61db971b + checksum/topology: d4293b3607dec297d688bfb0dfe5ffa78eb892015c926bf1d5b61500788468c4 labels: app: mig-faker component: mig-faker @@ -652,7 +652,7 @@ kind: Deployment metadata: name: kwok-gpu-device-plugin annotations: - checksum/topology: 63a9de7b0d41b449b9a0ba036e23ea7e55eb174d6a822ff9dc2531fe61db971b + checksum/topology: d4293b3607dec297d688bfb0dfe5ffa78eb892015c926bf1d5b61500788468c4 labels: app: kwok-gpu-device-plugin spec: @@ -664,7 +664,7 @@ spec: template: metadata: annotations: - checksum/topology: 63a9de7b0d41b449b9a0ba036e23ea7e55eb174d6a822ff9dc2531fe61db971b + checksum/topology: d4293b3607dec297d688bfb0dfe5ffa78eb892015c926bf1d5b61500788468c4 labels: app: kwok-gpu-device-plugin component: kwok-gpu-device-plugin @@ -770,7 +770,7 @@ kind: Deployment metadata: name: status-updater annotations: - checksum/topology: 63a9de7b0d41b449b9a0ba036e23ea7e55eb174d6a822ff9dc2531fe61db971b + checksum/topology: d4293b3607dec297d688bfb0dfe5ffa78eb892015c926bf1d5b61500788468c4 labels: app: status-updater spec: @@ -782,7 +782,7 @@ spec: template: metadata: annotations: - checksum/topology: 63a9de7b0d41b449b9a0ba036e23ea7e55eb174d6a822ff9dc2531fe61db971b + checksum/topology: d4293b3607dec297d688bfb0dfe5ffa78eb892015c926bf1d5b61500788468c4 labels: app: status-updater component: status-updater diff --git a/tools/ncp-local-cluster/apps/fake-gpu-operator/values.yaml b/tools/ncp-local-cluster/apps/fake-gpu-operator/values.yaml index 650f6ebe6..962e1a6d9 100644 --- a/tools/ncp-local-cluster/apps/fake-gpu-operator/values.yaml +++ b/tools/ncp-local-cluster/apps/fake-gpu-operator/values.yaml @@ -5,7 +5,10 @@ topology: nodePools: default: gpuProduct: NVIDIA-H100-SXM4-80GB - gpuCount: 128 + # Keep a one-GPU local instance schedulable after NVCA infrastructure + # overhead. Higher-density slices on k3d nodes suppress the required + # *_1x registration type. + gpuCount: 2 gpuMemory: 81559 nodePoolLabelKey: run.ai/simulated-gpu-node-pool migStrategy: mixed diff --git a/tools/ncp-local-cluster/k3d-config-control-plane.yaml b/tools/ncp-local-cluster/k3d-config-control-plane.yaml index 6af3c1414..6b0939f3a 100644 --- a/tools/ncp-local-cluster/k3d-config-control-plane.yaml +++ b/tools/ncp-local-cluster/k3d-config-control-plane.yaml @@ -40,6 +40,12 @@ ports: - port: ${CONTROL_PLANE_GRPC_WORKER_PORT}:10086 nodeFilters: - loadbalancer + - port: ${CONTROL_PLANE_LLM_GRPC_PORT}:50071 + nodeFilters: + - loadbalancer + - port: ${CONTROL_PLANE_LLM_QUIC_PORT}:50072/udp + nodeFilters: + - loadbalancer - port: ${CONTROL_PLANE_NATS_PORT}:4222 nodeFilters: - loadbalancer diff --git a/tools/ncp-local-cluster/tests/test-multicluster-make.sh b/tools/ncp-local-cluster/tests/test-multicluster-make.sh index 6cb014b89..daa486573 100755 --- a/tools/ncp-local-cluster/tests/test-multicluster-make.sh +++ b/tools/ncp-local-cluster/tests/test-multicluster-make.sh @@ -82,6 +82,9 @@ EOF assert_missing_docker_config_blocks_target start assert_missing_docker_config_blocks_target build-and-deploy-cluster +fake_gpu_count="$(yq -r '.topology.nodePools.default.gpuCount' "$ROOT_DIR/apps/fake-gpu-operator/values.yaml")" +assert_eq "2" "$fake_gpu_count" "fake GPU count" + print_directory_clusters="$(MAKEFLAGS=--print-directory; export MAKEFLAGS; run_make print-compute-clusters)" assert_eq "ncp-local-compute-1" "$print_directory_clusters" "print-directory compute cluster output" if ! grep -q '\$(MAKE) --no-print-directory -s print-compute-clusters' "$ROOT_DIR/Makefile"; then @@ -140,6 +143,12 @@ fi if ! grep -q '\${CONTROL_PLANE_GRPC_WORKER_PORT}:10086' "$ROOT_DIR/k3d-config-control-plane.yaml"; then fail "control-plane k3d config must expose CONTROL_PLANE_GRPC_WORKER_PORT to Gateway port 10086" fi +if ! grep -q '\${CONTROL_PLANE_LLM_GRPC_PORT}:50071' "$ROOT_DIR/k3d-config-control-plane.yaml"; then + fail "control-plane k3d config must expose CONTROL_PLANE_LLM_GRPC_PORT to the LLM TCP Gateway listener" +fi +if ! grep -q '\${CONTROL_PLANE_LLM_QUIC_PORT}:50072' "$ROOT_DIR/k3d-config-control-plane.yaml"; then + fail "control-plane k3d config must expose CONTROL_PLANE_LLM_QUIC_PORT to the LLM UDP Gateway listener" +fi if ! grep -q '10081:10081' "$ROOT_DIR/k3d-config.yaml"; then fail "single-cluster k3d config must expose the stack-owned grpc-gw TCP listener on host port 10081" fi @@ -171,6 +180,15 @@ fi if ! grep -R -q 'name: worker-tcp' "$ROOT_DIR/apps/envoy-gateway/gateway-grpc.yaml"; then fail "control-plane Gateway must define the grpc-proxy worker TCP listener" fi +if ! grep -R -q 'name: llm-grpc' "$ROOT_DIR/apps/envoy-gateway/gateway-grpc.yaml"; then + fail "control-plane Gateway must define the LLM worker TCP listener" +fi +if ! grep -R -q 'name: llm-quic' "$ROOT_DIR/apps/envoy-gateway/gateway-grpc.yaml"; then + fail "control-plane Gateway must define the LLM worker UDP listener" +fi +if ! awk '/name: llm-quic/{found=1; next} found && /protocol: UDP/{ok=1; exit} END{exit !ok}' "$ROOT_DIR/apps/envoy-gateway/gateway-grpc.yaml"; then + fail "LLM reverse-tunnel Gateway listener must use UDP" +fi if ! grep -q 'kubectl apply -k .*apps/envoy-gateway' "$ROOT_DIR/scripts/setup-gateway-api.sh"; then fail "gateway setup must apply the full envoy-gateway kustomization" fi @@ -206,6 +224,18 @@ fi if ! grep -q 'CONTROL_PLANE_GRPC_WORKER_PORT="$(CONTROL_PLANE_GRPC_WORKER_PORT)"' "$ROOT_DIR/Makefile"; then fail "Makefile must pass CONTROL_PLANE_GRPC_WORKER_PORT to the control-plane k3d config" fi +if ! grep -q 'CONTROL_PLANE_LLM_GRPC_PORT ?= 50071' "$ROOT_DIR/Makefile"; then + fail "Makefile must define the host port for the LLM worker TCP listener" +fi +if ! grep -q 'CONTROL_PLANE_LLM_QUIC_PORT ?= 50072' "$ROOT_DIR/Makefile"; then + fail "Makefile must define the host port for the LLM worker UDP listener" +fi +if ! grep -q 'CONTROL_PLANE_LLM_GRPC_PORT="$(CONTROL_PLANE_LLM_GRPC_PORT)"' "$ROOT_DIR/Makefile"; then + fail "Makefile must pass CONTROL_PLANE_LLM_GRPC_PORT to the control-plane k3d config" +fi +if ! grep -q 'CONTROL_PLANE_LLM_QUIC_PORT="$(CONTROL_PLANE_LLM_QUIC_PORT)"' "$ROOT_DIR/Makefile"; then + fail "Makefile must pass CONTROL_PLANE_LLM_QUIC_PORT to the control-plane k3d config" +fi if ! grep -q 'CONTROL_PLANE_LB_NATS_PORT=4222' "$ROOT_DIR/Makefile"; then fail "Makefile must pass the control-plane NATS container port to endpoint configuration" fi From 17096ec6720853cfe158130a133d2a8c160d5a7f Mon Sep 17 00:00:00 2001 From: Mike Camp Date: Fri, 21 Aug 2026 15:06:27 -0400 Subject: [PATCH 2/4] fix(compute-stack): inherit collector image repository --- .../helmfile.d/02-nvca.yaml.gotmpl | 6 +++--- .../tests/observability-profile.sh | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl b/deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl index e99565306..05322e129 100644 --- a/deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl +++ b/deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl @@ -91,6 +91,8 @@ helmDefaults: {{- $selfManaged := dig "selfManaged" dict $nvcaOp }} {{- $otelCollector := dig "otelCollector" dict $selfManaged }} {{- $otelCollectorEnabled := dig "enabled" $computeObservabilityEnabled $otelCollector }} +{{- $defaultOTelCollectorImageRepository := printf "%s/%s/nvcf-otel-collector" .Values.global.image.registry .Values.global.image.repository }} +{{- $otelCollectorImageRepository := dig "imageRepository" $defaultOTelCollectorImageRepository $otelCollector }} {{- $featureGateValues := dig "featureGateValues" list $selfManaged }} {{- $kaiSchedulerEnabled := dig "addons" "kaiScheduler" "enabled" false .Values }} @@ -224,9 +226,7 @@ releases: "dynamoOperatorEnabled" $dynamoOperatorEnabled) | nindent 10 }} otelCollector: enabled: {{ $otelCollectorEnabled }} - {{- with dig "imageRepository" "" $otelCollector }} - imageRepository: {{ . | quote }} - {{- end }} + imageRepository: {{ $otelCollectorImageRepository | quote }} {{- with dig "imageTag" "" $otelCollector }} imageTag: {{ . | quote }} {{- end }} diff --git a/deploy/stacks/nvcf-compute-plane/tests/observability-profile.sh b/deploy/stacks/nvcf-compute-plane/tests/observability-profile.sh index 6c08e9353..64c46935a 100755 --- a/deploy/stacks/nvcf-compute-plane/tests/observability-profile.sh +++ b/deploy/stacks/nvcf-compute-plane/tests/observability-profile.sh @@ -57,6 +57,14 @@ collector_image_tag() { ' "$1" } +collector_image_repository() { + awk ' + /^selfManaged:$/ { self_managed = 1; next } + self_managed && /^ otelCollector:$/ { collector = 1; next } + collector && /^ imageRepository:/ { gsub(/"/, "", $2); print $2; exit } + ' "$1" +} + operator_image_tag() { awk ' /^image:$/ { image = 1; next } @@ -93,6 +101,14 @@ test "$(operator_image_tag "$work_dir/default.yaml")" = "$expected_nvca_version" test "$(nvca_version "$work_dir/default.yaml")" = "$expected_nvca_version" || fail "default NVCA version is not $expected_nvca_version" +render_values compute "$work_dir/ncp-dev-default.yaml" \ + --state-values-set-string global.image.registry=nvcr.io \ + --state-values-set-string global.image.repository=0651155215864979/ncp-dev + +test "$(collector_image_repository "$work_dir/ncp-dev-default.yaml")" = \ + "nvcr.io/0651155215864979/ncp-dev/nvcf-otel-collector" || + fail "collector image repository did not inherit the global image repository" + render_values compute "$work_dir/compute-overrides.yaml" \ --state-values-set-string global.nvcaOperator.imageTag=operator-test-tag \ --state-values-set-string global.nvcaOperator.selfManaged.nvcaVersion=nvca-test-tag \ From ff99a9ed748f49b7ebc33857c184e0d37d87d424 Mon Sep 17 00:00:00 2001 From: Mike Camp Date: Fri, 21 Aug 2026 15:17:56 -0400 Subject: [PATCH 3/4] fix(compute-stack): disable self-hosted collector by default --- .../nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl | 2 +- .../nvcf-compute-plane/tests/observability-profile.sh | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl b/deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl index 05322e129..85b207eae 100644 --- a/deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl +++ b/deploy/stacks/nvcf-compute-plane/helmfile.d/02-nvca.yaml.gotmpl @@ -90,7 +90,7 @@ helmDefaults: {{- $nvcaOp := dig "nvcaOperator" dict .Values.global }} {{- $selfManaged := dig "selfManaged" dict $nvcaOp }} {{- $otelCollector := dig "otelCollector" dict $selfManaged }} -{{- $otelCollectorEnabled := dig "enabled" $computeObservabilityEnabled $otelCollector }} +{{- $otelCollectorEnabled := dig "enabled" false $otelCollector }} {{- $defaultOTelCollectorImageRepository := printf "%s/%s/nvcf-otel-collector" .Values.global.image.registry .Values.global.image.repository }} {{- $otelCollectorImageRepository := dig "imageRepository" $defaultOTelCollectorImageRepository $otelCollector }} diff --git a/deploy/stacks/nvcf-compute-plane/tests/observability-profile.sh b/deploy/stacks/nvcf-compute-plane/tests/observability-profile.sh index 64c46935a..6768b6cab 100755 --- a/deploy/stacks/nvcf-compute-plane/tests/observability-profile.sh +++ b/deploy/stacks/nvcf-compute-plane/tests/observability-profile.sh @@ -84,13 +84,9 @@ for profile in default disabled control compute all; do render_values "$profile" "$values" case "$profile" in - default|compute|all) - test "$(collector_enabled "$values")" = "true" || - fail "$profile profile did not enable the NVCA collector" - ;; - disabled|control) + default|disabled|control|compute|all) test "$(collector_enabled "$values")" = "false" || - fail "$profile profile enabled the NVCA collector" + fail "$profile profile enabled the self-managed NVCA collector by default" ;; esac done From 41e75a44b1e9a08f0217a5de7ff1f89ac8af46e2 Mon Sep 17 00:00:00 2001 From: Mike Camp Date: Fri, 21 Aug 2026 15:45:50 -0400 Subject: [PATCH 4/4] fix(stack): default managed LLM PKI image tag --- deploy/stacks/self-managed/global.yaml.gotmpl | 11 ++++----- .../self-managed/tests/llm-pki-release.sh | 24 +++++++++++++++++++ 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/deploy/stacks/self-managed/global.yaml.gotmpl b/deploy/stacks/self-managed/global.yaml.gotmpl index 25c7ef470..386b8f5de 100644 --- a/deploy/stacks/self-managed/global.yaml.gotmpl +++ b/deploy/stacks/self-managed/global.yaml.gotmpl @@ -1023,13 +1023,10 @@ llmRequestRouter: quicInsecure: false {{- if $managedIssuer }} {{- $allowedDomains := required "addons.llm.pki.allowedDomains is required when addons.llm.pki.clusterIssuer management is enabled" (dig "addons" "llm" "pki" "allowedDomains" "" .Values) }} - {{- /* Provisioning hook reuses the same image the k8s-openbao chart drives; - fall back to openbao.migrations.image.tag so operators set the version - in one place. */ -}} - {{- $pkiImageTag := dig "addons" "llm" "pki" "image" "tag" (dig "openbao" "migrations" "image" "tag" "" .Values) .Values }} - {{- if not $pkiImageTag }} - {{- fail "addons.llm.pki.image.tag (or openbao.migrations.image.tag) is required when addons.llm.pki.clusterIssuer management is enabled" }} - {{- end }} + {{- /* Provisioning reuses the OpenBao migrations image. Prefer an explicit + PKI tag, then the OpenBao migrations tag, and finally the compatible + stack default so a normal environment need not duplicate chart values. */ -}} + {{- $pkiImageTag := dig "addons" "llm" "pki" "image" "tag" (dig "openbao" "migrations" "image" "tag" "" .Values) .Values | default "0.16.2" }} pki: enabled: true namespace: {{ dig "addons" "llm" "pki" "namespace" "vault-system" .Values | quote }} diff --git a/deploy/stacks/self-managed/tests/llm-pki-release.sh b/deploy/stacks/self-managed/tests/llm-pki-release.sh index 26c5f6865..de46b652d 100755 --- a/deploy/stacks/self-managed/tests/llm-pki-release.sh +++ b/deploy/stacks/self-managed/tests/llm-pki-release.sh @@ -34,6 +34,25 @@ render_debug() { list --skip-charts --output json 2>&1 } +# The managed issuer hook needs the same OpenBao migrations image as the +# control-plane chart. An environment should not have to duplicate its tag. +render_without_pki_image_tag() { + HELMFILE_ENV=base HELMFILE_CACHE_HOME="$work_dir/helmfile-cache" helmfile \ + --file "$stack_dir/helmfile.d" \ + --environment default \ + --output-file-template "$work_dir/no-image-tag/{{ .Release.Name }}.yaml" \ + --state-values-set ingress.gatewayApi.controllerNamespace=envoy-gateway-system \ + --state-values-set ingress.gatewayApi.gateways.shared.name=shared-gw \ + --state-values-set ingress.gatewayApi.gateways.shared.namespace=envoy-gateway-system \ + --state-values-set ingress.gatewayApi.gateways.grpc.name=grpc-gw \ + --state-values-set ingress.gatewayApi.gateways.grpc.namespace=envoy-gateway-system \ + --state-values-set addons.llm.enabled=true \ + --state-values-set addons.llm.pki.enabled=true \ + --state-values-set-string addons.llm.pki.allowedDomains=cluster.local \ + --state-values-set-string 'addons.llm.pki.dnsNames[0]=llm-request-router.nvcf.svc.cluster.local' \ + write-values --selector name=llm-request-router 2>&1 +} + # The rendered state is the only place that carries both the release # declarations and their needs edges. `list` reports enablement but drops # needs, and `build` cannot run offline because it pulls every chart. Debug @@ -72,6 +91,11 @@ state_table() { awk -F '|' '{ key = ($1 == "" ? $2 : $1 "/" $2); print key "\t" $3 }' } +if ! render_without_pki_image_tag >"$work_dir/no-image-tag.log"; then + cat "$work_dir/no-image-tag.log" >&2 + fail "managed LLM PKI must render when addons.llm.pki.image.tag is omitted" +fi + if ! render_debug >"$work_dir/debug.log"; then cat "$work_dir/debug.log" >&2 fail "helmfile could not render helmfile.d"