From 79b7dcc56b1234c2f37c3fc217fd82010798bfc5 Mon Sep 17 00:00:00 2001 From: jcameron Date: Wed, 19 Aug 2026 16:10:53 -0300 Subject: [PATCH 1/3] feat(llm-routing): support per-replica worker endpoints Relates to #689 Signed-off-by: jcameron --- deploy/helm/llm-request-router/README.md | 47 ++++ .../llm-request-router/templates/_helpers.tpl | 35 +++ .../templates/deployment.yaml | 12 +- .../templates/service-per-pod.yaml | 49 ++++ .../llm-request-router/templates/service.yaml | 4 + .../llm-request-router/values.yaml | 21 +- .../scripts/check-multi-replica-render.sh | 110 +++++++++ .../self-managed/environments/base.yaml | 20 ++ deploy/stacks/self-managed/global.yaml.gotmpl | 14 ++ .../tests/llm-router-worker-address.sh | 49 ++++ .../nvcf-llm-multicluster-invocation.svg | 202 ++++++++-------- docs/user/llm-function-enablement.md | 106 +++++--- docs/user/llm-gateway.md | 16 +- .../pylon-lib/src/registration/tests.rs | 43 +++- .../src/control_plane/watch_stargates.rs | 89 ++++++- .../rust/stargate/crates/stargate/src/main.rs | 2 +- .../rust/stargate/docs/tunnel-transports.md | 45 +++- .../features/multi-cluster-helmfile.feature | 180 +++++++++++++- .../nvcf-compute-plane-local-bdd-multi.yaml | 2 +- .../self-managed-local-bdd-multi.yaml | 26 +- tests/bdd/fixtures_test.go | 16 ++ tests/bdd/godog_test.go | 26 +- tools/ncp-local-cluster/Makefile | 9 +- tools/ncp-local-cluster/README.md | 53 ++++ .../scripts/configure-llm-router-endpoints.sh | 228 ++++++++++++++++++ .../tests/test-multicluster-make.sh | 82 +++++++ 26 files changed, 1330 insertions(+), 156 deletions(-) create mode 100644 deploy/helm/llm-request-router/llm-request-router/templates/service-per-pod.yaml create mode 100755 tools/ncp-local-cluster/scripts/configure-llm-router-endpoints.sh diff --git a/deploy/helm/llm-request-router/README.md b/deploy/helm/llm-request-router/README.md index d1683c608..f0a75afbe 100644 --- a/deploy/helm/llm-request-router/README.md +++ b/deploy/helm/llm-request-router/README.md @@ -83,6 +83,53 @@ Important settings to review before deployment: The default values include development-oriented placeholders. Override them before using the chart in any shared or production environment. +## Split-Cluster Worker Access + +Remote pylons need one shared seed and one address per Stargate replica. The +shared Service is only the `WatchStargates` bootstrap address. After discovery, +each pylon registers directly with every returned replica over TCP and opens a +reverse QUIC tunnel to that same replica over UDP. + +Enable per-replica Services with: + +```yaml +llmRequestRouter: + service: + annotations: {} + externalAccess: + enabled: true + domain: router.region-a.example + service: + type: LoadBalancer + annotations: {} + discovery: + remoteStargateURLs: + - http://router-seed.region-b.example:50071 +``` + +For a StatefulSet pod named `llm-request-router-0`, this configuration +advertises these dial addresses: + +- TCP registration: `llm-request-router-0.router.region-a.example:50071` +- UDP reverse QUIC: `llm-request-router-0.router.region-a.example:50072` + +The external names are dial-only. The internal advertised hostname remains +the gRPC authority and the QUIC SNI. Keep the internal exact and wildcard names +in `certificate.dnsNames`; do not add the external domain to the router +certificate. + +The infrastructure provider must create the DNS records and transparently +forward both protocols to the matching per-pod Service. Do not terminate TLS, +change the HTTP/2 authority, or route several replicas behind one endpoint. +Use a region-unique external domain if `remoteStargateURLs` connects router +meshes from more than one region. The chart does not create DNS records or +provider load balancers beyond the requested Kubernetes Service type. + +In the self-managed stack, +`global.workerEndpoints.llmRequestRouterAddress` remains the single shared seed. +Configure `addons.llm.requestRouter.externalAccess` separately for the +per-replica paths. + ## Load Balancer Configuration The chart can pass a Stargate load-balancer config in either of two ways: diff --git a/deploy/helm/llm-request-router/llm-request-router/templates/_helpers.tpl b/deploy/helm/llm-request-router/llm-request-router/templates/_helpers.tpl index 512a69cea..b0e3e436f 100644 --- a/deploy/helm/llm-request-router/llm-request-router/templates/_helpers.tpl +++ b/deploy/helm/llm-request-router/llm-request-router/templates/_helpers.tpl @@ -149,6 +149,41 @@ comparing suffixes. {{- end -}} {{- end -}} +{{/* +External access gives every replica a distinct TCP and UDP dial address. The +internal advertised hostname remains the gRPC authority and QUIC certificate +identity, so the external domain must not be added to certificate SANs. +*/}} +{{- define "llm-request-router.validateExternalAccess" -}} +{{- $externalAccess := .Values.llmRequestRouter.externalAccess | default dict -}} +{{- if $externalAccess.enabled -}} +{{- $domain := $externalAccess.domain | default "" | toString | lower -}} +{{- if not $domain -}} +{{- fail "llmRequestRouter.externalAccess.domain is required when llmRequestRouter.externalAccess.enabled is true" -}} +{{- end -}} +{{- $labels := splitList "." $domain -}} +{{- $validDomain := and + (gt (len $domain) 0) + (le (len $domain) 253) + (not (hasPrefix "." $domain)) + (not (hasSuffix "." $domain)) -}} +{{- range $label := $labels -}} +{{- if not (regexMatch "^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$" $label) -}} +{{- $validDomain = false -}} +{{- end -}} +{{- end -}} +{{- if regexMatch "^[0-9]+$" (last $labels) -}} +{{- $validDomain = false -}} +{{- end -}} +{{- if not $validDomain -}} +{{- fail (printf "llmRequestRouter.externalAccess.domain %q is not a valid DNS name" $domain) -}} +{{- end -}} +{{- if not .Values.llmRequestRouter.transport.reverseTunnelListenAddr -}} +{{- fail "llmRequestRouter.transport.reverseTunnelListenAddr is required when llmRequestRouter.externalAccess.enabled is true" -}} +{{- end -}} +{{- end -}} +{{- end }} + {{- define "llm-request-router.serviceAccountName" -}} {{- if .Values.llmRequestRouter.serviceAccount.create }} {{- default (include "llm-request-router.fullname" .) .Values.llmRequestRouter.serviceAccount.name }} diff --git a/deploy/helm/llm-request-router/llm-request-router/templates/deployment.yaml b/deploy/helm/llm-request-router/llm-request-router/templates/deployment.yaml index 6d5059d5f..964dc743c 100644 --- a/deploy/helm/llm-request-router/llm-request-router/templates/deployment.yaml +++ b/deploy/helm/llm-request-router/llm-request-router/templates/deployment.yaml @@ -36,6 +36,8 @@ The identity guard lives here, not in certificate.yaml: existingSecret mode renders no Certificate, so a guard in that template would never run. */}} {{- include "llm-request-router.validateTlsIdentity" . }} +{{- $externalAccess := .Values.llmRequestRouter.externalAccess | default dict }} +{{- include "llm-request-router.validateExternalAccess" . }} {{- $advertisedHostnameTemplate := include "llm-request-router.advertisedHostnameTemplate" . }} spec: serviceName: {{ .Values.llmRequestRouter.service.headlessName }} @@ -87,6 +89,9 @@ spec: {{- with dig "discovery" "watchHeartbeatMs" "" .Values.llmRequestRouter }} - --watch-heartbeat-ms={{ . }} {{- end }} + {{- range (dig "discovery" "remoteStargateURLs" (list) .Values.llmRequestRouter) }} + - {{ printf "--remote-stargate-url=%s" . | quote }} + {{- end }} - --shutdown-drain-timeout-ms={{ .Values.llmRequestRouter.shutdown.drainTimeoutMs }} - --quic-connect-timeout-ms={{ .Values.llmRequestRouter.transport.quicConnectTimeoutMs }} - --quic-request-timeout-ms={{ .Values.llmRequestRouter.transport.quicRequestTimeoutMs }} @@ -95,7 +100,12 @@ spec: - --backend-connectivity=reverse - --reverse-tunnel-listen-addr={{ .Values.llmRequestRouter.transport.reverseTunnelListenAddr }} {{- end }} - {{- if and .Values.llmRequestRouter.transport.reverseTunnelListenAddr (gt $replicaCount 1) }} + {{- if $externalAccess.enabled }} + - --grpc-pylon-dial-addr={stargate_id}.{{ $externalAccess.domain }}:{{ .Values.llmRequestRouter.service.grpcPort }} + {{- end }} + {{- if and .Values.llmRequestRouter.transport.reverseTunnelListenAddr $externalAccess.enabled }} + - --reverse-tunnel-pylon-dial-addr=$(POD_NAME).{{ $externalAccess.domain }}:{{ .Values.llmRequestRouter.service.reverseTunnelPort }} + {{- else if and .Values.llmRequestRouter.transport.reverseTunnelListenAddr (gt $replicaCount 1) }} - --reverse-tunnel-pylon-dial-addr=$(POD_IP):{{ .Values.llmRequestRouter.service.reverseTunnelPort }} {{- end }} {{- if .Values.llmRequestRouter.transport.reverseTunnelConnectTimeoutMs }} diff --git a/deploy/helm/llm-request-router/llm-request-router/templates/service-per-pod.yaml b/deploy/helm/llm-request-router/llm-request-router/templates/service-per-pod.yaml new file mode 100644 index 000000000..85aa3b0a6 --- /dev/null +++ b/deploy/helm/llm-request-router/llm-request-router/templates/service-per-pod.yaml @@ -0,0 +1,49 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- $externalAccess := .Values.llmRequestRouter.externalAccess | default dict }} +{{- if $externalAccess.enabled }} +{{- include "llm-request-router.validateExternalAccess" . }} +{{- $service := $externalAccess.service | default dict }} +{{- $fullname := include "llm-request-router.fullname" . }} +{{- range $ordinal := until (int $.Values.llmRequestRouter.replicaCount) }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ printf "%s-%d" $fullname $ordinal }} + namespace: {{ include "llm-request-router.namespace" $ }} + labels: + {{- include "llm-request-router.labels" $ | nindent 4 }} + {{- with $service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + type: {{ $service.type | default "LoadBalancer" }} + selector: + {{- include "llm-request-router.selectorLabels" $ | nindent 4 }} + statefulset.kubernetes.io/pod-name: {{ printf "%s-%d" $fullname $ordinal }} + ports: + - name: grpc + port: {{ $.Values.llmRequestRouter.service.grpcPort }} + targetPort: grpc + protocol: TCP + - name: quic + port: {{ $.Values.llmRequestRouter.service.reverseTunnelPort }} + targetPort: reverse + protocol: UDP +{{- end }} +{{- end }} diff --git a/deploy/helm/llm-request-router/llm-request-router/templates/service.yaml b/deploy/helm/llm-request-router/llm-request-router/templates/service.yaml index 3a8c06604..b37df10df 100644 --- a/deploy/helm/llm-request-router/llm-request-router/templates/service.yaml +++ b/deploy/helm/llm-request-router/llm-request-router/templates/service.yaml @@ -20,6 +20,10 @@ metadata: namespace: {{ include "llm-request-router.namespace" . }} labels: {{- include "llm-request-router.labels" . | nindent 4 }} + {{- with .Values.llmRequestRouter.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} spec: type: {{ .Values.llmRequestRouter.service.type }} selector: diff --git a/deploy/helm/llm-request-router/llm-request-router/values.yaml b/deploy/helm/llm-request-router/llm-request-router/values.yaml index a0a3429dc..a5e568b2b 100644 --- a/deploy/helm/llm-request-router/llm-request-router/values.yaml +++ b/deploy/helm/llm-request-router/llm-request-router/values.yaml @@ -65,6 +65,7 @@ llmRequestRouter: service: type: ClusterIP + annotations: {} httpPort: 8000 grpcPort: 50071 metricsPort: 9090 @@ -121,8 +122,8 @@ llmRequestRouter: # suffixes). REQUIRED when enabled. Typically: # ",cluster.local" # so the signing role accepts both the customer's external DNS and - # in-cluster service names. Missing value → script aborts non-zero → - # entrypoint accumulator → Job fails on backoff exhaustion. + # in-cluster service names. Missing value -> script aborts non-zero -> + # entrypoint accumulator -> Job fails on backoff exhaustion. allowedDomains: "" # nvcf-openbao-migrations image. The chart picks up the same image # that the k8s-openbao Helm hook uses; supply registry/repository/tag @@ -154,6 +155,9 @@ llmRequestRouter: discovery: disableDnsDiscovery: false + # Additional Stargate WatchStargates endpoints. Each item renders as one + # --remote-stargate-url argument. + remoteStargateURLs: [] transport: quicConnectTimeoutMs: 2000 @@ -161,6 +165,19 @@ llmRequestRouter: reverseTunnelListenAddr: "0.0.0.0:50072" reverseTunnelConnectTimeoutMs: 10000 + # Per-replica TCP registration and UDP reverse-tunnel Services. Enable this + # when pylons run outside the router cluster. Each external hostname is + # ".". The provider owns DNS and transparent L4 + # forwarding for these names; the chart does not create DNS records. Use a + # region-unique domain. These names are dial-only and do not belong in the + # router certificate SANs. + externalAccess: + enabled: false + domain: "" + service: + type: LoadBalancer + annotations: {} + shutdown: drainTimeoutMs: 30000 diff --git a/deploy/helm/llm-request-router/scripts/check-multi-replica-render.sh b/deploy/helm/llm-request-router/scripts/check-multi-replica-render.sh index 08f5d4fb9..766d3a96e 100755 --- a/deploy/helm/llm-request-router/scripts/check-multi-replica-render.sh +++ b/deploy/helm/llm-request-router/scripts/check-multi-replica-render.sh @@ -40,6 +40,25 @@ statefulset_args() { yq -r 'select(.kind == "StatefulSet" and .metadata.name == "llm-request-router") | .spec.template.spec.containers[0].args[]' "${manifest}" } +per_pod_service_names() { + local manifest="$1" + yq -r 'select(.kind == "Service" and (.metadata.name | test("^llm-request-router-[0-9]+$"))) | .metadata.name' "${manifest}" \ + | grep -v '^---$' \ + | sort +} + +service_field() { + local manifest="$1" + local name="$2" + local expression="$3" + yq -r "select(.kind == \"Service\" and .metadata.name == \"${name}\") | (${expression})" "${manifest}" | head -n1 +} + +certificate_dns_names() { + local manifest="$1" + yq -r 'select(.kind == "Certificate") | .spec.dnsNames[]' "${manifest}" +} + default_manifest="${tmp_dir}/default.yaml" render "${default_manifest}" @@ -83,4 +102,95 @@ render "${custom_template_manifest}" \ custom_template_args="$(statefulset_args "${custom_template_manifest}")" printf '%s\n' "${custom_template_args}" | grep -qx -- '--reverse-tunnel-pylon-dial-addr=$(POD_IP):50072' || fail "custom multi-replica advertised hostname template missing reverse tunnel pylon dial addr" +[ -z "$(per_pod_service_names "${default_manifest}")" ] || fail "default render unexpectedly created per-pod Services" +if printf '%s\n' "${default_args}" | grep -q -- '--grpc-pylon-dial-addr'; then + fail "default render unexpectedly configured an external gRPC dial address" +fi + +external_domain="router.region.example" +external_manifest="${tmp_dir}/external-access.yaml" +render "${external_manifest}" \ + --set llmRequestRouter.replicaCount=2 \ + --set llmRequestRouter.service.type=NodePort \ + --set-string llmRequestRouter.service.annotations.shared=seed \ + --set llmRequestRouter.externalAccess.enabled=true \ + --set-string llmRequestRouter.externalAccess.domain="${external_domain}" \ + --set llmRequestRouter.externalAccess.service.type=NodePort \ + --set-string llmRequestRouter.externalAccess.service.annotations.scope=replica \ + --set-string 'llmRequestRouter.discovery.remoteStargateURLs[0]=https://watch-a.example:50071' \ + --set-string 'llmRequestRouter.discovery.remoteStargateURLs[1]=https://watch-b.example:50071' + +external_args="$(statefulset_args "${external_manifest}")" +printf '%s\n' "${external_args}" | grep -qx -- "--grpc-pylon-dial-addr={stargate_id}.${external_domain}:50071" || fail "external render missing templated per-replica gRPC dial address" +printf '%s\n' "${external_args}" | grep -qx -- "--reverse-tunnel-pylon-dial-addr=\$(POD_NAME).${external_domain}:50072" || fail "external render missing per-replica QUIC dial address" +[ "$(printf '%s\n' "${external_args}" | grep -cx -- '--remote-stargate-url=https://watch-a.example:50071')" = "1" ] || fail "first remote Stargate URL was not rendered exactly once" +[ "$(printf '%s\n' "${external_args}" | grep -cx -- '--remote-stargate-url=https://watch-b.example:50071')" = "1" ] || fail "second remote Stargate URL was not rendered exactly once" + +expected_services="$(printf 'llm-request-router-0\nllm-request-router-1\n')" +[ "$(per_pod_service_names "${external_manifest}")" = "${expected_services}" ] || fail "external render did not create exactly two per-pod Services" +[ "$(service_field "${external_manifest}" llm-request-router-1 '.spec.selector."statefulset.kubernetes.io/pod-name"')" = "llm-request-router-1" ] || fail "per-pod Service is not pinned to its StatefulSet replica" +[ "$(service_field "${external_manifest}" llm-request-router-1 '.spec.type')" = "NodePort" ] || fail "per-pod Service type override was not applied" +[ "$(service_field "${external_manifest}" llm-request-router-1 '.metadata.annotations.scope')" = "replica" ] || fail "per-pod Service annotations were not applied" +[ "$(service_field "${external_manifest}" llm-request-router-1 '.spec.ports[] | select(.name == "grpc") | .port')" = "50071" ] || fail "per-pod Service missing registration port 50071" +[ "$(service_field "${external_manifest}" llm-request-router-1 '.spec.ports[] | select(.name == "grpc") | .protocol')" = "TCP" ] || fail "per-pod registration port is not TCP" +[ "$(service_field "${external_manifest}" llm-request-router-1 '.spec.ports[] | select(.name == "quic") | .port')" = "50072" ] || fail "per-pod Service missing reverse-tunnel port 50072" +[ "$(service_field "${external_manifest}" llm-request-router-1 '.spec.ports[] | select(.name == "quic") | .protocol')" = "UDP" ] || fail "per-pod reverse-tunnel port is not UDP" +[ "$(service_field "${external_manifest}" llm-request-router '.metadata.annotations.shared')" = "seed" ] || fail "shared seed Service annotations were not applied" + +scaled_manifest="${tmp_dir}/external-access-scaled.yaml" +render "${scaled_manifest}" \ + --set llmRequestRouter.replicaCount=4 \ + --set llmRequestRouter.externalAccess.enabled=true \ + --set-string llmRequestRouter.externalAccess.domain="${external_domain}" +[ "$(per_pod_service_names "${scaled_manifest}" | wc -l | tr -d ' ')" = "4" ] || fail "per-pod Service count does not follow replicaCount" + +assert_render_fails() { + local description="$1" + local expected="$2" + shift 2 + local error_file="${tmp_dir}/external-invalid.err" + if helm template "${release}" "${chart_dir}" \ + --namespace "${namespace}" \ + --values "${chart_dir}/values.yaml" \ + "$@" \ + > "${tmp_dir}/external-invalid.yaml" 2> "${error_file}"; then + fail "${description} unexpectedly succeeded" + fi + grep -Fq "${expected}" "${error_file}" || fail "${description} did not return the expected guard message" +} + +assert_render_fails \ + "external access without a domain" \ + "llmRequestRouter.externalAccess.domain is required" \ + --set llmRequestRouter.externalAccess.enabled=true + +assert_render_fails \ + "external access with an invalid domain" \ + "is not a valid DNS name" \ + --set llmRequestRouter.externalAccess.enabled=true \ + --set-string llmRequestRouter.externalAccess.domain='not_a_domain.example' + +assert_render_fails \ + "external access without a reverse tunnel listener" \ + "llmRequestRouter.transport.reverseTunnelListenAddr is required" \ + --set llmRequestRouter.externalAccess.enabled=true \ + --set-string llmRequestRouter.externalAccess.domain="${external_domain}" \ + --set-string llmRequestRouter.transport.reverseTunnelListenAddr='' + +pki_manifest="${tmp_dir}/external-access-pki.yaml" +render "${pki_manifest}" \ + --set llmRequestRouter.replicaCount=2 \ + --set llmRequestRouter.externalAccess.enabled=true \ + --set-string llmRequestRouter.externalAccess.domain="${external_domain}" \ + --set llmRequestRouter.certificate.enabled=true \ + --set-string llmRequestRouter.certificate.issuerRef.name=test-issuer \ + --set-string 'llmRequestRouter.certificate.dnsNames[0]=llm-request-router.nvcf.svc.cluster.local' \ + --set-string 'llmRequestRouter.certificate.dnsNames[1]=*.llm-request-router-headless.nvcf.svc.cluster.local' +pki_dns_names="$(certificate_dns_names "${pki_manifest}")" +printf '%s\n' "${pki_dns_names}" | grep -qx -- 'llm-request-router.nvcf.svc.cluster.local' || fail "PKI render missing exact internal Service SAN" +printf '%s\n' "${pki_dns_names}" | grep -Fqx -- '*.llm-request-router-headless.nvcf.svc.cluster.local' || fail "PKI render missing wildcard internal headless Service SAN" +if printf '%s\n' "${pki_dns_names}" | grep -Fq -- "${external_domain}"; then + fail "PKI render unexpectedly added the external dial domain to certificate SANs" +fi + echo "multi-replica render checks passed" diff --git a/deploy/stacks/self-managed/environments/base.yaml b/deploy/stacks/self-managed/environments/base.yaml index 8d451b369..786279c3a 100644 --- a/deploy/stacks/self-managed/environments/base.yaml +++ b/deploy/stacks/self-managed/environments/base.yaml @@ -257,6 +257,26 @@ addons: # LLM addon: gateway + request router (stargate) for LLM function invocation llm: enabled: false + requestRouter: + replicaCount: 3 + # The shared Service is the single worker bootstrap address. It is + # separate from the per-replica external Services below. + service: + type: ClusterIP + annotations: {} + grpcPort: 50071 + discovery: + # Additional WatchStargates seeds, typically one or more per region. + remoteStargateURLs: [] + # Provider-owned DNS and transparent L4 forwarding expose one TCP and + # UDP endpoint per StatefulSet replica when workers are remote. Use a + # region-unique domain. It is dial-only and is not a certificate SAN. + externalAccess: + enabled: false + domain: "" + service: + type: LoadBalancer + annotations: {} # QUIC TLS certificate for the request router (Stargate). Disabled by # default; opt in per env. # diff --git a/deploy/stacks/self-managed/global.yaml.gotmpl b/deploy/stacks/self-managed/global.yaml.gotmpl index 5f4238f2f..3a991034b 100644 --- a/deploy/stacks/self-managed/global.yaml.gotmpl +++ b/deploy/stacks/self-managed/global.yaml.gotmpl @@ -834,8 +834,22 @@ llmRequestRouter: enabled: {{ dig "addons" "llm" "enabled" false .Values }} fullnameOverride: llm-request-router replicaCount: {{ dig "addons" "llm" "requestRouter" "replicaCount" 3 .Values }} + {{- $llmRequestRouterService := dig "addons" "llm" "requestRouter" "service" dict .Values }} service: + {{- with omit $llmRequestRouterService "grpcPort" }} + {{- toYaml . | nindent 4 }} + {{- end }} grpcPort: {{ $llmRequestRouterGrpcPort }} + {{- with dig "addons" "llm" "requestRouter" "externalAccess" dict .Values }} + externalAccess: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- $remoteStargateURLs := dig "addons" "llm" "requestRouter" "discovery" "remoteStargateURLs" (list) .Values }} + {{- if gt (len $remoteStargateURLs) 0 }} + discovery: + remoteStargateURLs: + {{- toYaml $remoteStargateURLs | nindent 6 }} + {{- end }} {{- if .Values.global.imagePullSecrets }} imagePullSecrets: {{- toYaml .Values.global.imagePullSecrets | nindent 4 }} diff --git a/deploy/stacks/self-managed/tests/llm-router-worker-address.sh b/deploy/stacks/self-managed/tests/llm-router-worker-address.sh index 55677e208..0cf009af3 100755 --- a/deploy/stacks/self-managed/tests/llm-router-worker-address.sh +++ b/deploy/stacks/self-managed/tests/llm-router-worker-address.sh @@ -346,4 +346,53 @@ if grep -Fq "$staged_worker_address" "$work_dir/disabled-api-values.yaml"; then fail "disabled LLM supplied a staged worker address to the API chart" fi +split_seed_address='llm-request-router.nvcf.svc.cluster.local:50071' +printf '%s\n' \ + 'global:' \ + ' workerEndpoints:' \ + " llmRequestRouterAddress: '$split_seed_address'" \ + 'addons:' \ + ' llm:' \ + ' enabled: true' \ + ' requestRouter:' \ + ' service:' \ + ' type: NodePort' \ + ' annotations:' \ + ' shared: seed' \ + ' externalAccess:' \ + ' enabled: true' \ + ' domain: nvcf-llm-router.svc.cluster.local' \ + ' service:' \ + ' type: NodePort' \ + ' annotations:' \ + ' scope: replica' \ + ' discovery:' \ + ' remoteStargateURLs:' \ + ' - https://watch-a.example:50071' \ + ' - https://watch-b.example:50071' \ + >"$environment_file" +render_api_values "$work_dir/split-router-api-values.yaml" >/dev/null +assert_remote_config_address "$work_dir/split-router-api-values.yaml" \ + "$split_seed_address" || + fail "split-cluster configuration changed the shared worker seed address" +split_values="$work_dir/split-router-api-values.yaml" +yq -e '.llmRequestRouter.service.type == "NodePort"' "$split_values" >/dev/null || + fail "shared seed Service type was not passed through the stack" +yq -e '.llmRequestRouter.service.annotations.shared == "seed"' "$split_values" >/dev/null || + fail "shared seed Service annotations were not passed through the stack" +yq -e '.llmRequestRouter.externalAccess.enabled == true' "$split_values" >/dev/null || + fail "external access enablement was not passed through the stack" +yq -e '.llmRequestRouter.externalAccess.domain == "nvcf-llm-router.svc.cluster.local"' "$split_values" >/dev/null || + fail "external access domain was not passed through the stack" +yq -e '.llmRequestRouter.externalAccess.service.type == "NodePort"' "$split_values" >/dev/null || + fail "per-replica Service type was not passed through the stack" +yq -e '.llmRequestRouter.externalAccess.service.annotations.scope == "replica"' "$split_values" >/dev/null || + fail "per-replica Service annotations were not passed through the stack" +yq -e '.llmRequestRouter.discovery.remoteStargateURLs | length == 2' "$split_values" >/dev/null || + fail "remote Stargate URL count was not passed through the stack" +yq -e '.llmRequestRouter.discovery.remoteStargateURLs[0] == "https://watch-a.example:50071"' "$split_values" >/dev/null || + fail "first remote Stargate URL was not passed through the stack" +yq -e '.llmRequestRouter.discovery.remoteStargateURLs[1] == "https://watch-b.example:50071"' "$split_values" >/dev/null || + fail "second remote Stargate URL was not passed through the stack" + echo "llm-router-worker-address: all checks passed" diff --git a/docs/user/images/nvcf-llm-multicluster-invocation.svg b/docs/user/images/nvcf-llm-multicluster-invocation.svg index c468080c0..7a6072964 100644 --- a/docs/user/images/nvcf-llm-multicluster-invocation.svg +++ b/docs/user/images/nvcf-llm-multicluster-invocation.svg @@ -1,6 +1,6 @@ - NVCF LLM multi-cluster invocation path - Global DNS or an optional custom front door selects a regional public LLM/HTTP endpoint. Each region has LLM API Gateway, LLM Request Router, NATS usage sync, and LLM worker gateways in GPU cluster groups, with NATS usage chatter and request-router paths to worker gateways in other clusters. + NVCF LLM split-cluster invocation and worker routing + Two LLM API Gateways and two Stargate replicas run in a control cluster. Pylons in a compute cluster use one shared seed for discovery, then distinct provider-owned TCP and UDP addresses for every Stargate while preserving internal certificate identities. - - + + + + + + + + - LLM Multi-Cluster Invocation - LLM traffic uses the OpenAI-compatible gateway path and worker gateway tunnels to regional model services. - - - Global DNS / Optional Custom Front Door - regional selection for llm.invocation.<domain> - - - Region A - LLM invocation services - - - Region B - LLM invocation services - - - Public LLM/HTTP - llm.invocation - - - LLM API - Gateway - auth and validation - - - LLM Request - Router - Stargate peer - - - NATS Usage - gateway state sync - - - GPU Cluster Group A - - Worker Gateway + Model A - - Worker Gateway + Model B - - - Public LLM/HTTP - llm.invocation - - - LLM API - Gateway - auth and validation - - - LLM Request - Router - Stargate peer - - - NATS Usage - gateway state sync - - - GPU Cluster Group B - - Worker Gateway + Model C - - Worker Gateway + Model D - - - - - - - - - - - - - - - NATS cross-cluster usage chatter - - - remote worker gateway paths + LLM Split-Cluster Routing + Shared discovery seed, per-replica dial paths, and internal QUIC certificate identity + + + Control cluster + + + Mac client + llm.invocation + + + 2 x LLM API + Gateway + OpenAI-compatible HTTP + + + Stargate 0 + llm-request-router-0 + TCP 50071 / UDP 50072 + + + Stargate 1 + llm-request-router-1 + TCP 50071 / UDP 50072 + + + Shared seed Service + llm-request-router.nvcf + WatchStargates / TCP 50071 + + + Internal identity + llm-request-router-0. + llm-request-router-headless.nvcf... + llm-request-router-1. + llm-request-router-headless.nvcf... + HTTP/2 authority + QUIC SNI + + + Provider-owned DNS + transparent L4 + + + Replica 0 dial name + llm-request-router-0. + router.region-a.example + TCP 50071 + UDP 50072 + + + Replica 1 dial name + llm-request-router-1. + router.region-a.example + TCP 50071 + UDP 50072 + + + + + + + + + + + Compute cluster + + + Function replica 0 + pylon + mock OpenAI server + GPU + registers with both Stargates + + + Function replica 1 + pylon + mock OpenAI server + GPU + registers with both Stargates + + + + 1. Discover through shared seed + + + + + + 2. Register over TCP, then form trusted reverse QUIC over UDP + + External names select the replica. Internal advertised names remain the certificate identity and are the only router SANs. diff --git a/docs/user/llm-function-enablement.md b/docs/user/llm-function-enablement.md index 28995c9ce..36ccbca02 100644 --- a/docs/user/llm-function-enablement.md +++ b/docs/user/llm-function-enablement.md @@ -39,10 +39,12 @@ cert-manager, or one you issue yourself and supply in a pre-created Secret. Each compute plane receives the public root CA certificate and uses the combined system and private trust bundle in the `llm-worker` sidecar. -The request-router address configured for the compute plane must use a DNS name -listed in the certificate SANs. For a single-cluster deployment, use -`llm-request-router.nvcf.svc.cluster.local:50071`. Use an address reachable from -each compute cluster when the control and compute planes use separate networks. +The request-router address configured for the compute plane is the shared TCP +discovery seed. For a single-cluster deployment, use +`llm-request-router.nvcf.svc.cluster.local:50071`. In a split-cluster +deployment, use a seed that every worker can resolve and reach. Per-replica +external dial names are separate from the internal hostname used as the QUIC +certificate identity. ### Managed OpenBao issuer @@ -193,12 +195,12 @@ stack-managed issuance. Rendering fails if any of them is set in this mode, so a configuration that expects the stack to issue a certificate cannot be mistaken for one that expects you to. -The certificate must carry a SAN covering the router's advertised hostname and -the address workers connect to. At the default single-replica configuration -that is `llm-request-router.nvcf.svc.cluster.local`. At higher replica counts -the router advertises per-pod headless names, so use a leftmost wildcard such -as `*.llm-request-router-headless.nvcf.svc.cluster.local`. Include any external -name set in `global.workerEndpoints.llmRequestRouterAddress`. The stack cannot +The certificate must carry a SAN covering the router's internal advertised +hostname. At the default single-replica configuration that is +`llm-request-router.nvcf.svc.cluster.local`. At higher replica counts the router +advertises per-pod headless names, so use a leftmost wildcard such as +`*.llm-request-router-headless.nvcf.svc.cluster.local`. Per-replica external +names are dial-only and must not be added to the certificate. The stack cannot read your Secret at render time, so it validates neither the SANs nor the expiry. A certificate that does not cover the advertised hostname fails at worker connection time, not at install time. @@ -266,18 +268,19 @@ nvcf-cli self-hosted \ --region ``` -The exporter cannot infer the worker-facing request-router endpoint. Add the -reachable `host:port` before validating or registering the profile: +The exporter cannot infer the worker-facing shared seed. Add the reachable +`host:port` before validating or registering the profile: ```yaml controlPlane: addons: llm: - requestRouterAddress: llm-router.example.com:443 + requestRouterAddress: llm-request-router.nvcf.svc.cluster.local:50071 ``` -Use the endpoint that compute-plane workers can resolve and reach. Its -hostname must match a SAN on the request-router certificate. +Use the endpoint that compute-plane workers can resolve and reach. In a split +cluster, this can be a selectorless Service alias backed by the control-cluster +seed endpoint. Registration renders this field as `agent.llm.requestRouterAddress` in the compute-plane values. That is operator configuration, not a runtime fallback @@ -289,25 +292,69 @@ from the `LLM_REQUEST_ROUTER_ADDRESS` variable in its launch environment, with the workload, translation rejects the launch instead of falling back to the registered address. -Add that hostname to `addons.llm.pki.dnsNames`. The list accepts any number of -additional names; the stack only requires that one entry covers the router's -advertised hostname. For the managed issuer, also extend `allowedDomains` with -the parent domain, because the OpenBao signing role allows subdomains and -wildcards but not bare domains. To issue for `llm-router.example.com`, use: +Keep only internal advertised identities in `addons.llm.pki.dnsNames`. A +multi-replica router normally uses: ```yaml addons: llm: pki: - allowedDomains: nvcf.svc.cluster.local,example.com + allowedDomains: cluster.local dnsNames: - llm-request-router.nvcf.svc.cluster.local - "*.llm-request-router-headless.nvcf.svc.cluster.local" - - llm-router.example.com ``` -`allowedDomains: llm-router.example.com` does not work for that name. The role -sets `allow_bare_domains=false`, so the entry must be the parent domain. +Do not add the shared seed alias or the per-replica external domain unless one +of those names is also the router's internal advertised identity. + +### Split-cluster replica endpoints + +Use per-replica endpoints when the compute cluster cannot reach +control-cluster pod DNS or pod IPs directly. The worker path has three address +classes: + +| Address | Purpose | Identity | +| --- | --- | --- | +| Shared seed | Initial `WatchStargates` call on TCP 50071 | Dial-only | +| Per-replica TCP | Direct registration with each Stargate on TCP 50071 | Internal advertised hostname remains the HTTP/2 authority | +| Per-replica UDP | Reverse QUIC tunnel to the registering Stargate on UDP 50072 | Internal advertised hostname remains the QUIC SNI | + +Configure the self-managed stack like this: + +```yaml +global: + workerEndpoints: + llmRequestRouterAddress: llm-request-router.nvcf.svc.cluster.local:50071 + +addons: + llm: + requestRouter: + replicaCount: 2 + service: + annotations: {} + externalAccess: + enabled: true + domain: router.region-a.example + service: + type: LoadBalancer + annotations: {} + discovery: + remoteStargateURLs: [] +``` + +The chart creates one Service per StatefulSet ordinal. For +`llm-request-router-0`, workers dial +`llm-request-router-0.router.region-a.example:50071` for registration and the +same host on UDP 50072 for reverse QUIC. The internal name +`llm-request-router-0.llm-request-router-headless.nvcf.svc.cluster.local` +remains the authority and QUIC SNI. + +The infrastructure provider owns the external DNS records and transparent L4 +load balancers. Each name must forward TCP and UDP to the matching replica. +Do not terminate TLS or place several replicas behind one external name. Use a +region-unique domain when `discovery.remoteStargateURLs` joins regional router +meshes. The stack does not create provider DNS records. The managed export reads the public root CA certificate from `services/all/pki/root` in the stack's OpenBao service and calculates the @@ -504,8 +551,10 @@ kubectl -n nvcf-backend get pod \ The worker args must contain `--stargate-address=llm-request-router.nvcf.svc.cluster.local:50071`, or the -configured routable DNS name, and must not contain `--quic-insecure`. The -address hostname must match a certificate SAN. The environment must contain: +configured shared seed, and must not contain `--quic-insecure`. In a +split-cluster deployment, router logs should show a distinct registration and +reverse tunnel for every replica. The internal advertised hostname used as +QUIC SNI must match a certificate SAN. The environment must contain: ```text STARGATE_TLS_CERT_PATH=/etc/ssl/certs/ca-certificates.crt @@ -677,8 +726,9 @@ For transport TLS failures, check: - Unknown issuer: inspect the `Certificate` Ready condition and verify `issuerRef.kind`, `issuerRef.name`, and the issuer namespace. A namespaced `Issuer` must be in `nvcf`. -- SAN mismatch: compare the hostname in `--stargate-address` with the SANs in - `Secret/stargate-quic-tls`. Do not replace the hostname with an IP address. +- SAN mismatch: compare the internal advertised hostname returned by Stargate + with the SANs in `Secret/stargate-quic-tls`. The shared seed and external + dial names are not the QUIC identity. - Expired or not-yet-valid certificate: inspect the certificate dates and the cluster clock. Renew the certificate and restart the request-router StatefulSet. diff --git a/docs/user/llm-gateway.md b/docs/user/llm-gateway.md index 863aed2b7..605b14b80 100644 --- a/docs/user/llm-gateway.md +++ b/docs/user/llm-gateway.md @@ -38,14 +38,14 @@ The function container must expose the declared OpenAI-compatible paths on its i ### Multi-Cluster View -In a global deployment, DNS or a custom front door selects a regional -`llm.invocation.` endpoint. The LLM Gateway can use NATS for -cross-cluster usage-state chatter. LLM worker request streams still use the -request router and worker gateway path, not NATS worker streams. The -worker-gateway arrows show that routers can target local or remote worker -gateways when those workers are registered into the router mesh. - -![LLM multi-cluster invocation path](images/nvcf-llm-multicluster-invocation.svg) +In a split-cluster deployment, the LLM API Gateway and request routers run in +the control cluster while pylons and model servers run in a compute cluster. +Pylons use one shared router seed for discovery, then connect to distinct TCP +registration and UDP reverse-tunnel addresses for every router replica. The +provider-owned addresses are dial-only. The internal router hostname remains +the gRPC authority and QUIC certificate identity. + +![LLM split-cluster invocation path](images/nvcf-llm-multicluster-invocation.svg) ## Function Configuration diff --git a/src/libraries/rust/stargate/crates/pylon-lib/src/registration/tests.rs b/src/libraries/rust/stargate/crates/pylon-lib/src/registration/tests.rs index cba367b0f..f6a1ae76a 100644 --- a/src/libraries/rust/stargate/crates/pylon-lib/src/registration/tests.rs +++ b/src/libraries/rust/stargate/crates/pylon-lib/src/registration/tests.rs @@ -310,10 +310,51 @@ fn reverse_tunnel_config_uses_registration_upstream_and_preserves_forwarding() { #[test] fn stargate_grpc_endpoint_rejects_empty_authority_and_formats_dial_overrides() { assert!(StargateGrpcEndpoint::new(" ", "stargate-grpc-lb:443").is_none()); + let endpoint = grpc_endpoint_with_dial("router-a:50071", "stargate-grpc-lb:443"); assert_eq!( - grpc_endpoint_with_dial("router-a:50071", "stargate-grpc-lb:443").to_string(), + endpoint.to_string(), "router-a:50071 via stargate-grpc-lb:443" ); + assert_eq!(endpoint.dial_endpoint(), "http://stargate-grpc-lb:443"); + assert_eq!(endpoint.authority_endpoint(), "http://router-a:50071"); +} + +#[test] +fn regional_dial_addresses_keep_duplicate_stargate_ids_distinct() { + let region_a = watch_endpoint_snapshot_from_response( + "seed.region-a:50071", + WatchStargatesResponse { + stargates: vec![stargate_info( + "llm-request-router-0", + "llm-request-router-0.headless.nvcf.svc.cluster.local:50071", + "llm-request-router-0.router.region-a.example:50071", + )], + watch_stargate_urls: Vec::new(), + }, + ); + let region_b = watch_endpoint_snapshot_from_response( + "seed.region-b:50071", + WatchStargatesResponse { + stargates: vec![stargate_info( + "llm-request-router-0", + "llm-request-router-0.headless.nvcf.svc.cluster.local:50071", + "llm-request-router-0.router.region-b.example:50071", + )], + watch_stargate_urls: Vec::new(), + }, + ); + + let routers = active_registration_routers([®ion_a, ®ion_b]); + + assert_eq!(routers.len(), 2); + assert!(routers.contains(&grpc_endpoint_with_dial( + "llm-request-router-0.headless.nvcf.svc.cluster.local:50071", + "llm-request-router-0.router.region-a.example:50071", + ))); + assert!(routers.contains(&grpc_endpoint_with_dial( + "llm-request-router-0.headless.nvcf.svc.cluster.local:50071", + "llm-request-router-0.router.region-b.example:50071", + ))); } #[test] diff --git a/src/libraries/rust/stargate/crates/stargate/src/control_plane/watch_stargates.rs b/src/libraries/rust/stargate/crates/stargate/src/control_plane/watch_stargates.rs index aac57468b..7c6d5ce16 100644 --- a/src/libraries/rust/stargate/crates/stargate/src/control_plane/watch_stargates.rs +++ b/src/libraries/rust/stargate/crates/stargate/src/control_plane/watch_stargates.rs @@ -20,7 +20,7 @@ use std::time::{Duration, Instant}; use futures::{Stream, stream}; use tokio::sync::watch; use tonic::Status; -use tracing::debug; +use tracing::{debug, warn}; use url::Url; use stargate_proto::pb::{StargateInfo, WatchStargatesResponse}; @@ -28,6 +28,8 @@ use stargate_proto::pb::{StargateInfo, WatchStargatesResponse}; use crate::discovery::Discovery; use stargate_runtime::CriticalTaskGroup; +const STARGATE_ID_DIAL_ADDR_TOKEN: &str = "{stargate_id}"; + pub(super) struct WatchStargatesPublisherConfig { pub(super) advertise_addr: SocketAddr, pub(super) discovery_dns_name: String, @@ -113,8 +115,24 @@ fn build_watch_stargates_response( .map(str::trim) .filter(|addr| !addr.is_empty()) { - for stargate in &mut stargates { - stargate.grpc_pylon_dial_addr = grpc_pylon_dial_addr.to_string(); + if grpc_pylon_dial_addr.contains(STARGATE_ID_DIAL_ADDR_TOKEN) { + stargates.retain_mut(|stargate| { + let stargate_id = stargate.stargate_id.trim(); + if stargate_id.is_empty() { + warn!( + advertise_addr = %stargate.advertise_addr, + "omitting Stargate without an ID from templated pylon dial addresses" + ); + return false; + } + stargate.grpc_pylon_dial_addr = + grpc_pylon_dial_addr.replace(STARGATE_ID_DIAL_ADDR_TOKEN, stargate_id); + true + }); + } else { + for stargate in &mut stargates { + stargate.grpc_pylon_dial_addr = grpc_pylon_dial_addr.to_string(); + } } } WatchStargatesResponse { @@ -298,6 +316,71 @@ mod tests { ); } + #[test] + fn watch_stargates_response_renders_per_stargate_grpc_pylon_dial_addrs() { + let response = build_watch_stargates_response( + vec![ + stargate( + "llm-request-router-1", + "llm-request-router-1.headless:50071", + "", + ), + stargate( + "llm-request-router-0", + "llm-request-router-0.headless:50071", + "", + ), + ], + &[], + Some("{stargate_id}.router.region-a.example:50071"), + ); + + let dial_addrs: Vec<&str> = response + .stargates + .iter() + .map(|info| info.grpc_pylon_dial_addr.as_str()) + .collect(); + assert_eq!( + dial_addrs, + vec![ + "llm-request-router-0.router.region-a.example:50071", + "llm-request-router-1.router.region-a.example:50071", + ] + ); + } + + #[test] + fn watch_stargates_response_ignores_empty_grpc_pylon_dial_addr() { + let mut info = stargate("stargate-0", "stargate-0.region-a:50071", ""); + info.grpc_pylon_dial_addr = "existing.region-a:50071".to_string(); + + let response = build_watch_stargates_response(vec![info], &[], Some(" ")); + + assert_eq!( + response.stargates[0].grpc_pylon_dial_addr, + "existing.region-a:50071" + ); + } + + #[test] + fn watch_stargates_response_omits_missing_id_from_templated_dial_addrs() { + let response = build_watch_stargates_response( + vec![ + stargate("", "10.0.0.1:50071", ""), + stargate("stargate-1", "10.0.0.2:50071", ""), + ], + &[], + Some("{stargate_id}.router.region-a.example:50071"), + ); + + assert_eq!(response.stargates.len(), 1); + assert_eq!(response.stargates[0].stargate_id, "stargate-1"); + assert_eq!( + response.stargates[0].grpc_pylon_dial_addr, + "stargate-1.router.region-a.example:50071" + ); + } + #[test] fn remote_watch_urls_are_normalized_and_filter_self_endpoints() { let excluded = local_watch_endpoint_keys( diff --git a/src/libraries/rust/stargate/crates/stargate/src/main.rs b/src/libraries/rust/stargate/crates/stargate/src/main.rs index da89e9e0e..ee4b9cb0d 100644 --- a/src/libraries/rust/stargate/crates/stargate/src/main.rs +++ b/src/libraries/rust/stargate/crates/stargate/src/main.rs @@ -82,7 +82,7 @@ struct Args { value_name = "URL" )] remote_stargate_url: Vec, - /// Optional TCP load-balancer dial address for pylons; per-pod addresses remain the advertised gRPC authority/SNI identity. + /// Optional TCP dial address for pylons. Supports `{stargate_id}` for per-Stargate addresses; advertised addresses remain the gRPC authority. #[arg(long, value_name = "ADDR")] grpc_pylon_dial_addr: Option, /// Backend hostname template supporting `{pod_name}` and `{namespace}`; its rendered host is the pylon gRPC authority and reverse QUIC SNI. diff --git a/src/libraries/rust/stargate/docs/tunnel-transports.md b/src/libraries/rust/stargate/docs/tunnel-transports.md index 959c518c6..eb8977187 100644 --- a/src/libraries/rust/stargate/docs/tunnel-transports.md +++ b/src/libraries/rust/stargate/docs/tunnel-transports.md @@ -82,15 +82,37 @@ GKE overlay uses the Terraform-managed shared internal VIP `ip-us-central1-stargate-backend` (`10.69.170.115`) with TCP `443` for gRPC registration/watch and UDP `8080` for Raw QUIC reverse tunnels. -Remote backend clusters that reach Stargate through split internal load -balancers should point pylon `--stargate-address` at the gRPC/TCP endpoint. -Stargate should set `--grpc-pylon-dial-addr` to the same gRPC/TCP endpoint so -`StargateInfo.grpc_pylon_dial_addr` tells pylon where to dial while -`advertise_addr` remains the per-pod gRPC authority/SNI routing identity. -Stargate should also set `--reverse-tunnel-pylon-dial-addr` to the QUIC/UDP -endpoint so `InferenceServerAck.reverse_tunnel_pylon_dial_addr` tells pylon -where to dial while `reverse_tunnel_target` remains the per-pod QUIC -SNI/routing identity. +### Split-Cluster Replica Addressing + +Point pylon `--stargate-address` at one worker-reachable TCP seed. That seed is +used for `WatchStargates`; it is not the address used for every registration. + +`--grpc-pylon-dial-addr` controls the TCP dial address published in each +`StargateInfo`: + +- A literal address preserves the shared-address behavior. +- An address containing `{stargate_id}` is rendered for each discovered + Stargate. Entries without a Stargate ID are omitted. +- An empty value leaves the override disabled. + +Pylon dials `StargateInfo.grpc_pylon_dial_addr` but retains +`StargateInfo.advertise_addr` as the HTTP/2 authority. After registration, +pylon dials `InferenceServerAck.reverse_tunnel_pylon_dial_addr` over UDP but +retains `reverse_tunnel_target` as the QUIC SNI and routing identity. This lets +the dial path use provider-owned names while the certificate covers only the +internal advertised identity. + +For example, a router can use: + +```text +--grpc-pylon-dial-addr={stargate_id}.router.region-a.example:50071 +--reverse-tunnel-pylon-dial-addr=llm-request-router-0.router.region-a.example:50072 +``` + +The provider must give every replica distinct TCP and UDP endpoints and use +transparent L4 forwarding. It must not terminate TLS, rewrite authority, or +combine replicas behind an SNI demultiplexer. Use a region-unique external +domain when remote watch URLs join multiple regional router meshes. When a reverse-tunnel dial address resolves to more than one socket address, pylon and the WebTransport router try IPv4 candidates first for compatibility, @@ -99,8 +121,9 @@ candidates sequentially and bind each QUIC client endpoint in the matching address family. This is deterministic failover, not a racing Happy Eyeballs strategy. -The local overlay mirrors the split with ClusterIP Services whose service ports -(`443` and `8080`) differ from the router pod ports (`50071` and `50072`). +The local k3d overlay exposes control-cluster router Services as NodePorts and +creates selectorless compute-cluster aliases. Each per-replica alias has its +own ClusterIP while keeping public ports `50071` and `50072`. ### Development-Only Built-In Peer Relay diff --git a/tests/bdd/features/multi-cluster-helmfile.feature b/tests/bdd/features/multi-cluster-helmfile.feature index 54f1b44ae..e245743d9 100644 --- a/tests/bdd/features/multi-cluster-helmfile.feature +++ b/tests/bdd/features/multi-cluster-helmfile.feature @@ -77,7 +77,7 @@ Feature: Install a local multi-cluster NVCF stack with Helmfile | nvca-operator | | cert-manager | - @control-plane @llm-gateway + @control-plane @llm-gateway @split-cluster-llm @llm-pki Scenario: Operator installs the control plane through Helmfile on the control-plane cluster When I run command "make -C deploy/stacks/self-managed install HELMFILE_ENV=local-bdd" Then the command exit code should be 0 @@ -157,6 +157,31 @@ Feature: Install a local multi-cluster NVCF stack with Helmfile """ Then the command exit code should be 0 + # Keep the Helmfile path values-driven. Read only the public CA from the + # cert-manager Secret, compute its canonical fingerprint, and merge the + # trust settings into the compute environment authored in the Background. + When I run command: + """ + set -euo pipefail + kubectl --context k3d-ncp-local-cp wait certificate/stargate-quic-tls -n nvcf --for=condition=Ready --timeout=5m + trust_dir=deploy/stacks/nvcf-compute-plane/out/bdd-split-llm + mkdir -p "$trust_dir" + rm -f "$trust_dir"/certificate-*.pem "$trust_dir/certificate-hashes" + encoded_ca="$(kubectl --context k3d-ncp-local-cp get secret/stargate-quic-tls -n nvcf -o jsonpath='{.data.ca\.crt}')" + test -n "$encoded_ca" + printf '%s' "$encoded_ca" | openssl base64 -d -A >"$trust_dir/ca.pem" + openssl x509 -in "$trust_dir/ca.pem" -noout -subject -issuer + awk -v output_dir="$trust_dir" '/-----BEGIN CERTIFICATE-----/ { certificate++; in_certificate=1 } in_certificate { print > (output_dir "/certificate-" certificate ".pem") } /-----END CERTIFICATE-----/ { in_certificate=0 }' "$trust_dir/ca.pem" + for certificate_file in "$trust_dir"/certificate-*.pem; do + openssl x509 -in "$certificate_file" -outform DER | openssl dgst -sha256 -r | awk '{print $1}' + done | sort -u >"$trust_dir/certificate-hashes" + trust_fingerprint="$({ printf 'nvcf-trust-bundle-v1\n'; cat "$trust_dir/certificate-hashes"; } | openssl dgst -sha256 -r | awk '{print "sha256:" $1}')" + export TRUST_BUNDLE_PEM="$(sed -n '/-----BEGIN CERTIFICATE-----/,/-----END CERTIFICATE-----/p' "$trust_dir/ca.pem")" + export TRUST_BUNDLE_FINGERPRINT="$trust_fingerprint" + yq -i '.agentConfig.mergeConfig = (((.agentConfig.mergeConfig | from_yaml) * {"workload": {"stargateQUICInsecure": false, "transportTLS": {"trustMode": "bundle", "trustBundleFingerprint": strenv(TRUST_BUNDLE_FINGERPRINT), "trustBundlePem": strenv(TRUST_BUNDLE_PEM)}}}) | to_yaml)' deploy/stacks/nvcf-compute-plane/environments/local-bdd.yaml + """ + Then the command exit code should be 0 + Rule: Helmfile registers and installs NVCA on the compute cluster Background: @@ -167,7 +192,7 @@ Feature: Install a local multi-cluster NVCF stack with Helmfile # the compute-reachable endpoints, creates the pull secrets, and # installs the control plane. Do not repeat that setup here. - @nvca-registration + @nvca-registration @split-cluster-llm @llm-pki Scenario: Operator registers the compute cluster and installs the NVCA operator there # nvcf-cli cluster register auto-discovers the target cluster's # OIDC issuer + JWKS by running a probe Job in the CURRENT @@ -209,6 +234,16 @@ Feature: Install a local multi-cluster NVCF stack with Helmfile """ Then the command exit code should be 0 + When I run command: + """ + set -euo pipefail + expected_fingerprint="$({ printf 'nvcf-trust-bundle-v1\n'; cat deploy/stacks/nvcf-compute-plane/out/bdd-split-llm/certificate-hashes; } | openssl dgst -sha256 -r | awk '{print "sha256:" $1}')" + merge_config="$(kubectl --context k3d-ncp-local-compute-1 get configmap/agent-config-merge -n nvca-operator -o jsonpath='{.data.config\.yaml}')" + test "$(printf '%s' "$merge_config" | yq -r '.workload.transportTLS.trustMode')" = bundle + test "$(printf '%s' "$merge_config" | yq -r '.workload.transportTLS.trustBundleFingerprint')" = "$expected_fingerprint" + """ + Then the command exit code should be 0 + When I run command "helm list -n nvca-operator --kube-context k3d-ncp-local-compute-1 -o json" Then the json output should contain rows: | name | namespace | status | @@ -293,3 +328,144 @@ Feature: Install a local multi-cluster NVCF stack with Helmfile """ Then the command exit code should be 0 And the command output should contain "bdd-grpc-echo" + + # This tagged scenario uses the local implementation chart and image. It + # depends on the control-plane and compute-plane scenarios above, including + # the managed OpenBao trust bundle in the compute environment. + @function-lifecycle @split-cluster-llm @llm-pki + Scenario: Operator invokes two mock LLM replicas through two split-cluster Stargates + When I run command: + """ + docker build --file src/libraries/rust/stargate/Dockerfile --target stargate-runtime --tag nvcf-stargate-per-replica:bdd src/libraries/rust/stargate + """ + Then the command exit code should be 0 + + When I run command: + """ + k3d image import nvcf-stargate-per-replica:bdd --cluster ncp-local-cp + """ + Then the command exit code should be 0 + + When I run command: + """ + helm upgrade llm-request-router deploy/helm/llm-request-router/llm-request-router --namespace nvcf --kube-context k3d-ncp-local-cp --reuse-values --set-string llmRequestRouter.image.registry= --set-string llmRequestRouter.image.repository=nvcf-stargate-per-replica --set-string llmRequestRouter.image.tag=bdd --wait --timeout 10m + """ + Then the command exit code should be 0 + + When I run command: + """ + make -C tools/ncp-local-cluster configure-compute-llm-router-endpoints CONTROL_PLANE_CLUSTER_NAME=ncp-local-cp COMPUTE_CLUSTER_NAME=ncp-local-compute-1 + """ + Then the command exit code should be 0 + + When I run command: + """ + kubectl --context k3d-ncp-local-cp rollout status statefulset/llm-request-router -n nvcf --timeout=10m + kubectl --context k3d-ncp-local-cp rollout status deployment/llm-api-gateway -n nvcf --timeout=10m + test "$(kubectl --context k3d-ncp-local-cp get statefulset llm-request-router -n nvcf -o jsonpath='{.status.readyReplicas}')" = "2" + test "$(kubectl --context k3d-ncp-local-cp get deployment llm-api-gateway -n nvcf -o jsonpath='{.status.readyReplicas}')" = "2" + kubectl --context k3d-ncp-local-cp wait certificate/stargate-quic-tls -n nvcf --for=condition=Ready --timeout=5m + kubectl --context k3d-ncp-local-cp get statefulset/llm-request-router -n nvcf -o json | jq -e '.spec.template.spec.containers[] | select(.name == "llm-request-router") | (.args | index("--grpc-pylon-dial-addr={stargate_id}.nvcf-llm-router.svc.cluster.local:50071")) != null and (.args | index("--reverse-tunnel-pylon-dial-addr=$(POD_NAME).nvcf-llm-router.svc.cluster.local:50072")) != null' >/dev/null + kubectl --context k3d-ncp-local-cp get certificate/stargate-quic-tls -n nvcf -o json | jq -e '(.spec.dnsNames | index("llm-request-router.nvcf.svc.cluster.local")) != null and (.spec.dnsNames | index("*.llm-request-router-headless.nvcf.svc.cluster.local")) != null and all(.spec.dnsNames[]; contains("nvcf-llm-router.svc.cluster.local") | not)' >/dev/null + """ + Then the command exit code should be 0 + + When I run command: + """ + docker build --file src/libraries/rust/stargate/Dockerfile --target mock-dynamo-runtime --tag nvcf-mock-dynamo-per-replica:bdd src/libraries/rust/stargate + k3d image import nvcf-mock-dynamo-per-replica:bdd --cluster ncp-local-compute-1 + """ + Then the command exit code should be 0 + + When I run command: + """ + ${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml function create --name bdd-split-mock-llm --image nvcf-mock-dynamo-per-replica:bdd --container-args "--http-listen-addr=0.0.0.0:8000 --model-name=dummy-model --num-tokens=2 --token-delay-ms=0" --inference-url /v1/chat/completions --inference-port 8000 --health-uri /health --health-port 8000 --health-timeout PT30S --function-type LLM --llm-model "name=dummy-model,uris=/v1/chat/completions|/v1/responses|/v1/embeddings,routingMethod=round_robin,tokenRateLimit=1000-S" + """ + Then the command exit code should be 0 + + When I run command: + """ + ${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml function deploy create --gpu H100 --instance-type NCP.GPU.H100_8x --backend ncp-local-compute-1 --regions us-west-1 --min-instances 2 --max-instances 2 --timeout 900 + """ + Then the command exit code should be 0 + + When I run command: + """ + set -euo pipefail + pods="$(kubectl --context k3d-ncp-local-compute-1 get pods -n nvcf-backend -o json | jq -r '.items[] | select(any(.spec.containers[]; .name == "llm-worker")) | .metadata.name')" + test "$(printf '%s\n' "$pods" | sed '/^$/d' | wc -l | tr -d ' ')" = "2" + while IFS= read -r pod; do + test -n "$pod" + kubectl --context k3d-ncp-local-compute-1 wait "pod/$pod" -n nvcf-backend --for=condition=Ready --timeout=10m + kubectl --context k3d-ncp-local-compute-1 get "pod/$pod" -n nvcf-backend -o json | jq -e '.spec.containers[] | select(.name == "llm-worker") | ((.args | index("--quic-insecure")) == null) and (any(.env[]?; .name == "STARGATE_TLS_CERT_PATH"))' >/dev/null + router_count="$(kubectl --context k3d-ncp-local-compute-1 logs "pod/$pod" -n nvcf-backend -c llm-worker | grep 'reverse tunnel connected' | sed -n 's/.*router_addr=\([^ ]*\).*/\1/p' | sort -u | wc -l | tr -d ' ')" + test "$router_count" -eq 2 + done <<<"$pods" + """ + Then the command exit code should be 0 + + When I run command: + """ + ${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml api-key generate --description bdd-split-mock-llm --for function --scopes invoke_function,list_functions,queue_details,list_functions_details >/dev/null + """ + Then the command exit code should be 0 + + When I run command: + """ + ${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml function invoke --model-name dummy-model --inference-url /v1/chat/completions --request-body '{"messages":[{"role":"user","content":"split cluster smoke"}],"stream":false}' --timeout 30 + """ + Then the command exit code should be 0 + And the command output should contain "choices" + And the command output should contain "/dummy-model" + + When I run command: + """ + set -euo pipefail + for request_number in 1 2 3 4 5 6; do + ${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml function invoke --model-name dummy-model --inference-url /v1/chat/completions --request-body "{\"messages\":[{\"role\":\"user\",\"content\":\"split cluster request $request_number\"}],\"stream\":false}" --timeout 30 + done + """ + Then the command exit code should be 0 + + When I run command: + """ + set -euo pipefail + pods="$(kubectl --context k3d-ncp-local-compute-1 get pods -n nvcf-backend -o json | jq -r '.items[] | select(any(.spec.containers[]; .name == "llm-worker")) | .metadata.name')" + backends_with_chat=0 + forward_pid="" + cleanup_forward() { + if [ -n "$forward_pid" ]; then + kill "$forward_pid" >/dev/null 2>&1 || true + wait "$forward_pid" >/dev/null 2>&1 || true + fi + } + trap cleanup_forward EXIT + for pod in $pods; do + log_file="$(mktemp)" + kubectl --context k3d-ncp-local-compute-1 port-forward "pod/$pod" -n nvcf-backend 28090:8000 >"$log_file" 2>&1 & + forward_pid=$! + ready=false + for attempt in 1 2 3 4 5 6 7 8 9 10; do + if curl --silent --fail http://127.0.0.1:28090/health >/dev/null 2>&1; then + ready=true + break + fi + sleep 1 + done + if [ "$ready" != true ]; then + kill "$forward_pid" || true + wait "$forward_pid" || true + sed -n '1,40p' "$log_file" >&2 + exit 1 + fi + chat_count="$(curl --silent --fail http://127.0.0.1:28090/test-control | jq '[.counters[]? | select(.endpoint == "chat_completions" and .request_class == "api_gateway") | .count] | add // 0')" + cleanup_forward + forward_pid="" + if [ "$chat_count" -gt 0 ]; then + backends_with_chat=$((backends_with_chat + 1)) + fi + printf '%s chat_requests=%s\n' "$pod" "$chat_count" + done + test "$backends_with_chat" -eq 2 + """ + Then the command exit code should be 0 diff --git a/tests/bdd/fixtures/nvcf-compute-plane-local-bdd-multi.yaml b/tests/bdd/fixtures/nvcf-compute-plane-local-bdd-multi.yaml index a1d897af6..d4f0cb4f1 100644 --- a/tests/bdd/fixtures/nvcf-compute-plane-local-bdd-multi.yaml +++ b/tests/bdd/fixtures/nvcf-compute-plane-local-bdd-multi.yaml @@ -42,4 +42,4 @@ agentConfig: validationPolicy: name: Unrestricted workload: - stargateQUICInsecure: true + stargateQUICInsecure: false diff --git a/tests/bdd/fixtures/self-managed-local-bdd-multi.yaml b/tests/bdd/fixtures/self-managed-local-bdd-multi.yaml index 333422ef5..23a2dd6e6 100644 --- a/tests/bdd/fixtures/self-managed-local-bdd-multi.yaml +++ b/tests/bdd/fixtures/self-managed-local-bdd-multi.yaml @@ -33,6 +33,7 @@ global: workerEndpoints: essServiceURL: http://ess-api.ess.svc.cluster.local:8080 invocationServiceURL: http://invocation.nvcf.svc.cluster.local:8080 + llmRequestRouterAddress: llm-request-router.nvcf.svc.cluster.local:50071 cassandra: replicaCount: 1 @@ -59,18 +60,39 @@ api: addons: llm: enabled: true + pki: + enabled: true + allowedDomains: cluster.local + dnsNames: + - llm-request-router.nvcf.svc.cluster.local + - "*.llm-request-router-headless.nvcf.svc.cluster.local" gateway: - replicaCount: 1 + replicaCount: 2 auth: grpcInsecure: true metrics: serviceMonitor: enabled: false requestRouter: - replicaCount: 1 + replicaCount: 2 + service: + type: NodePort + externalAccess: + enabled: true + domain: nvcf-llm-router.svc.cluster.local + service: + type: NodePort metrics: serviceMonitor: enabled: false + loadBalancer: + config: | + { + "default": "round-robin", + "request_algorithms": { + "round-robin": "round-robin" + } + } grpcproxy: workerConnectBaseURL: http://grpc.nvcf.svc.cluster.local:10086 diff --git a/tests/bdd/fixtures_test.go b/tests/bdd/fixtures_test.go index 3a5dd51ca..29426159a 100644 --- a/tests/bdd/fixtures_test.go +++ b/tests/bdd/fixtures_test.go @@ -157,6 +157,11 @@ func TestSelfManagedLocalBDDMultiFixtureWiresGRPCWorkerCallback(t *testing.T) { fixture := string(fixtureBytes) for _, want := range []string{ "workerConnectBaseURL: http://grpc.nvcf.svc.cluster.local:10086", + "llmRequestRouterAddress: llm-request-router.nvcf.svc.cluster.local:50071", + "allowedDomains: cluster.local", + `"*.llm-request-router-headless.nvcf.svc.cluster.local"`, + "domain: nvcf-llm-router.svc.cluster.local", + "type: NodePort", "chart: ../../../helm/gateway-routes/chart", `version: ""`, "grpcWorker:", @@ -169,6 +174,17 @@ func TestSelfManagedLocalBDDMultiFixtureWiresGRPCWorkerCallback(t *testing.T) { } } +func TestComputePlaneLocalBDDMultiFixtureRequiresTrustedStargateQUIC(t *testing.T) { + fixtureBytes, err := os.ReadFile("fixtures/nvcf-compute-plane-local-bdd-multi.yaml") + if err != nil { + t.Fatalf("read multi-cluster compute fixture: %v", err) + } + fixture := string(fixtureBytes) + if !strings.Contains(fixture, "stargateQUICInsecure: false") { + t.Fatal("multi-cluster compute fixture must require trusted Stargate QUIC") + } +} + func TestNVCTTaskSmokeUsesTaskSimpleSample(t *testing.T) { for _, path := range []string{ "../../examples/task-samples/task-simple-sample/Dockerfile", diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go index 56da25ad9..c5d2a0b5e 100644 --- a/tests/bdd/godog_test.go +++ b/tests/bdd/godog_test.go @@ -846,6 +846,12 @@ func TestMultiClusterHelmfileFeatureFileWiresToSteps(t *testing.T) { ExitCode: 0, Stdout: "Function invocation completed!\n\nResponse:\n{\"message\":\"bdd-grpc-echo\"}\n", }, + "/usr/bin/nvcf-cli --config /repo-root-placeholder/tests/bdd/fixtures/nvcf-cli-local.yaml function invoke" + + " --model-name dummy-model --inference-url /v1/chat/completions" + + " --request-body '{\"messages\":[{\"role\":\"user\",\"content\":\"split cluster smoke\"}],\"stream\":false}' --timeout 30": { + ExitCode: 0, + Stdout: "{\"object\":\"chat.completion\",\"model\":\"function-id/dummy-model\",\"choices\":[]}\n", + }, "tests/bdd/scripts/run-nvct-task-smoke.sh": { ExitCode: 0, Stdout: "Task bdd-nvct-task-smoke status: COMPLETED\n", @@ -1174,6 +1180,7 @@ func seedHelmfileLocalBDDMultiFixture(t *testing.T, repoRoot string) { workerEndpoints: essServiceURL: http://ess-api.ess.svc.cluster.local:8080 invocationServiceURL: http://invocation.nvcf.svc.cluster.local:8080 + llmRequestRouterAddress: llm-request-router.nvcf.svc.cluster.local:50071 nvcaOperator: selfManaged: icmsServiceURL: http://api.sis.svc.cluster.local:8080 @@ -1182,6 +1189,23 @@ func seedHelmfileLocalBDDMultiFixture(t *testing.T, repoRoot string) { addons: llm: enabled: true + pki: + enabled: true + allowedDomains: cluster.local + dnsNames: + - llm-request-router.nvcf.svc.cluster.local + - "*.llm-request-router-headless.nvcf.svc.cluster.local" + gateway: + replicaCount: 2 + requestRouter: + replicaCount: 2 + service: + type: NodePort + externalAccess: + enabled: true + domain: nvcf-llm-router.svc.cluster.local + service: + type: NodePort grpcproxy: workerConnectBaseURL: http://grpc.nvcf.svc.cluster.local:10086 ingress: @@ -1211,7 +1235,7 @@ agentConfig: validationPolicy: name: Unrestricted workload: - stargateQUICInsecure: true + stargateQUICInsecure: false `) } diff --git a/tools/ncp-local-cluster/Makefile b/tools/ncp-local-cluster/Makefile index 354da8727..acecf7520 100644 --- a/tools/ncp-local-cluster/Makefile +++ b/tools/ncp-local-cluster/Makefile @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -.PHONY: help clean build test test-coverage-html test-manual start stop destroy destroy-all-ncp-local status ensure-cluster ensure-context ensure-docker-config validate-compute-clusters print-compute-clusters start-control-plane deploy-control-plane-addons deploy-control-plane-endpoints build-and-deploy-control-plane-cluster destroy-control-plane start-compute-plane deploy-compute-plane-addons configure-compute-control-plane-dns deploy-compute-control-plane-endpoints build-and-deploy-compute-plane-cluster destroy-compute-plane build-and-deploy-multicluster destroy-multicluster test-multicluster-make setup-gateway-api setup-metallb check-gateway-api deploy-sample deploy-nginx wait-for-nginx wait-for-gateway validate-gateway cleanup-nginx wait-for-deployment validate-deployment cleanup-sample build-and-deploy-cluster build-csi-smb deploy-csi-smb wait-for-csi-smb build-fake-gpu-operator deploy-fake-gpu-operator wait-for-fake-gpu-operator deploy-prometheus-crds uninstall-prometheus-crds deploy-kube-state-metrics wait-for-kube-state-metrics uninstall-kube-state-metrics build-credential-provider-multiarch +.PHONY: help clean build test test-coverage-html test-manual start stop destroy destroy-all-ncp-local status ensure-cluster ensure-context ensure-docker-config validate-compute-clusters print-compute-clusters start-control-plane deploy-control-plane-addons deploy-control-plane-endpoints build-and-deploy-control-plane-cluster destroy-control-plane start-compute-plane deploy-compute-plane-addons configure-compute-control-plane-dns deploy-compute-control-plane-endpoints configure-compute-llm-router-endpoints build-and-deploy-compute-plane-cluster destroy-compute-plane build-and-deploy-multicluster destroy-multicluster test-multicluster-make setup-gateway-api setup-metallb check-gateway-api deploy-sample deploy-nginx wait-for-nginx wait-for-gateway validate-gateway cleanup-nginx wait-for-deployment validate-deployment cleanup-sample build-and-deploy-cluster build-csi-smb deploy-csi-smb wait-for-csi-smb build-fake-gpu-operator deploy-fake-gpu-operator wait-for-fake-gpu-operator deploy-prometheus-crds uninstall-prometheus-crds deploy-kube-state-metrics wait-for-kube-state-metrics uninstall-kube-state-metrics build-credential-provider-multiarch # === Cluster Configuration === CLUSTER_NAME := ncp-local @@ -256,6 +256,13 @@ deploy-compute-control-plane-endpoints: ## Deploy compute-cluster aliases for co bash ./scripts/configure-control-plane-endpoints.sh @echo "OK Compute-plane endpoint aliases deployed for $(CONTROL_PLANE_DOMAIN)" +configure-compute-llm-router-endpoints: ## Configure split-cluster LLM seed and per-replica aliases after control install + $(call require,docker,See https://docs.docker.com/get-docker/) + $(call require,kubectl,See https://kubernetes.io/docs/tasks/tools/) + @CONTROL_PLANE_CLUSTER_NAME="$(CONTROL_PLANE_CLUSTER_NAME)" \ + COMPUTE_CLUSTER_NAME="$(COMPUTE_CLUSTER_NAME)" \ + bash ./scripts/configure-llm-router-endpoints.sh + build-and-deploy-compute-plane-cluster: clean build start-compute-plane deploy-compute-plane-addons ## Build and deploy one local compute-plane cluster @echo "========== COMPLETED ==========" @echo "OK Compute-plane cluster $(COMPUTE_CLUSTER_NAME) is up and ready." diff --git a/tools/ncp-local-cluster/README.md b/tools/ncp-local-cluster/README.md index baf13fa56..19c9534ba 100644 --- a/tools/ncp-local-cluster/README.md +++ b/tools/ncp-local-cluster/README.md @@ -33,6 +33,7 @@ Key targets include: * `make build-and-deploy-multicluster`: Build one control-plane cluster and one or more compute-plane clusters. * `make build-and-deploy-control-plane-cluster`: Build only the local control-plane cluster. * `make build-and-deploy-compute-plane-cluster`: Build one local compute-plane cluster. +* `make configure-compute-llm-router-endpoints`: Configure post-install split-cluster LLM router aliases. * `make clean`: Remove built binaries and test coverage files. * `make build-credential-provider-multiarch`: Build `linux/amd64` and `linux/arm64` binaries locally without publishing. @@ -154,6 +155,58 @@ make build-and-deploy-multicluster \ CONTROL_PLANE_NATS_PORT=14222 ``` +### Split-cluster LLM router aliases + +Configure the control-plane stack with two router replicas, NodePort Services, +and the compute alias DNS suffix: + +```yaml +global: + workerEndpoints: + llmRequestRouterAddress: llm-request-router.nvcf.svc.cluster.local:50071 + +addons: + llm: + requestRouter: + replicaCount: 2 + service: + type: NodePort + externalAccess: + enabled: true + domain: nvcf-llm-router.svc.cluster.local + service: + type: NodePort +``` + +After the control-plane chart creates the Services, configure one compute +cluster: + +```sh +make configure-compute-llm-router-endpoints \ + CONTROL_PLANE_CLUSTER_NAME=ncp-local-cp \ + COMPUTE_CLUSTER_NAME=ncp-local-compute-1 +``` + +The target discovers the StatefulSet replica count and allocated TCP and UDP +NodePorts. It attaches the control server node to the compute Docker network, +creates the shared `llm-request-router.nvcf` seed alias, and creates one +selectorless Service and Endpoints pair per router in the `nvcf-llm-router` +namespace. Every per-replica Service has a distinct ClusterIP and exposes TCP +50071 plus UDP 50072. + +Preview the generated aliases without changing Docker or Kubernetes state: + +```sh +CONTROL_PLANE_NODE_IP=172.20.0.10 \ +LLM_REQUEST_ROUTER_REPLICAS=2 \ +LLM_REQUEST_ROUTER_SHARED_GRPC_NODE_PORT=31071 \ +LLM_REQUEST_ROUTER_NAMESPACE=nvcf \ +bash scripts/configure-llm-router-endpoints.sh --dry-run +``` + +Dry-run still discovers per-replica NodePorts from the control context. The +ordinary API, SIS, ReVal, NATS, invocation, and gRPC aliases are unchanged. + Use a custom local control-plane DNS suffix: ```sh diff --git a/tools/ncp-local-cluster/scripts/configure-llm-router-endpoints.sh b/tools/ncp-local-cluster/scripts/configure-llm-router-endpoints.sh new file mode 100755 index 000000000..41977def4 --- /dev/null +++ b/tools/ncp-local-cluster/scripts/configure-llm-router-endpoints.sh @@ -0,0 +1,228 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +usage() { + cat >&2 <<'EOF' +Usage: configure-llm-router-endpoints.sh [--dry-run] + +Expose a control-cluster LLM request router to one compute k3d cluster. Run +this after the control-plane chart creates the shared and per-replica NodePort +Services. Dry-run discovers the topology and prints the compute aliases without +connecting Docker networks or applying Kubernetes resources. + +Important environment variables: + CONTROL_PLANE_CLUSTER_NAME Default: ncp-local-cp + COMPUTE_CLUSTER_NAME Default: ncp-local-compute-1 + CONTROL_PLANE_CONTEXT Default: k3d-$CONTROL_PLANE_CLUSTER_NAME + COMPUTE_CONTEXT Default: k3d-$COMPUTE_CLUSTER_NAME + CONTROL_PLANE_NODE_CONTAINER Default: k3d-$CONTROL_PLANE_CLUSTER_NAME-server-0 + CONTROL_PLANE_NODE_IP Optional discovery override + LLM_REQUEST_ROUTER_REPLICAS Optional StatefulSet replica override + LLM_REQUEST_ROUTER_ALIAS_NAMESPACE Default: nvcf-llm-router +EOF +} + +dry_run=false +while [ "$#" -gt 0 ]; do + case "$1" in + --dry-run) + dry_run=true + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "ERROR: unknown argument: $1" >&2 + usage + exit 1 + ;; + esac +done + +control_cluster="${CONTROL_PLANE_CLUSTER_NAME:-ncp-local-cp}" +compute_cluster="${COMPUTE_CLUSTER_NAME:-ncp-local-compute-1}" +control_context="${CONTROL_PLANE_CONTEXT:-k3d-${control_cluster}}" +compute_context="${COMPUTE_CONTEXT:-k3d-${compute_cluster}}" +control_node="${CONTROL_PLANE_NODE_CONTAINER:-k3d-${control_cluster}-server-0}" +compute_network="${COMPUTE_DOCKER_NETWORK:-k3d-${compute_cluster}}" +control_namespace="${LLM_REQUEST_ROUTER_NAMESPACE:-nvcf}" +alias_namespace="${LLM_REQUEST_ROUTER_ALIAS_NAMESPACE:-nvcf-llm-router}" +router_name="${LLM_REQUEST_ROUTER_NAME:-llm-request-router}" + +require_positive_integer() { + local name="$1" + local value="$2" + case "$value" in + ''|*[!0-9]*) + echo "ERROR: ${name} must be a positive integer, got '${value}'" >&2 + exit 1 + ;; + esac + if [ "$value" -lt 1 ]; then + echo "ERROR: ${name} must be a positive integer, got '${value}'" >&2 + exit 1 + fi +} + +require_node_port() { + local service_name="$1" + local port_name="$2" + local value="$3" + case "$value" in + ''|*[!0-9]*) + echo "ERROR: ${service_name}/${port_name} has no allocated NodePort" >&2 + exit 1 + ;; + esac + if [ "$value" -lt 1 ] || [ "$value" -gt 65535 ]; then + echo "ERROR: ${service_name}/${port_name} returned invalid NodePort '${value}'" >&2 + exit 1 + fi +} + +discover_node_port() { + local service_name="$1" + local port_name="$2" + kubectl --context "$control_context" --namespace "$control_namespace" \ + get service "$service_name" \ + -o "jsonpath={.spec.ports[?(@.name==\"${port_name}\")].nodePort}" +} + +replicas="${LLM_REQUEST_ROUTER_REPLICAS:-}" +if [ -z "$replicas" ]; then + replicas="$(kubectl --context "$control_context" --namespace "$control_namespace" \ + get statefulset "$router_name" -o 'jsonpath={.spec.replicas}')" +fi +require_positive_integer LLM_REQUEST_ROUTER_REPLICAS "$replicas" + +node_ip="${CONTROL_PLANE_NODE_IP:-}" +if [ -z "$node_ip" ]; then + node_ip="$(docker network inspect "$compute_network" \ + --format '{{range .Containers}}{{if eq .Name "'"${control_node}"'"}}{{.IPv4Address}}{{end}}{{end}}')" + if [ -z "$node_ip" ] && [ "$dry_run" = false ]; then + docker network connect "$compute_network" "$control_node" + node_ip="$(docker network inspect "$compute_network" \ + --format '{{range .Containers}}{{if eq .Name "'"${control_node}"'"}}{{.IPv4Address}}{{end}}{{end}}')" + fi + node_ip="${node_ip%%/*}" +fi +if [ -z "$node_ip" ] || [ "$node_ip" = "" ]; then + echo "ERROR: unable to discover ${control_node} on ${compute_network}" >&2 + if [ "$dry_run" = true ]; then + echo "Set CONTROL_PLANE_NODE_IP for an offline dry-run." >&2 + fi + exit 1 +fi + +shared_grpc_node_port="${LLM_REQUEST_ROUTER_SHARED_GRPC_NODE_PORT:-$(discover_node_port "$router_name" grpc)}" +require_node_port "$router_name" grpc "$shared_grpc_node_port" + +declare -a grpc_node_ports=() +declare -a quic_node_ports=() +for ((ordinal = 0; ordinal < replicas; ordinal++)); do + service_name="${router_name}-${ordinal}" + grpc_node_port="$(discover_node_port "$service_name" grpc)" + quic_node_port="$(discover_node_port "$service_name" quic)" + require_node_port "$service_name" grpc "$grpc_node_port" + require_node_port "$service_name" quic "$quic_node_port" + grpc_node_ports+=("$grpc_node_port") + quic_node_ports+=("$quic_node_port") +done + +echo "LLM router endpoint source: ${control_node} (${node_ip})" >&2 +echo "Shared seed: ${router_name}.${control_namespace}.svc.cluster.local:50071 -> ${node_ip}:${shared_grpc_node_port}/TCP" >&2 +echo "Per-replica dial domain: ${alias_namespace}.svc.cluster.local" >&2 + +render_aliases() { + cat <"$llm_fake_bin/docker" <<'EOF' +#!/usr/bin/env bash +if [ "$1" = "network" ] && [ "$2" = "inspect" ]; then + printf '%s\n' '172.29.0.5/16' + exit 0 +fi +echo "unexpected docker command: $*" >&2 +exit 42 +EOF +cat >"$llm_fake_bin/kubectl" <<'EOF' +#!/usr/bin/env bash +args="$*" +if [[ "$args" == *" get statefulset llm-request-router "* ]]; then + printf '%s' "${MOCK_LLM_REPLICAS:-2}" + exit 0 +fi +if [[ "$args" =~ get\ service\ (llm-request-router(-[0-9]+)?)\ ]]; then + service_name="${BASH_REMATCH[1]}" + if [ "$service_name" = "llm-request-router" ]; then + printf '%s' '31071' + exit 0 + fi + ordinal="${service_name##*-}" + if [[ "$args" == *'name=="grpc"'* ]]; then + printf '%s' "$((32071 + ordinal * 10))" + exit 0 + fi + if [[ "$args" == *'name=="quic"'* ]]; then + printf '%s' "$((32072 + ordinal * 10))" + exit 0 + fi +fi +echo "unexpected kubectl command: $*" >&2 +exit 42 +EOF +chmod +x "$llm_fake_bin/docker" "$llm_fake_bin/kubectl" +llm_aliases_yaml="$(MOCK_LLM_REPLICAS=2 PATH="$llm_fake_bin:$PATH" \ + CONTROL_PLANE_CLUSTER_NAME=ncp-test-cp \ + COMPUTE_CLUSTER_NAME=ncp-test-compute \ + "$ROOT_DIR/scripts/configure-llm-router-endpoints.sh" --dry-run)" +if ! grep -q 'ip: 172.29.0.5' <<<"$llm_aliases_yaml"; then + fail "LLM endpoint dry-run must discover the control node on the compute Docker network" +fi +if ! grep -A12 'name: llm-request-router$' <<<"$llm_aliases_yaml" | grep -q 'port: 31071'; then + fail "LLM endpoint dry-run must discover the shared seed TCP NodePort" +fi +for ordinal in 0 1; do + if ! grep -q "name: llm-request-router-${ordinal}" <<<"$llm_aliases_yaml"; then + fail "LLM endpoint dry-run missing replica ${ordinal} aliases" + fi +done +if [ "$(grep -c '^kind: Service$' <<<"$llm_aliases_yaml")" -ne 3 ]; then + fail "LLM endpoint dry-run must generate one shared and two per-replica Services" +fi +for port in 32071 32072 32081 32082; do + if ! grep -q "port: ${port}" <<<"$llm_aliases_yaml"; then + fail "LLM endpoint dry-run missing discovered NodePort ${port}" + fi +done +if ! grep -q 'protocol: TCP' <<<"$llm_aliases_yaml" || ! grep -q 'protocol: UDP' <<<"$llm_aliases_yaml"; then + fail "LLM endpoint aliases must preserve TCP registration and UDP QUIC protocols" +fi +if grep -q 'selector:' <<<"$llm_aliases_yaml"; then + fail "compute LLM aliases must be selectorless Services backed by explicit Endpoints" +fi + +scaled_llm_aliases_yaml="$(MOCK_LLM_REPLICAS=3 PATH="$llm_fake_bin:$PATH" \ + CONTROL_PLANE_NODE_IP=172.29.0.6 \ + "$ROOT_DIR/scripts/configure-llm-router-endpoints.sh" --dry-run)" +if [ "$(grep -c '^kind: Service$' <<<"$scaled_llm_aliases_yaml")" -ne 4 ]; then + fail "LLM endpoint aliases must follow request-router replica-count changes" +fi +if ! grep -q 'name: llm-request-router-2' <<<"$scaled_llm_aliases_yaml"; then + fail "scaled LLM endpoint dry-run missing the third replica" +fi +rm -rf "$llm_fake_bin" + fake_bin="$(mktemp -d)" cat >"$fake_bin/docker" <<'EOF' #!/usr/bin/env bash From db035f5b32a6f0bdd292a0745191c7a1ab81070d Mon Sep 17 00:00:00 2001 From: jcameron Date: Wed, 19 Aug 2026 17:15:22 -0300 Subject: [PATCH 2/3] test(llm-routing): match deployed router container Signed-off-by: jcameron --- tests/bdd/features/multi-cluster-helmfile.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bdd/features/multi-cluster-helmfile.feature b/tests/bdd/features/multi-cluster-helmfile.feature index e245743d9..68ffb83b7 100644 --- a/tests/bdd/features/multi-cluster-helmfile.feature +++ b/tests/bdd/features/multi-cluster-helmfile.feature @@ -365,7 +365,7 @@ Feature: Install a local multi-cluster NVCF stack with Helmfile test "$(kubectl --context k3d-ncp-local-cp get statefulset llm-request-router -n nvcf -o jsonpath='{.status.readyReplicas}')" = "2" test "$(kubectl --context k3d-ncp-local-cp get deployment llm-api-gateway -n nvcf -o jsonpath='{.status.readyReplicas}')" = "2" kubectl --context k3d-ncp-local-cp wait certificate/stargate-quic-tls -n nvcf --for=condition=Ready --timeout=5m - kubectl --context k3d-ncp-local-cp get statefulset/llm-request-router -n nvcf -o json | jq -e '.spec.template.spec.containers[] | select(.name == "llm-request-router") | (.args | index("--grpc-pylon-dial-addr={stargate_id}.nvcf-llm-router.svc.cluster.local:50071")) != null and (.args | index("--reverse-tunnel-pylon-dial-addr=$(POD_NAME).nvcf-llm-router.svc.cluster.local:50072")) != null' >/dev/null + kubectl --context k3d-ncp-local-cp get statefulset/llm-request-router -n nvcf -o json | jq -e '.spec.template.spec.containers[] | select(any(.args[]?; startswith("--grpc-pylon-dial-addr="))) | (.args | index("--grpc-pylon-dial-addr={stargate_id}.nvcf-llm-router.svc.cluster.local:50071")) != null and (.args | index("--reverse-tunnel-pylon-dial-addr=$(POD_NAME).nvcf-llm-router.svc.cluster.local:50072")) != null' >/dev/null kubectl --context k3d-ncp-local-cp get certificate/stargate-quic-tls -n nvcf -o json | jq -e '(.spec.dnsNames | index("llm-request-router.nvcf.svc.cluster.local")) != null and (.spec.dnsNames | index("*.llm-request-router-headless.nvcf.svc.cluster.local")) != null and all(.spec.dnsNames[]; contains("nvcf-llm-router.svc.cluster.local") | not)' >/dev/null """ Then the command exit code should be 0 From 288550b6139c8a27c6ddc3593adb275a97261425 Mon Sep 17 00:00:00 2001 From: jcameron Date: Wed, 19 Aug 2026 17:36:10 -0300 Subject: [PATCH 3/3] test(llm-routing): require manifest-backed mock image Signed-off-by: jcameron --- tests/bdd/README.md | 8 +++++++- tests/bdd/features/multi-cluster-helmfile.feature | 13 ++++++++++--- tests/bdd/godog_test.go | 1 + 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/tests/bdd/README.md b/tests/bdd/README.md index a16cfcaaf..6321adc18 100644 --- a/tests/bdd/README.md +++ b/tests/bdd/README.md @@ -84,8 +84,11 @@ BDD_CLEANUP_MODE=topology-multi \ # Multi-cluster Helmfile feature: control-plane install on # k3d-ncp-local-cp followed by compute-plane register-cluster + install # on k3d-ncp-local-compute-1. Same secrets as the single-cluster -# Helmfile feature. +# Helmfile feature. NVCF_BDD_MOCK_DYNAMO_TAG must name an existing +# mock-dynamo registry manifest; the feature builds and imports local bytes +# under that tag. NGC_API_KEY= SAMPLE_NGC_ORG= SAMPLE_NGC_TEAM= \ + NVCF_BDD_MOCK_DYNAMO_TAG= \ go test -run '^TestMultiClusterHelmfile$' -timeout 90m -v ``` @@ -164,6 +167,9 @@ If a new step is genuinely needed: invocations resolve. - For the Helmfile feature: `NGC_API_KEY`, `SAMPLE_NGC_ORG`, `SAMPLE_NGC_TEAM` env vars set. +- For the multi-cluster LLM scenario: `NVCF_BDD_MOCK_DYNAMO_TAG` names an + existing `mock-dynamo` manifest in the sample NGC repository. The function + API validates the manifest before the locally imported image is deployed. - For the upstream-image feature: outbound cluster access to `docker.io/natsio` and `docker.io/alpine`. diff --git a/tests/bdd/features/multi-cluster-helmfile.feature b/tests/bdd/features/multi-cluster-helmfile.feature index 68ffb83b7..fbfa686ad 100644 --- a/tests/bdd/features/multi-cluster-helmfile.feature +++ b/tests/bdd/features/multi-cluster-helmfile.feature @@ -334,6 +334,11 @@ Feature: Install a local multi-cluster NVCF stack with Helmfile # the managed OpenBao trust bundle in the compute environment. @function-lifecycle @split-cluster-llm @llm-pki Scenario: Operator invokes two mock LLM replicas through two split-cluster Stargates + # Function creation validates the registry manifest even when k3d has + # local image bytes. The operator supplies an existing mock-dynamo tag; + # the build and import below replace only the local cluster image. + Given environment variable "NVCF_BDD_MOCK_DYNAMO_TAG" is set + When I run command: """ docker build --file src/libraries/rust/stargate/Dockerfile --target stargate-runtime --tag nvcf-stargate-per-replica:bdd src/libraries/rust/stargate @@ -372,14 +377,16 @@ Feature: Install a local multi-cluster NVCF stack with Helmfile When I run command: """ - docker build --file src/libraries/rust/stargate/Dockerfile --target mock-dynamo-runtime --tag nvcf-mock-dynamo-per-replica:bdd src/libraries/rust/stargate - k3d image import nvcf-mock-dynamo-per-replica:bdd --cluster ncp-local-compute-1 + MOCK_IMAGE="nvcr.io/$SAMPLE_NGC_ORG/$SAMPLE_NGC_TEAM/mock-dynamo:$NVCF_BDD_MOCK_DYNAMO_TAG" + docker build --file src/libraries/rust/stargate/Dockerfile --target mock-dynamo-runtime --tag "$MOCK_IMAGE" src/libraries/rust/stargate + k3d image import "$MOCK_IMAGE" --cluster ncp-local-compute-1 """ Then the command exit code should be 0 When I run command: """ - ${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml function create --name bdd-split-mock-llm --image nvcf-mock-dynamo-per-replica:bdd --container-args "--http-listen-addr=0.0.0.0:8000 --model-name=dummy-model --num-tokens=2 --token-delay-ms=0" --inference-url /v1/chat/completions --inference-port 8000 --health-uri /health --health-port 8000 --health-timeout PT30S --function-type LLM --llm-model "name=dummy-model,uris=/v1/chat/completions|/v1/responses|/v1/embeddings,routingMethod=round_robin,tokenRateLimit=1000-S" + MOCK_IMAGE="nvcr.io/$SAMPLE_NGC_ORG/$SAMPLE_NGC_TEAM/mock-dynamo:$NVCF_BDD_MOCK_DYNAMO_TAG" + ${NVCF_CLI} --config ${REPO_ROOT}/tests/bdd/fixtures/nvcf-cli-local.yaml function create --name bdd-split-mock-llm --image "$MOCK_IMAGE" --container-args "--http-listen-addr=0.0.0.0:8000 --model-name=dummy-model --num-tokens=2 --token-delay-ms=0" --inference-url /v1/chat/completions --inference-port 8000 --health-uri /health --health-port 8000 --health-timeout PT30S --function-type LLM --llm-model "name=dummy-model,uris=/v1/chat/completions|/v1/responses|/v1/embeddings,routingMethod=round_robin,tokenRateLimit=1000-S" """ Then the command exit code should be 0 diff --git a/tests/bdd/godog_test.go b/tests/bdd/godog_test.go index c5d2a0b5e..821e71ea4 100644 --- a/tests/bdd/godog_test.go +++ b/tests/bdd/godog_test.go @@ -831,6 +831,7 @@ func TestMultiClusterHelmfileFeatureFileWiresToSteps(t *testing.T) { t.Setenv("NGC_API_KEY", "test-key") t.Setenv("SAMPLE_NGC_ORG", "test-org") t.Setenv("SAMPLE_NGC_TEAM", "test-team") + t.Setenv("NVCF_BDD_MOCK_DYNAMO_TAG", "existing-manifest-tag") t.Setenv("NVCF_CLI", "/usr/bin/nvcf-cli") t.Setenv("REPO_ROOT", "/repo-root-placeholder") suite := newWiringSuite(t, newFakeRunner(map[string]harness.Result{