diff --git a/.github/workflows/chart-test.yml b/.github/workflows/chart-test.yml new file mode 100644 index 00000000..64825839 --- /dev/null +++ b/.github/workflows/chart-test.yml @@ -0,0 +1,44 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: chart-test + +on: + push: + branches: + - "main" + paths: + - "charts/**" + - ".github/workflows/chart-test.yml" + pull_request: + branches: [main] + paths: + - "charts/**" + - ".github/workflows/chart-test.yml" + workflow_dispatch: + +permissions: + contents: read + +jobs: + chart_test: + runs-on: ubuntu-22.04 + name: Chart lint and unit tests + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + # helm-lint already runs in test.yaml + - name: Run chart unit tests + run: make helm-test diff --git a/Makefile b/Makefile index 58fc5b4c..f2e4ca78 100644 --- a/Makefile +++ b/Makefile @@ -118,7 +118,7 @@ kind-logs: ./development/kind/run.sh -l kind-down: - kind delete cluster --name sam-kind + ./development/kind/run.sh -d .PHONY: kind-local-node kind-local-node: @@ -194,9 +194,21 @@ helm-lint: fi; \ $$HELM_BIN lint ./charts/sam-mesh +# render the chart to bin/chart/ for inspection; pass extra flags via ARGS, e.g. ARGS="--set gateway.enabled=true" +.PHONY: helm-template +helm-template: + rm -rf bin/chart + helm template sam-mesh ./charts/sam-mesh --output-dir bin/chart $(ARGS) + lint: fmt helm-lint hack/lint.sh +# fast chart template checks; no cluster needed +.PHONY: helm-test +helm-test: + @helm plugin list 2>/dev/null | grep -q '^unittest' || helm plugin install https://github.com/helm-unittest/helm-unittest + helm unittest charts/sam-mesh + .PHONY: verify verify: ./hack/verify-generated.sh diff --git a/charts/sam-mesh/README.md b/charts/sam-mesh/README.md index e271237b..600e7330 100644 --- a/charts/sam-mesh/README.md +++ b/charts/sam-mesh/README.md @@ -1,8 +1,8 @@ # sam-mesh Helm chart -Deploys a self-contained SAM mesh (control plane, router, console, and -optionally an in-cluster Postgres + Dex) for local development, testing, or -self-hosting your own hub. +Deploys a self-contained SAM mesh (control plane, router, console, and an +in-cluster Postgres) for local development, testing, or self-hosting your +own mesh. > For large-scale production deployments (GKE/EKS/AKS) using externally > managed Postgres/DNS/OIDC, see the @@ -12,9 +12,17 @@ self-hosting your own hub. ## Install ```bash -helm upgrade --install sam-mesh ./charts/sam-mesh --namespace sam --create-namespace +helm upgrade --install sam-mesh ./charts/sam-mesh --namespace sam --create-namespace \ + --set controlPlane.oidcIssuer= ``` +`controlPlane.oidcIssuer` is required: the chart bundles no identity +provider, and the control plane refuses to start without an issuer. Point it +at your own OIDC provider (Google, Okta, a Dex you run, the cluster's own +issuer for ServiceAccount Workload Identity Federation, …). The `kind` dev +environment (`make kind-up`) deploys its own throwaway Dex from +`development/kind/dex.yaml` and wires it in for you. + At the end of `helm install`/`helm upgrade`, the chart prints the exact `kubectl` command to retrieve your generated secrets (see below) — read the NOTES output before doing anything else. @@ -59,9 +67,53 @@ administrator to approve each enrollment via `/admin/enrollments` before a node can join — see the [Control Plane Configuration guide](https://sam-mesh.dev/docs/user/control-plane-configuration/#6-headless-node-enrollment-bootstrap-token-flow). -## Dex (`dex.enabled`) +## Gateway API (`gateway.enabled`) + +Disabled by default. When enabled the chart creates one `Gateway` fronting +the mesh, with one `HTTPRoute`. `gateway.className` is then **required**, +with no default, because the right GatewayClass is provider-specific +(`cloud-provider-kind` in kind, `gke-l7-global-external-managed` on GKE, +`istio`, `envoy-gateway`, …). + +The route exposes only the control plane's enrollment surface (`/register`, +`/info`, `/keys`, `/routers/lease`, `/policies`, `/enroll`, `/enroll/status`, +`/refresh`) and the console under `gateway.consolePath`; everything else, +including `/admin` and `/user`, is unrouted. `gateway.adminRoute: true` +additionally routes `/admin` — a dev convenience, leave it off in production. + +For the console, the bare prefix (`/console`) is answered with a 302 to +`/console/`, and a `URLRewrite` filter strips the prefix before the request +reaches the console. `URLRewrite` is **Extended** (not core) Gateway API +conformance, so the provider must support it. Set `gateway.consolePath: ""` +to leave the console unrouted. + +`listeners`, `hostnames`, `addresses` and `annotations` are passed through to +the Gateway API objects verbatim, so anything the spec allows is expressible. +They default to one plain-HTTP listener on port 80 matching every host, which +suits a local cluster. For example, on GKE: + +```yaml +gateway: + enabled: true + className: gke-l7-global-external-managed + listeners: + - name: https + protocol: HTTPS + port: 443 + tls: + certificateRefs: + - name: sam-mesh-tls + allowedRoutes: + namespaces: + from: Same + hostnames: [sam.example.com] + addresses: + - type: NamedAddress + value: sam-cp-ip +``` + +## OIDC login for the console -Disabled by default. The bundled Dex is only meant for local/dev OIDC login -(username/password test users); real deployments should point -`controlPlane.oidcIssuer` at your own identity provider instead of enabling -this. +There is no bundled Dex. Point `controlPlane.oidcIssuer` at your identity +provider and register `https:///auth/callback` as +a redirect URI for the OIDC client the control plane reports. diff --git a/charts/sam-mesh/templates/NOTES.txt b/charts/sam-mesh/templates/NOTES.txt index 62b01ef9..545eed5c 100644 --- a/charts/sam-mesh/templates/NOTES.txt +++ b/charts/sam-mesh/templates/NOTES.txt @@ -31,5 +31,13 @@ valid identity is enrolled automatically without manual admin approval. Set it to false for production deployments where you want to review enrollments. {{- end }} +{{- if and .Values.router.hostPort (gt (int .Values.router.replicaCount) 1) }} + +WARNING: router.hostPort={{ .Values.router.hostPort }} with replicaCount={{ .Values.router.replicaCount }}. +Each router binds that port on its node, so at most one router runs per node. +With fewer than {{ .Values.router.replicaCount }} schedulable nodes (after router.nodeSelector), the extra +pods stay Pending with "node(s) didn't have free ports". +{{- end }} + See charts/sam-mesh/README.md for more details on configuring OIDC, TLS, and production values. diff --git a/charts/sam-mesh/templates/bootstrap-job.yaml b/charts/sam-mesh/templates/bootstrap-job.yaml index b49f3238..1365c176 100644 --- a/charts/sam-mesh/templates/bootstrap-job.yaml +++ b/charts/sam-mesh/templates/bootstrap-job.yaml @@ -12,9 +12,15 @@ metadata: labels: {{- include "sam-mesh.labels" . | nindent 4 }} rules: +# resourceNames cannot restrict create (the name is unknown at admission), +# so it gets a bare rule; everything else is pinned to the router-token secret. - apiGroups: [""] resources: ["secrets"] - verbs: ["get", "create", "delete", "patch", "update"] + verbs: ["create"] +- apiGroups: [""] + resources: ["secrets"] + resourceNames: ["{{ include "sam-mesh.fullname" . }}-router-token"] + verbs: ["get", "patch", "update"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding @@ -49,10 +55,27 @@ spec: spec: serviceAccountName: {{ include "sam-mesh.fullname" . }}-bootstrap-sa restartPolicy: OnFailure + securityContext: + runAsNonRoot: true + # curlimages/curl's USER is non-numeric; runAsNonRoot needs an explicit uid. + runAsUser: 100 + seccompProfile: + type: RuntimeDefault containers: - name: bootstrap image: curlimages/curl:8.6.0 imagePullPolicy: {{ .Values.global.imagePullPolicy }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + cpu: 100m + memory: 64Mi env: - name: NAMESPACE value: {{ .Release.Namespace | quote }} @@ -81,9 +104,8 @@ spec: {{- $bindings := .Values.bootstrap.bindings -}} {{- if not $bindings -}} {{- $defaultRouterSA := printf "user:system:serviceaccount:%s:%s-router-sa" .Release.Namespace (include "sam-mesh.fullname" .) -}} - {{- $bindings = list + {{- $bindings = list (dict "role" "sam:role:router" "members" (list "group:routers" $defaultRouterSA)) - (dict "role" "sam-admin" "members" (list (printf "user:system:serviceaccount:%s:node-a-sa" .Release.Namespace) (printf "user:system:serviceaccount:%s:node-b-sa" .Release.Namespace) (printf "user:system:serviceaccount:%s:node-c-sa" .Release.Namespace) (printf "user:system:serviceaccount:%s:local-node-sa" .Release.Namespace))) (dict "role" "sam:role:sambox" "members" (list (printf "user:system:serviceaccount:%s:sam-box-sa" .Release.Namespace))) (dict "role" "sam:role:node" "members" (list "group:data-scientist" "group:users")) -}} @@ -96,7 +118,7 @@ spec: {"name": "sam-admin", "allowed_services": ["*"], "allowed_targets": ["*"]}, {"name": "sam:role:sambox", "allowed_services": ["*"], "allowed_targets": ["*"]}, {"name": "sam:role:router", "allowed_services": ["*"], "allowed_targets": ["*"]}, - {"name": "sam:role:node", "allowed_services": ["mcp://calculator", "mcp://db-agent", "mcp://http-tool", "mcp://stdio-tool", "system://sam.catalog"], "allowed_targets": ["*"]} + {"name": "sam:role:node", "allowed_services": {{ toJson .Values.bootstrap.nodeServices }}, "allowed_targets": ["*"]} ], "bindings": {{ toJson $bindings }} }' \ diff --git a/charts/sam-mesh/templates/console-deployment.yaml b/charts/sam-mesh/templates/console-deployment.yaml index c3f22ce3..8d77457b 100644 --- a/charts/sam-mesh/templates/console-deployment.yaml +++ b/charts/sam-mesh/templates/console-deployment.yaml @@ -12,13 +12,45 @@ spec: app: {{ include "sam-mesh.fullname" . }}-console template: metadata: + annotations: + # Secrets land as env vars, which are not live-reloaded; roll the pods on change. + checksum/secrets: {{ include (print $.Template.BasePath "/secrets.yaml") . | sha256sum }} labels: app: {{ include "sam-mesh.fullname" . }}-console spec: + {{- with .Values.console.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.console.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.console.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.console.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} initContainers: - name: wait-control-plane image: curlimages/curl:8.6.0 imagePullPolicy: {{ .Values.global.imagePullPolicy }} + securityContext: + # curlimages/curl's USER is non-numeric; runAsNonRoot needs an explicit uid. + runAsUser: 100 + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + cpu: 100m + memory: 64Mi command: ["/bin/sh", "-c"] args: - | @@ -30,8 +62,12 @@ spec: done containers: - name: sam-console - image: "{{ .Values.console.image.repository }}:{{ .Values.global.imageTag }}" + image: "{{ .Values.console.image.repository }}:{{ .Values.console.image.tag | default .Values.global.imageTag }}" imagePullPolicy: {{ .Values.global.imagePullPolicy }} + {{- with .Values.console.securityContext }} + securityContext: + {{- toYaml . | nindent 10 }} + {{- end }} env: - name: SAM_ADMIN_TOKEN valueFrom: diff --git a/charts/sam-mesh/templates/control-plane-deployment.yaml b/charts/sam-mesh/templates/control-plane-deployment.yaml index 9897cc55..96d76d8d 100644 --- a/charts/sam-mesh/templates/control-plane-deployment.yaml +++ b/charts/sam-mesh/templates/control-plane-deployment.yaml @@ -6,18 +6,49 @@ metadata: {{- include "sam-mesh.labels" . | nindent 4 }} spec: replicas: {{ .Values.controlPlane.replicaCount }} + {{- if .Values.controlPlane.hostPort }} + # hostPort: a surged pod can't bind a port the old pod still holds, so the + # default maxSurge=1 deadlocks the rollout on port-saturated nodes. + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + {{- end }} selector: matchLabels: app: {{ include "sam-mesh.fullname" . }}-control-plane template: metadata: + annotations: + # Secrets land as env vars, which are not live-reloaded; roll the pods on change. + checksum/secrets: {{ include (print $.Template.BasePath "/secrets.yaml") . | sha256sum }} labels: app: {{ include "sam-mesh.fullname" . }}-control-plane spec: + {{- with .Values.controlPlane.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.controlPlane.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.controlPlane.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.controlPlane.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} containers: - name: sam-control-plane - image: "{{ .Values.controlPlane.image.repository }}:{{ .Values.global.imageTag }}" + image: "{{ .Values.controlPlane.image.repository }}:{{ .Values.controlPlane.image.tag | default .Values.global.imageTag }}" imagePullPolicy: {{ .Values.global.imagePullPolicy }} + {{- with .Values.controlPlane.securityContext }} + securityContext: + {{- toYaml . | nindent 10 }} + {{- end }} env: - name: DB_PASSWORD valueFrom: diff --git a/charts/sam-mesh/templates/db-statefulset.yaml b/charts/sam-mesh/templates/db-statefulset.yaml index 9a585a8e..2e635355 100644 --- a/charts/sam-mesh/templates/db-statefulset.yaml +++ b/charts/sam-mesh/templates/db-statefulset.yaml @@ -16,6 +16,18 @@ spec: labels: app: {{ include "sam-mesh.fullname" . }}-db spec: + {{- with .Values.database.postgres.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.database.postgres.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.database.postgres.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} containers: - name: postgres image: "{{ .Values.database.postgres.image.repository }}:{{ .Values.database.postgres.image.tag }}" @@ -34,6 +46,14 @@ spec: - containerPort: 5432 name: postgres protocol: TCP + readinessProbe: + exec: + command: ["pg_isready", "-U", {{ .Values.database.postgres.user | quote }}, "-d", {{ .Values.database.postgres.database | quote }}] + periodSeconds: 5 + livenessProbe: + exec: + command: ["pg_isready", "-U", {{ .Values.database.postgres.user | quote }}, "-d", {{ .Values.database.postgres.database | quote }}] + periodSeconds: 15 resources: {{- toYaml .Values.database.postgres.resources | nindent 10 }} volumeMounts: diff --git a/charts/sam-mesh/templates/dex-deployment.yaml b/charts/sam-mesh/templates/dex-deployment.yaml deleted file mode 100644 index 8f6a65a7..00000000 --- a/charts/sam-mesh/templates/dex-deployment.yaml +++ /dev/null @@ -1,86 +0,0 @@ -{{- if .Values.dex.enabled -}} -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "sam-mesh.fullname" . }}-dex-config - labels: - {{- include "sam-mesh.labels" . | nindent 4 }} -data: - config.yaml: | - issuer: http://{{ include "sam-mesh.fullname" . }}-dex:5556/dex - storage: - type: memory - web: - http: 0.0.0.0:5556 - oauth2: - skipApprovalScreen: true - staticClients: - - id: sam-console - redirectURIs: - - http://localhost:9092/auth/callback - - http://127.0.0.1:9092/auth/callback - name: 'SAM Console' - public: true - enablePasswordDB: true - staticPasswords: - - email: admin@example.com - # bcrypt hash for 'password' - hash: "$2a$10$TUOgJVdtu0AL6Y6TcNBQXOSjRWRKhIkC0kXDzN9kOkPB4hg2aFPE2" - username: admin - userID: "0898fa90-3453-4311-8566-020953250212" ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "sam-mesh.fullname" . }}-dex - labels: - {{- include "sam-mesh.labels" . | nindent 4 }} -spec: - replicas: 1 - selector: - matchLabels: - app: {{ include "sam-mesh.fullname" . }}-dex - template: - metadata: - labels: - app: {{ include "sam-mesh.fullname" . }}-dex - spec: - nodeSelector: - sam-role: control-plane - containers: - - name: dex - image: "{{ .Values.dex.image.repository }}:{{ .Values.dex.image.tag }}" - imagePullPolicy: {{ .Values.global.imagePullPolicy }} - args: - - dex - - serve - - /etc/dex/cfg/config.yaml - ports: - - containerPort: 5556 - name: http - volumeMounts: - - name: config - mountPath: /etc/dex/cfg - volumes: - - name: config - configMap: - name: {{ include "sam-mesh.fullname" . }}-dex-config ---- -apiVersion: v1 -kind: Service -metadata: - name: {{ include "sam-mesh.fullname" . }}-dex - labels: - {{- include "sam-mesh.labels" . | nindent 4 }} -spec: - type: NodePort - selector: - app: {{ include "sam-mesh.fullname" . }}-dex - ports: - - name: http - port: 5556 - targetPort: 5556 - {{- if .Values.dex.nodePort }} - nodePort: {{ .Values.dex.nodePort }} - {{- end }} -{{- end }} diff --git a/charts/sam-mesh/templates/gateway.yaml b/charts/sam-mesh/templates/gateway.yaml new file mode 100644 index 00000000..8d6105e3 --- /dev/null +++ b/charts/sam-mesh/templates/gateway.yaml @@ -0,0 +1,102 @@ +{{- if .Values.gateway.enabled -}} +{{- $fullName := include "sam-mesh.fullname" . -}} +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: {{ $fullName }}-gateway + labels: + {{- include "sam-mesh.labels" . | nindent 4 }} + {{- with .Values.gateway.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + gatewayClassName: {{ required "gateway.className is required when gateway.enabled is true" .Values.gateway.className }} + {{- with .Values.gateway.addresses }} + addresses: + {{- toYaml . | nindent 2 }} + {{- end }} + listeners: + {{- toYaml .Values.gateway.listeners | nindent 2 }} +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: {{ $fullName }}-route + labels: + {{- include "sam-mesh.labels" . | nindent 4 }} +spec: + parentRefs: + - name: {{ $fullName }}-gateway + {{- with .Values.gateway.hostnames }} + hostnames: + {{- toYaml . | nindent 2 }} + {{- end }} + rules: + # The enrollment surface, matching the cloud's allow-list exactly. + - matches: + - path: + type: Exact + value: /register + - path: + type: Exact + value: /info + - path: + type: Exact + value: /keys + - path: + type: Exact + value: /routers/lease + - path: + type: Exact + value: /policies + - path: + type: Exact + value: /enroll + - path: + type: Exact + value: /enroll/status + - path: + type: Exact + value: /refresh + backendRefs: + - name: {{ $fullName }}-control-plane + port: {{ .Values.controlPlane.service.port }} + {{- if .Values.gateway.adminRoute }} + - matches: + - path: + type: PathPrefix + value: /admin + backendRefs: + - name: {{ $fullName }}-control-plane + port: {{ .Values.controlPlane.service.port }} + {{- end }} + {{- if and .Values.console.enabled .Values.gateway.consolePath }} + # The cloud deployment's console shape: the bare prefix 302s to /, and + # URLRewrite (Extended conformance) strips it before the console sees the request. + - matches: + - path: + type: Exact + value: {{ .Values.gateway.consolePath }} + filters: + - type: RequestRedirect + requestRedirect: + path: + type: ReplaceFullPath + replaceFullPath: {{ .Values.gateway.consolePath }}/ + statusCode: 302 + - matches: + - path: + type: PathPrefix + value: {{ .Values.gateway.consolePath }}/ + filters: + - type: URLRewrite + urlRewrite: + path: + type: ReplacePrefixMatch + replacePrefixMatch: / + backendRefs: + - name: {{ $fullName }}-console + port: {{ .Values.console.service.port }} + {{- end }} +{{- end }} diff --git a/charts/sam-mesh/templates/router-service.yaml b/charts/sam-mesh/templates/router-service.yaml index 0735681b..708fc098 100644 --- a/charts/sam-mesh/templates/router-service.yaml +++ b/charts/sam-mesh/templates/router-service.yaml @@ -11,9 +11,6 @@ spec: - name: p2p-tcp port: {{ .Values.router.service.port }} targetPort: 4501 - {{- if and (eq .Values.router.service.type "NodePort") .Values.router.service.nodePort }} - nodePort: {{ .Values.router.service.nodePort }} - {{- end }} protocol: TCP - name: p2p-udp port: {{ .Values.router.service.port }} diff --git a/charts/sam-mesh/templates/router-statefulset.yaml b/charts/sam-mesh/templates/router-statefulset.yaml index 94def123..b4c51f39 100644 --- a/charts/sam-mesh/templates/router-statefulset.yaml +++ b/charts/sam-mesh/templates/router-statefulset.yaml @@ -24,6 +24,18 @@ spec: app: {{ include "sam-mesh.fullname" . }}-router spec: serviceAccountName: {{ include "sam-mesh.fullname" . }}-router-sa + {{- with .Values.router.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.router.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.router.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.router.podSecurityContext }} securityContext: {{- toYaml . | nindent 8 }} @@ -32,6 +44,19 @@ spec: - name: wait-control-plane image: curlimages/curl:8.6.0 imagePullPolicy: {{ .Values.global.imagePullPolicy }} + securityContext: + # curlimages/curl's USER is non-numeric; runAsNonRoot needs an explicit uid. + runAsUser: 100 + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + cpu: 100m + memory: 64Mi command: ["/bin/sh", "-c"] args: - | @@ -43,8 +68,12 @@ spec: done containers: - name: sam-router - image: "{{ .Values.router.image.repository }}:{{ .Values.global.imageTag }}" + image: "{{ .Values.router.image.repository }}:{{ .Values.router.image.tag | default .Values.global.imageTag }}" imagePullPolicy: {{ .Values.global.imagePullPolicy }} + {{- with .Values.router.securityContext }} + securityContext: + {{- toYaml . | nindent 10 }} + {{- end }} ports: - containerPort: 4501 name: p2p-tcp @@ -76,6 +105,13 @@ spec: name: {{ include "sam-mesh.fullname" . }}-router-token key: token {{- end }} + {{- if .Values.router.hostPort }} + # kubelet expands $(HOST_IP) in the args below. + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- end }} args: - "--control-plane=http://{{ include "sam-mesh.fullname" . }}-control-plane:{{ .Values.controlPlane.service.port }}" - "--listen=/ip4/0.0.0.0/tcp/4501" @@ -84,11 +120,16 @@ spec: {{- range .Values.router.externalAddrs }} - "--external-addr={{ . }}" {{- end }} + {{- else if .Values.router.hostPort }} + - "--external-addr=/ip4/$(HOST_IP)/tcp/{{ .Values.router.hostPort }}" + - "--external-addr=/ip4/$(HOST_IP)/udp/{{ .Values.router.hostPort }}/quic-v1" {{- else }} - - "--external-addr=/dns4/{{ include "sam-mesh.fullname" . }}-router/tcp/4501" + - "--external-addr=/dns4/{{ include "sam-mesh.fullname" . }}-router/tcp/{{ .Values.router.service.port }}" {{- end }} - "--keys-path=/data/router.key" + {{- if .Values.router.allowLoopback }} - "--allow-loopback" + {{- end }} - "--log-level={{ .Values.router.logLevel }}" {{- if .Values.router.useOidcToken }} - "--jwt-path=/var/run/secrets/tokens/sam-token" @@ -103,10 +144,8 @@ spec: mountPath: /var/run/secrets/tokens readOnly: true {{- end }} - volumes: - - name: router-data - emptyDir: {} {{- if .Values.router.useOidcToken }} + volumes: - name: sam-token projected: sources: @@ -115,4 +154,15 @@ spec: expirationSeconds: 3600 audience: "sam-control-plane-audience" {{- end }} + volumeClaimTemplates: + - metadata: + name: router-data + spec: + accessModes: [ "ReadWriteOnce" ] + {{- if .Values.router.storageClass }} + storageClassName: {{ .Values.router.storageClass | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.router.storageSize | quote }} {{- end }} diff --git a/charts/sam-mesh/tests/bootstrap-job_test.yaml b/charts/sam-mesh/tests/bootstrap-job_test.yaml new file mode 100644 index 00000000..621e0dc7 --- /dev/null +++ b/charts/sam-mesh/tests/bootstrap-job_test.yaml @@ -0,0 +1,34 @@ +suite: bootstrap job +templates: + - templates/bootstrap-job.yaml +release: + name: sam-mesh +tests: + - it: role only touches the router-token secret + documentSelector: + path: kind + value: Role + asserts: + - equal: + path: rules[0].verbs + value: [create] + - equal: + path: rules[1].resourceNames + value: [sam-mesh-router-token] + - equal: + path: rules[1].verbs + value: [get, patch, update] + - lengthEqual: + path: rules + count: 2 + + - it: job pod runs as the numeric curl uid with resources set + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: spec.template.spec.securityContext.runAsUser + value: 100 + - exists: + path: spec.template.spec.containers[0].resources.limits diff --git a/charts/sam-mesh/tests/console-deployment_test.yaml b/charts/sam-mesh/tests/console-deployment_test.yaml new file mode 100644 index 00000000..4b4c7426 --- /dev/null +++ b/charts/sam-mesh/tests/console-deployment_test.yaml @@ -0,0 +1,35 @@ +suite: console deployment +templates: + # secrets.yaml must be loaded for the checksum/secrets include to resolve + - templates/console-deployment.yaml + - templates/secrets.yaml +release: + name: sam-mesh +tests: + - it: pins the numeric uids for the app and the curl init container + template: templates/console-deployment.yaml + asserts: + - equal: + path: spec.template.spec.securityContext.runAsUser + value: 65532 + - equal: + path: spec.template.spec.initContainers[0].securityContext.runAsUser + value: 100 + + - it: rolls the pods when the shared secret changes + template: templates/console-deployment.yaml + set: + controlPlane.adminToken: aaa + database.postgres.password: p + asserts: + - matchRegex: + path: spec.template.metadata.annotations["checksum/secrets"] + pattern: ^[0-9a-f]{64}$ + + - it: console.enabled=false renders nothing + template: templates/console-deployment.yaml + set: + console.enabled: false + asserts: + - hasDocuments: + count: 0 diff --git a/charts/sam-mesh/tests/control-plane-deployment_test.yaml b/charts/sam-mesh/tests/control-plane-deployment_test.yaml new file mode 100644 index 00000000..238f010b --- /dev/null +++ b/charts/sam-mesh/tests/control-plane-deployment_test.yaml @@ -0,0 +1,77 @@ +suite: control plane deployment +templates: + # secrets.yaml must be loaded for the checksum/secrets include to resolve + - templates/control-plane-deployment.yaml + - templates/secrets.yaml +release: + name: sam-mesh +tests: + - it: pins the numeric distroless uid so runAsNonRoot is verifiable + template: templates/control-plane-deployment.yaml + asserts: + - equal: + path: spec.template.spec.securityContext.runAsNonRoot + value: true + - equal: + path: spec.template.spec.securityContext.runAsUser + value: 65532 + + - it: rolls the pods when the shared secret changes + template: templates/control-plane-deployment.yaml + set: + controlPlane.adminToken: aaa + database.postgres.password: p + asserts: + - matchRegex: + path: spec.template.metadata.annotations["checksum/secrets"] + pattern: ^[0-9a-f]{64}$ + + - it: hostPort disables surge so rollouts can rebind the node port + template: templates/control-plane-deployment.yaml + set: + controlPlane.hostPort: 8080 + asserts: + - equal: + path: spec.strategy.rollingUpdate.maxSurge + value: 0 + - equal: + path: spec.strategy.rollingUpdate.maxUnavailable + value: 1 + + - it: no strategy override without hostPort + template: templates/control-plane-deployment.yaml + asserts: + - notExists: + path: spec.strategy + + - it: uses the default image tag when none is pinned + template: templates/control-plane-deployment.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: sam-control-plane:local + + - it: external postgres points the DSN at the configured host + template: templates/control-plane-deployment.yaml + set: + database.postgres.deployInternal: false + database.postgres.host: ext-db + asserts: + - matchRegex: + path: spec.template.spec.containers[0].args[2] + pattern: "@ext-db:5432" + + - it: scheduling knobs pass through to the pod spec + template: templates/control-plane-deployment.yaml + set: + controlPlane.nodeSelector.disk: ssd + controlPlane.tolerations: + - key: dedicated + asserts: + - equal: + path: spec.template.spec.nodeSelector.disk + value: ssd + - contains: + path: spec.template.spec.tolerations + content: + key: dedicated diff --git a/charts/sam-mesh/tests/db-statefulset_test.yaml b/charts/sam-mesh/tests/db-statefulset_test.yaml new file mode 100644 index 00000000..7d3090e7 --- /dev/null +++ b/charts/sam-mesh/tests/db-statefulset_test.yaml @@ -0,0 +1,37 @@ +suite: db statefulset +templates: + - templates/db-statefulset.yaml +release: + name: sam-mesh +tests: + - it: probes postgres with pg_isready + documentSelector: + path: kind + value: StatefulSet + asserts: + - equal: + path: spec.template.spec.containers[0].readinessProbe.exec.command[0] + value: pg_isready + - equal: + path: spec.template.spec.containers[0].livenessProbe.exec.command[0] + value: pg_isready + + - it: scheduling knobs pass through to the pod spec + set: + database.postgres.tolerations: + - key: dedicated + documentSelector: + path: kind + value: StatefulSet + asserts: + - contains: + path: spec.template.spec.tolerations + content: + key: dedicated + + - it: external postgres renders nothing + set: + database.postgres.deployInternal: false + asserts: + - hasDocuments: + count: 0 diff --git a/charts/sam-mesh/tests/gateway_test.yaml b/charts/sam-mesh/tests/gateway_test.yaml new file mode 100644 index 00000000..23b69ced --- /dev/null +++ b/charts/sam-mesh/tests/gateway_test.yaml @@ -0,0 +1,62 @@ +suite: gateway +templates: + - templates/gateway.yaml +release: + name: sam-mesh +tests: + - it: disabled by default renders nothing + asserts: + - hasDocuments: + count: 0 + + - it: enabling it requires gateway.className + set: + gateway.enabled: true + asserts: + - failedTemplate: + errorPattern: gateway.className is required when gateway.enabled is true + + - it: routes enrollment plus the console redirect and rewrite by default + set: + gateway.enabled: true + gateway.className: x + documentSelector: + path: kind + value: HTTPRoute + asserts: + # enrollment surface + consolePath 302 + consolePath URLRewrite + - lengthEqual: + path: spec.rules + count: 3 + + - it: empty consolePath leaves the console unrouted + set: + gateway.enabled: true + gateway.className: x + gateway.consolePath: "" + documentSelector: + path: kind + value: HTTPRoute + asserts: + - lengthEqual: + path: spec.rules + count: 1 + + - it: adminRoute adds the /admin rule + set: + gateway.enabled: true + gateway.className: x + gateway.adminRoute: true + documentSelector: + path: kind + value: HTTPRoute + asserts: + - lengthEqual: + path: spec.rules + count: 4 + - contains: + path: spec.rules[1].matches + content: + path: + type: PathPrefix + value: /admin diff --git a/charts/sam-mesh/tests/router-statefulset_test.yaml b/charts/sam-mesh/tests/router-statefulset_test.yaml new file mode 100644 index 00000000..a29bc394 --- /dev/null +++ b/charts/sam-mesh/tests/router-statefulset_test.yaml @@ -0,0 +1,133 @@ +suite: router statefulset +templates: + - templates/router-statefulset.yaml +release: + # collapses fullname to "sam-mesh" (see _helpers.tpl) + name: sam-mesh +tests: + - it: pins the numeric distroless uid so runAsNonRoot is verifiable + documentSelector: + path: kind + value: StatefulSet + asserts: + - equal: + path: spec.template.spec.securityContext.runAsNonRoot + value: true + - equal: + path: spec.template.spec.securityContext.runAsUser + value: 65532 + - equal: + path: spec.template.spec.initContainers[0].securityContext.runAsUser + value: 100 + + - it: announces the service DNS address when hostPort is unset + documentSelector: + path: kind + value: StatefulSet + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --external-addr=/dns4/sam-mesh-router/tcp/4501 + - notExists: + path: spec.template.spec.containers[0].ports[0].hostPort + + - it: announced DNS address follows router.service.port + set: + router.service.port: 9999 + documentSelector: + path: kind + value: StatefulSet + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --external-addr=/dns4/sam-mesh-router/tcp/9999 + + - it: hostPort binds both protocols and announces the node IP + set: + router.hostPort: 4501 + documentSelector: + path: kind + value: StatefulSet + asserts: + - equal: + path: spec.template.spec.containers[0].ports[0].hostPort + value: 4501 + - equal: + path: spec.template.spec.containers[0].ports[1].hostPort + value: 4501 + - contains: + path: spec.template.spec.containers[0].args + content: --external-addr=/ip4/$(HOST_IP)/tcp/4501 + - contains: + path: spec.template.spec.containers[0].args + content: --external-addr=/ip4/$(HOST_IP)/udp/4501/quic-v1 + + - it: externalAddrs overrides the derived announcement + set: + router.hostPort: 4501 + router.externalAddrs: + - /dns4/x/tcp/1 + documentSelector: + path: kind + value: StatefulSet + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --external-addr=/dns4/x/tcp/1 + - notContains: + path: spec.template.spec.containers[0].args + content: --external-addr=/ip4/$(HOST_IP)/tcp/4501 + + - it: keeps the key on a PVC, not emptyDir + documentSelector: + path: kind + value: StatefulSet + asserts: + - equal: + path: spec.volumeClaimTemplates[0].metadata.name + value: router-data + - equal: + path: spec.volumeClaimTemplates[0].spec.resources.requests.storage + value: 1Gi + - notExists: + path: spec.volumeClaimTemplates[0].spec.storageClassName + - notExists: + path: spec.template.spec.volumes + + - it: storageClass sets the PVC storageClassName + set: + router.storageClass: fast-ssd + documentSelector: + path: kind + value: StatefulSet + asserts: + - equal: + path: spec.volumeClaimTemplates[0].spec.storageClassName + value: fast-ssd + + - it: useOidcToken swaps the bootstrap token for a projected JWT + set: + router.useOidcToken: true + documentSelector: + path: kind + value: StatefulSet + asserts: + - contains: + path: spec.template.spec.containers[0].args + content: --jwt-path=/var/run/secrets/tokens/sam-token + - notContains: + path: spec.template.spec.containers[0].args + content: --bootstrap-token=$(BOOTSTRAP_TOKEN) + - exists: + path: spec.template.spec.volumes[0].projected + + - it: image.tag overrides global.imageTag for the router only + set: + router.image.tag: v9 + documentSelector: + path: kind + value: StatefulSet + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: sam-router:v9 diff --git a/charts/sam-mesh/values.yaml b/charts/sam-mesh/values.yaml index 96386274..4ae7afc2 100644 --- a/charts/sam-mesh/values.yaml +++ b/charts/sam-mesh/values.yaml @@ -2,10 +2,34 @@ global: imageTag: local imagePullPolicy: IfNotPresent +# One Gateway fronting the mesh: the control plane's enrollment surface plus the console +# under gateway.consolePath. +gateway: + enabled: false + # Required when enabled. No default: the GatewayClass is provider-specific, e.g. + # cloud-provider-kind in kind, gke-l7-global-external-managed on GKE. + className: "" + # Routes the control plane's /admin API. Dev only. + adminRoute: false + # External prefix the console is served under. Empty leaves the console unrouted. + consolePath: /console + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: Same + hostnames: [] + addresses: [] + annotations: {} + controlPlane: replicaCount: 2 image: repository: sam-control-plane + # Overrides global.imageTag for this image only. + tag: "" logLevel: info # Leave empty to auto-generate a random admin token on first install (kept # stable across upgrades). Set explicitly (e.g. via --set) to pin a known value. @@ -14,8 +38,24 @@ controlPlane: # Only skips TLS verification for the OIDC issuer; leave false unless # pointing at a local/dev issuer without a valid certificate. insecureSkipTlsVerify: false - oidcIssuer: "http://sam-mesh-dex:5556/dex" + # Required: the chart bundles no identity provider. Point it at your OIDC issuer + # (comma-separated for more than one); the control plane refuses to start without one. + oidcIssuer: "" allowedAudiences: "sam-mesh-audience,sam-control-plane-audience" + nodeSelector: {} + tolerations: [] + affinity: {} + podSecurityContext: + runAsNonRoot: true + # Dockerfiles say "USER nonroot" by name; runAsNonRoot needs the numeric uid. + runAsUser: 65532 + runAsGroup: 65532 + seccompProfile: + type: RuntimeDefault + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] resources: requests: cpu: 100m @@ -43,6 +83,9 @@ database: port: 5432 sslmode: disable storageSize: 1Gi + nodeSelector: {} + tolerations: [] + affinity: {} resources: requests: cpu: 100m @@ -55,12 +98,37 @@ router: enabled: true replicaCount: 1 podSecurityContext: + runAsNonRoot: true + # Dockerfiles say "USER nonroot" by name; runAsNonRoot needs the numeric uid. + runAsUser: 65532 + runAsGroup: 65532 + seccompProfile: + type: RuntimeDefault fsGroup: 65532 fsGroupChangePolicy: "OnRootMismatch" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] image: repository: sam-router + # Overrides global.imageTag for this image only. + tag: "" useOidcToken: false logLevel: info + # Binds 4501 TCP+UDP on the router's own node and announces /ip4//... for both + # transports. One router per node. + hostPort: null + nodeSelector: {} + tolerations: [] + affinity: {} + # Overrides the announced multiaddrs; derived from hostPort when left empty. + externalAddrs: [] + allowLoopback: false + # PVC for /data/router.key, so the libp2p peer ID survives rescheduling. + storageSize: 1Gi + # PVC StorageClass; cluster default when null. + storageClass: null resources: requests: cpu: 100m @@ -71,13 +139,28 @@ router: service: type: ClusterIP port: 4501 - nodePort: null console: enabled: true replicaCount: 1 image: repository: sam-console + # Overrides global.imageTag for this image only. + tag: "" + nodeSelector: {} + tolerations: [] + affinity: {} + podSecurityContext: + runAsNonRoot: true + # Dockerfiles say "USER nonroot" by name; runAsNonRoot needs the numeric uid. + runAsUser: 65532 + runAsGroup: 65532 + seccompProfile: + type: RuntimeDefault + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] resources: requests: cpu: 50m @@ -90,20 +173,7 @@ console: port: 8081 nodePort: null -dex: - enabled: false - nodePort: 30556 - image: - repository: dexidp/dex - tag: v2.39.0 - google: - clientId: "" - clientSecret: "" - github: - clientId: "" - clientSecret: "" - cliOAuthSecret: "" - bootstrap: bindings: [] - + # Services sam:role:node may call + nodeServices: ["*"] diff --git a/development/examples/calc-mcp/sam-node-config.yaml b/development/examples/calc-mcp/sam-node-config.yaml index fad29c57..1abd60e8 100644 --- a/development/examples/calc-mcp/sam-node-config.yaml +++ b/development/examples/calc-mcp/sam-node-config.yaml @@ -1,8 +1,6 @@ version: "v1alpha1" attenuation: - # Allow cross-node MCP only when the caller's biscuit carries a - # node($peer) fact — i.e. it was issued by our control plane upon enrollment. - policies: + policies: [] services: - type: "mcp" name: "calculator" diff --git a/development/examples/everything-mcp/sam-node-config.yaml b/development/examples/everything-mcp/sam-node-config.yaml index af7e33f7..3107e5df 100644 --- a/development/examples/everything-mcp/sam-node-config.yaml +++ b/development/examples/everything-mcp/sam-node-config.yaml @@ -1,8 +1,6 @@ version: "v1alpha1" attenuation: - policies: - - 'allow if service("mcp", "everything");' - - 'allow if service("system", "sam.catalog");' + policies: [] services: - type: "mcp" name: "everything" diff --git a/development/examples/greeter-mcp/requirements.txt b/development/examples/greeter-mcp/requirements.txt index c413b608..7ba9cdc1 100644 --- a/development/examples/greeter-mcp/requirements.txt +++ b/development/examples/greeter-mcp/requirements.txt @@ -1 +1 @@ -mcp>=1.0.0 +mcp[cli]>=1.0.0,<2.0.0 diff --git a/development/examples/greeter-mcp/sam-node-config.yaml b/development/examples/greeter-mcp/sam-node-config.yaml index 4b52f7b2..12722585 100644 --- a/development/examples/greeter-mcp/sam-node-config.yaml +++ b/development/examples/greeter-mcp/sam-node-config.yaml @@ -1,6 +1,6 @@ version: "v1alpha1" attenuation: - policies: + policies: [] services: - type: "mcp" name: "greeter" diff --git a/development/kind/dex.yaml b/development/kind/dex.yaml new file mode 100644 index 00000000..b068d31b --- /dev/null +++ b/development/kind/dex.yaml @@ -0,0 +1,107 @@ +# Dev-only Dex for the kind mesh. Dex is an independent component, so it lives here, not +# in the Helm chart. run.sh applies this with envsubst, filling ${DEX_ISSUER} and +# ${CONSOLE_REDIRECT_URI} once the gateway addresses exist. +apiVersion: v1 +kind: ConfigMap +metadata: + name: sam-mesh-dex-config +data: + config.yaml: | + issuer: ${DEX_ISSUER} + storage: + type: memory + web: + http: 0.0.0.0:5556 + oauth2: + skipApprovalScreen: true + staticClients: + - id: sam-console + redirectURIs: + - ${CONSOLE_REDIRECT_URI} + name: 'SAM Console' + public: true + enablePasswordDB: true + staticPasswords: + # The control plane auto-registers a first-time OIDC login with the "user" role, so + # this account is not an admin however it is named. + - email: user@example.com + # bcrypt hash for 'password' + hash: "$2a$10$TUOgJVdtu0AL6Y6TcNBQXOSjRWRKhIkC0kXDzN9kOkPB4hg2aFPE2" + username: user + userID: "0898fa90-3453-4311-8566-020953250212" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: sam-mesh-dex +spec: + replicas: 1 + selector: + matchLabels: + app: sam-mesh-dex + template: + metadata: + labels: + app: sam-mesh-dex + spec: + nodeSelector: + sam-role: control-plane + containers: + - name: dex + image: dexidp/dex:v2.39.0 + args: + - dex + - serve + - /etc/dex/cfg/config.yaml + ports: + - containerPort: 5556 + name: http + volumeMounts: + - name: config + mountPath: /etc/dex/cfg + volumes: + - name: config + configMap: + name: sam-mesh-dex-config +--- +apiVersion: v1 +kind: Service +metadata: + name: sam-mesh-dex +spec: + selector: + app: sam-mesh-dex + ports: + - name: http + port: 5556 + targetPort: 5556 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: sam-mesh-dex-gateway +spec: + gatewayClassName: cloud-provider-kind + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: Same +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: sam-mesh-dex-route +spec: + parentRefs: + - name: sam-mesh-dex-gateway + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - name: sam-mesh-dex + port: 5556 diff --git a/development/kind/kind-config.yaml b/development/kind/kind-config.yaml index 58f08211..80906475 100644 --- a/development/kind/kind-config.yaml +++ b/development/kind/kind-config.yaml @@ -2,21 +2,6 @@ kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 nodes: - role: control-plane - # Expose the control plane and router to the host (via the sam-control-plane-nodeport NodePort) so a - # locally-built sam-node can enroll: host:9090->HTTP, host:4001->libp2p TCP. - extraPortMappings: - - containerPort: 30090 - hostPort: 9090 - protocol: TCP - - containerPort: 30401 - hostPort: 4001 - protocol: TCP - - containerPort: 30081 - hostPort: 9092 - protocol: TCP - - containerPort: 30556 - hostPort: 5556 - protocol: TCP - role: worker labels: sam-role: control-plane @@ -26,12 +11,3 @@ nodes: - role: worker labels: sam-role: node-b - - role: worker - labels: - sam-role: node-c - - role: worker - labels: - sam-role: node-d - - role: worker - labels: - sam-role: node-e diff --git a/development/kind/mesh-config.e2e.yaml b/development/kind/mesh-config.e2e.yaml index 3f8bf904..86753d6c 100644 --- a/development/kind/mesh-config.e2e.yaml +++ b/development/kind/mesh-config.e2e.yaml @@ -2,6 +2,3 @@ # mcp://calculator/add. Select it with MESH_CONFIG=development/kind/mesh-config.e2e.yaml. node-a: node-b: calc-mcp -node-c: -node-d: -node-e: diff --git a/development/kind/mesh-config.yaml b/development/kind/mesh-config.yaml index 6f6e4587..58430ebb 100644 --- a/development/kind/mesh-config.yaml +++ b/development/kind/mesh-config.yaml @@ -1,11 +1,8 @@ # node -> service. A blank value means a bare node (no service, e.g. a caller). # Set a value to host a service on that node; the value is a folder path under # development/examples/, e.g: -# node-b: calc-mcp -# node-c: code-reviewer-pool/reviewer +# node-a: calc-mcp +# node-b: code-reviewer-pool/reviewer # All nodes ship bare by default — assign services to suit what you're testing. node-a: node-b: -node-c: -node-d: -node-e: diff --git a/development/kind/run-local-node.sh b/development/kind/run-local-node.sh index f56a31a2..469d746b 100755 --- a/development/kind/run-local-node.sh +++ b/development/kind/run-local-node.sh @@ -1,8 +1,7 @@ #!/usr/bin/env bash -# Enroll a locally-built ./bin/sam-node into the kind mesh for local testing. -# The control plane (exposed via sam-control-plane-nodeport + kind extraPortMappings) is reached at -# 127.0.0.1:9090 (HTTP enroll) and 127.0.0.1:4001 (libp2p TCP); the node relays -# peer traffic through the router. Extra args pass through, e.g. to host a service: +# Enroll a locally-built ./bin/sam-node into the kind mesh the way a real external node joins: +# a bootstrap token over the control plane's gateway address, peer traffic through the router's +# node-IP multiaddrs. Extra args pass through, e.g. to host a service: # ARGS="--config development/examples/calc-mcp/sam-node-config.yaml" set -euo pipefail @@ -14,25 +13,38 @@ PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" cd "${PROJECT_ROOT}" # Prereqs -for bin in kubectl kind; do +for bin in kubectl kind curl jq; do command -v "$bin" >/dev/null 2>&1 || { echo "missing prerequisite: $bin" >&2; exit 1; } done kind get clusters 2>/dev/null | grep -qx "${CLUSTER}" || { echo "kind cluster '${CLUSTER}' not found; start it first: make kind-up" >&2; exit 1; } [[ -x ./bin/sam-node ]] || { echo "./bin/sam-node not found; build it first: make build" >&2; exit 1; } -# Identity + token -kubectl --context "${KCTX}" -n "${NAMESPACE}" create serviceaccount local-node-sa \ - >/dev/null 2>&1 || true -JWT="$(kubectl --context "${KCTX}" -n "${NAMESPACE}" create token local-node-sa \ - --audience=sam-mesh-audience --duration=1h)" +MAIN_IP="$(kubectl --context "${KCTX}" -n "${NAMESPACE}" get gateway sam-mesh-gateway \ + -o jsonpath='{.status.addresses[0].value}' 2>/dev/null || true)" +[[ -n "${MAIN_IP}" ]] || { + echo "gateway 'sam-mesh-gateway' has no LoadBalancer address; is cloud-provider-kind running?" >&2 + exit 1; } +CONTROL_PLANE_URL="http://${MAIN_IP}" -echo "Enrolling local ./bin/sam-node into the mesh control plane at http://127.0.0.1:9090…" +# Mint an enrollment credential the way an operator would, through the dev-routed /admin API. +ADMIN_TOKEN="$(kubectl --context "${KCTX}" -n "${NAMESPACE}" get secret sam-mesh-secrets \ + -o jsonpath='{.data.admin-token}' | base64 -d)" +TOKEN_RESPONSE="$(curl -s -X POST "${CONTROL_PLANE_URL}/admin/bootstrap-tokens" \ + -H "Authorization: Bearer ${ADMIN_TOKEN}" \ + -H "Content-Type: application/json" \ + -d '{"role":"sam:role:node","ttl_hours":1,"max_usages":1,"description":"local sam-node"}' || true)" +BOOTSTRAP_TOKEN="$(printf '%s' "${TOKEN_RESPONSE}" | jq -r '.token // empty' 2>/dev/null || true)" +[[ -n "${BOOTSTRAP_TOKEN}" ]] || { + echo "could not mint a bootstrap token at ${CONTROL_PLANE_URL}/admin/bootstrap-tokens: ${TOKEN_RESPONSE}" >&2 + exit 1; } + +echo "Enrolling local ./bin/sam-node into the mesh control plane at ${CONTROL_PLANE_URL}…" echo " MCP/sidecar API on 127.0.0.1:9099" export SAM_API_TOKEN=devtoken exec ./bin/sam-node run \ - --control-plane http://127.0.0.1:9090 \ - --jwt "${JWT}" \ + --control-plane "${CONTROL_PLANE_URL}" \ + --bootstrap-token "${BOOTSTRAP_TOKEN}" \ --listen /ip4/0.0.0.0/tcp/0 \ --bind-addr 127.0.0.1:9099 \ --discovery-interval 200ms \ diff --git a/development/kind/run.sh b/development/kind/run.sh index 3712b28a..e8f1e178 100755 --- a/development/kind/run.sh +++ b/development/kind/run.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash -# Kind dev mesh: a control plane and router plus the nodes declared in mesh-config.yaml, each pinned -# to its own k8s node, with live per-pod logs in named tmux panes. +# Kind dev mesh: a control plane and router plus the nodes from mesh-config.yaml, each pinned to its +# own k8s node, with live per-pod logs in named tmux panes. Control plane, console and Dex are reached +# over Gateway API LoadBalancer addresses from cloud-provider-kind (started here); the router at its node IP. set -euo pipefail CLUSTER="sam-kind" @@ -10,6 +11,11 @@ KCTX="kind-${CLUSTER}" IMAGE_TAG="local" CONTROL_PLANE_URL="http://sam-mesh-control-plane:8080" HELM="helm" +# Serves the gateway LoadBalancer IPs. It installs the Gateway API CRDs and the +# cloud-provider-kind GatewayClass itself, so the cluster needs no CRD step. +CPK_CONTAINER="cloud-provider-kind" +CPK_IMAGE="registry.k8s.io/cloud-provider-kind/cloud-controller-manager:v0.11.1" +CONSOLE_BASE_PATH="/console" @@ -39,6 +45,77 @@ check_prereqs() { fi } +# cloud-provider-kind names each gateway's envoy container after a hash of the gateway, so +# one left behind by a deleted cluster is adopted by the next run and reported Programmed +# while its config stream is dead and it serves nothing. +remove_lb_containers() { + local containers + containers="$(docker ps -aq -f "label=io.x-k8s.cloud-provider-kind.cluster=${CLUSTER}")" + if [[ -n "${containers}" ]]; then + # shellcheck disable=SC2086 # one container id per argument + docker rm -f ${containers} >/dev/null + fi +} + +# Must run after the cluster, so the 'kind' docker network exists. +start_cloud_provider_kind() { + remove_lb_containers + if [[ -n "$(docker ps -q -f "name=^${CPK_CONTAINER}$")" ]]; then + echo "cloud-provider-kind already running" + return + fi + docker rm -f "${CPK_CONTAINER}" >/dev/null 2>&1 || true + docker run -d --name "${CPK_CONTAINER}" --network kind \ + -v /var/run/docker.sock:/var/run/docker.sock "${CPK_IMAGE}" >/dev/null +} + +# Stop cloud-provider-kind before its containers, or it recreates them, and remove them before +# deleting the cluster, or deleting it makes cloud-provider-kind race us to the same containers. +teardown() { + docker rm -f "${CPK_CONTAINER}" >/dev/null 2>&1 || true + remove_lb_containers + kind delete cluster --name "${CLUSTER}" +} + +# gateway_ip : the LoadBalancer address cloud-provider-kind assigned, waited for. +gateway_ip() { + local gateway="$1" ip="" + for _ in $(seq 1 60); do + ip="$(kubectl --context "${KCTX}" -n "${NAMESPACE}" get gateway "${gateway}" \ + -o jsonpath='{.status.addresses[0].value}' 2>/dev/null || true)" + [[ -n "$ip" ]] && { echo "$ip"; return 0; } + sleep 2 + done + echo "gateway '${gateway}' never got a LoadBalancer address; is cloud-provider-kind running?" >&2 + return 1 +} + +# deploy_chart [extra --set flags]: later flags win, so phase 2 overrides phase 1. +# TLS verification is skipped because one issuer is the kind cluster's own API server, +# served with a self-signed cert. +deploy_chart() { + "${HELM}" --kube-context "${KCTX}" upgrade --install sam-mesh "${PROJECT_ROOT}/charts/sam-mesh" --timeout 10m \ + --namespace "${NAMESPACE}" \ + --set global.imageTag="${IMAGE_TAG}" \ + --set controlPlane.oidcIssuer="${CONTROL_PLANE_ISSUERS//,/\\,}" \ + --set controlPlane.allowedAudiences="${ALLOWED_AUDIENCES//,/\\,}" \ + --set controlPlane.insecureSkipTlsVerify=true \ + --set gateway.enabled=true \ + --set gateway.className=cloud-provider-kind \ + --set gateway.adminRoute=true \ + --set router.hostPort=4501 \ + --set router.nodeSelector.sam-role=control-plane \ + --set router.useOidcToken=true \ + "$@" +} + +# apply_dex : Dex is an independent component, deployed +# from dex.yaml rather than the chart, with the URLs of the moment substituted in. +apply_dex() { + DEX_ISSUER="$1" CONSOLE_REDIRECT_URI="$2" \ + envsubst '${DEX_ISSUER} ${CONSOLE_REDIRECT_URI}' < "${SCRIPT_DIR}/dex.yaml" \ + | kubectl --context "${KCTX}" -n "${NAMESPACE}" apply -f - +} render_and_apply() { local node="$1" svc="$2" @@ -107,8 +184,8 @@ read_mesh_nodes() { ### MAIN ### -if [[ $# -gt 0 && "$1" != "-s" && "$1" != "-l" ]]; then - echo "usage: $(basename "$0") [-s]" >&2 +if [[ $# -gt 0 && "$1" != "-s" && "$1" != "-l" && "$1" != "-d" ]]; then + echo "usage: $(basename "$0") [-s|-l|-d]" >&2 exit 1 fi @@ -118,11 +195,19 @@ if [[ "${1:-}" == "-l" ]]; then exit 0 fi +if [[ "${1:-}" == "-d" ]]; then + teardown + exit 0 +fi + check_prereqs "${1:-}" echo "== Creating kind cluster '${CLUSTER}' ==" kind create cluster --name "${CLUSTER}" --config "${SCRIPT_DIR}/kind-config.yaml" +echo "== Starting cloud-provider-kind ==" +start_cloud_provider_kind + echo "== Building sam images ==" make docker-build-control-plane docker-build-router docker-build-node docker-build-sam-console echo "== Loading sam images into kind ==" @@ -134,48 +219,68 @@ read_mesh_nodes ISSUER="$(kubectl --context "${KCTX}" get --raw /.well-known/openid-configuration | jq -r .issuer)" [[ -n "$ISSUER" ]] || { echo "could not determine cluster OIDC issuer" >&2; exit 1; } -OIDC_ISSUER="${OIDC_ISSUER:-http://sam-mesh-dex:5556/dex}" -OIDC_CLIENT_ID="${OIDC_CLIENT_ID:-sam-console}" -OIDC_CLIENT_SECRET="${OIDC_CLIENT_SECRET:-}" -OIDC_REDIRECT_URL="${OIDC_REDIRECT_URL:-}" - +# Dex's own address isn't known until its gateway has one, so phase 1 trusts only the +# cluster issuer and phase 2 prepends Dex. CONTROL_PLANE_ISSUERS="${ISSUER}" -if [[ -n "${OIDC_ISSUER}" ]]; then - CONTROL_PLANE_ISSUERS="${OIDC_ISSUER},${ISSUER}" -fi -ALLOWED_AUDIENCES="sam-mesh-audience,sam-control-plane-audience" -if [[ -n "${OIDC_CLIENT_ID}" ]]; then - ALLOWED_AUDIENCES="${OIDC_CLIENT_ID},${ALLOWED_AUDIENCES}" -fi +# The first audience is what the control plane reports as the OIDC client id, so it has to +# match Dex's static client. +ALLOWED_AUDIENCES="sam-console,sam-mesh-audience,sam-control-plane-audience" -export NAMESPACE ISSUER IMAGE_TAG OIDC_ISSUER OIDC_CLIENT_ID OIDC_CLIENT_SECRET OIDC_REDIRECT_URL CONTROL_PLANE_ISSUERS ALLOWED_AUDIENCES +# Only the node template's envsubst reads these from the environment. +export NAMESPACE IMAGE_TAG echo "== Applying namespace and RBAC cluster rules ==" envsubst '${NAMESPACE}' < "${SCRIPT_DIR}/00-namespace-rbac.yaml" | kubectl --context "${KCTX}" apply -f - echo "== Deploying SAM Mesh via Helm ==" -"${HELM}" --kube-context "${KCTX}" upgrade --install sam-mesh "${PROJECT_ROOT}/charts/sam-mesh" --timeout 10m \ - --namespace "${NAMESPACE}" \ - --set global.imageTag="${IMAGE_TAG}" \ - --set controlPlane.oidcIssuer="${CONTROL_PLANE_ISSUERS//,/\\,}" \ - --set controlPlane.allowedAudiences="${ALLOWED_AUDIENCES//,/\\,}" \ - --set controlPlane.service.type=NodePort \ - --set controlPlane.service.nodePort=30090 \ - --set router.service.type=NodePort \ - --set router.service.nodePort=30401 \ - --set router.useOidcToken=true \ - --set router.externalAddrs[0]="/dns4/sam-mesh-router/tcp/4501" \ - --set router.externalAddrs[1]="/ip4/127.0.0.1/tcp/4001" \ - --set console.service.type=NodePort \ - --set console.service.nodePort=30081 \ - --set dex.enabled=true \ - --set controlPlane.insecureSkipTlsVerify=true # ISSUER is the kind cluster's own API server, served with a self-signed cert +deploy_chart + +# The real URLs aren't known until Dex's gateway exists, so deploy it with placeholders +# first and rewire it below. +echo "== Deploying Dex ==" +apply_dex "http://sam-mesh-dex:5556/dex" "http://127.0.0.1${CONSOLE_BASE_PATH}/auth/callback" + +# The OIDC URLs are the gateway addresses, which only exist once the gateways do — so +# resolve them, then redeploy with the URLs everything must agree on. +echo "== Waiting for gateway LoadBalancer addresses ==" +MAIN_IP="$(gateway_ip sam-mesh-gateway)" +DEX_IP="$(gateway_ip sam-mesh-dex-gateway)" +CONSOLE_URL="http://${MAIN_IP}${CONSOLE_BASE_PATH}/" +echo "control plane: http://${MAIN_IP} console: ${CONSOLE_URL} dex: http://${DEX_IP}" + +OIDC_ISSUER="http://${DEX_IP}/dex" + +echo "== Wiring the OIDC URLs into Dex ==" +apply_dex "${OIDC_ISSUER}" "http://${MAIN_IP}${CONSOLE_BASE_PATH}/auth/callback" +# A config-only change doesn't roll the pods, and Dex reads its config at startup. +kubectl --context "${KCTX}" -n "${NAMESPACE}" rollout restart deployment/sam-mesh-dex +kubectl --context "${KCTX}" -n "${NAMESPACE}" rollout status deployment/sam-mesh-dex --timeout=180s + +# The control plane discovers the issuer at startup and refuses to start if it can't, so +# prove a pod can reach the gateway address before pinning the mesh to it. +echo "== Checking Dex discovery from inside the cluster ==" +# A pod left behind by an interrupted run fails the next one with AlreadyExists, which +# reads exactly like a routing failure. +kubectl --context "${KCTX}" -n "${NAMESPACE}" delete pod dex-discovery-check --ignore-not-found >/dev/null +kubectl --context "${KCTX}" -n "${NAMESPACE}" run dex-discovery-check \ + --rm -i --restart=Never --quiet --image=curlimages/curl:8.6.0 -- \ + curl -sf --retry 10 --retry-delay 2 --retry-connrefused --connect-timeout 5 --max-time 20 \ + "${OIDC_ISSUER}/.well-known/openid-configuration" >/dev/null || { + echo "cannot reach ${OIDC_ISSUER} from inside the cluster; pods must be able to route to the cloud-provider-kind LoadBalancer addresses" >&2 + exit 1 + } + +echo "== Wiring the OIDC URLs into the control plane ==" +CONTROL_PLANE_ISSUERS="${OIDC_ISSUER},${ISSUER}" +deploy_chart echo "== Waiting for database to be ready ==" kubectl --context "${KCTX}" -n "${NAMESPACE}" wait --for=condition=ready --timeout=180s pod -l app=sam-mesh-db echo "== Waiting for control plane to be ready ==" -kubectl --context "${KCTX}" -n "${NAMESPACE}" wait --for=condition=available --timeout=180s deployment/sam-mesh-control-plane +# rollout status, not wait --for=available: every pod must serve the Dex issuer before +# the console reads it below. +kubectl --context "${KCTX}" -n "${NAMESPACE}" rollout status deployment/sam-mesh-control-plane --timeout=180s # Policy seeding is handled automatically by the Helm chart bootstrap job, we just wait for it to complete. echo "== Waiting for bootstrap job to complete ==" @@ -183,10 +288,12 @@ kubectl --context "${KCTX}" -n "${NAMESPACE}" wait --for=condition=complete --ti echo "== Waiting for router to be ready ==" kubectl --context "${KCTX}" -n "${NAMESPACE}" wait --for=condition=ready --timeout=180s pod -l app=sam-mesh-router -echo "== Waiting for console to be ready ==" -kubectl --context "${KCTX}" -n "${NAMESPACE}" wait --for=condition=available --timeout=180s deployment/sam-mesh-console -echo "== Waiting for Dex to be ready ==" -kubectl --context "${KCTX}" -n "${NAMESPACE}" wait --for=condition=available --timeout=180s deployment/sam-mesh-dex + +# The console discovers the issuer from the control plane's /info once, at startup, so +# restart it now that the control plane serves the Dex issuer. +echo "== Restarting the console with the final issuer ==" +kubectl --context "${KCTX}" -n "${NAMESPACE}" rollout restart deployment/sam-mesh-console +kubectl --context "${KCTX}" -n "${NAMESPACE}" rollout status deployment/sam-mesh-console --timeout=180s echo "== Applying sam-nodes ==" @@ -201,13 +308,16 @@ for node in "${NODES[@]}"; do done echo -echo "Mesh up. To call a node's MCP API, port-forward it in another shell, e.g.:" -echo " kubectl --context ${KCTX} -n ${NAMESPACE} port-forward deploy/node-a 9091:8080" -echo "then:" -echo " ./bin/mcp-client -url http://127.0.0.1:9091/mcp -token devtoken -tool find_remote_tools -args '{}'" -echo "" -echo "You can access the SAM Web Console at:" -echo " http://localhost:9092/" +echo "Mesh up." +echo " console: ${CONSOLE_URL}" +echo " control plane: http://${MAIN_IP}" +echo " dex: ${OIDC_ISSUER}" +echo +echo "To drive the mesh, enroll a local node in another shell (it stays in the foreground):" +echo " make build && make kind-local-node" +echo "then call its MCP API on 127.0.0.1:9099:" +echo " ./bin/mcp-client -url http://127.0.0.1:9099/mcp -token devtoken -tool get_mesh_info -args '{}'" +echo " ./bin/mcp-client -url http://127.0.0.1:9099/mcp -token devtoken -tool find_remote_tools -args '{}'" if [[ "${1:-}" != "-s" ]]; then show_cluster_logs diff --git a/site/content/docs/development/kubernetes-deployment.md b/site/content/docs/development/kubernetes-deployment.md index 19f682ee..9f36f6ac 100644 --- a/site/content/docs/development/kubernetes-deployment.md +++ b/site/content/docs/development/kubernetes-deployment.md @@ -2,7 +2,7 @@ title: "Kubernetes Deployment and Local Testing Guide" linkTitle: "Kubernetes Deployment and Local Testing Guide" --- -This guide explains how to deploy the SAM control plane and router in a Kubernetes cluster and how to test it locally with `kind` — using the bundled `make kind-*` targets for a one-command mesh, or a manual setup with `cloud-provider-kind`. +This guide explains how to deploy the SAM control plane and router in a Kubernetes cluster and how to test it locally with `kind` — using the bundled `make kind-*` targets for a one-command mesh, or a manual setup. Both paths need `cloud-provider-kind`. > [!TIP] > This guide focuses on local development sandboxing. For production-grade Kubernetes deployments (GKE, EKS, AKS), see the [Production Kubernetes Deployment](../../user/kubernetes-deployment/) guide. @@ -19,19 +19,26 @@ The repository ships a one-command local mesh under `development/kind/`, driven make kind-up ``` -This creates a `sam-kind` cluster (one control-plane plus workers for the control plane, router, and `node-a` through `node-e`), builds the `sam-control-plane:local`, `sam-router:local`, and `sam-node:local` images, loads them into the cluster, and deploys: +This creates a `sam-kind` cluster (one control-plane plus three workers — one labeled `sam-role: control-plane`, and one each for `node-a` and `node-b`), builds the `sam-control-plane:local`, `sam-router:local`, and `sam-node:local` images, loads them into the cluster, and deploys: - The **control plane**, configured to trust the cluster's own OIDC issuer. -- Five **nodes** declared in `development/kind/mesh-config.yaml` (`node-a` through `node-e`), all **bare** by default — assign services to suit what you're testing. +- Two **nodes** declared in `development/kind/mesh-config.yaml` (`node-a` and `node-b`), both **bare** by default — assign services to suit what you're testing. -Nodes authenticate to the control plane via **Workload Identity Federation** (projected ServiceAccount tokens), so no static secrets or mock OIDC provider are needed. The control plane is exposed to the host on `127.0.0.1:9090` (HTTP enroll) and `127.0.0.1:4001` (libp2p) via a NodePort and the cluster's `extraPortMappings` — `cloud-provider-kind` is not required. +In-cluster nodes authenticate to the control plane via **Workload Identity Federation** (projected ServiceAccount tokens), so no static secrets or mock OIDC provider are needed. + +The mesh is exposed through Gateway API LoadBalancer addresses, so there are no port-forwards and no `extraPortMappings`. `run.sh` runs **`cloud-provider-kind`** as a container (a hard prerequisite — it needs the docker socket) to serve those addresses, and prints them when the mesh is up: + +- The **control plane** on its own address, routing the 8 exact enrollment paths, plus a dev-only `/admin` route. +- The **console** at `/console/` on that same address, shaped like the production deployment: `/console` 302s to `/console/`, and a `URLRewrite` filter strips the prefix before the console sees the request. +- **Dex** on its own address, deployed from `development/kind/dex.yaml` rather than the chart — Dex is an independent component the chart no longer bundles. +- The **router** at its own node's IP on port 4501, TCP **and** QUIC, announced from `status.hostIP`. Once everything is up, `make kind-up` opens a tmux session with live per-pod logs (control plane, router and each node in its own pane). Manage the mesh with: ```bash make kind-up ARGS=-s # bring the mesh up without attaching the log view make kind-logs # (re)attach the live-logs tmux session -make kind-down # delete the sam-kind cluster +make kind-down # delete the sam-kind cluster and stop cloud-provider-kind ``` ### Mesh Layout (`mesh-config.yaml`) @@ -40,17 +47,16 @@ The nodes that make up the dev mesh are declared in `development/kind/mesh-confi ```yaml # node -> service. A blank value means a bare node (no service, e.g. a caller). -# The service value is a folder path under development/examples/, e.g: -# node-b: calc-mcp -# node-c: code-reviewer-pool/reviewer +# Set a value to host a service on that node; the value is a folder path under +# development/examples/, e.g: +# node-a: calc-mcp +# node-b: code-reviewer-pool/reviewer +# All nodes ship bare by default — assign services to suit what you're testing. node-a: node-b: -node-c: -node-d: -node-e: ``` -- The key is the node's name. The cluster currently ships with a control plane and router plus these **five** agent nodes, all bare by default; each is pinned to a matching worker via the `sam-role` labels in `kind-config.yaml`. +- The key is the node's name. The cluster currently ships with a control plane and router plus these **two** agent nodes, both bare by default; each is pinned to a matching worker via the `sam-role` labels in `kind-config.yaml`. - A **blank** value is a bare node — a `sam-node` with no local service, useful as a caller/consumer. - A **non-blank** value is a folder name under `development/examples/`. That service is built and deployed as a **sidecar** next to the node, and the node is configured to advertise it to the mesh. @@ -75,12 +81,12 @@ A service is any backend a node advertises to the mesh. Its kind is set by the ` ``` The sidecar and `sam-node` share the pod's network, so `target_url` is always `127.0.0.1:`, where `` matches the port your service listens on. -2. **Assign it to a node** in `mesh-config.yaml` — set the value on any free node slot (`node-a` through `node-e`): +2. **Assign it to a node** in `mesh-config.yaml` — set the value on a free node slot (`node-a` or `node-b`): ```yaml node-a: my-mcp ``` > [!NOTE] - > There are five node slots because `kind-config.yaml` defines five workers labeled `sam-role: node-a|node-b|node-c|node-d|node-e`. To host more than five services at once, add a matching labeled worker there too. + > There are two node slots because `kind-config.yaml` defines two workers labeled `sam-role: node-a|node-b`. To host more than two services at once, add a matching labeled worker there too. 3. **Recreate the cluster** so the new service is built and deployed: ```bash @@ -104,7 +110,7 @@ make build # produce ./bin/sam-node make kind-local-node ``` -This mints a ServiceAccount token and runs `./bin/sam-node` against the control plane at `127.0.0.1:9090`, exposing its MCP API on `127.0.0.1:9099` with the API token `devtoken`. Extra flags pass through via `ARGS`, e.g. to host an example service: +This mints a bootstrap token through the control plane's `/admin` API and runs `./bin/sam-node` against the control plane's gateway address — the same credential and path a real external node uses — exposing its MCP API on `127.0.0.1:9099` with the API token `devtoken`. Extra flags pass through via `ARGS`, e.g. to host an example service: ```bash make kind-local-node ARGS="--config development/examples/calc-mcp/sam-node-config.yaml" diff --git a/site/content/docs/use-cases/gemini-buddy.md b/site/content/docs/use-cases/gemini-buddy.md index dbb8abed..b309328b 100644 --- a/site/content/docs/use-cases/gemini-buddy.md +++ b/site/content/docs/use-cases/gemini-buddy.md @@ -86,9 +86,6 @@ Host the buddy on one node in `development/kind/mesh-config.yaml`: ```yaml node-a: # bare node (orchestrator entry) node-b: gemini-buddy-mcp # the buddy -node-c: -node-d: -node-e: ``` ### 3. Bring the mesh up and start a local orchestrator node diff --git a/site/content/docs/use-cases/warm-agent-pool.md b/site/content/docs/use-cases/warm-agent-pool.md index 0f9461e0..f191bd7e 100644 --- a/site/content/docs/use-cases/warm-agent-pool.md +++ b/site/content/docs/use-cases/warm-agent-pool.md @@ -108,7 +108,23 @@ before building (a free key is fine for the demo). ### 2. Mesh layout -The layout ships in `development/kind/mesh-config.yaml`: +This use case needs five node slots, and the cluster ships with two (`node-a` +and `node-b`). Add the three missing workers to +`development/kind/kind-config.yaml` first: + +```yaml + - role: worker + labels: + sam-role: node-c + - role: worker + labels: + sam-role: node-d + - role: worker + labels: + sam-role: node-e +``` + +Then write this layout into `development/kind/mesh-config.yaml`: ```yaml node-a: # bare node (orchestrator entry) diff --git a/tests/e2e/lib/container_mesh.bash b/tests/e2e/lib/container_mesh.bash index 8f066084..baf633fb 100644 --- a/tests/e2e/lib/container_mesh.bash +++ b/tests/e2e/lib/container_mesh.bash @@ -363,19 +363,31 @@ if [[ -z "${MESH_HELPERS_LOADED:-}" ]]; then fi fi - "${helm_bin}" --kube-context="${KUBECONTEXT}" upgrade --install sam ./charts/sam-mesh \ - --namespace default \ - --set fullnameOverride="sam" \ - --set global.imageTag="local" \ - --set controlPlane.oidcIssuer="${ISSUERS//,/\\,}" \ - --set controlPlane.allowedAudiences="sam-mesh-audience\,sam-control-plane-audience" \ - --set controlPlane.insecureSkipTlsVerify=true \ - --set controlPlane.adminToken="super-secret-admin-token" \ - --set controlPlane.replicaCount=2 \ - --set controlPlane.hostPort=8080 \ - --set router.useOidcToken=false \ - --set router.hostPort=4501 \ + # router.externalAddrs: nodes resolve sam-router per-network via --add-host, so the router must + # announce that name. Announcing a node IP instead is unroutable from an isolated test network. + # bootstrap.nodeServices: policy.bats asserts the control plane denies an ungranted service, so + # this lane pins the grants it tests rather than inheriting the chart default. + local helm_args=(--kube-context="${KUBECONTEXT}" upgrade --install sam ./charts/sam-mesh + --namespace default + --set fullnameOverride="sam" + --set global.imageTag="local" + --set controlPlane.oidcIssuer="${ISSUERS//,/\\,}" + --set controlPlane.allowedAudiences="sam-mesh-audience\,sam-control-plane-audience" + --set controlPlane.insecureSkipTlsVerify=true + --set controlPlane.adminToken="super-secret-admin-token" + --set controlPlane.replicaCount=2 + --set controlPlane.hostPort=8080 + --set router.useOidcToken=false + --set router.hostPort=4501 --set console.enabled=false + --set 'router.externalAddrs={/dns4/sam-router/tcp/4501}' + --set 'bootstrap.nodeServices={mcp://calculator,mcp://db-agent,mcp://http-tool,mcp://stdio-tool,system://sam.catalog}') + if ! "${helm_bin}" "${helm_args[@]}"; then + # The reused cluster may hold StatefulSets whose immutable spec (e.g. + # volumeClaimTemplates) changed; drop them (PVCs survive) and retry. + kubectl --context="${KUBECONTEXT}" delete statefulset sam-router sam-db --ignore-not-found + "${helm_bin}" "${helm_args[@]}" + fi mesh_wait_for_rollout statefulset/sam-db mesh_wait_for_rollout deployment/sam-control-plane