From 317ba8894a43bf9645173d4e681de02e009b75e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9C=88=E5=8F=AF?= Date: Tue, 10 Nov 2026 15:01:44 +0800 Subject: [PATCH 1/4] fix(deploy): verify Dragonfly Registry access after rollouts --- .github/workflows/ci.yml | 1 + deploy/README.md | 5 + deploy/registry-connectivity/.dockerignore | 5 + deploy/registry-connectivity/Dockerfile | 7 + deploy/registry-connectivity/README.md | 141 ++++++++++ deploy/registry-connectivity/chart/Chart.yaml | 5 + .../chart/templates/resources.yaml | 187 ++++++++++++ .../registry-connectivity/chart/values.yaml | 37 +++ deploy/registry-connectivity/check.py | 266 ++++++++++++++++++ deploy/registry-connectivity/requirements.txt | 15 + .../registry-connectivity/tests/test_chart.py | 71 +++++ .../registry-connectivity/tests/test_check.py | 204 ++++++++++++++ 12 files changed, 944 insertions(+) create mode 100644 deploy/registry-connectivity/.dockerignore create mode 100644 deploy/registry-connectivity/Dockerfile create mode 100644 deploy/registry-connectivity/README.md create mode 100644 deploy/registry-connectivity/chart/Chart.yaml create mode 100644 deploy/registry-connectivity/chart/templates/resources.yaml create mode 100644 deploy/registry-connectivity/chart/values.yaml create mode 100644 deploy/registry-connectivity/check.py create mode 100644 deploy/registry-connectivity/requirements.txt create mode 100644 deploy/registry-connectivity/tests/test_chart.py create mode 100644 deploy/registry-connectivity/tests/test_check.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 66215efb..f39f7c0c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -193,6 +193,7 @@ jobs: python deploy/tests/kubernetes/test_node_lifecycle.py -v python deploy/tests/kubernetes/test_runc_filesystems.py -v python deploy/akernel/charts/core/tests/test_adx_chart.py -v + python -m unittest discover -s deploy/registry-connectivity/tests -v python deploy/akernel/charts/core/tests/test_node_directories.py -v python deploy/akernel/charts/core/tests/test_shared_checkpoint_storage.py -v python deploy/akernel/charts/core/tests/test_prepare_checkpoint_storage.py -v diff --git a/deploy/README.md b/deploy/README.md index 7d640bd2..05ccae05 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -17,6 +17,11 @@ when a shared filesystem is independently required. For one deployment serving both native and E2B SDKs with version-independent application logs, see [the dual SDK/logging profile](docs/dual-sdk-and-application-logs.md). +For private Registries serving Nydus through Dragonfly, install the +[Registry connectivity guard](registry-connectivity/README.md). It maintains +namespace-and-Pod ingress selectors and verifies all five seeds after Registry +moves or rollouts without changing Registry placement or storage. + ## Aggregate sandbox memory budget The direct Helm chart enables the cgroup v2 aggregate sandbox memory policy by diff --git a/deploy/registry-connectivity/.dockerignore b/deploy/registry-connectivity/.dockerignore new file mode 100644 index 00000000..6647548a --- /dev/null +++ b/deploy/registry-connectivity/.dockerignore @@ -0,0 +1,5 @@ +__pycache__ +**/__pycache__ +tests +chart +README.md diff --git a/deploy/registry-connectivity/Dockerfile b/deploy/registry-connectivity/Dockerfile new file mode 100644 index 00000000..037d0c42 --- /dev/null +++ b/deploy/registry-connectivity/Dockerfile @@ -0,0 +1,7 @@ +FROM python:3.13-slim@sha256:e48f59cdba69a3cbd9ed2cf0f348d90d45a8d350d98134cf1070bc106ee9ed90 +WORKDIR /app +COPY --chmod=0444 requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt +COPY --chmod=0444 check.py ./check.py +USER 65532:65532 +ENTRYPOINT ["python", "/app/check.py"] diff --git a/deploy/registry-connectivity/README.md b/deploy/registry-connectivity/README.md new file mode 100644 index 00000000..930fc3e7 --- /dev/null +++ b/deploy/registry-connectivity/README.md @@ -0,0 +1,141 @@ +# Registry connectivity guard + +Install this companion Helm release alongside a private Registry that uses +Bearer token authentication and serves Nydus images. It does not own or change +the Registry Deployment, storage, resources or placement. Keep it installed +when moving or upgrading the Registry. + +## Network boundary + +The NetworkPolicy selects Registry Pods and permits TCP 5000 (Registry) and +5001 (token authentication) from **one peer containing both selectors**: + +- namespace label `kubernetes.io/metadata.name: dragonfly-system`; +- Pod labels `app: dragonfly` and `component: seed-client`. + +These selectors are an AND, not separate namespace-wide / Pod-wide allowances. +Existing ingress policies remain additive. CIDRs alone are insufficient when +the CNI enforces source Pod identity across node pools. Change the selectors +and container ports to match the deployed services; service-facing ports may +differ from container ports. This policy grants no access to unrelated +Registry management ports. It does not configure source-side egress policies. + +## Automatic verification + +A CronJob checks Registry generation, ready Pod UIDs, container identities and +restart counts, plus the identities of the five named seed Pods, every minute. +A change triggers a full check after the rollout becomes ready. An unchanged +deployment is checked again every ten minutes. A change during verification +invalidates the result and is retried. Missing/unready seeds fail the check. + +Each of the five seeds must: + +1. Receive an unauthenticated Registry Bearer challenge. +2. Obtain a scoped pull token from an explicitly trusted token-service origin. +3. Fetch the configured immutable manifest and verify its SHA-256. +4. Fetch its actual Nydus bootstrap layer directly from the Registry; verify + digest and size, and read the regular `image/image.boot` tar member. +5. Read the same layer through the seed's local Dragonfly HTTP proxy and verify + identical content. + +Only a complete five-seed success advances `-state`. A failed Job +logs redacted per-seed results and exits nonzero. Subsequent scheduled Jobs +retry; this is a post-rollout check, not a Registry readiness gate or automatic +rollback. Monitor failed Jobs and the age of the state ConfigMap's `checked_at`. +History keeps two successful and three failed Jobs; completed Jobs expire after +one day. A successful `unchanged` Job alone is not a new network verification. + +The checker uses the Kubernetes API and execs only in the five configured seed +Pods. Seeds require `sh`, `head`, `curl` and `base64`, but do not require Python. +No seed files or process state are changed. Secret contents, Basic credentials +and Bearer tokens pass through exec stdin, never command arguments or logs. +The checker ServiceAccount cannot read Secrets. It mounts one existing Secret +containing a pull-only credential. Its exec permission is restricted to the +five Pod names; assess that permission as part of the deployment trust boundary. + +## Prepare the checker and probe + +Build `Dockerfile` with an approved remote BuildKit builder and publish the +checker image; pin its manifest digest in Helm values. The client running +Buildx must reach both the exporter Registry and its token-service realm. +Forwarding only BuildKit gRPC does not provide token-service connectivity. +Do not build images on an operator workstation when remote builds are required. + +Create a small real Nydus probe using the deployed Nydus-capable BuildKit: + +```dockerfile +FROM scratch +COPY probe.txt /probe.txt +``` + +Use its native image exporter with `oci-mediatypes=true,compression=nydus` and +publish to a dedicated probe repository. Pin a **single-platform Nydus manifest +digest**, not an index. Retain that manifest, bootstrap and referenced blobs; +exclude this fixture from task-image garbage collection. The checker does not +create artifacts, retain task images, or modify Registry garbage collection. +Registry backup/restore must preserve this fixture and its pull authorization. + +Provision a Secret in the Registry namespace through the platform's Secret +manager, with a `credentials.json` key containing `username` and `password`. +Use a pull-only identity, preferably restricted to the probe repository. Never +put credentials in Helm values, command arguments or version control. The +checker does not need the token signer's private key or a push credential. + +## Deploy + +Supply operator-owned values (keep private addresses out of Git): + +```yaml +image: registry.example/operations/registry-connectivity@sha256: +registry: + deployment: registry + podSelector: {app: managed-registry} + url: https://registry.example + allowedAuthOrigins: [https://registry-auth.example] + probeRepository: operations/nydus-connectivity + probeDigest: sha256: + pullSecret: registry-connectivity-pull +# Defaults select the five dragonfly-seed-client-{0..4} Pods. +``` + +Configure `imagePullSecrets`, placement and tolerations as needed. Registry +and token-service certificate validation uses curl's native trust in each seed; +private HTTPS endpoints need their trusted CA installed there. Plain HTTP is +supported for deployments whose network/security policy permits it. The +checker never adds an insecure TLS flag. + +```bash +helm upgrade --install registry-connectivity ./deploy/registry-connectivity/chart \ + --namespace -f /secure/operator-values.yaml +kubectl -n create job \ + --from=cronjob/registry-connectivity registry-connectivity-initial +kubectl -n logs job/registry-connectivity-initial +``` + +If a policy with the configured name already exists, compare its full spec +before adopting that policy into this release. Do not adopt or replace the +Registry Deployment/PVC as part of installing the guard. Preserve other +operators' placement and resource settings. + +After moving or rolling the Registry, inspect the next Job and confirm a new +`verified` event, **five** `passed: true` results, matching bootstrap hashes and +a fresh `checked_at` in `registry-connectivity-state`. To force an immediate +recheck without restarting production services: + +```bash +kubectl -n create job \ + --from=cronjob/registry-connectivity registry-connectivity-manual +# A periodic check runs automatically even if no Registry identity changed. +# To bypass the ten-minute reuse window, add FORCE_CHECK=1 to a copy of the +# CronJob's Job template before creating that manual Job. +``` + +Validate changes with: + +```bash +python3 -m unittest discover -s deploy/registry-connectivity/tests -v +``` + +Also run `helm lint` with complete operator/example values and review the +rendered policy. Test a rollout trigger using a disposable watched Deployment +rather than restarting a shared Registry solely to exercise the checker. diff --git a/deploy/registry-connectivity/chart/Chart.yaml b/deploy/registry-connectivity/chart/Chart.yaml new file mode 100644 index 00000000..9f39482e --- /dev/null +++ b/deploy/registry-connectivity/chart/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +name: registry-connectivity +version: 0.1.0 +description: Restricted Dragonfly Registry ingress and authenticated Nydus rollout checks +type: application diff --git a/deploy/registry-connectivity/chart/templates/resources.yaml b/deploy/registry-connectivity/chart/templates/resources.yaml new file mode 100644 index 00000000..08ec748e --- /dev/null +++ b/deploy/registry-connectivity/chart/templates/resources.yaml @@ -0,0 +1,187 @@ +{{- $image := required "image must specify the checker image digest" .Values.image -}} +{{- if not (regexMatch "@sha256:[0-9a-f]{64}$" $image) -}} +{{- fail "image must be pinned by sha256 digest" -}} +{{- end -}} +{{- if or (ne (len .Values.dragonfly.seedNames) 5) (ne (len (uniq .Values.dragonfly.seedNames)) 5) -}} +{{- fail "exactly five distinct Dragonfly seedNames are required" -}} +{{- end -}} +{{- if or (empty .Values.dragonfly.podSelector) (empty .Values.registry.podSelector) (empty .Values.registry.allowedAuthOrigins) -}} +{{- fail "nonempty Registry/Dragonfly Pod selectors and allowedAuthOrigins are required" -}} +{{- end }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ .Values.networkPolicy.name }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: registry-connectivity + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/part-of: akernel +spec: + podSelector: + matchLabels: + {{- toYaml .Values.registry.podSelector | nindent 6 }} + policyTypes: [Ingress] + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ .Values.dragonfly.namespace | quote }} + podSelector: + matchLabels: + {{- toYaml .Values.dragonfly.podSelector | nindent 14 }} + ports: + {{- range .Values.registry.ports }} + - protocol: TCP + port: {{ . }} + {{- end }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} +rules: + - apiGroups: [apps] + resources: [deployments] + resourceNames: [{{ .Values.registry.deployment | quote }}] + verbs: [get] + - apiGroups: [''] + resources: [pods] + verbs: [get, list] + - apiGroups: [''] + resources: [configmaps] + resourceNames: [{{ printf "%s-state" .Release.Name | quote }}] + verbs: [get, patch] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ .Release.Name }} +subjects: + - kind: ServiceAccount + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ printf "%s-%s" .Release.Namespace .Release.Name }} + namespace: {{ .Values.dragonfly.namespace }} +rules: + - apiGroups: [''] + resources: [pods] + verbs: [get, list] + - apiGroups: [''] + resources: [pods/exec] + resourceNames: + {{- toYaml .Values.dragonfly.seedNames | nindent 6 }} + verbs: [get, create] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ printf "%s-%s" .Release.Namespace .Release.Name }} + namespace: {{ .Values.dragonfly.namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ printf "%s-%s" .Release.Namespace .Release.Name }} +subjects: + - kind: ServiceAccount + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-config + namespace: {{ .Release.Namespace }} +data: + config.json: | + {{- dict "namespace" .Release.Namespace "deployment" .Values.registry.deployment "registrySelector" .Values.registry.podSelector "registry" (required "registry.url is required" .Values.registry.url) "allowedAuthOrigins" .Values.registry.allowedAuthOrigins "repository" (required "registry.probeRepository is required" .Values.registry.probeRepository) "digest" (required "registry.probeDigest is required" .Values.registry.probeDigest) "seedNamespace" .Values.dragonfly.namespace "seedNames" .Values.dragonfly.seedNames "seedSelector" .Values.dragonfly.podSelector "seedContainer" .Values.dragonfly.container "seedProxy" .Values.dragonfly.proxy "stateConfigMap" (printf "%s-state" .Release.Name) "recheckSeconds" .Values.recheckSeconds | toJson | nindent 4 }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-state + namespace: {{ .Release.Namespace }} +data: + state.json: '{}' +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} +spec: + schedule: {{ .Values.schedule | quote }} + concurrencyPolicy: Forbid + startingDeadlineSeconds: 90 + successfulJobsHistoryLimit: 2 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + backoffLimit: 0 + activeDeadlineSeconds: 240 + ttlSecondsAfterFinished: 86400 + template: + spec: + serviceAccountName: {{ .Release.Name }} + restartPolicy: Never + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: {type: RuntimeDefault} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 12 }} + {{- end }} + containers: + - name: check + image: {{ $image | quote }} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: {drop: [ALL]} + env: + - name: PYTHONDONTWRITEBYTECODE + value: '1' + - name: NO_PROXY + value: '*' + - name: no_proxy + value: '*' + resources: + {{- toYaml .Values.resources | nindent 16 }} + volumeMounts: + - {name: config, mountPath: /config, readOnly: true} + - {name: credentials, mountPath: /credentials, readOnly: true} + volumes: + - name: config + configMap: + name: {{ .Release.Name }}-config + - name: credentials + secret: + secretName: {{ required "registry.pullSecret is required" .Values.registry.pullSecret }} + defaultMode: 0440 diff --git a/deploy/registry-connectivity/chart/values.yaml b/deploy/registry-connectivity/chart/values.yaml new file mode 100644 index 00000000..a7adc7ef --- /dev/null +++ b/deploy/registry-connectivity/chart/values.yaml @@ -0,0 +1,37 @@ +image: "" # Required immutable checker image (repository@sha256:...). +imagePullSecrets: [] +nodeSelector: {} +tolerations: [] +registry: + deployment: registry + podSelector: + app: managed-registry + url: "" + allowedAuthOrigins: [] + ports: [5000, 5001] + # A retained, single-platform Nydus manifest (not an image index). + probeRepository: "" + probeDigest: "" + # Existing Secret with credentials.json: {"username": ..., "password": ...}. + # Use a pull-only identity restricted to the probe repository. + pullSecret: "" +dragonfly: + namespace: dragonfly-system + podSelector: + app: dragonfly + component: seed-client + container: seed-client + seedNames: + - dragonfly-seed-client-0 + - dragonfly-seed-client-1 + - dragonfly-seed-client-2 + - dragonfly-seed-client-3 + - dragonfly-seed-client-4 + proxy: http://127.0.0.1:4001 +networkPolicy: + name: registry-dragonfly-seed-ingress +schedule: '* * * * *' +recheckSeconds: 600 +resources: + requests: {cpu: 50m, memory: 96Mi} + limits: {cpu: '1', memory: 256Mi} diff --git a/deploy/registry-connectivity/check.py b/deploy/registry-connectivity/check.py new file mode 100644 index 00000000..f195c5b0 --- /dev/null +++ b/deploy/registry-connectivity/check.py @@ -0,0 +1,266 @@ +"""Verify Registry authentication and Nydus reads from every Dragonfly seed. + +Credentials are supplied to remote curl over exec stdin, never command arguments. +Only digests, sizes, timings and fixed diagnostic messages are recorded. +""" +from __future__ import annotations + +import base64 +import concurrent.futures +import hashlib +import io +import json +import os +import re +import tarfile +import time +from datetime import datetime, timezone +from pathlib import Path +from urllib.parse import urlencode, urlsplit + +LIMIT = 32 * 1024 * 1024 +MARKER = b"\n__AKERNEL_HTTP_STATUS__" +BOOTSTRAP = "containerd.io/snapshot/nydus-bootstrap" + + +class CheckFailed(RuntimeError): + """A credential-free diagnostic safe for the Job log.""" + + +def origin(url): + parsed = urlsplit(url) + if parsed.scheme not in ("http", "https") or not parsed.hostname or parsed.username or parsed.password: + raise CheckFailed("invalid endpoint") + return parsed.scheme, parsed.hostname, parsed.port or (443 if parsed.scheme == "https" else 80) + + +def challenge(headers, allowed_origins): + value = headers.get("www-authenticate", "") + if not value.lower().startswith("bearer "): + raise CheckFailed("Registry did not return a Bearer challenge") + fields = dict(re.findall(r'(\w+)="([^"\r\n]*)"', value)) + if not fields.get("service") or origin(fields.get("realm", "")) not in {origin(u) for u in allowed_origins}: + raise CheckFailed("token realm is not explicitly trusted") + if urlsplit(fields["realm"]).query or urlsplit(fields["realm"]).fragment: + raise CheckFailed("token realm must not contain query or fragment") + return fields["realm"], fields["service"] + + +def decode_response(encoded): + try: + raw = base64.b64decode(encoded, validate=False) + response, code = raw.rsplit(MARKER, 1) + status = int(code.strip()) + # curl may include proxy CONNECT / HTTP interim header blocks. + while True: + head, body = response.split(b"\r\n\r\n", 1) + lines = head.decode("iso-8859-1").split("\r\n") + if not lines[0].startswith("HTTP/"): + raise ValueError("missing status") + headers = {k.lower(): v.strip() for k, v in (line.split(":", 1) for line in lines[1:] if ":" in line)} + if body.startswith(b"HTTP/") and ("connection established" in lines[0].lower() or lines[0].split()[1] == "100"): + response = body + continue + break + if status == 0 or len(body) > LIMIT: + raise ValueError("failed or too large") + return status, headers, body + except (ValueError, IndexError) as error: + raise CheckFailed("seed HTTP request failed or returned an invalid response") from error + + +def digest_bytes(data, expected): + if not re.fullmatch(r"sha256:[0-9a-f]{64}", expected or ""): + raise CheckFailed("an immutable sha256 digest is required") + if "sha256:" + hashlib.sha256(data).hexdigest() != expected: + raise CheckFailed("content digest mismatch") + + +def bootstrap_descriptor(manifest): + layers = manifest.get("layers", []) + if not layers or BOOTSTRAP not in layers[-1].get("annotations", {}): + raise CheckFailed("probe is not a Nydus manifest with a bootstrap layer") + layer = layers[-1] + if not isinstance(layer.get("size"), int) or not 0 < layer["size"] <= LIMIT: + raise CheckFailed("bootstrap layer size is invalid") + return layer + + +def bootstrap_contents(layer, descriptor): + digest_bytes(layer, descriptor["digest"]) + if len(layer) != descriptor["size"]: + raise CheckFailed("bootstrap layer size mismatch") + try: + with tarfile.open(fileobj=io.BytesIO(layer), mode="r:*") as archive: + member = archive.getmember("image/image.boot") + if not member.isfile() or not 0 < member.size <= LIMIT: + raise CheckFailed("bootstrap is not a bounded regular file") + data = archive.extractfile(member).read(LIMIT + 1) + if len(data) != member.size: + raise CheckFailed("bootstrap file is truncated") + return {"sha256": hashlib.sha256(data).hexdigest(), "bytes": len(data)} + except (tarfile.TarError, KeyError, OSError) as error: + raise CheckFailed("cannot read image/image.boot from bootstrap layer") from error + + +class SeedHTTP: + def __init__(self, api, namespace, pod, container, timeout=30): + self.api, self.namespace, self.pod, self.container, self.timeout = api, namespace, pod, container, timeout + + def get(self, url, headers=None, proxy=None): + from kubernetes import client + from kubernetes.stream import stream + + # Fixed-length stdin gives curl EOF without depending on a particular + # Kubernetes exec websocket protocol's half-close implementation. + options = {"url": url, "connect-timeout": "5", "max-time": str(self.timeout), "max-filesize": str(LIMIT), "noproxy": "" if proxy else "*", "proxy": proxy or ""} + config = "".join(f"{key} = {json.dumps(value)}\n" for key, value in options.items()) + config += "".join(f"header = {json.dumps(key + ': ' + value)}\n" for key, value in (headers or {}).items()) + payload = config.encode() + command = ["/bin/sh", "-c", 'head -c "$1" | curl --config - --silent --show-error --include --write-out "\\n__AKERNEL_HTTP_STATUS__%{http_code}\\n" | base64', "registry-check", str(len(payload))] + # Kubernetes stream temporarily replaces ApiClient.request. A client + # shared by concurrent seed checks would corrupt HTTP/exec dispatch. + isolated = client.ApiClient(configuration=self.api.api_client.configuration) + connection = None + output = [] + size = 0 + deadline = time.monotonic() + self.timeout + 15 + try: + connection = stream(client.CoreV1Api(isolated).connect_get_namespaced_pod_exec, self.pod, self.namespace, container=self.container, command=command, stdin=True, stdout=True, stderr=True, tty=False, _preload_content=False, _request_timeout=10) + connection.write_stdin(config) + while connection.is_open(): + if time.monotonic() >= deadline: + raise CheckFailed("seed exec deadline exceeded") + connection.update(timeout=1) + while connection.peek_stdout(): + chunk = connection.read_stdout() + size += len(chunk) + if size > LIMIT * 2: + raise CheckFailed("seed response exceeds limit") + output.append(chunk) + if connection.peek_stderr(): + connection.read_stderr() # Never expose upstream text/credentials. + return decode_response("".join(output)) + finally: + if connection is not None: + connection.close() + isolated.close() + + +def verify_seed(http, cfg, credential): + start = time.monotonic() + registry = cfg["registry"].rstrip("/") + status, headers, _ = http.get(registry + "/v2/") + if status != 401: + raise CheckFailed("anonymous Registry access did not require authentication") + realm, service = challenge(headers, cfg["allowedAuthOrigins"]) + scope = "repository:" + cfg["repository"] + ":pull" + basic = base64.b64encode((credential["username"] + ":" + credential["password"]).encode()).decode() + status, _, body = http.get(realm + "?" + urlencode({"service": service, "scope": scope}), {"Authorization": "Basic " + basic}) + if status != 200: + raise CheckFailed("pull token acquisition failed") + try: + token_data = json.loads(body) + token = token_data.get("token") or token_data.get("access_token") + if not isinstance(token, str) or not token: + raise ValueError("missing token") + except (ValueError, TypeError) as error: + raise CheckFailed("invalid pull token response") from error + auth = {"Authorization": "Bearer " + token, "Accept": "application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json"} + root = registry + "/v2/" + cfg["repository"] + status, _, body = http.get(root + "/manifests/" + cfg["digest"], auth) + if status != 200: + raise CheckFailed("authenticated manifest read failed") + digest_bytes(body, cfg["digest"]) + try: + descriptor = bootstrap_descriptor(json.loads(body)) + except (ValueError, TypeError) as error: + raise CheckFailed("invalid manifest JSON") from error + url = root + "/blobs/" + descriptor["digest"] + status, _, body = http.get(url, auth) + if status != 200: + raise CheckFailed("authenticated bootstrap read failed") + result = bootstrap_contents(body, descriptor) + status, _, body = http.get(url, auth, proxy=cfg.get("seedProxy", "http://127.0.0.1:4001")) + if status != 200: + raise CheckFailed("Dragonfly proxy bootstrap read failed") + if bootstrap_contents(body, descriptor) != result: + raise CheckFailed("Dragonfly proxy bootstrap differs from Registry") + return {"seed": http.pod, "passed": True, "bootstrap": result, "layer_digest": descriptor["digest"], "seconds": round(time.monotonic() - start, 3)} + + +def ready(pod): + return not pod.metadata.deletion_timestamp and any(c.type == "Ready" and c.status == "True" for c in (pod.status.conditions or [])) + + +def fingerprint(api, apps, cfg): + deployment = apps.read_namespaced_deployment(cfg["deployment"], cfg["namespace"], _request_timeout=20) + spec, status = deployment.spec, deployment.status + count = spec.replicas or 0 + if count < 1 or (status.observed_generation or 0) < deployment.metadata.generation or (status.updated_replicas or 0) != count or (status.available_replicas or 0) != count or (status.replicas or 0) != count: + raise CheckFailed("Registry rollout is not ready") + selector = ",".join(f"{k}={v}" for k, v in sorted(cfg["registrySelector"].items())) + pods = [p for p in api.list_namespaced_pod(cfg["namespace"], label_selector=selector, _request_timeout=20).items if not p.metadata.deletion_timestamp] + if len(pods) != count or not all(ready(p) for p in pods): + raise CheckFailed("Registry Pods are not ready") + selector = ",".join(f"{k}={v}" for k, v in sorted(cfg["seedSelector"].items())) + seeds = api.list_namespaced_pod(cfg["seedNamespace"], label_selector=selector, _request_timeout=20).items + selected = {p.metadata.name: p for p in seeds if p.metadata.name in cfg["seedNames"]} + if len(cfg["seedNames"]) != 5 or len(set(cfg["seedNames"])) != 5 or set(selected) != set(cfg["seedNames"]) or not all(ready(p) for p in selected.values()): + raise CheckFailed("all five configured Dragonfly seeds must be present and Ready") + identities = {"deployment": [deployment.metadata.uid, deployment.metadata.generation], "registry": sorted((p.metadata.uid, [(s.name, s.image_id, s.restart_count) for s in (p.status.container_statuses or [])]) for p in pods), "seeds": sorted((name, p.metadata.uid, [(s.name, s.image_id, s.restart_count) for s in (p.status.container_statuses or [])]) for name, p in selected.items()), "config": cfg} + return hashlib.sha256(json.dumps(identities, sort_keys=True).encode()).hexdigest() + + +def execute(api, apps, cfg, credential, state, force=False, now=None, http_factory=SeedHTTP): + now = time.time() if now is None else now + before = fingerprint(api, apps, cfg) + if not force and state.get("fingerprint") == before and now - state.get("checked_epoch", 0) < cfg.get("recheckSeconds", 600): + return None + results = [] + def one(name): + try: + return verify_seed(http_factory(api, cfg["seedNamespace"], name, cfg["seedContainer"]), cfg, credential) + except CheckFailed as error: + return {"seed": name, "passed": False, "error": str(error)} + except Exception: + return {"seed": name, "passed": False, "error": "seed exec or HTTP transport failed"} + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as pool: + results = list(pool.map(one, cfg["seedNames"])) + if not all(r["passed"] for r in results): + print(json.dumps({"event": "verification_failed", "seeds": results}), flush=True) + raise CheckFailed("one or more Dragonfly seeds failed; success state was not advanced") + if len({r["bootstrap"]["sha256"] for r in results}) != 1: + raise CheckFailed("seed bootstrap digests disagree") + if fingerprint(api, apps, cfg) != before: + raise CheckFailed("Registry or seed identity changed during validation; retry required") + return {"fingerprint": before, "checked_epoch": now, "checked_at": datetime.fromtimestamp(now, timezone.utc).isoformat(), "manifest_digest": cfg["digest"], "seeds": results} + + +def main(): + from kubernetes import client, config + config.load_incluster_config() + api, apps = client.CoreV1Api(), client.AppsV1Api() + cfg = json.loads(Path(os.environ.get("CONFIG_FILE", "/config/config.json")).read_text()) + credential = json.loads(Path("/credentials/credentials.json").read_text()) + cm = api.read_namespaced_config_map(cfg["stateConfigMap"], cfg["namespace"], _request_timeout=20) + state = json.loads((cm.data or {}).get("state.json", "{}")) + result = execute(api, apps, cfg, credential, state, force=os.environ.get("FORCE_CHECK") == "1") + if result is None: + print(json.dumps({"event": "unchanged", "last_success": state.get("checked_at")}), flush=True) + return + # resourceVersion prevents overlapping manual/hook Jobs overwriting state. + api.patch_namespaced_config_map(cfg["stateConfigMap"], cfg["namespace"], {"metadata": {"resourceVersion": cm.metadata.resource_version}, "data": {"state.json": json.dumps(result, sort_keys=True)}}, _request_timeout=20) + print(json.dumps({"event": "verified", **result}), flush=True) + + +if __name__ == "__main__": + try: + main() + except CheckFailed as error: + print(json.dumps({"event": "failed", "error": str(error)}), flush=True) + raise SystemExit(1) + except Exception: + # Kubernetes/HTTP exceptions may include response bodies and credentials. + print(json.dumps({"event": "failed", "error": "verification API or configuration error"}), flush=True) + raise SystemExit(1) diff --git a/deploy/registry-connectivity/requirements.txt b/deploy/registry-connectivity/requirements.txt new file mode 100644 index 00000000..720ed308 --- /dev/null +++ b/deploy/registry-connectivity/requirements.txt @@ -0,0 +1,15 @@ +# Pin the complete checker dependency set for reproducible operator images. +certifi==2026.10.23 +charset-normalizer==3.5.2 +durationpy==0.11 +idna==3.20 +kubernetes==35.0.0 +oauthlib==4.0.1 +python-dateutil==4.0.0 +PyYAML==6.0.3 +requests==2.34.2 +requests-oauthlib==2.0.0 +six==1.17.0 +tzdata==2026.6 +urllib3==2.8.0 +websocket-client==1.9.2 diff --git a/deploy/registry-connectivity/tests/test_chart.py b/deploy/registry-connectivity/tests/test_chart.py new file mode 100644 index 00000000..ee0e0ae8 --- /dev/null +++ b/deploy/registry-connectivity/tests/test_chart.py @@ -0,0 +1,71 @@ +"""Render and check security boundaries without a cluster.""" +import json +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + +import yaml + +CHART = Path(__file__).resolve().parents[1] / "chart" +HELM = shutil.which("helm") or str(Path.home() / ".local/bin/helm") + + +class ChartTests(unittest.TestCase): + def values(self): + return {"image": "registry.example/check@sha256:" + "a" * 64, "registry": {"url": "https://registry.example", "allowedAuthOrigins": ["https://auth.example"], "probeRepository": "operations/nydus", "probeDigest": "sha256:" + "b" * 64, "pullSecret": "probe-pull"}} + + def render(self, values): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "values.json" + path.write_text(json.dumps(values)) + return subprocess.run([HELM, "template", "guard", str(CHART), "--namespace", "registry", "-f", str(path)], text=True, capture_output=True) + + def objects(self): + result = self.render(self.values()) + self.assertEqual(result.returncode, 0, result.stderr) + return list(yaml.safe_load_all(result.stdout)) + + def test_namespace_and_pod_labels_share_one_peer_and_only_data_auth_ports(self): + policy = next(o for o in self.objects() if o["kind"] == "NetworkPolicy") + rule = policy["spec"]["ingress"][0] + self.assertEqual(len(rule["from"]), 1) + self.assertEqual(rule["from"][0], {"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "dragonfly-system"}}, "podSelector": {"matchLabels": {"app": "dragonfly", "component": "seed-client"}}}) + self.assertEqual(rule["ports"], [{"protocol": "TCP", "port": 5000}, {"protocol": "TCP", "port": 5001}]) + + def test_exec_restricted_to_exact_five_seeds_and_no_secret_read_permission(self): + roles = [o for o in self.objects() if o["kind"] == "Role"] + rules = [r for role in roles for r in role["rules"]] + self.assertFalse(any("secrets" in r["resources"] for r in rules)) + execution = next(r for r in rules if "pods/exec" in r["resources"]) + self.assertEqual(execution["resourceNames"], [f"dragonfly-seed-client-{i}" for i in range(5)]) + + def test_checker_is_bounded_nonroot_and_cannot_overlap_scheduled_checks(self): + cron = next(o for o in self.objects() if o["kind"] == "CronJob") + self.assertEqual(cron["spec"]["schedule"], "* * * * *") + self.assertEqual(cron["spec"]["concurrencyPolicy"], "Forbid") + job = cron["spec"]["jobTemplate"]["spec"] + self.assertEqual(job["activeDeadlineSeconds"], 240) + pod = job["template"]["spec"] + self.assertTrue(pod["securityContext"]["runAsNonRoot"]) + self.assertTrue(pod["containers"][0]["securityContext"]["readOnlyRootFilesystem"]) + secret = next(v for v in pod["volumes"] if "secret" in v) + self.assertEqual(secret["secret"]["secretName"], "probe-pull") + + def test_chart_does_not_take_over_registry_or_storage(self): + kinds = {o["kind"] for o in self.objects()} + self.assertFalse(kinds.intersection({"Deployment", "PersistentVolumeClaim", "Service", "Secret"})) + + def test_empty_selectors_or_auth_origins_and_unpinned_image_fail_closed(self): + for field in ["podSelector", "allowedAuthOrigins"]: + values = self.values() + values["registry"][field] = None + self.assertNotEqual(self.render(values).returncode, 0) + values = self.values() + values["image"] = "registry.example/check:latest" + self.assertNotEqual(self.render(values).returncode, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/deploy/registry-connectivity/tests/test_check.py b/deploy/registry-connectivity/tests/test_check.py new file mode 100644 index 00000000..55a09e2c --- /dev/null +++ b/deploy/registry-connectivity/tests/test_check.py @@ -0,0 +1,204 @@ +import base64 +import contextlib +import hashlib +import importlib.util +import io +import json +from pathlib import Path +from types import SimpleNamespace as NS +import tarfile +import unittest +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location("registry_check", Path(__file__).resolve().parents[1] / "check.py") +check = importlib.util.module_from_spec(spec) +spec.loader.exec_module(check) + + +def sha(data): + return "sha256:" + hashlib.sha256(data).hexdigest() + + +def fixture(): + output = io.BytesIO() + with tarfile.open(fileobj=output, mode="w:gz") as archive: + data = b"Nydus-bootstrap-probe" + member = tarfile.TarInfo("image/image.boot") + member.size = len(data) + archive.addfile(member, io.BytesIO(data)) + layer = output.getvalue() + manifest = json.dumps({"layers": [{"digest": sha(layer), "size": len(layer), "annotations": {check.BOOTSTRAP: "true"}}]}).encode() + return layer, manifest + + +def pod(name, uid=None, is_ready=True): + return NS(metadata=NS(name=name, uid=uid or name, deletion_timestamp=None), status=NS(conditions=[NS(type="Ready", status="True" if is_ready else "False")], container_statuses=[NS(name="test", image_id="sha256:1", restart_count=0)])) + + +def configuration(): + return {"namespace": "registry", "deployment": "registry", "registrySelector": {"app": "registry"}, "seedNamespace": "dragonfly", "seedSelector": {"app": "dragonfly", "component": "seed-client"}, "seedContainer": "seed-client", "seedNames": [f"seed-{i}" for i in range(5)], "registry": "http://registry.test:5000", "allowedAuthOrigins": ["http://auth.test:5001"], "repository": "health/probe", "digest": sha(fixture()[1])} + + +class API: + def __init__(self): + self.registry = [pod("registry-0")] + self.seeds = [pod(f"seed-{i}") for i in range(5)] + self.generation = 1 + + def read_namespaced_deployment(self, *args, **kwargs): + return NS(metadata=NS(uid="deployment", generation=self.generation), spec=NS(replicas=1), status=NS(observed_generation=self.generation, updated_replicas=1, available_replicas=1, replicas=1)) + + def list_namespaced_pod(self, namespace, **kwargs): + return NS(items=self.registry if namespace == "registry" else self.seeds) + + +class CheckerTests(unittest.TestCase): + def test_requires_trusted_token_origin_before_sending_credentials(self): + with self.assertRaisesRegex(check.CheckFailed, "not explicitly trusted"): + check.challenge({"www-authenticate": 'Bearer realm="http://other.test/auth",service="registry"'}, ["http://auth.test:5001"]) + + def test_response_decodes_binary_and_status(self): + data = b"\x00\xff\xfe" + encoded = base64.b64encode(b"HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\n\r\n" + data + check.MARKER + b"200\n") + status, headers, body = check.decode_response(encoded) + self.assertEqual((status, body), (200, data)) + self.assertEqual(headers["content-type"], "application/octet-stream") + + def test_missing_curl_status_is_failure(self): + with self.assertRaises(check.CheckFailed): + check.decode_response(base64.b64encode(b"curl failed")) + + def test_content_is_verified_and_real_bootstrap_is_read(self): + layer, manifest = fixture() + result = check.bootstrap_contents(layer, check.bootstrap_descriptor(json.loads(manifest))) + self.assertEqual(result["sha256"], hashlib.sha256(b"Nydus-bootstrap-probe").hexdigest()) + with self.assertRaisesRegex(check.CheckFailed, "digest mismatch"): + check.bootstrap_contents(layer + b"x", check.bootstrap_descriptor(json.loads(manifest))) + + def test_oci_without_nydus_bootstrap_is_rejected(self): + with self.assertRaises(check.CheckFailed): + check.bootstrap_descriptor({"layers": [{"size": 1}]}) + + def test_authenticated_read_and_dragonfly_proxy_are_both_required(self): + layer, manifest = fixture() + calls = [] + class HTTP: + pod = "seed-0" + def get(self, url, headers=None, proxy=None): + calls.append((url, headers, proxy)) + if url.endswith("/v2/"): + return 401, {"www-authenticate": 'Bearer realm="http://auth.test:5001/auth",service="registry"'}, b"" + if url.startswith("http://auth.test"): + return 200, {}, b'{"token":"secret-token"}' + if "/manifests/" in url: + return 200, {}, manifest + return 200, {}, layer + result = check.verify_seed(HTTP(), configuration(), {"username": "pull", "password": "secret-password"}) + self.assertTrue(result["passed"]) + self.assertEqual(len(calls), 5) + self.assertEqual(calls[-1][2], "http://127.0.0.1:4001") + self.assertNotIn("secret", json.dumps(result)) + self.assertTrue(calls[1][1]["Authorization"].startswith("Basic ")) + self.assertEqual(calls[2][1]["Authorization"], "Bearer secret-token") + + def test_exec_uses_isolated_api_clients_and_keeps_credentials_off_argv(self): + clients, commands = [], [] + class Client: + def __init__(self, configuration): + self.configuration, self.closed = configuration, False + clients.append(self) + def close(self): + self.closed = True + class Core: + def __init__(self, client): + self.api_client = client + def connect_get_namespaced_pod_exec(self): + pass + class WS: + opened, unread = True, True + def write_stdin(self, config): + self.config = config + def is_open(self): + return self.opened + def update(self, **kwargs): + self.opened = False + def peek_stdout(self): + return self.unread + def read_stdout(self): + self.unread = False + return base64.b64encode(b"HTTP/1.1 200 OK\r\n\r\nbody" + check.MARKER + b"200\n").decode() + def peek_stderr(self): + return False + def close(self): + self.opened = False + shared = NS(api_client=NS(configuration=object())) + def stream(method, *args, **kwargs): + self.assertIsNot(method.__self__.api_client, shared.api_client) + commands.append(kwargs["command"]) + return WS() + modules = {"kubernetes": NS(client=NS(ApiClient=Client, CoreV1Api=Core)), "kubernetes.stream": NS(stream=stream)} + with patch.dict("sys.modules", modules): + for name in ["seed-0", "seed-1"]: + result = check.SeedHTTP(shared, "df", name, "seed").get("http://registry.test/v2/", {"Authorization": "Bearer secret-token"}) + self.assertEqual(result[0], 200) + self.assertEqual(len(clients), 2) + self.assertTrue(all(c.closed for c in clients)) + self.assertNotIn("secret-token", repr(commands)) + + def test_migration_rollout_and_seed_replacement_change_fingerprint(self): + api, cfg = API(), configuration() + first = check.fingerprint(api, api, cfg) + api.registry[0].metadata.uid = "migrated" + self.assertNotEqual(check.fingerprint(api, api, cfg), first) + second = check.fingerprint(api, api, cfg) + api.generation += 1 + self.assertNotEqual(check.fingerprint(api, api, cfg), second) + second = check.fingerprint(api, api, cfg) + api.seeds[0].metadata.uid = "replacement" + self.assertNotEqual(check.fingerprint(api, api, cfg), second) + + def test_missing_or_unready_seed_is_not_a_success(self): + api, cfg = API(), configuration() + api.seeds.pop() + with self.assertRaises(check.CheckFailed): + check.fingerprint(api, api, cfg) + api.seeds.append(pod("seed-4", is_ready=False)) + with self.assertRaises(check.CheckFailed): + check.fingerprint(api, api, cfg) + + def test_unchanged_state_skips_until_periodic_recheck(self): + api, cfg = API(), configuration() + state = {"fingerprint": check.fingerprint(api, api, cfg), "checked_epoch": 100} + with patch.object(check, "verify_seed", side_effect=AssertionError("unexpected network call")): + self.assertIsNone(check.execute(api, api, cfg, {}, state, now=101)) + + def test_every_seed_must_pass_and_errors_are_redacted(self): + api, cfg = API(), configuration() + def verify(http, *args): + if http.pod == "seed-4": + raise RuntimeError("secret-password") + return {"seed": http.pod, "passed": True, "bootstrap": {"sha256": "one"}} + factory = lambda api, namespace, name, container: NS(pod=name) + output = io.StringIO() + with patch.object(check, "verify_seed", side_effect=verify), contextlib.redirect_stdout(output): + with self.assertRaises(check.CheckFailed): + check.execute(api, api, cfg, {}, {}, http_factory=factory) + self.assertNotIn("secret-password", output.getvalue()) + self.assertIn('"passed": false', output.getvalue()) + + def test_rollout_during_check_never_advances_success_state(self): + api, cfg = API(), configuration() + with patch.object(check, "fingerprint", side_effect=["before", "after"]), patch.object(check, "verify_seed", return_value={"passed": True, "bootstrap": {"sha256": "one"}}): + with self.assertRaisesRegex(check.CheckFailed, "identity changed"): + check.execute(api, api, cfg, {}, {}) + + def test_five_successes_form_a_new_state(self): + api, cfg = API(), configuration() + with patch.object(check, "verify_seed", return_value={"passed": True, "bootstrap": {"sha256": "one"}}): + state = check.execute(api, api, cfg, {}, {}, now=100) + self.assertEqual(len(state["seeds"]), 5) + self.assertEqual(state["fingerprint"], check.fingerprint(api, api, cfg)) + + +if __name__ == "__main__": + unittest.main() From 311da3628ea1fc55cdd95607842eaae2efcf63b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9C=88=E5=8F=AF?= Date: Tue, 10 Nov 2026 15:48:49 +0800 Subject: [PATCH 2/4] fix(deploy): harden Registry guard policy and transport checks --- deploy/registry-connectivity/README.md | 97 ++++++++-- .../chart/templates/resources.yaml | 25 ++- .../registry-connectivity/chart/values.yaml | 13 ++ deploy/registry-connectivity/check.py | 76 ++++++-- .../registry-connectivity/tests/test_chart.py | 31 +++- .../registry-connectivity/tests/test_check.py | 8 +- .../tests/test_transport.py | 167 ++++++++++++++++++ 7 files changed, 378 insertions(+), 39 deletions(-) create mode 100644 deploy/registry-connectivity/tests/test_transport.py diff --git a/deploy/registry-connectivity/README.md b/deploy/registry-connectivity/README.md index 930fc3e7..5703de9f 100644 --- a/deploy/registry-connectivity/README.md +++ b/deploy/registry-connectivity/README.md @@ -7,8 +7,17 @@ when moving or upgrading the Registry. ## Network boundary -The NetworkPolicy selects Registry Pods and permits TCP 5000 (Registry) and -5001 (token authentication) from **one peer containing both selectors**: +NetworkPolicy creation is **off by default**. Before setting +`networkPolicy.enabled=true`, review every current Registry client, including +BuildKit, runtime nodes and direct push/pull clients. Existing policies must +already permit their traffic. Set `networkPolicy.existingClientsReviewed=true` +only after that review. The first ingress policy selecting a Pod makes it +ingress-isolated: five passing seed checks do not prove other clients retain +access. Keep both flags explicit when upgrading an installation that already +owns this policy; omitting them would remove that Helm-managed policy. + +When enabled, the NetworkPolicy selects Registry Pods and permits TCP 5000 +(Registry) and 5001 (token authentication) from **one peer containing both selectors**: - namespace label `kubernetes.io/metadata.name: dragonfly-system`; - Pod labels `app: dragonfly` and `component: seed-client`. @@ -18,7 +27,9 @@ Existing ingress policies remain additive. CIDRs alone are insufficient when the CNI enforces source Pod identity across node pools. Change the selectors and container ports to match the deployed services; service-facing ports may differ from container ports. This policy grants no access to unrelated -Registry management ports. It does not configure source-side egress policies. +Registry management ports. Empty port lists and invalid port numbers are rejected; +they must never render as an unrestricted `ports: null`. The policy does not +configure source-side egress policies. ## Automatic verification @@ -35,8 +46,16 @@ Each of the five seeds must: 3. Fetch the configured immutable manifest and verify its SHA-256. 4. Fetch its actual Nydus bootstrap layer directly from the Registry; verify digest and size, and read the regular `image/image.boot` tar member. -5. Read the same layer through the seed's local Dragonfly HTTP proxy and verify - identical content. +5. When proxy checking is enabled, read the same layer through the seed's local + Dragonfly HTTP proxy and verify identical content. Direct-only results + explicitly report `proxy_checked: false` and do not validate the proxy path. + +Every curl transfer must exit successfully, even if HTTP 200 and the expected +content bytes were received. Responses and bootstrap files are capped at 1 MiB, +decompressed tar archives at 2 MiB, with at most two seeds checked concurrently. +Keep the dedicated probe small; the default 256 MiB Job is not intended to +inspect arbitrary task-image bootstraps. Oversized probes fail without advancing +success state. A peak-RSS regression exercises the allowed bounds. Only a complete five-seed success advances `-state`. A failed Job logs redacted per-seed results and exits nonzero. Subsequent scheduled Jobs @@ -48,7 +67,9 @@ one day. A successful `unchanged` Job alone is not a new network verification. The checker uses the Kubernetes API and execs only in the five configured seed Pods. Seeds require `sh`, `head`, `curl` and `base64`, but do not require Python. No seed files or process state are changed. Secret contents, Basic credentials -and Bearer tokens pass through exec stdin, never command arguments or logs. +and Bearer tokens pass through exec stdin and are omitted from command arguments +and **checker** logs. Other components have independent logging behavior; +credential safety of their logs must be verified separately. The checker ServiceAccount cannot read Secrets. It mounts one existing Secret containing a pull-only credential. Its exec permission is restricted to the five Pod names; assess that permission as part of the deployment trust boundary. @@ -81,6 +102,50 @@ Use a pull-only identity, preferably restricted to the probe repository. Never put credentials in Helm values, command arguments or version control. The checker does not need the token signer's private key or a push credential. +## Proxy logging and HTTPS prerequisites + +Dragonfly client v1.3.0 logs complete proxy requests, including Authorization, +at INFO level. Checker log redaction does not protect those logs. Before +sending a real pull token through the proxy: + +1. Disable request-dump logging (for unmodified v1.3.0, `--log-level=warn` removes + the INFO/DEBUG dumps), or use a verified redacting implementation. Coordinate + this with the owner of Dragonfly; this chart does not roll the seed Pods. +2. Send a **dummy**, non-production Authorization value through each seed's + proxy. Search for that marker in console output, daemon log files and any + collected logs after allowing the logging pipeline to flush. Do not use a + real token to test redaction. Repeat after logging/version changes. +3. Only after verifying the log sinks, set + `dragonfly.proxyLoggingVerified=true`. Both Helm and the checker reject + enabled proxy probes without this explicit verification. If it cannot yet + be verified, use `dragonfly.proxyCheckEnabled=false` for direct-only checks + or `suspend=true` to pause the guard. + +For an HTTPS Registry, curl uses CONNECT to the **HTTP** Dragonfly proxy. +Dragonfly terminates TLS and signs a replacement endpoint certificate. Configure +`proxy.server.caCert` and `proxy.server.caKey` in Dragonfly with its signing CA; +its default ephemeral self-signed certificate is unsuitable for this verified +probe. Configure Dragonfly's upstream Registry trust separately, for example +`proxy.registryMirror.cert` in v1.3.0. + +Use these paths **inside every seed**, provisioned by the platform: + +- `registry.caFile`: curl's CA bundle for a direct private HTTPS Registry. +- `registry.authCAFile`: curl's CA bundle for the private HTTPS token service. +- `dragonfly.proxyCAFile`: the Dragonfly signing CA bundle, explicitly required + for HTTPS proxy checks. This is curl's target `cacert` for the intercepted + connection, not `proxy-cacert` (the proxy transport itself is HTTP). + +Direct requests can use curl's native trusted roots when their CA path is empty. +Trusting only the Registry CA does not trust Dragonfly's generated certificate. +No insecure TLS flag is added. These settings concern Registry traffic inside +seeds; SDK `AKERNEL_VERIFY_TLS` controls the separate AKernel API connection and +is neither required nor consumed by this chart. Using an HTTP API endpoint does +not require that SDK setting; it does not enable HTTPS or change the URL scheme. + +Sources: [Dragonfly proxy logging and CONNECT handling](https://github.com/dragonflyoss/client/blob/v1.3.0/dragonfly-client/src/proxy/mod.rs), +[proxy and Registry CA configuration](https://github.com/dragonflyoss/client/blob/v1.3.0/dragonfly-client-config/src/dfdaemon.rs). + ## Deploy Supply operator-owned values (keep private addresses out of Git): @@ -95,14 +160,20 @@ registry: probeRepository: operations/nydus-connectivity probeDigest: sha256: pullSecret: registry-connectivity-pull +dragonfly: + # Start with direct-only checks until proxy logging/trust has been verified. + proxyCheckEnabled: false # Defaults select the five dragonfly-seed-client-{0..4} Pods. +# NetworkPolicy is enabled separately after auditing existing client ingress: +# networkPolicy: {enabled: true, existingClientsReviewed: true} ``` -Configure `imagePullSecrets`, placement and tolerations as needed. Registry -and token-service certificate validation uses curl's native trust in each seed; -private HTTPS endpoints need their trusted CA installed there. Plain HTTP is -supported for deployments whose network/security policy permits it. The -checker never adds an insecure TLS flag. +Configure `imagePullSecrets`, placement and tolerations as needed. Enable proxy +checks only after completing the prerequisites above; for HTTPS add +`dragonfly.proxyCAFile`. Plain HTTP is supported where deployment policy permits +it. The chart creates the state ConfigMap without `data.state.json`; the Job +owns that field, so Helm server-side apply must not initialize/reset it on an +upgrade. Retain the state through subsequent releases. ```bash helm upgrade --install registry-connectivity ./deploy/registry-connectivity/chart \ @@ -130,7 +201,9 @@ kubectl -n create job \ # CronJob's Job template before creating that manual Job. ``` -Validate changes with: +The local transport regressions require `curl` and `openssl` in addition to +Python, PyYAML and Helm. They exercise truncated HTTP and HTTPS CONNECT without +changing a cluster. Validate changes with: ```bash python3 -m unittest discover -s deploy/registry-connectivity/tests -v diff --git a/deploy/registry-connectivity/chart/templates/resources.yaml b/deploy/registry-connectivity/chart/templates/resources.yaml index 08ec748e..613b6c64 100644 --- a/deploy/registry-connectivity/chart/templates/resources.yaml +++ b/deploy/registry-connectivity/chart/templates/resources.yaml @@ -8,6 +8,24 @@ {{- if or (empty .Values.dragonfly.podSelector) (empty .Values.registry.podSelector) (empty .Values.registry.allowedAuthOrigins) -}} {{- fail "nonempty Registry/Dragonfly Pod selectors and allowedAuthOrigins are required" -}} {{- end }} +{{- if empty .Values.registry.ports -}} +{{- fail "registry.ports must be a nonempty list of explicit TCP ports" -}} +{{- end -}} +{{- range .Values.registry.ports -}} +{{- if or (not (regexMatch "^[1-9][0-9]*$" (toString .))) (gt (int .) 65535) -}} +{{- fail "registry.ports must contain integers between 1 and 65535" -}} +{{- end -}} +{{- end -}} +{{- if and .Values.dragonfly.proxyCheckEnabled (not .Values.dragonfly.proxyLoggingVerified) -}} +{{- fail "proxy checks require dragonfly.proxyLoggingVerified after credential-safe logging is verified with dummy credentials" -}} +{{- end -}} +{{- if and .Values.dragonfly.proxyCheckEnabled (hasPrefix "https://" .Values.registry.url) (empty .Values.dragonfly.proxyCAFile) -}} +{{- fail "HTTPS proxy checks require dragonfly.proxyCAFile inside each seed" -}} +{{- end -}} +{{- if .Values.networkPolicy.enabled -}} +{{- if not .Values.networkPolicy.existingClientsReviewed -}} +{{- fail "review existing ingress coverage for BuildKit and direct clients before enabling networkPolicy.existingClientsReviewed" -}} +{{- end }} apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: @@ -36,6 +54,7 @@ spec: port: {{ . }} {{- end }} --- +{{- end }} apiVersion: v1 kind: ServiceAccount metadata: @@ -110,15 +129,14 @@ metadata: namespace: {{ .Release.Namespace }} data: config.json: | - {{- dict "namespace" .Release.Namespace "deployment" .Values.registry.deployment "registrySelector" .Values.registry.podSelector "registry" (required "registry.url is required" .Values.registry.url) "allowedAuthOrigins" .Values.registry.allowedAuthOrigins "repository" (required "registry.probeRepository is required" .Values.registry.probeRepository) "digest" (required "registry.probeDigest is required" .Values.registry.probeDigest) "seedNamespace" .Values.dragonfly.namespace "seedNames" .Values.dragonfly.seedNames "seedSelector" .Values.dragonfly.podSelector "seedContainer" .Values.dragonfly.container "seedProxy" .Values.dragonfly.proxy "stateConfigMap" (printf "%s-state" .Release.Name) "recheckSeconds" .Values.recheckSeconds | toJson | nindent 4 }} + {{- dict "namespace" .Release.Namespace "deployment" .Values.registry.deployment "registrySelector" .Values.registry.podSelector "registry" (required "registry.url is required" .Values.registry.url) "allowedAuthOrigins" .Values.registry.allowedAuthOrigins "repository" (required "registry.probeRepository is required" .Values.registry.probeRepository) "digest" (required "registry.probeDigest is required" .Values.registry.probeDigest) "seedNamespace" .Values.dragonfly.namespace "seedNames" .Values.dragonfly.seedNames "seedSelector" .Values.dragonfly.podSelector "seedContainer" .Values.dragonfly.container "seedProxy" .Values.dragonfly.proxy "proxyCheckEnabled" .Values.dragonfly.proxyCheckEnabled "proxyLoggingVerified" .Values.dragonfly.proxyLoggingVerified "registryCAFile" .Values.registry.caFile "authCAFile" .Values.registry.authCAFile "proxyCAFile" .Values.dragonfly.proxyCAFile "stateConfigMap" (printf "%s-state" .Release.Name) "recheckSeconds" .Values.recheckSeconds | toJson | nindent 4 }} --- apiVersion: v1 kind: ConfigMap metadata: name: {{ .Release.Name }}-state namespace: {{ .Release.Namespace }} -data: - state.json: '{}' +# state.json belongs to the Job; Helm must never initialize/reset this field. --- apiVersion: batch/v1 kind: CronJob @@ -127,6 +145,7 @@ metadata: namespace: {{ .Release.Namespace }} spec: schedule: {{ .Values.schedule | quote }} + suspend: {{ .Values.suspend }} concurrencyPolicy: Forbid startingDeadlineSeconds: 90 successfulJobsHistoryLimit: 2 diff --git a/deploy/registry-connectivity/chart/values.yaml b/deploy/registry-connectivity/chart/values.yaml index a7adc7ef..711ede55 100644 --- a/deploy/registry-connectivity/chart/values.yaml +++ b/deploy/registry-connectivity/chart/values.yaml @@ -9,6 +9,9 @@ registry: url: "" allowedAuthOrigins: [] ports: [5000, 5001] + # CA bundle paths INSIDE each seed, not paths on the checker/client. + caFile: "" + authCAFile: "" # A retained, single-platform Nydus manifest (not an image index). probeRepository: "" probeDigest: "" @@ -28,9 +31,19 @@ dragonfly: - dragonfly-seed-client-3 - dragonfly-seed-client-4 proxy: http://127.0.0.1:4001 + proxyCheckEnabled: true + # Enable only after disabling/redacting request dumps and testing a dummy + # credential against all proxy log sinks (console, files and collectors). + proxyLoggingVerified: false + # HTTPS CONNECT is terminated by Dragonfly: curl needs this signing CA. + proxyCAFile: "" networkPolicy: + enabled: false + # Verify existing policies preserve BuildKit/direct-client ingress first. + existingClientsReviewed: false name: registry-dragonfly-seed-ingress schedule: '* * * * *' +suspend: false recheckSeconds: 600 resources: requests: {cpu: 50m, memory: 96Mi} diff --git a/deploy/registry-connectivity/check.py b/deploy/registry-connectivity/check.py index f195c5b0..a7fbcba4 100644 --- a/deploy/registry-connectivity/check.py +++ b/deploy/registry-connectivity/check.py @@ -8,6 +8,7 @@ import base64 import concurrent.futures import hashlib +import gzip import io import json import os @@ -18,8 +19,12 @@ from pathlib import Path from urllib.parse import urlencode, urlsplit -LIMIT = 32 * 1024 * 1024 +# A dedicated retained probe is tiny. Bound both transport and decompressed +# content instead of accepting arbitrary task-image bootstraps in this Job. +LIMIT = 1024 * 1024 +CONCURRENCY = 2 MARKER = b"\n__AKERNEL_HTTP_STATUS__" +EXIT_MARKER = b"\n__AKERNEL_CURL_EXIT__" BOOTSTRAP = "containerd.io/snapshot/nydus-bootstrap" @@ -47,9 +52,14 @@ def challenge(headers, allowed_origins): def decode_response(encoded): + if len(encoded) > 2 * LIMIT: + raise CheckFailed("seed response exceeds limit") try: raw = base64.b64decode(encoded, validate=False) - response, code = raw.rsplit(MARKER, 1) + response, exit_code = raw.rsplit(EXIT_MARKER, 1) + if int(exit_code.strip()) != 0: + raise CheckFailed("seed curl transfer failed") + response, code = response.rsplit(MARKER, 1) status = int(code.strip()) # curl may include proxy CONNECT / HTTP interim header blocks. while True: @@ -62,13 +72,17 @@ def decode_response(encoded): response = body continue break - if status == 0 or len(body) > LIMIT: + if status == 0 or len(body) > LIMIT or len(head) > 64 * 1024: raise ValueError("failed or too large") return status, headers, body except (ValueError, IndexError) as error: raise CheckFailed("seed HTTP request failed or returned an invalid response") from error +def curl_command(length): + return ["/bin/sh", "-c", 'head -c "$1" | { curl --config - --silent --show-error --suppress-connect-headers --include --write-out "\\n__AKERNEL_HTTP_STATUS__%{http_code}\\n"; result=$?; printf "\\n__AKERNEL_CURL_EXIT__%s\\n" "$result"; } | base64', "registry-check", str(length)] + + def digest_bytes(data, expected): if not re.fullmatch(r"sha256:[0-9a-f]{64}", expected or ""): raise CheckFailed("an immutable sha256 digest is required") @@ -91,7 +105,15 @@ def bootstrap_contents(layer, descriptor): if len(layer) != descriptor["size"]: raise CheckFailed("bootstrap layer size mismatch") try: - with tarfile.open(fileobj=io.BytesIO(layer), mode="r:*") as archive: + # Limit decompression before tar parsing, including skipped members. + if layer.startswith(b"\x1f\x8b"): + with gzip.GzipFile(fileobj=io.BytesIO(layer)) as compressed: + archive_data = compressed.read(2 * LIMIT + 1) + else: + archive_data = layer + if len(archive_data) > 2 * LIMIT: + raise CheckFailed("bootstrap archive exceeds decompression limit") + with tarfile.open(fileobj=io.BytesIO(archive_data), mode="r:") as archive: member = archive.getmember("image/image.boot") if not member.isfile() or not 0 < member.size <= LIMIT: raise CheckFailed("bootstrap is not a bounded regular file") @@ -99,7 +121,7 @@ def bootstrap_contents(layer, descriptor): if len(data) != member.size: raise CheckFailed("bootstrap file is truncated") return {"sha256": hashlib.sha256(data).hexdigest(), "bytes": len(data)} - except (tarfile.TarError, KeyError, OSError) as error: + except (tarfile.TarError, KeyError, OSError, EOFError) as error: raise CheckFailed("cannot read image/image.boot from bootstrap layer") from error @@ -107,17 +129,21 @@ class SeedHTTP: def __init__(self, api, namespace, pod, container, timeout=30): self.api, self.namespace, self.pod, self.container, self.timeout = api, namespace, pod, container, timeout - def get(self, url, headers=None, proxy=None): + def get(self, url, headers=None, proxy=None, ca_file=None): from kubernetes import client from kubernetes.stream import stream # Fixed-length stdin gives curl EOF without depending on a particular # Kubernetes exec websocket protocol's half-close implementation. options = {"url": url, "connect-timeout": "5", "max-time": str(self.timeout), "max-filesize": str(LIMIT), "noproxy": "" if proxy else "*", "proxy": proxy or ""} + if proxy and urlsplit(proxy).scheme != "http": + raise CheckFailed("Dragonfly probe requires an HTTP CONNECT proxy") + if ca_file: + options["cacert"] = ca_file config = "".join(f"{key} = {json.dumps(value)}\n" for key, value in options.items()) config += "".join(f"header = {json.dumps(key + ': ' + value)}\n" for key, value in (headers or {}).items()) payload = config.encode() - command = ["/bin/sh", "-c", 'head -c "$1" | curl --config - --silent --show-error --include --write-out "\\n__AKERNEL_HTTP_STATUS__%{http_code}\\n" | base64', "registry-check", str(len(payload))] + command = curl_command(len(payload)) # Kubernetes stream temporarily replaces ApiClient.request. A client # shared by concurrent seed checks would corrupt HTTP/exec dispatch. isolated = client.ApiClient(configuration=self.api.api_client.configuration) @@ -147,16 +173,25 @@ def get(self, url, headers=None, proxy=None): isolated.close() +def validate_probe_security(cfg): + if cfg.get("proxyCheckEnabled", True): + if cfg.get("proxyLoggingVerified") is not True: + raise CheckFailed("proxy checks require verified credential-safe Dragonfly logging") + if urlsplit(cfg["registry"]).scheme == "https" and not cfg.get("proxyCAFile"): + raise CheckFailed("HTTPS proxy checks require the Dragonfly signing CA bundle path in each seed") + + def verify_seed(http, cfg, credential): + validate_probe_security(cfg) start = time.monotonic() registry = cfg["registry"].rstrip("/") - status, headers, _ = http.get(registry + "/v2/") + status, headers, _ = http.get(registry + "/v2/", ca_file=cfg.get("registryCAFile")) if status != 401: raise CheckFailed("anonymous Registry access did not require authentication") realm, service = challenge(headers, cfg["allowedAuthOrigins"]) scope = "repository:" + cfg["repository"] + ":pull" basic = base64.b64encode((credential["username"] + ":" + credential["password"]).encode()).decode() - status, _, body = http.get(realm + "?" + urlencode({"service": service, "scope": scope}), {"Authorization": "Basic " + basic}) + status, _, body = http.get(realm + "?" + urlencode({"service": service, "scope": scope}), {"Authorization": "Basic " + basic}, ca_file=cfg.get("authCAFile")) if status != 200: raise CheckFailed("pull token acquisition failed") try: @@ -168,7 +203,7 @@ def verify_seed(http, cfg, credential): raise CheckFailed("invalid pull token response") from error auth = {"Authorization": "Bearer " + token, "Accept": "application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json"} root = registry + "/v2/" + cfg["repository"] - status, _, body = http.get(root + "/manifests/" + cfg["digest"], auth) + status, _, body = http.get(root + "/manifests/" + cfg["digest"], auth, ca_file=cfg.get("registryCAFile")) if status != 200: raise CheckFailed("authenticated manifest read failed") digest_bytes(body, cfg["digest"]) @@ -177,16 +212,18 @@ def verify_seed(http, cfg, credential): except (ValueError, TypeError) as error: raise CheckFailed("invalid manifest JSON") from error url = root + "/blobs/" + descriptor["digest"] - status, _, body = http.get(url, auth) + status, _, body = http.get(url, auth, ca_file=cfg.get("registryCAFile")) if status != 200: raise CheckFailed("authenticated bootstrap read failed") result = bootstrap_contents(body, descriptor) - status, _, body = http.get(url, auth, proxy=cfg.get("seedProxy", "http://127.0.0.1:4001")) - if status != 200: - raise CheckFailed("Dragonfly proxy bootstrap read failed") - if bootstrap_contents(body, descriptor) != result: - raise CheckFailed("Dragonfly proxy bootstrap differs from Registry") - return {"seed": http.pod, "passed": True, "bootstrap": result, "layer_digest": descriptor["digest"], "seconds": round(time.monotonic() - start, 3)} + proxy_checked = cfg.get("proxyCheckEnabled", True) + if proxy_checked: + status, _, body = http.get(url, auth, proxy=cfg.get("seedProxy", "http://127.0.0.1:4001"), ca_file=cfg.get("proxyCAFile")) + if status != 200: + raise CheckFailed("Dragonfly proxy bootstrap read failed") + if bootstrap_contents(body, descriptor) != result: + raise CheckFailed("Dragonfly proxy bootstrap differs from Registry") + return {"seed": http.pod, "passed": True, "proxy_checked": proxy_checked, "bootstrap": result, "layer_digest": descriptor["digest"], "seconds": round(time.monotonic() - start, 3)} def ready(pod): @@ -213,6 +250,7 @@ def fingerprint(api, apps, cfg): def execute(api, apps, cfg, credential, state, force=False, now=None, http_factory=SeedHTTP): + validate_probe_security(cfg) now = time.time() if now is None else now before = fingerprint(api, apps, cfg) if not force and state.get("fingerprint") == before and now - state.get("checked_epoch", 0) < cfg.get("recheckSeconds", 600): @@ -225,7 +263,7 @@ def one(name): return {"seed": name, "passed": False, "error": str(error)} except Exception: return {"seed": name, "passed": False, "error": "seed exec or HTTP transport failed"} - with concurrent.futures.ThreadPoolExecutor(max_workers=5) as pool: + with concurrent.futures.ThreadPoolExecutor(max_workers=CONCURRENCY) as pool: results = list(pool.map(one, cfg["seedNames"])) if not all(r["passed"] for r in results): print(json.dumps({"event": "verification_failed", "seeds": results}), flush=True) @@ -250,7 +288,7 @@ def main(): print(json.dumps({"event": "unchanged", "last_success": state.get("checked_at")}), flush=True) return # resourceVersion prevents overlapping manual/hook Jobs overwriting state. - api.patch_namespaced_config_map(cfg["stateConfigMap"], cfg["namespace"], {"metadata": {"resourceVersion": cm.metadata.resource_version}, "data": {"state.json": json.dumps(result, sort_keys=True)}}, _request_timeout=20) + api.patch_namespaced_config_map(cfg["stateConfigMap"], cfg["namespace"], {"metadata": {"resourceVersion": cm.metadata.resource_version}, "data": {"state.json": json.dumps(result, sort_keys=True)}}, field_manager="registry-connectivity-check", _request_timeout=20) print(json.dumps({"event": "verified", **result}), flush=True) diff --git a/deploy/registry-connectivity/tests/test_chart.py b/deploy/registry-connectivity/tests/test_chart.py index ee0e0ae8..96360bf4 100644 --- a/deploy/registry-connectivity/tests/test_chart.py +++ b/deploy/registry-connectivity/tests/test_chart.py @@ -14,7 +14,7 @@ class ChartTests(unittest.TestCase): def values(self): - return {"image": "registry.example/check@sha256:" + "a" * 64, "registry": {"url": "https://registry.example", "allowedAuthOrigins": ["https://auth.example"], "probeRepository": "operations/nydus", "probeDigest": "sha256:" + "b" * 64, "pullSecret": "probe-pull"}} + return {"image": "registry.example/check@sha256:" + "a" * 64, "registry": {"url": "https://registry.example", "allowedAuthOrigins": ["https://auth.example"], "probeRepository": "operations/nydus", "probeDigest": "sha256:" + "b" * 64, "pullSecret": "probe-pull"}, "networkPolicy": {"enabled": True, "existingClientsReviewed": True}, "dragonfly": {"proxyLoggingVerified": True, "proxyCAFile": "/trust/dragonfly-ca.pem"}} def render(self, values): with tempfile.TemporaryDirectory() as tmp: @@ -53,6 +53,35 @@ def test_checker_is_bounded_nonroot_and_cannot_overlap_scheduled_checks(self): secret = next(v for v in pod["volumes"] if "secret" in v) self.assertEqual(secret["secret"]["secretName"], "probe-pull") + def test_policy_is_opt_in_and_requires_existing_client_review(self): + values = self.values() + values.pop("networkPolicy") + result = self.render(values) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(any(o and o["kind"] == "NetworkPolicy" for o in yaml.safe_load_all(result.stdout))) + values["networkPolicy"] = {"enabled": True} + self.assertNotEqual(self.render(values).returncode, 0) + + def test_empty_or_invalid_port_lists_are_rejected(self): + for ports in [[], None, [0], [65536], ["all"], [-1]]: + values = self.values() + values["registry"]["ports"] = ports + self.assertNotEqual(self.render(values).returncode, 0, repr(ports)) + + def test_state_configmap_does_not_own_job_written_data(self): + state = next(o for o in self.objects() if o["kind"] == "ConfigMap" and o["metadata"]["name"] == "guard-state") + self.assertNotIn("data", state) + + def test_proxy_credential_logging_and_https_ca_are_explicit(self): + values = self.values() + values["dragonfly"]["proxyLoggingVerified"] = False + self.assertNotEqual(self.render(values).returncode, 0) + values["dragonfly"]["proxyLoggingVerified"] = True + values["dragonfly"]["proxyCAFile"] = "" + self.assertNotEqual(self.render(values).returncode, 0) + values["dragonfly"]["proxyCheckEnabled"] = False + self.assertEqual(self.render(values).returncode, 0) + def test_chart_does_not_take_over_registry_or_storage(self): kinds = {o["kind"] for o in self.objects()} self.assertFalse(kinds.intersection({"Deployment", "PersistentVolumeClaim", "Service", "Secret"})) diff --git a/deploy/registry-connectivity/tests/test_check.py b/deploy/registry-connectivity/tests/test_check.py index 55a09e2c..27f4f5ac 100644 --- a/deploy/registry-connectivity/tests/test_check.py +++ b/deploy/registry-connectivity/tests/test_check.py @@ -36,7 +36,7 @@ def pod(name, uid=None, is_ready=True): def configuration(): - return {"namespace": "registry", "deployment": "registry", "registrySelector": {"app": "registry"}, "seedNamespace": "dragonfly", "seedSelector": {"app": "dragonfly", "component": "seed-client"}, "seedContainer": "seed-client", "seedNames": [f"seed-{i}" for i in range(5)], "registry": "http://registry.test:5000", "allowedAuthOrigins": ["http://auth.test:5001"], "repository": "health/probe", "digest": sha(fixture()[1])} + return {"namespace": "registry", "deployment": "registry", "registrySelector": {"app": "registry"}, "seedNamespace": "dragonfly", "seedSelector": {"app": "dragonfly", "component": "seed-client"}, "seedContainer": "seed-client", "seedNames": [f"seed-{i}" for i in range(5)], "registry": "http://registry.test:5000", "allowedAuthOrigins": ["http://auth.test:5001"], "proxyLoggingVerified": True, "repository": "health/probe", "digest": sha(fixture()[1])} class API: @@ -59,7 +59,7 @@ def test_requires_trusted_token_origin_before_sending_credentials(self): def test_response_decodes_binary_and_status(self): data = b"\x00\xff\xfe" - encoded = base64.b64encode(b"HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\n\r\n" + data + check.MARKER + b"200\n") + encoded = base64.b64encode(b"HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\n\r\n" + data + check.MARKER + b"200\n" + check.EXIT_MARKER + b"0\n") status, headers, body = check.decode_response(encoded) self.assertEqual((status, body), (200, data)) self.assertEqual(headers["content-type"], "application/octet-stream") @@ -84,7 +84,7 @@ def test_authenticated_read_and_dragonfly_proxy_are_both_required(self): calls = [] class HTTP: pod = "seed-0" - def get(self, url, headers=None, proxy=None): + def get(self, url, headers=None, proxy=None, ca_file=None): calls.append((url, headers, proxy)) if url.endswith("/v2/"): return 401, {"www-authenticate": 'Bearer realm="http://auth.test:5001/auth",service="registry"'}, b"" @@ -126,7 +126,7 @@ def peek_stdout(self): return self.unread def read_stdout(self): self.unread = False - return base64.b64encode(b"HTTP/1.1 200 OK\r\n\r\nbody" + check.MARKER + b"200\n").decode() + return base64.b64encode(b"HTTP/1.1 200 OK\r\n\r\nbody" + check.MARKER + b"200\n" + check.EXIT_MARKER + b"0\n").decode() def peek_stderr(self): return False def close(self): diff --git a/deploy/registry-connectivity/tests/test_transport.py b/deploy/registry-connectivity/tests/test_transport.py new file mode 100644 index 00000000..38b212fb --- /dev/null +++ b/deploy/registry-connectivity/tests/test_transport.py @@ -0,0 +1,167 @@ +"""Exercise the actual seed shell command against local HTTP servers.""" +import base64 +import http.server +import json +import subprocess +import sys +import threading +import unittest + +from test_check import check, sha + + +class TransportTests(unittest.TestCase): + def test_real_curl_rejects_200_with_truncated_content_length(self): + body = b"expected-bytes-with-the-correct-content-digest" + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header("Content-Length", str(len(body) + 10)) + self.end_headers() + self.wfile.write(body) + self.close_connection = True + def log_message(self, *args): + pass + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + config = 'url = "http://127.0.0.1:%s/probe"\nmax-time = "3"\nnoproxy = "*"\n' % server.server_port + result = subprocess.run(check.curl_command(len(config.encode())), input=config.encode(), capture_output=True, timeout=5) + self.assertEqual(result.returncode, 0) # base64 itself succeeds. + raw = base64.b64decode(result.stdout) + self.assertIn(body, raw) + self.assertIn(check.MARKER + b"200", raw) + self.assertIn(check.EXIT_MARKER + b"18", raw) + check.digest_bytes(body, sha(body)) # Content digest alone is insufficient. + with self.assertRaisesRegex(check.CheckFailed, "curl transfer failed"): + check.decode_response(result.stdout) + finally: + server.shutdown() + server.server_close() + thread.join() + + def test_https_connect_response_headers_do_not_become_bootstrap_bytes(self): + import select + import socket + import socketserver + import ssl + import tempfile + from pathlib import Path + with tempfile.TemporaryDirectory() as tmp: + cert, key = Path(tmp) / "ca.pem", Path(tmp) / "key.pem" + subprocess.run(["openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes", "-keyout", str(key), "-out", str(cert), "-days", "1", "-subj", "/CN=localhost", "-addext", "subjectAltName=DNS:localhost"], capture_output=True, check=True, timeout=10) + body = b"actual-bootstrap-content" + class Registry(http.server.BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + def log_message(self, *args): + pass + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Registry) + tls = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + tls.load_cert_chain(str(cert), str(key)) + server.socket = tls.wrap_socket(server.socket, server_side=True) + class Proxy(socketserver.StreamRequestHandler): + def handle(self): + request = self.rfile.readline() + if not request.startswith(b"CONNECT "): + return + while self.rfile.readline().strip(): + pass + with socket.create_connection(("127.0.0.1", server.server_port), timeout=3) as upstream: + # Dragonfly can return OK rather than 'Connection established'. + self.wfile.write(b"HTTP/1.1 200 OK\r\n\r\n") + self.wfile.flush() + while True: + readable, _, _ = select.select([self.connection, upstream], [], [], 3) + if not readable: + return + for stream in readable: + data = stream.recv(65536) + if not data: + return + (upstream if stream is self.connection else self.connection).sendall(data) + proxy = socketserver.ThreadingTCPServer(("127.0.0.1", 0), Proxy) + proxy.daemon_threads = True + workers = [threading.Thread(target=s.serve_forever, daemon=True) for s in [server, proxy]] + for worker in workers: + worker.start() + try: + config = f'url = "https://localhost:{server.server_port}/blob"\nproxy = "http://127.0.0.1:{proxy.server_address[1]}"\nnoproxy = ""\nmax-time = "3"\n' + # No insecure flag: a missing CA must produce a curl failure. + failed = subprocess.run(check.curl_command(len(config.encode())), input=config.encode(), capture_output=True, timeout=5) + with self.assertRaisesRegex(check.CheckFailed, "curl transfer failed"): + check.decode_response(failed.stdout) + config += "cacert = " + json.dumps(str(cert)) + "\n" + result = subprocess.run(check.curl_command(len(config.encode())), input=config.encode(), capture_output=True, timeout=5, check=True) + status, _, received = check.decode_response(result.stdout) + self.assertEqual((status, received), (200, body)) + finally: + for instance in [server, proxy]: + instance.shutdown() + instance.server_close() + for worker in workers: + worker.join() + + def test_missing_exit_marker_never_counts_as_success(self): + with self.assertRaises(check.CheckFailed): + check.decode_response(base64.b64encode(b"HTTP/1.1 200 OK\r\n\r\nbody" + check.MARKER + b"200\n")) + + def test_oversized_compressed_archive_fails_before_tar_parsing(self): + import gzip + data = gzip.compress(b"x" * (2 * check.LIMIT + 1)) + descriptor = {"digest": sha(data), "size": len(data)} + with self.assertRaisesRegex(check.CheckFailed, "decompression limit"): + check.bootstrap_contents(data, descriptor) + + def test_proxy_logging_ack_is_required_before_any_network_request(self): + from test_check import configuration + cfg = configuration() + cfg["proxyLoggingVerified"] = False + class HTTP: + def get(self, *args, **kwargs): + raise AssertionError("must not send credentials") + with self.assertRaisesRegex(check.CheckFailed, "credential-safe"): + check.verify_seed(HTTP(), cfg, {}) + + def test_https_proxy_requires_its_own_ca_even_if_registry_ca_exists(self): + from test_check import configuration + cfg = configuration() + cfg.update(registry="https://registry.test", registryCAFile="/trust/registry.pem") + with self.assertRaisesRegex(check.CheckFailed, "Dragonfly signing CA"): + check.validate_probe_security(cfg) + cfg["proxyCAFile"] = "/trust/dragonfly.pem" + check.validate_probe_security(cfg) + + def test_allowed_cap_and_concurrency_fit_default_memory_limit(self): + # Fresh process isolates peak RSS from other tests. Exercise worst-case + # uncompressed layer buffering/decoding at the actual concurrency cap. + code = '''import base64,concurrent.futures,hashlib,io,resource,sys,tarfile,threading +sys.path.insert(0,sys.argv[1]);from test_check import check,sha +barrier=threading.Barrier(check.CONCURRENCY) +def probe(_): + out=io.BytesIO() + with tarfile.open(fileobj=out,mode='w:') as tar: + member=tarfile.TarInfo('image/image.boot');member.size=check.LIMIT-16384;tar.addfile(member,io.BytesIO(b'x'*member.size)) + layer=out.getvalue();assert len(layer)<=check.LIMIT + raw=b'HTTP/1.1 200 OK\\r\\n\\r\\n'+layer+check.MARKER+b'200\\n'+check.EXIT_MARKER+b'0\\n' + encoded=base64.b64encode(raw).decode();chunks=[encoded[i:i+16384] for i in range(0,len(encoded),16384)] + barrier.wait() + _,_,body=check.decode_response(''.join(chunks));check.bootstrap_contents(body,{'digest':sha(layer),'size':len(layer)}) +with concurrent.futures.ThreadPoolExecutor(max_workers=check.CONCURRENCY) as pool:list(pool.map(probe,range(check.CONCURRENCY))) +peak=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss +print(peak//1024 if sys.platform=='darwin' else peak) +''' + from pathlib import Path + result = subprocess.run([sys.executable, "-c", code, str(Path(__file__).parent)], capture_output=True, text=True, timeout=20, check=True) + peak_kib = int(result.stdout.strip()) + self.assertLess(peak_kib, 192 * 1024) + self.assertEqual(check.LIMIT, 1024 * 1024) + self.assertEqual(check.CONCURRENCY, 2) + + +if __name__ == "__main__": + unittest.main() From daf07fdf98ae23acc115f755335372b8f7d33b6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9C=88=E5=8F=AF?= Date: Tue, 10 Nov 2026 15:55:12 +0800 Subject: [PATCH 3/4] fix(deploy): support configured seeds and safe blob redirects --- deploy/README.md | 2 +- deploy/registry-connectivity/README.md | 38 +++- .../chart/templates/resources.yaml | 6 +- .../registry-connectivity/chart/values.yaml | 2 + deploy/registry-connectivity/check.py | 51 ++++- .../registry-connectivity/tests/test_chart.py | 15 ++ .../registry-connectivity/tests/test_check.py | 35 +++- .../tests/test_redirects.py | 188 ++++++++++++++++++ 8 files changed, 311 insertions(+), 26 deletions(-) create mode 100644 deploy/registry-connectivity/tests/test_redirects.py diff --git a/deploy/README.md b/deploy/README.md index 05ccae05..2ef6c545 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -19,7 +19,7 @@ application logs, see [the dual SDK/logging profile](docs/dual-sdk-and-applicati For private Registries serving Nydus through Dragonfly, install the [Registry connectivity guard](registry-connectivity/README.md). It maintains -namespace-and-Pod ingress selectors and verifies all five seeds after Registry +namespace-and-Pod ingress selectors and verifies all configured seeds after Registry moves or rollouts without changing Registry placement or storage. ## Aggregate sandbox memory budget diff --git a/deploy/registry-connectivity/README.md b/deploy/registry-connectivity/README.md index 5703de9f..e5fe11b9 100644 --- a/deploy/registry-connectivity/README.md +++ b/deploy/registry-connectivity/README.md @@ -12,7 +12,7 @@ NetworkPolicy creation is **off by default**. Before setting BuildKit, runtime nodes and direct push/pull clients. Existing policies must already permit their traffic. Set `networkPolicy.existingClientsReviewed=true` only after that review. The first ingress policy selecting a Pod makes it -ingress-isolated: five passing seed checks do not prove other clients retain +ingress-isolated: passing seed checks do not prove other clients retain access. Keep both flags explicit when upgrading an installation that already owns this policy; omitting them would remove that Helm-managed policy. @@ -34,12 +34,17 @@ configure source-side egress policies. ## Automatic verification A CronJob checks Registry generation, ready Pod UIDs, container identities and -restart counts, plus the identities of the five named seed Pods, every minute. +restart counts, plus the identities of all configured seed Pods, every minute. A change triggers a full check after the rollout becomes ready. An unchanged deployment is checked again every ten minutes. A change during verification invalidates the result and is retried. Missing/unready seeds fail the check. -Each of the five seeds must: +Set `dragonfly.seedNames` to the deployment's nonempty, duplicate-free Pod-name +list. Three- and five-seed profiles are both supported; five is only the example +default. All configured seeds must be Ready and pass; exec RBAC is generated +from this same list. The guard does not require expanding a seed pool. + +Each configured seed must: 1. Receive an unauthenticated Registry Bearer challenge. 2. Obtain a scoped pull token from an explicitly trusted token-service origin. @@ -50,6 +55,23 @@ Each of the five seeds must: Dragonfly HTTP proxy and verify identical content. Direct-only results explicitly report `proxy_checked: false` and do not validate the proxy path. +Proxy reads may be served from an existing Dragonfly cache entry. They verify +proxy availability and matching content, not a fresh origin fetch or cold-cache +performance; the separate direct Registry read still validates origin access. + +Blob reads support at most five HTTP(S) redirects. Every hop is a separate curl +request, so intermediate redirect bodies cannot enter bootstrap validation. +Same-origin redirects retain authorization; crossing an origin removes +Authorization, Proxy-Authorization and Cookie permanently for that chain, +including any return to the original origin. Signed URL queries are preserved +without being recorded in checker results. HTTPS-to-HTTP downgrades, credentialed +URLs, loops, non-HTTP protocols and excessive redirects are rejected. The final +layer still must match its immutable digest and size. Redirects use the same +Dragonfly proxy for proxied checks. `registry.redirectCAFile` optionally trusts +private object-storage origins during direct reads; otherwise native roots are +used. An HTTPS redirect target also requires `dragonfly.proxyCAFile` when +proxy checking is enabled, even when the initial Registry URL is HTTP. + Every curl transfer must exit successfully, even if HTTP 200 and the expected content bytes were received. Responses and bootstrap files are capped at 1 MiB, decompressed tar archives at 2 MiB, with at most two seeds checked concurrently. @@ -57,14 +79,14 @@ Keep the dedicated probe small; the default 256 MiB Job is not intended to inspect arbitrary task-image bootstraps. Oversized probes fail without advancing success state. A peak-RSS regression exercises the allowed bounds. -Only a complete five-seed success advances `-state`. A failed Job +Only a complete all-seed success advances `-state`. A failed Job logs redacted per-seed results and exits nonzero. Subsequent scheduled Jobs retry; this is a post-rollout check, not a Registry readiness gate or automatic rollback. Monitor failed Jobs and the age of the state ConfigMap's `checked_at`. History keeps two successful and three failed Jobs; completed Jobs expire after one day. A successful `unchanged` Job alone is not a new network verification. -The checker uses the Kubernetes API and execs only in the five configured seed +The checker uses the Kubernetes API and execs only in the configured seed Pods. Seeds require `sh`, `head`, `curl` and `base64`, but do not require Python. No seed files or process state are changed. Secret contents, Basic credentials and Bearer tokens pass through exec stdin and are omitted from command arguments @@ -72,7 +94,7 @@ and **checker** logs. Other components have independent logging behavior; credential safety of their logs must be verified separately. The checker ServiceAccount cannot read Secrets. It mounts one existing Secret containing a pull-only credential. Its exec permission is restricted to the -five Pod names; assess that permission as part of the deployment trust boundary. +configured Pod names; assess that permission as part of the deployment trust boundary. ## Prepare the checker and probe @@ -112,7 +134,7 @@ sending a real pull token through the proxy: the INFO/DEBUG dumps), or use a verified redacting implementation. Coordinate this with the owner of Dragonfly; this chart does not roll the seed Pods. 2. Send a **dummy**, non-production Authorization value through each seed's - proxy. Search for that marker in console output, daemon log files and any + proxy. For redirects also use a dummy signed-query marker. Search for that marker in console output, daemon log files and any collected logs after allowing the logging pipeline to flush. Do not use a real token to test redaction. Repeat after logging/version changes. 3. Only after verifying the log sinks, set @@ -189,7 +211,7 @@ Registry Deployment/PVC as part of installing the guard. Preserve other operators' placement and resource settings. After moving or rolling the Registry, inspect the next Job and confirm a new -`verified` event, **five** `passed: true` results, matching bootstrap hashes and +`verified` event, one `passed: true` result per configured seed, matching bootstrap hashes and a fresh `checked_at` in `registry-connectivity-state`. To force an immediate recheck without restarting production services: diff --git a/deploy/registry-connectivity/chart/templates/resources.yaml b/deploy/registry-connectivity/chart/templates/resources.yaml index 613b6c64..3f91747f 100644 --- a/deploy/registry-connectivity/chart/templates/resources.yaml +++ b/deploy/registry-connectivity/chart/templates/resources.yaml @@ -2,8 +2,8 @@ {{- if not (regexMatch "@sha256:[0-9a-f]{64}$" $image) -}} {{- fail "image must be pinned by sha256 digest" -}} {{- end -}} -{{- if or (ne (len .Values.dragonfly.seedNames) 5) (ne (len (uniq .Values.dragonfly.seedNames)) 5) -}} -{{- fail "exactly five distinct Dragonfly seedNames are required" -}} +{{- if or (empty .Values.dragonfly.seedNames) (ne (len (uniq .Values.dragonfly.seedNames)) (len .Values.dragonfly.seedNames)) -}} +{{- fail "a nonempty list of distinct Dragonfly seedNames is required" -}} {{- end -}} {{- if or (empty .Values.dragonfly.podSelector) (empty .Values.registry.podSelector) (empty .Values.registry.allowedAuthOrigins) -}} {{- fail "nonempty Registry/Dragonfly Pod selectors and allowedAuthOrigins are required" -}} @@ -129,7 +129,7 @@ metadata: namespace: {{ .Release.Namespace }} data: config.json: | - {{- dict "namespace" .Release.Namespace "deployment" .Values.registry.deployment "registrySelector" .Values.registry.podSelector "registry" (required "registry.url is required" .Values.registry.url) "allowedAuthOrigins" .Values.registry.allowedAuthOrigins "repository" (required "registry.probeRepository is required" .Values.registry.probeRepository) "digest" (required "registry.probeDigest is required" .Values.registry.probeDigest) "seedNamespace" .Values.dragonfly.namespace "seedNames" .Values.dragonfly.seedNames "seedSelector" .Values.dragonfly.podSelector "seedContainer" .Values.dragonfly.container "seedProxy" .Values.dragonfly.proxy "proxyCheckEnabled" .Values.dragonfly.proxyCheckEnabled "proxyLoggingVerified" .Values.dragonfly.proxyLoggingVerified "registryCAFile" .Values.registry.caFile "authCAFile" .Values.registry.authCAFile "proxyCAFile" .Values.dragonfly.proxyCAFile "stateConfigMap" (printf "%s-state" .Release.Name) "recheckSeconds" .Values.recheckSeconds | toJson | nindent 4 }} + {{- dict "namespace" .Release.Namespace "deployment" .Values.registry.deployment "registrySelector" .Values.registry.podSelector "registry" (required "registry.url is required" .Values.registry.url) "allowedAuthOrigins" .Values.registry.allowedAuthOrigins "repository" (required "registry.probeRepository is required" .Values.registry.probeRepository) "digest" (required "registry.probeDigest is required" .Values.registry.probeDigest) "seedNamespace" .Values.dragonfly.namespace "seedNames" .Values.dragonfly.seedNames "seedSelector" .Values.dragonfly.podSelector "seedContainer" .Values.dragonfly.container "seedProxy" .Values.dragonfly.proxy "proxyCheckEnabled" .Values.dragonfly.proxyCheckEnabled "proxyLoggingVerified" .Values.dragonfly.proxyLoggingVerified "registryCAFile" .Values.registry.caFile "authCAFile" .Values.registry.authCAFile "redirectCAFile" .Values.registry.redirectCAFile "proxyCAFile" .Values.dragonfly.proxyCAFile "stateConfigMap" (printf "%s-state" .Release.Name) "recheckSeconds" .Values.recheckSeconds | toJson | nindent 4 }} --- apiVersion: v1 kind: ConfigMap diff --git a/deploy/registry-connectivity/chart/values.yaml b/deploy/registry-connectivity/chart/values.yaml index 711ede55..77ea6ffc 100644 --- a/deploy/registry-connectivity/chart/values.yaml +++ b/deploy/registry-connectivity/chart/values.yaml @@ -12,6 +12,8 @@ registry: # CA bundle paths INSIDE each seed, not paths on the checker/client. caFile: "" authCAFile: "" + # Optional private object-storage CA for cross-origin blob redirects. + redirectCAFile: "" # A retained, single-platform Nydus manifest (not an image index). probeRepository: "" probeDigest: "" diff --git a/deploy/registry-connectivity/check.py b/deploy/registry-connectivity/check.py index a7fbcba4..35d12167 100644 --- a/deploy/registry-connectivity/check.py +++ b/deploy/registry-connectivity/check.py @@ -17,12 +17,13 @@ import time from datetime import datetime, timezone from pathlib import Path -from urllib.parse import urlencode, urlsplit +from urllib.parse import urlencode, urlsplit, urljoin # A dedicated retained probe is tiny. Bound both transport and decompressed # content instead of accepting arbitrary task-image bootstraps in this Job. LIMIT = 1024 * 1024 CONCURRENCY = 2 +MAX_REDIRECTS = 5 MARKER = b"\n__AKERNEL_HTTP_STATUS__" EXIT_MARKER = b"\n__AKERNEL_CURL_EXIT__" BOOTSTRAP = "containerd.io/snapshot/nydus-bootstrap" @@ -181,6 +182,41 @@ def validate_probe_security(cfg): raise CheckFailed("HTTPS proxy checks require the Dragonfly signing CA bundle path in each seed") +def read_bootstrap(http, cfg, url, headers, proxy=None): + """Follow bounded blob redirects without forwarding cross-origin secrets. + + Each hop is a separate curl request: intermediate headers/bodies cannot be + mistaken for the final layer. Credentials are never restored after leaving + the initial origin, even if a later redirect returns to it. + """ + initial_origin = origin(url) + forwarded = dict(headers) + visited = set() + for hop in range(MAX_REDIRECTS + 1): + if url in visited: + raise CheckFailed("bootstrap redirect loop detected") + visited.add(url) + if proxy: + ca_file = cfg.get("proxyCAFile") + if urlsplit(url).scheme == "https" and not ca_file: + raise CheckFailed("HTTPS proxy redirect requires the Dragonfly signing CA bundle") + else: + ca_file = cfg.get("registryCAFile") if origin(url) == initial_origin else cfg.get("redirectCAFile") + status, response_headers, body = http.get(url, forwarded, proxy=proxy, ca_file=ca_file) + if status == 200: + return body + if status not in (301, 302, 303, 307, 308) or not response_headers.get("location"): + raise CheckFailed("authenticated bootstrap read failed") + target = urljoin(url, response_headers["location"]) + target_origin = origin(target) # HTTP(S) only, with no URL credentials. + if urlsplit(url).scheme == "https" and urlsplit(target).scheme != "https": + raise CheckFailed("bootstrap redirect would downgrade HTTPS") + if origin(url) != target_origin: + forwarded = {key: value for key, value in forwarded.items() if key.lower() not in ("authorization", "proxy-authorization", "cookie")} + url = target + raise CheckFailed("bootstrap redirect limit exceeded") + + def verify_seed(http, cfg, credential): validate_probe_security(cfg) start = time.monotonic() @@ -212,15 +248,11 @@ def verify_seed(http, cfg, credential): except (ValueError, TypeError) as error: raise CheckFailed("invalid manifest JSON") from error url = root + "/blobs/" + descriptor["digest"] - status, _, body = http.get(url, auth, ca_file=cfg.get("registryCAFile")) - if status != 200: - raise CheckFailed("authenticated bootstrap read failed") + body = read_bootstrap(http, cfg, url, auth) result = bootstrap_contents(body, descriptor) proxy_checked = cfg.get("proxyCheckEnabled", True) if proxy_checked: - status, _, body = http.get(url, auth, proxy=cfg.get("seedProxy", "http://127.0.0.1:4001"), ca_file=cfg.get("proxyCAFile")) - if status != 200: - raise CheckFailed("Dragonfly proxy bootstrap read failed") + body = read_bootstrap(http, cfg, url, auth, proxy=cfg.get("seedProxy", "http://127.0.0.1:4001")) if bootstrap_contents(body, descriptor) != result: raise CheckFailed("Dragonfly proxy bootstrap differs from Registry") return {"seed": http.pod, "passed": True, "proxy_checked": proxy_checked, "bootstrap": result, "layer_digest": descriptor["digest"], "seconds": round(time.monotonic() - start, 3)} @@ -243,8 +275,9 @@ def fingerprint(api, apps, cfg): selector = ",".join(f"{k}={v}" for k, v in sorted(cfg["seedSelector"].items())) seeds = api.list_namespaced_pod(cfg["seedNamespace"], label_selector=selector, _request_timeout=20).items selected = {p.metadata.name: p for p in seeds if p.metadata.name in cfg["seedNames"]} - if len(cfg["seedNames"]) != 5 or len(set(cfg["seedNames"])) != 5 or set(selected) != set(cfg["seedNames"]) or not all(ready(p) for p in selected.values()): - raise CheckFailed("all five configured Dragonfly seeds must be present and Ready") + names = cfg["seedNames"] + if not names or len(set(names)) != len(names) or set(selected) != set(names) or not all(ready(p) for p in selected.values()): + raise CheckFailed("all configured Dragonfly seeds must be distinct, present and Ready") identities = {"deployment": [deployment.metadata.uid, deployment.metadata.generation], "registry": sorted((p.metadata.uid, [(s.name, s.image_id, s.restart_count) for s in (p.status.container_statuses or [])]) for p in pods), "seeds": sorted((name, p.metadata.uid, [(s.name, s.image_id, s.restart_count) for s in (p.status.container_statuses or [])]) for name, p in selected.items()), "config": cfg} return hashlib.sha256(json.dumps(identities, sort_keys=True).encode()).hexdigest() diff --git a/deploy/registry-connectivity/tests/test_chart.py b/deploy/registry-connectivity/tests/test_chart.py index 96360bf4..88a093da 100644 --- a/deploy/registry-connectivity/tests/test_chart.py +++ b/deploy/registry-connectivity/tests/test_chart.py @@ -82,6 +82,21 @@ def test_proxy_credential_logging_and_https_ca_are_explicit(self): values["dragonfly"]["proxyCheckEnabled"] = False self.assertEqual(self.render(values).returncode, 0) + def test_configured_three_and_five_seed_topologies_and_bad_lists(self): + for count in [3, 5]: + values = self.values() + names = [f"seed-{i}" for i in range(count)] + values["dragonfly"]["seedNames"] = names + result = self.render(values) + self.assertEqual(result.returncode, 0, result.stderr) + roles = [o for o in yaml.safe_load_all(result.stdout) if o["kind"] == "Role"] + rule = next(r for role in roles for r in role["rules"] if "pods/exec" in r["resources"]) + self.assertEqual(rule["resourceNames"], names) + for names in [[], ["seed-0", "seed-0"]]: + values = self.values() + values["dragonfly"]["seedNames"] = names + self.assertNotEqual(self.render(values).returncode, 0) + def test_chart_does_not_take_over_registry_or_storage(self): kinds = {o["kind"] for o in self.objects()} self.assertFalse(kinds.intersection({"Deployment", "PersistentVolumeClaim", "Service", "Secret"})) diff --git a/deploy/registry-connectivity/tests/test_check.py b/deploy/registry-connectivity/tests/test_check.py index 27f4f5ac..040f1b9a 100644 --- a/deploy/registry-connectivity/tests/test_check.py +++ b/deploy/registry-connectivity/tests/test_check.py @@ -1,6 +1,7 @@ import base64 import contextlib import hashlib +import gzip import importlib.util import io import json @@ -21,11 +22,12 @@ def sha(data): def fixture(): output = io.BytesIO() - with tarfile.open(fileobj=output, mode="w:gz") as archive: - data = b"Nydus-bootstrap-probe" - member = tarfile.TarInfo("image/image.boot") - member.size = len(data) - archive.addfile(member, io.BytesIO(data)) + with gzip.GzipFile(fileobj=output, mode="wb", mtime=0) as compressed: + with tarfile.open(fileobj=compressed, mode="w:") as archive: + data = b"Nydus-bootstrap-probe" + member = tarfile.TarInfo("image/image.boot") + member.size = len(data) + archive.addfile(member, io.BytesIO(data)) layer = output.getvalue() manifest = json.dumps({"layers": [{"digest": sha(layer), "size": len(layer), "annotations": {check.BOOTSTRAP: "true"}}]}).encode() return layer, manifest @@ -157,6 +159,29 @@ def test_migration_rollout_and_seed_replacement_change_fingerprint(self): api.seeds[0].metadata.uid = "replacement" self.assertNotEqual(check.fingerprint(api, api, cfg), second) + def test_configured_three_and_five_seeds_all_pass_and_fail_closed(self): + for count in [3, 5]: + api, cfg = API(), configuration() + api.seeds = api.seeds[:count] + cfg["seedNames"] = cfg["seedNames"][:count] + check.fingerprint(api, api, cfg) + with patch.object(check, "verify_seed", return_value={"passed": True, "bootstrap": {"sha256": "one"}}): + result = check.execute(api, api, cfg, {}, {}) + self.assertEqual(len(result["seeds"]), count) + api.seeds[-1].status.conditions[0].status = "False" + with self.assertRaises(check.CheckFailed): + check.fingerprint(api, api, cfg) + api.seeds.pop() + with self.assertRaises(check.CheckFailed): + check.fingerprint(api, api, cfg) + api, cfg = API(), configuration() + cfg["seedNames"] = ["seed-0", "seed-0"] + with self.assertRaises(check.CheckFailed): + check.fingerprint(api, api, cfg) + cfg["seedNames"] = [] + with self.assertRaises(check.CheckFailed): + check.fingerprint(api, api, cfg) + def test_missing_or_unready_seed_is_not_a_success(self): api, cfg = API(), configuration() api.seeds.pop() diff --git a/deploy/registry-connectivity/tests/test_redirects.py b/deploy/registry-connectivity/tests/test_redirects.py new file mode 100644 index 00000000..17aa0326 --- /dev/null +++ b/deploy/registry-connectivity/tests/test_redirects.py @@ -0,0 +1,188 @@ +"""Registry redirects through the real curl/SeedHTTP path; only exec is local.""" +import contextlib +import http.client +import http.server +import json +import subprocess +import threading +from types import SimpleNamespace as NS +from urllib.parse import urlsplit +import unittest +from unittest.mock import patch + +from test_check import check, configuration, fixture + + +@contextlib.contextmanager +def server(handler): + instance = http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler) + thread = threading.Thread(target=instance.serve_forever, daemon=True) + thread.start() + try: + yield instance + finally: + instance.shutdown() + instance.server_close() + thread.join() + + +@contextlib.contextmanager +def local_exec(): + class Client: + def __init__(self, configuration): + self.configuration = configuration + def close(self): + pass + class Core: + def __init__(self, client): + pass + def connect_get_namespaced_pod_exec(self): + pass + class WS: + opened, unread = True, True + def __init__(self, command): + self.command = command + def write_stdin(self, config): + self.result = subprocess.run(self.command, input=config.encode(), capture_output=True, timeout=10) + def is_open(self): + return self.opened + def update(self, **kwargs): + self.opened = False + def peek_stdout(self): + return self.unread + def read_stdout(self): + self.unread = False + return self.result.stdout.decode() + def peek_stderr(self): + return False + def close(self): + self.opened = False + modules = {"kubernetes": NS(client=NS(ApiClient=Client, CoreV1Api=Core)), "kubernetes.stream": NS(stream=lambda method, *args, **kwargs: WS(kwargs["command"]))} + with patch.dict("sys.modules", modules): + yield check.SeedHTTP(NS(api_client=NS(configuration=None)), "test", "seed", "proxy") + + +class RedirectTests(unittest.TestCase): + def verify(self, proxied, cross_origin=False, loop=False, limit=False): + layer, manifest = fixture() + events = [] + dummy = "dummy-redirect-pull-token" + class Objects(http.server.BaseHTTPRequestHandler): + def do_GET(self): + events.append(("object", self.path, self.headers.get("Authorization"))) + self.send_response(200) + self.send_header("Content-Length", str(len(layer))) + self.end_headers() + self.wfile.write(layer) + def log_message(self, *args): + pass + with server(Objects) as objects: + class Registry(http.server.BaseHTTPRequestHandler): + def do_GET(self): + events.append(("registry", self.path, self.headers.get("Authorization"))) + if self.path == "/v2/": + self.send_response(401) + self.send_header("WWW-Authenticate", f'Bearer realm="http://127.0.0.1:{self.server.server_port}/auth",service="registry"') + self.end_headers() + return + if self.path.startswith("/auth?"): + body = json.dumps({"token": dummy}).encode() + elif "/manifests/" in self.path: + body = manifest + elif self.path.startswith("/objects/") and not limit: + body = layer + else: + target = f"http://127.0.0.1:{objects.server_port}/objects/blob?signature=DUMMY-SIGNED-QUERY" if cross_origin else "/objects/blob" + if loop: + target = self.path + if limit: + target = "/objects/" + str(len(events)) + self.send_response(307) + self.send_header("Location", target) + self.send_header("Content-Length", "0") + self.end_headers() + return + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + def log_message(self, *args): + pass + class Proxy(http.server.BaseHTTPRequestHandler): + def do_GET(self): + target = urlsplit(self.path) + with contextlib.closing(http.client.HTTPConnection(target.hostname, target.port, timeout=3)) as upstream: + upstream.request("GET", target.path + ("?" + target.query if target.query else ""), headers=dict(self.headers)) + response = upstream.getresponse() + body = response.read() + self.send_response(response.status) + if response.getheader("Location"): + self.send_header("Location", response.getheader("Location")) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + def log_message(self, *args): + pass + with server(Registry) as registry, server(Proxy) as proxy, local_exec() as seed_http: + cfg = configuration() + cfg.update(registry=f"http://127.0.0.1:{registry.server_port}", allowedAuthOrigins=[f"http://127.0.0.1:{registry.server_port}"], seedProxy=f"http://127.0.0.1:{proxy.server_port}", proxyCheckEnabled=proxied) + if loop or limit: + with self.assertRaisesRegex(check.CheckFailed, "redirect (loop|limit)"): + check.verify_seed(seed_http, cfg, {"username": "dummy", "password": "dummy"}) + self.assertLessEqual(len(events), check.MAX_REDIRECTS + 4) + else: + result = check.verify_seed(seed_http, cfg, {"username": "dummy", "password": "dummy"}) + self.assertTrue(result["passed"]) + self.assertEqual(result["proxy_checked"], proxied) + objects_seen = [e for e in events if e[0] == "object"] + if cross_origin: + self.assertEqual(len(objects_seen), 2 if proxied else 1) + self.assertTrue(all(e[2] is None for e in objects_seen)) + self.assertTrue(all("signature=DUMMY-SIGNED-QUERY" in e[1] for e in objects_seen)) + else: + completed = [e for e in events if e[1] == "/objects/blob"] + self.assertTrue(completed) + self.assertTrue(all(e[2] == "Bearer " + dummy for e in completed)) + + def test_same_origin_direct_and_proxy_redirects(self): + for proxy in [False, True]: + with self.subTest(proxy=proxy): + self.verify(proxy) + + def test_cross_origin_strips_authorization_direct_and_proxy(self): + for proxy in [False, True]: + with self.subTest(proxy=proxy): + self.verify(proxy, cross_origin=True) + + def test_redirect_loop_and_limit_fail(self): + self.verify(False, loop=True) + self.verify(False, limit=True) + + def test_cross_origin_secrets_never_return_and_ca_is_per_origin(self): + calls = [] + responses = [(307, {"location": "https://objects.test/blob?sig=dummy"}, b""), (302, {"location": "https://registry.test/final"}, b""), (200, {}, b"ok")] + class HTTP: + def get(self, url, headers, **kwargs): + calls.append((url, dict(headers), kwargs.get("ca_file"))) + return responses.pop(0) + result = check.read_bootstrap(HTTP(), {"registryCAFile": "/trust/registry.pem", "redirectCAFile": "/trust/objects.pem"}, "https://registry.test/blob", {"Authorization": "Bearer dummy", "Cookie": "dummy", "Accept": "blob"}) + self.assertEqual(result, b"ok") + self.assertEqual([c[2] for c in calls], ["/trust/registry.pem", "/trust/objects.pem", "/trust/registry.pem"]) + self.assertEqual(calls[1][1], {"Accept": "blob"}) + self.assertEqual(calls[2][1], {"Accept": "blob"}) + + def test_https_downgrade_and_non_http_redirects_fail_before_next_request(self): + for location in ["http://objects.test/blob", "file:///etc/passwd", "https://user:password@objects.test/blob"]: + class HTTP: + calls = 0 + def get(self, *args, **kwargs): + self.calls += 1 + return 307, {"location": location}, b"" + http = HTTP() + with self.assertRaises(check.CheckFailed): + check.read_bootstrap(http, {}, "https://registry.test/blob", {"Authorization": "Bearer dummy"}) + self.assertEqual(http.calls, 1) + + +if __name__ == "__main__": + unittest.main() From f5f6c22c96b4b15c8fc557002cdeb2eca60f9074 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9C=88=E5=8F=AF?= Date: Wed, 11 Nov 2026 17:22:50 +0800 Subject: [PATCH 4/4] test(deploy): trim redundant Registry guard coverage --- deploy/registry-connectivity/README.md | 3 +- .../registry-connectivity/tests/test_chart.py | 23 +---- .../registry-connectivity/tests/test_check.py | 96 +------------------ .../tests/test_redirects.py | 69 +++++++------ .../tests/test_transport.py | 50 ++-------- 5 files changed, 49 insertions(+), 192 deletions(-) diff --git a/deploy/registry-connectivity/README.md b/deploy/registry-connectivity/README.md index e5fe11b9..f6f9441d 100644 --- a/deploy/registry-connectivity/README.md +++ b/deploy/registry-connectivity/README.md @@ -77,7 +77,8 @@ content bytes were received. Responses and bootstrap files are capped at 1 MiB, decompressed tar archives at 2 MiB, with at most two seeds checked concurrently. Keep the dedicated probe small; the default 256 MiB Job is not intended to inspect arbitrary task-image bootstraps. Oversized probes fail without advancing -success state. A peak-RSS regression exercises the allowed bounds. +success state. Size and decompression boundary regressions run in CI; peak-RSS +qualification is a separate environment-dependent check. Only a complete all-seed success advances `-state`. A failed Job logs redacted per-seed results and exits nonzero. Subsequent scheduled Jobs diff --git a/deploy/registry-connectivity/tests/test_chart.py b/deploy/registry-connectivity/tests/test_chart.py index 88a093da..0c58217a 100644 --- a/deploy/registry-connectivity/tests/test_chart.py +++ b/deploy/registry-connectivity/tests/test_chart.py @@ -34,25 +34,6 @@ def test_namespace_and_pod_labels_share_one_peer_and_only_data_auth_ports(self): self.assertEqual(rule["from"][0], {"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "dragonfly-system"}}, "podSelector": {"matchLabels": {"app": "dragonfly", "component": "seed-client"}}}) self.assertEqual(rule["ports"], [{"protocol": "TCP", "port": 5000}, {"protocol": "TCP", "port": 5001}]) - def test_exec_restricted_to_exact_five_seeds_and_no_secret_read_permission(self): - roles = [o for o in self.objects() if o["kind"] == "Role"] - rules = [r for role in roles for r in role["rules"]] - self.assertFalse(any("secrets" in r["resources"] for r in rules)) - execution = next(r for r in rules if "pods/exec" in r["resources"]) - self.assertEqual(execution["resourceNames"], [f"dragonfly-seed-client-{i}" for i in range(5)]) - - def test_checker_is_bounded_nonroot_and_cannot_overlap_scheduled_checks(self): - cron = next(o for o in self.objects() if o["kind"] == "CronJob") - self.assertEqual(cron["spec"]["schedule"], "* * * * *") - self.assertEqual(cron["spec"]["concurrencyPolicy"], "Forbid") - job = cron["spec"]["jobTemplate"]["spec"] - self.assertEqual(job["activeDeadlineSeconds"], 240) - pod = job["template"]["spec"] - self.assertTrue(pod["securityContext"]["runAsNonRoot"]) - self.assertTrue(pod["containers"][0]["securityContext"]["readOnlyRootFilesystem"]) - secret = next(v for v in pod["volumes"] if "secret" in v) - self.assertEqual(secret["secret"]["secretName"], "probe-pull") - def test_policy_is_opt_in_and_requires_existing_client_review(self): values = self.values() values.pop("networkPolicy") @@ -90,7 +71,9 @@ def test_configured_three_and_five_seed_topologies_and_bad_lists(self): result = self.render(values) self.assertEqual(result.returncode, 0, result.stderr) roles = [o for o in yaml.safe_load_all(result.stdout) if o["kind"] == "Role"] - rule = next(r for role in roles for r in role["rules"] if "pods/exec" in r["resources"]) + rules = [r for role in roles for r in role["rules"]] + self.assertFalse(any("secrets" in r["resources"] for r in rules)) + rule = next(r for r in rules if "pods/exec" in r["resources"]) self.assertEqual(rule["resourceNames"], names) for names in [[], ["seed-0", "seed-0"]]: values = self.values() diff --git a/deploy/registry-connectivity/tests/test_check.py b/deploy/registry-connectivity/tests/test_check.py index 040f1b9a..a30d93f7 100644 --- a/deploy/registry-connectivity/tests/test_check.py +++ b/deploy/registry-connectivity/tests/test_check.py @@ -1,4 +1,3 @@ -import base64 import contextlib import hashlib import gzip @@ -59,17 +58,6 @@ def test_requires_trusted_token_origin_before_sending_credentials(self): with self.assertRaisesRegex(check.CheckFailed, "not explicitly trusted"): check.challenge({"www-authenticate": 'Bearer realm="http://other.test/auth",service="registry"'}, ["http://auth.test:5001"]) - def test_response_decodes_binary_and_status(self): - data = b"\x00\xff\xfe" - encoded = base64.b64encode(b"HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\n\r\n" + data + check.MARKER + b"200\n" + check.EXIT_MARKER + b"0\n") - status, headers, body = check.decode_response(encoded) - self.assertEqual((status, body), (200, data)) - self.assertEqual(headers["content-type"], "application/octet-stream") - - def test_missing_curl_status_is_failure(self): - with self.assertRaises(check.CheckFailed): - check.decode_response(base64.b64encode(b"curl failed")) - def test_content_is_verified_and_real_bootstrap_is_read(self): layer, manifest = fixture() result = check.bootstrap_contents(layer, check.bootstrap_descriptor(json.loads(manifest))) @@ -81,72 +69,6 @@ def test_oci_without_nydus_bootstrap_is_rejected(self): with self.assertRaises(check.CheckFailed): check.bootstrap_descriptor({"layers": [{"size": 1}]}) - def test_authenticated_read_and_dragonfly_proxy_are_both_required(self): - layer, manifest = fixture() - calls = [] - class HTTP: - pod = "seed-0" - def get(self, url, headers=None, proxy=None, ca_file=None): - calls.append((url, headers, proxy)) - if url.endswith("/v2/"): - return 401, {"www-authenticate": 'Bearer realm="http://auth.test:5001/auth",service="registry"'}, b"" - if url.startswith("http://auth.test"): - return 200, {}, b'{"token":"secret-token"}' - if "/manifests/" in url: - return 200, {}, manifest - return 200, {}, layer - result = check.verify_seed(HTTP(), configuration(), {"username": "pull", "password": "secret-password"}) - self.assertTrue(result["passed"]) - self.assertEqual(len(calls), 5) - self.assertEqual(calls[-1][2], "http://127.0.0.1:4001") - self.assertNotIn("secret", json.dumps(result)) - self.assertTrue(calls[1][1]["Authorization"].startswith("Basic ")) - self.assertEqual(calls[2][1]["Authorization"], "Bearer secret-token") - - def test_exec_uses_isolated_api_clients_and_keeps_credentials_off_argv(self): - clients, commands = [], [] - class Client: - def __init__(self, configuration): - self.configuration, self.closed = configuration, False - clients.append(self) - def close(self): - self.closed = True - class Core: - def __init__(self, client): - self.api_client = client - def connect_get_namespaced_pod_exec(self): - pass - class WS: - opened, unread = True, True - def write_stdin(self, config): - self.config = config - def is_open(self): - return self.opened - def update(self, **kwargs): - self.opened = False - def peek_stdout(self): - return self.unread - def read_stdout(self): - self.unread = False - return base64.b64encode(b"HTTP/1.1 200 OK\r\n\r\nbody" + check.MARKER + b"200\n" + check.EXIT_MARKER + b"0\n").decode() - def peek_stderr(self): - return False - def close(self): - self.opened = False - shared = NS(api_client=NS(configuration=object())) - def stream(method, *args, **kwargs): - self.assertIsNot(method.__self__.api_client, shared.api_client) - commands.append(kwargs["command"]) - return WS() - modules = {"kubernetes": NS(client=NS(ApiClient=Client, CoreV1Api=Core)), "kubernetes.stream": NS(stream=stream)} - with patch.dict("sys.modules", modules): - for name in ["seed-0", "seed-1"]: - result = check.SeedHTTP(shared, "df", name, "seed").get("http://registry.test/v2/", {"Authorization": "Bearer secret-token"}) - self.assertEqual(result[0], 200) - self.assertEqual(len(clients), 2) - self.assertTrue(all(c.closed for c in clients)) - self.assertNotIn("secret-token", repr(commands)) - def test_migration_rollout_and_seed_replacement_change_fingerprint(self): api, cfg = API(), configuration() first = check.fingerprint(api, api, cfg) @@ -168,6 +90,7 @@ def test_configured_three_and_five_seeds_all_pass_and_fail_closed(self): with patch.object(check, "verify_seed", return_value={"passed": True, "bootstrap": {"sha256": "one"}}): result = check.execute(api, api, cfg, {}, {}) self.assertEqual(len(result["seeds"]), count) + self.assertEqual(result["fingerprint"], check.fingerprint(api, api, cfg)) api.seeds[-1].status.conditions[0].status = "False" with self.assertRaises(check.CheckFailed): check.fingerprint(api, api, cfg) @@ -182,15 +105,6 @@ def test_configured_three_and_five_seeds_all_pass_and_fail_closed(self): with self.assertRaises(check.CheckFailed): check.fingerprint(api, api, cfg) - def test_missing_or_unready_seed_is_not_a_success(self): - api, cfg = API(), configuration() - api.seeds.pop() - with self.assertRaises(check.CheckFailed): - check.fingerprint(api, api, cfg) - api.seeds.append(pod("seed-4", is_ready=False)) - with self.assertRaises(check.CheckFailed): - check.fingerprint(api, api, cfg) - def test_unchanged_state_skips_until_periodic_recheck(self): api, cfg = API(), configuration() state = {"fingerprint": check.fingerprint(api, api, cfg), "checked_epoch": 100} @@ -217,13 +131,5 @@ def test_rollout_during_check_never_advances_success_state(self): with self.assertRaisesRegex(check.CheckFailed, "identity changed"): check.execute(api, api, cfg, {}, {}) - def test_five_successes_form_a_new_state(self): - api, cfg = API(), configuration() - with patch.object(check, "verify_seed", return_value={"passed": True, "bootstrap": {"sha256": "one"}}): - state = check.execute(api, api, cfg, {}, {}, now=100) - self.assertEqual(len(state["seeds"]), 5) - self.assertEqual(state["fingerprint"], check.fingerprint(api, api, cfg)) - - if __name__ == "__main__": unittest.main() diff --git a/deploy/registry-connectivity/tests/test_redirects.py b/deploy/registry-connectivity/tests/test_redirects.py index 17aa0326..bd5f8630 100644 --- a/deploy/registry-connectivity/tests/test_redirects.py +++ b/deploy/registry-connectivity/tests/test_redirects.py @@ -28,21 +28,16 @@ def server(handler): @contextlib.contextmanager def local_exec(): - class Client: - def __init__(self, configuration): - self.configuration = configuration - def close(self): - pass - class Core: - def __init__(self, client): - pass - def connect_get_namespaced_pod_exec(self): - pass class WS: opened, unread = True, True def __init__(self, command): self.command = command def write_stdin(self, config): + for line in config.splitlines(): + if line.startswith("header ="): + header = json.loads(line.split("=", 1)[1]) + if header.lower().startswith("authorization:"): + assert header not in " ".join(self.command) self.result = subprocess.run(self.command, input=config.encode(), capture_output=True, timeout=10) def is_open(self): return self.opened @@ -57,13 +52,14 @@ def peek_stderr(self): return False def close(self): self.opened = False - modules = {"kubernetes": NS(client=NS(ApiClient=Client, CoreV1Api=Core)), "kubernetes.stream": NS(stream=lambda method, *args, **kwargs: WS(kwargs["command"]))} + api = NS(ApiClient=lambda **kwargs: NS(close=lambda: None), CoreV1Api=lambda client: NS(connect_get_namespaced_pod_exec=None)) + modules = {"kubernetes": NS(client=api), "kubernetes.stream": NS(stream=lambda method, *args, **kwargs: WS(kwargs["command"]))} with patch.dict("sys.modules", modules): yield check.SeedHTTP(NS(api_client=NS(configuration=None)), "test", "seed", "proxy") class RedirectTests(unittest.TestCase): - def verify(self, proxied, cross_origin=False, loop=False, limit=False): + def verify(self, proxied, cross_origin=False): layer, manifest = fixture() events = [] dummy = "dummy-redirect-pull-token" @@ -89,14 +85,10 @@ def do_GET(self): body = json.dumps({"token": dummy}).encode() elif "/manifests/" in self.path: body = manifest - elif self.path.startswith("/objects/") and not limit: + elif self.path.startswith("/objects/"): body = layer else: target = f"http://127.0.0.1:{objects.server_port}/objects/blob?signature=DUMMY-SIGNED-QUERY" if cross_origin else "/objects/blob" - if loop: - target = self.path - if limit: - target = "/objects/" + str(len(events)) self.send_response(307) self.send_header("Location", target) self.send_header("Content-Length", "0") @@ -126,23 +118,22 @@ def log_message(self, *args): with server(Registry) as registry, server(Proxy) as proxy, local_exec() as seed_http: cfg = configuration() cfg.update(registry=f"http://127.0.0.1:{registry.server_port}", allowedAuthOrigins=[f"http://127.0.0.1:{registry.server_port}"], seedProxy=f"http://127.0.0.1:{proxy.server_port}", proxyCheckEnabled=proxied) - if loop or limit: - with self.assertRaisesRegex(check.CheckFailed, "redirect (loop|limit)"): - check.verify_seed(seed_http, cfg, {"username": "dummy", "password": "dummy"}) - self.assertLessEqual(len(events), check.MAX_REDIRECTS + 4) + result = check.verify_seed(seed_http, cfg, {"username": "dummy", "password": "dummy"}) + self.assertTrue(result["passed"]) + self.assertEqual(result["proxy_checked"], proxied) + self.assertNotIn(dummy, json.dumps(result)) + auth_requests = [e for e in events if e[1].startswith("/auth?")] + self.assertTrue(auth_requests) + self.assertTrue(all(e[2].startswith("Basic ") for e in auth_requests)) + objects_seen = [e for e in events if e[0] == "object"] + if cross_origin: + self.assertEqual(len(objects_seen), 2 if proxied else 1) + self.assertTrue(all(e[2] is None for e in objects_seen)) + self.assertTrue(all("signature=DUMMY-SIGNED-QUERY" in e[1] for e in objects_seen)) else: - result = check.verify_seed(seed_http, cfg, {"username": "dummy", "password": "dummy"}) - self.assertTrue(result["passed"]) - self.assertEqual(result["proxy_checked"], proxied) - objects_seen = [e for e in events if e[0] == "object"] - if cross_origin: - self.assertEqual(len(objects_seen), 2 if proxied else 1) - self.assertTrue(all(e[2] is None for e in objects_seen)) - self.assertTrue(all("signature=DUMMY-SIGNED-QUERY" in e[1] for e in objects_seen)) - else: - completed = [e for e in events if e[1] == "/objects/blob"] - self.assertTrue(completed) - self.assertTrue(all(e[2] == "Bearer " + dummy for e in completed)) + completed = [e for e in events if e[1] == "/objects/blob"] + self.assertTrue(completed) + self.assertTrue(all(e[2] == "Bearer " + dummy for e in completed)) def test_same_origin_direct_and_proxy_redirects(self): for proxy in [False, True]: @@ -155,8 +146,16 @@ def test_cross_origin_strips_authorization_direct_and_proxy(self): self.verify(proxy, cross_origin=True) def test_redirect_loop_and_limit_fail(self): - self.verify(False, loop=True) - self.verify(False, limit=True) + for loop in [True, False]: + class HTTP: + calls = 0 + def get(self, url, *args, **kwargs): + self.calls += 1 + return 307, {"location": url if loop else f"/next/{self.calls}"}, b"" + http = HTTP() + with self.assertRaisesRegex(check.CheckFailed, "redirect (loop|limit)"): + check.read_bootstrap(http, {}, "http://registry.test/blob", {}) + self.assertLessEqual(http.calls, check.MAX_REDIRECTS + 1) def test_cross_origin_secrets_never_return_and_ca_is_per_origin(self): calls = [] diff --git a/deploy/registry-connectivity/tests/test_transport.py b/deploy/registry-connectivity/tests/test_transport.py index 38b212fb..1ce40512 100644 --- a/deploy/registry-connectivity/tests/test_transport.py +++ b/deploy/registry-connectivity/tests/test_transport.py @@ -3,7 +3,6 @@ import http.server import json import subprocess -import sys import threading import unittest @@ -117,51 +116,20 @@ def test_oversized_compressed_archive_fails_before_tar_parsing(self): with self.assertRaisesRegex(check.CheckFailed, "decompression limit"): check.bootstrap_contents(data, descriptor) - def test_proxy_logging_ack_is_required_before_any_network_request(self): + def test_proxy_logging_and_https_trust_are_required_before_network(self): from test_check import configuration - cfg = configuration() - cfg["proxyLoggingVerified"] = False class HTTP: def get(self, *args, **kwargs): raise AssertionError("must not send credentials") - with self.assertRaisesRegex(check.CheckFailed, "credential-safe"): - check.verify_seed(HTTP(), cfg, {}) - - def test_https_proxy_requires_its_own_ca_even_if_registry_ca_exists(self): - from test_check import configuration - cfg = configuration() - cfg.update(registry="https://registry.test", registryCAFile="/trust/registry.pem") - with self.assertRaisesRegex(check.CheckFailed, "Dragonfly signing CA"): - check.validate_probe_security(cfg) - cfg["proxyCAFile"] = "/trust/dragonfly.pem" - check.validate_probe_security(cfg) - - def test_allowed_cap_and_concurrency_fit_default_memory_limit(self): - # Fresh process isolates peak RSS from other tests. Exercise worst-case - # uncompressed layer buffering/decoding at the actual concurrency cap. - code = '''import base64,concurrent.futures,hashlib,io,resource,sys,tarfile,threading -sys.path.insert(0,sys.argv[1]);from test_check import check,sha -barrier=threading.Barrier(check.CONCURRENCY) -def probe(_): - out=io.BytesIO() - with tarfile.open(fileobj=out,mode='w:') as tar: - member=tarfile.TarInfo('image/image.boot');member.size=check.LIMIT-16384;tar.addfile(member,io.BytesIO(b'x'*member.size)) - layer=out.getvalue();assert len(layer)<=check.LIMIT - raw=b'HTTP/1.1 200 OK\\r\\n\\r\\n'+layer+check.MARKER+b'200\\n'+check.EXIT_MARKER+b'0\\n' - encoded=base64.b64encode(raw).decode();chunks=[encoded[i:i+16384] for i in range(0,len(encoded),16384)] - barrier.wait() - _,_,body=check.decode_response(''.join(chunks));check.bootstrap_contents(body,{'digest':sha(layer),'size':len(layer)}) -with concurrent.futures.ThreadPoolExecutor(max_workers=check.CONCURRENCY) as pool:list(pool.map(probe,range(check.CONCURRENCY))) -peak=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss -print(peak//1024 if sys.platform=='darwin' else peak) -''' - from pathlib import Path - result = subprocess.run([sys.executable, "-c", code, str(Path(__file__).parent)], capture_output=True, text=True, timeout=20, check=True) - peak_kib = int(result.stdout.strip()) - self.assertLess(peak_kib, 192 * 1024) - self.assertEqual(check.LIMIT, 1024 * 1024) - self.assertEqual(check.CONCURRENCY, 2) + for overrides in [{"proxyLoggingVerified": False}, {"registry": "https://registry.test", "registryCAFile": "/trust/registry.pem"}]: + with self.subTest(config=overrides), self.assertRaises(check.CheckFailed): + check.verify_seed(HTTP(), {**configuration(), **overrides}, {}) + def test_oversized_response_is_rejected(self): + raw = b"HTTP/1.1 200 OK\r\n\r\n" + b"x" * (check.LIMIT + 1) + raw += check.MARKER + b"200\n" + check.EXIT_MARKER + b"0\n" + with self.assertRaises(check.CheckFailed): + check.decode_response(base64.b64encode(raw)) if __name__ == "__main__": unittest.main()