diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 66215efb..f39f7c0c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -193,6 +193,7 @@ jobs: python deploy/tests/kubernetes/test_node_lifecycle.py -v python deploy/tests/kubernetes/test_runc_filesystems.py -v python deploy/akernel/charts/core/tests/test_adx_chart.py -v + python -m unittest discover -s deploy/registry-connectivity/tests -v python deploy/akernel/charts/core/tests/test_node_directories.py -v python deploy/akernel/charts/core/tests/test_shared_checkpoint_storage.py -v python deploy/akernel/charts/core/tests/test_prepare_checkpoint_storage.py -v diff --git a/deploy/README.md b/deploy/README.md index 7d640bd2..2ef6c545 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -17,6 +17,11 @@ when a shared filesystem is independently required. For one deployment serving both native and E2B SDKs with version-independent application logs, see [the dual SDK/logging profile](docs/dual-sdk-and-application-logs.md). +For private Registries serving Nydus through Dragonfly, install the +[Registry connectivity guard](registry-connectivity/README.md). It maintains +namespace-and-Pod ingress selectors and verifies all configured seeds after Registry +moves or rollouts without changing Registry placement or storage. + ## Aggregate sandbox memory budget The direct Helm chart enables the cgroup v2 aggregate sandbox memory policy by diff --git a/deploy/registry-connectivity/.dockerignore b/deploy/registry-connectivity/.dockerignore new file mode 100644 index 00000000..6647548a --- /dev/null +++ b/deploy/registry-connectivity/.dockerignore @@ -0,0 +1,5 @@ +__pycache__ +**/__pycache__ +tests +chart +README.md diff --git a/deploy/registry-connectivity/Dockerfile b/deploy/registry-connectivity/Dockerfile new file mode 100644 index 00000000..037d0c42 --- /dev/null +++ b/deploy/registry-connectivity/Dockerfile @@ -0,0 +1,7 @@ +FROM python:3.13-slim@sha256:e48f59cdba69a3cbd9ed2cf0f348d90d45a8d350d98134cf1070bc106ee9ed90 +WORKDIR /app +COPY --chmod=0444 requirements.txt ./ +RUN pip install --no-cache-dir -r requirements.txt +COPY --chmod=0444 check.py ./check.py +USER 65532:65532 +ENTRYPOINT ["python", "/app/check.py"] diff --git a/deploy/registry-connectivity/README.md b/deploy/registry-connectivity/README.md new file mode 100644 index 00000000..f6f9441d --- /dev/null +++ b/deploy/registry-connectivity/README.md @@ -0,0 +1,237 @@ +# Registry connectivity guard + +Install this companion Helm release alongside a private Registry that uses +Bearer token authentication and serves Nydus images. It does not own or change +the Registry Deployment, storage, resources or placement. Keep it installed +when moving or upgrading the Registry. + +## Network boundary + +NetworkPolicy creation is **off by default**. Before setting +`networkPolicy.enabled=true`, review every current Registry client, including +BuildKit, runtime nodes and direct push/pull clients. Existing policies must +already permit their traffic. Set `networkPolicy.existingClientsReviewed=true` +only after that review. The first ingress policy selecting a Pod makes it +ingress-isolated: passing seed checks do not prove other clients retain +access. Keep both flags explicit when upgrading an installation that already +owns this policy; omitting them would remove that Helm-managed policy. + +When enabled, the NetworkPolicy selects Registry Pods and permits TCP 5000 +(Registry) and 5001 (token authentication) from **one peer containing both selectors**: + +- namespace label `kubernetes.io/metadata.name: dragonfly-system`; +- Pod labels `app: dragonfly` and `component: seed-client`. + +These selectors are an AND, not separate namespace-wide / Pod-wide allowances. +Existing ingress policies remain additive. CIDRs alone are insufficient when +the CNI enforces source Pod identity across node pools. Change the selectors +and container ports to match the deployed services; service-facing ports may +differ from container ports. This policy grants no access to unrelated +Registry management ports. Empty port lists and invalid port numbers are rejected; +they must never render as an unrestricted `ports: null`. The policy does not +configure source-side egress policies. + +## Automatic verification + +A CronJob checks Registry generation, ready Pod UIDs, container identities and +restart counts, plus the identities of all configured seed Pods, every minute. +A change triggers a full check after the rollout becomes ready. An unchanged +deployment is checked again every ten minutes. A change during verification +invalidates the result and is retried. Missing/unready seeds fail the check. + +Set `dragonfly.seedNames` to the deployment's nonempty, duplicate-free Pod-name +list. Three- and five-seed profiles are both supported; five is only the example +default. All configured seeds must be Ready and pass; exec RBAC is generated +from this same list. The guard does not require expanding a seed pool. + +Each configured seed must: + +1. Receive an unauthenticated Registry Bearer challenge. +2. Obtain a scoped pull token from an explicitly trusted token-service origin. +3. Fetch the configured immutable manifest and verify its SHA-256. +4. Fetch its actual Nydus bootstrap layer directly from the Registry; verify + digest and size, and read the regular `image/image.boot` tar member. +5. When proxy checking is enabled, read the same layer through the seed's local + Dragonfly HTTP proxy and verify identical content. Direct-only results + explicitly report `proxy_checked: false` and do not validate the proxy path. + +Proxy reads may be served from an existing Dragonfly cache entry. They verify +proxy availability and matching content, not a fresh origin fetch or cold-cache +performance; the separate direct Registry read still validates origin access. + +Blob reads support at most five HTTP(S) redirects. Every hop is a separate curl +request, so intermediate redirect bodies cannot enter bootstrap validation. +Same-origin redirects retain authorization; crossing an origin removes +Authorization, Proxy-Authorization and Cookie permanently for that chain, +including any return to the original origin. Signed URL queries are preserved +without being recorded in checker results. HTTPS-to-HTTP downgrades, credentialed +URLs, loops, non-HTTP protocols and excessive redirects are rejected. The final +layer still must match its immutable digest and size. Redirects use the same +Dragonfly proxy for proxied checks. `registry.redirectCAFile` optionally trusts +private object-storage origins during direct reads; otherwise native roots are +used. An HTTPS redirect target also requires `dragonfly.proxyCAFile` when +proxy checking is enabled, even when the initial Registry URL is HTTP. + +Every curl transfer must exit successfully, even if HTTP 200 and the expected +content bytes were received. Responses and bootstrap files are capped at 1 MiB, +decompressed tar archives at 2 MiB, with at most two seeds checked concurrently. +Keep the dedicated probe small; the default 256 MiB Job is not intended to +inspect arbitrary task-image bootstraps. Oversized probes fail without advancing +success state. Size and decompression boundary regressions run in CI; peak-RSS +qualification is a separate environment-dependent check. + +Only a complete all-seed success advances `-state`. A failed Job +logs redacted per-seed results and exits nonzero. Subsequent scheduled Jobs +retry; this is a post-rollout check, not a Registry readiness gate or automatic +rollback. Monitor failed Jobs and the age of the state ConfigMap's `checked_at`. +History keeps two successful and three failed Jobs; completed Jobs expire after +one day. A successful `unchanged` Job alone is not a new network verification. + +The checker uses the Kubernetes API and execs only in the configured seed +Pods. Seeds require `sh`, `head`, `curl` and `base64`, but do not require Python. +No seed files or process state are changed. Secret contents, Basic credentials +and Bearer tokens pass through exec stdin and are omitted from command arguments +and **checker** logs. Other components have independent logging behavior; +credential safety of their logs must be verified separately. +The checker ServiceAccount cannot read Secrets. It mounts one existing Secret +containing a pull-only credential. Its exec permission is restricted to the +configured Pod names; assess that permission as part of the deployment trust boundary. + +## Prepare the checker and probe + +Build `Dockerfile` with an approved remote BuildKit builder and publish the +checker image; pin its manifest digest in Helm values. The client running +Buildx must reach both the exporter Registry and its token-service realm. +Forwarding only BuildKit gRPC does not provide token-service connectivity. +Do not build images on an operator workstation when remote builds are required. + +Create a small real Nydus probe using the deployed Nydus-capable BuildKit: + +```dockerfile +FROM scratch +COPY probe.txt /probe.txt +``` + +Use its native image exporter with `oci-mediatypes=true,compression=nydus` and +publish to a dedicated probe repository. Pin a **single-platform Nydus manifest +digest**, not an index. Retain that manifest, bootstrap and referenced blobs; +exclude this fixture from task-image garbage collection. The checker does not +create artifacts, retain task images, or modify Registry garbage collection. +Registry backup/restore must preserve this fixture and its pull authorization. + +Provision a Secret in the Registry namespace through the platform's Secret +manager, with a `credentials.json` key containing `username` and `password`. +Use a pull-only identity, preferably restricted to the probe repository. Never +put credentials in Helm values, command arguments or version control. The +checker does not need the token signer's private key or a push credential. + +## Proxy logging and HTTPS prerequisites + +Dragonfly client v1.3.0 logs complete proxy requests, including Authorization, +at INFO level. Checker log redaction does not protect those logs. Before +sending a real pull token through the proxy: + +1. Disable request-dump logging (for unmodified v1.3.0, `--log-level=warn` removes + the INFO/DEBUG dumps), or use a verified redacting implementation. Coordinate + this with the owner of Dragonfly; this chart does not roll the seed Pods. +2. Send a **dummy**, non-production Authorization value through each seed's + proxy. For redirects also use a dummy signed-query marker. Search for that marker in console output, daemon log files and any + collected logs after allowing the logging pipeline to flush. Do not use a + real token to test redaction. Repeat after logging/version changes. +3. Only after verifying the log sinks, set + `dragonfly.proxyLoggingVerified=true`. Both Helm and the checker reject + enabled proxy probes without this explicit verification. If it cannot yet + be verified, use `dragonfly.proxyCheckEnabled=false` for direct-only checks + or `suspend=true` to pause the guard. + +For an HTTPS Registry, curl uses CONNECT to the **HTTP** Dragonfly proxy. +Dragonfly terminates TLS and signs a replacement endpoint certificate. Configure +`proxy.server.caCert` and `proxy.server.caKey` in Dragonfly with its signing CA; +its default ephemeral self-signed certificate is unsuitable for this verified +probe. Configure Dragonfly's upstream Registry trust separately, for example +`proxy.registryMirror.cert` in v1.3.0. + +Use these paths **inside every seed**, provisioned by the platform: + +- `registry.caFile`: curl's CA bundle for a direct private HTTPS Registry. +- `registry.authCAFile`: curl's CA bundle for the private HTTPS token service. +- `dragonfly.proxyCAFile`: the Dragonfly signing CA bundle, explicitly required + for HTTPS proxy checks. This is curl's target `cacert` for the intercepted + connection, not `proxy-cacert` (the proxy transport itself is HTTP). + +Direct requests can use curl's native trusted roots when their CA path is empty. +Trusting only the Registry CA does not trust Dragonfly's generated certificate. +No insecure TLS flag is added. These settings concern Registry traffic inside +seeds; SDK `AKERNEL_VERIFY_TLS` controls the separate AKernel API connection and +is neither required nor consumed by this chart. Using an HTTP API endpoint does +not require that SDK setting; it does not enable HTTPS or change the URL scheme. + +Sources: [Dragonfly proxy logging and CONNECT handling](https://github.com/dragonflyoss/client/blob/v1.3.0/dragonfly-client/src/proxy/mod.rs), +[proxy and Registry CA configuration](https://github.com/dragonflyoss/client/blob/v1.3.0/dragonfly-client-config/src/dfdaemon.rs). + +## Deploy + +Supply operator-owned values (keep private addresses out of Git): + +```yaml +image: registry.example/operations/registry-connectivity@sha256: +registry: + deployment: registry + podSelector: {app: managed-registry} + url: https://registry.example + allowedAuthOrigins: [https://registry-auth.example] + probeRepository: operations/nydus-connectivity + probeDigest: sha256: + pullSecret: registry-connectivity-pull +dragonfly: + # Start with direct-only checks until proxy logging/trust has been verified. + proxyCheckEnabled: false +# Defaults select the five dragonfly-seed-client-{0..4} Pods. +# NetworkPolicy is enabled separately after auditing existing client ingress: +# networkPolicy: {enabled: true, existingClientsReviewed: true} +``` + +Configure `imagePullSecrets`, placement and tolerations as needed. Enable proxy +checks only after completing the prerequisites above; for HTTPS add +`dragonfly.proxyCAFile`. Plain HTTP is supported where deployment policy permits +it. The chart creates the state ConfigMap without `data.state.json`; the Job +owns that field, so Helm server-side apply must not initialize/reset it on an +upgrade. Retain the state through subsequent releases. + +```bash +helm upgrade --install registry-connectivity ./deploy/registry-connectivity/chart \ + --namespace -f /secure/operator-values.yaml +kubectl -n create job \ + --from=cronjob/registry-connectivity registry-connectivity-initial +kubectl -n logs job/registry-connectivity-initial +``` + +If a policy with the configured name already exists, compare its full spec +before adopting that policy into this release. Do not adopt or replace the +Registry Deployment/PVC as part of installing the guard. Preserve other +operators' placement and resource settings. + +After moving or rolling the Registry, inspect the next Job and confirm a new +`verified` event, one `passed: true` result per configured seed, matching bootstrap hashes and +a fresh `checked_at` in `registry-connectivity-state`. To force an immediate +recheck without restarting production services: + +```bash +kubectl -n create job \ + --from=cronjob/registry-connectivity registry-connectivity-manual +# A periodic check runs automatically even if no Registry identity changed. +# To bypass the ten-minute reuse window, add FORCE_CHECK=1 to a copy of the +# CronJob's Job template before creating that manual Job. +``` + +The local transport regressions require `curl` and `openssl` in addition to +Python, PyYAML and Helm. They exercise truncated HTTP and HTTPS CONNECT without +changing a cluster. Validate changes with: + +```bash +python3 -m unittest discover -s deploy/registry-connectivity/tests -v +``` + +Also run `helm lint` with complete operator/example values and review the +rendered policy. Test a rollout trigger using a disposable watched Deployment +rather than restarting a shared Registry solely to exercise the checker. diff --git a/deploy/registry-connectivity/chart/Chart.yaml b/deploy/registry-connectivity/chart/Chart.yaml new file mode 100644 index 00000000..9f39482e --- /dev/null +++ b/deploy/registry-connectivity/chart/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +name: registry-connectivity +version: 0.1.0 +description: Restricted Dragonfly Registry ingress and authenticated Nydus rollout checks +type: application diff --git a/deploy/registry-connectivity/chart/templates/resources.yaml b/deploy/registry-connectivity/chart/templates/resources.yaml new file mode 100644 index 00000000..3f91747f --- /dev/null +++ b/deploy/registry-connectivity/chart/templates/resources.yaml @@ -0,0 +1,206 @@ +{{- $image := required "image must specify the checker image digest" .Values.image -}} +{{- if not (regexMatch "@sha256:[0-9a-f]{64}$" $image) -}} +{{- fail "image must be pinned by sha256 digest" -}} +{{- end -}} +{{- if or (empty .Values.dragonfly.seedNames) (ne (len (uniq .Values.dragonfly.seedNames)) (len .Values.dragonfly.seedNames)) -}} +{{- fail "a nonempty list of distinct Dragonfly seedNames is required" -}} +{{- end -}} +{{- if or (empty .Values.dragonfly.podSelector) (empty .Values.registry.podSelector) (empty .Values.registry.allowedAuthOrigins) -}} +{{- fail "nonempty Registry/Dragonfly Pod selectors and allowedAuthOrigins are required" -}} +{{- end }} +{{- if empty .Values.registry.ports -}} +{{- fail "registry.ports must be a nonempty list of explicit TCP ports" -}} +{{- end -}} +{{- range .Values.registry.ports -}} +{{- if or (not (regexMatch "^[1-9][0-9]*$" (toString .))) (gt (int .) 65535) -}} +{{- fail "registry.ports must contain integers between 1 and 65535" -}} +{{- end -}} +{{- end -}} +{{- if and .Values.dragonfly.proxyCheckEnabled (not .Values.dragonfly.proxyLoggingVerified) -}} +{{- fail "proxy checks require dragonfly.proxyLoggingVerified after credential-safe logging is verified with dummy credentials" -}} +{{- end -}} +{{- if and .Values.dragonfly.proxyCheckEnabled (hasPrefix "https://" .Values.registry.url) (empty .Values.dragonfly.proxyCAFile) -}} +{{- fail "HTTPS proxy checks require dragonfly.proxyCAFile inside each seed" -}} +{{- end -}} +{{- if .Values.networkPolicy.enabled -}} +{{- if not .Values.networkPolicy.existingClientsReviewed -}} +{{- fail "review existing ingress coverage for BuildKit and direct clients before enabling networkPolicy.existingClientsReviewed" -}} +{{- end }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ .Values.networkPolicy.name }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: registry-connectivity + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/part-of: akernel +spec: + podSelector: + matchLabels: + {{- toYaml .Values.registry.podSelector | nindent 6 }} + policyTypes: [Ingress] + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ .Values.dragonfly.namespace | quote }} + podSelector: + matchLabels: + {{- toYaml .Values.dragonfly.podSelector | nindent 14 }} + ports: + {{- range .Values.registry.ports }} + - protocol: TCP + port: {{ . }} + {{- end }} +--- +{{- end }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} +rules: + - apiGroups: [apps] + resources: [deployments] + resourceNames: [{{ .Values.registry.deployment | quote }}] + verbs: [get] + - apiGroups: [''] + resources: [pods] + verbs: [get, list] + - apiGroups: [''] + resources: [configmaps] + resourceNames: [{{ printf "%s-state" .Release.Name | quote }}] + verbs: [get, patch] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ .Release.Name }} +subjects: + - kind: ServiceAccount + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ printf "%s-%s" .Release.Namespace .Release.Name }} + namespace: {{ .Values.dragonfly.namespace }} +rules: + - apiGroups: [''] + resources: [pods] + verbs: [get, list] + - apiGroups: [''] + resources: [pods/exec] + resourceNames: + {{- toYaml .Values.dragonfly.seedNames | nindent 6 }} + verbs: [get, create] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ printf "%s-%s" .Release.Namespace .Release.Name }} + namespace: {{ .Values.dragonfly.namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ printf "%s-%s" .Release.Namespace .Release.Name }} +subjects: + - kind: ServiceAccount + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-config + namespace: {{ .Release.Namespace }} +data: + config.json: | + {{- dict "namespace" .Release.Namespace "deployment" .Values.registry.deployment "registrySelector" .Values.registry.podSelector "registry" (required "registry.url is required" .Values.registry.url) "allowedAuthOrigins" .Values.registry.allowedAuthOrigins "repository" (required "registry.probeRepository is required" .Values.registry.probeRepository) "digest" (required "registry.probeDigest is required" .Values.registry.probeDigest) "seedNamespace" .Values.dragonfly.namespace "seedNames" .Values.dragonfly.seedNames "seedSelector" .Values.dragonfly.podSelector "seedContainer" .Values.dragonfly.container "seedProxy" .Values.dragonfly.proxy "proxyCheckEnabled" .Values.dragonfly.proxyCheckEnabled "proxyLoggingVerified" .Values.dragonfly.proxyLoggingVerified "registryCAFile" .Values.registry.caFile "authCAFile" .Values.registry.authCAFile "redirectCAFile" .Values.registry.redirectCAFile "proxyCAFile" .Values.dragonfly.proxyCAFile "stateConfigMap" (printf "%s-state" .Release.Name) "recheckSeconds" .Values.recheckSeconds | toJson | nindent 4 }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-state + namespace: {{ .Release.Namespace }} +# state.json belongs to the Job; Helm must never initialize/reset this field. +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} +spec: + schedule: {{ .Values.schedule | quote }} + suspend: {{ .Values.suspend }} + concurrencyPolicy: Forbid + startingDeadlineSeconds: 90 + successfulJobsHistoryLimit: 2 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + backoffLimit: 0 + activeDeadlineSeconds: 240 + ttlSecondsAfterFinished: 86400 + template: + spec: + serviceAccountName: {{ .Release.Name }} + restartPolicy: Never + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: {type: RuntimeDefault} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 12 }} + {{- end }} + containers: + - name: check + image: {{ $image | quote }} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: {drop: [ALL]} + env: + - name: PYTHONDONTWRITEBYTECODE + value: '1' + - name: NO_PROXY + value: '*' + - name: no_proxy + value: '*' + resources: + {{- toYaml .Values.resources | nindent 16 }} + volumeMounts: + - {name: config, mountPath: /config, readOnly: true} + - {name: credentials, mountPath: /credentials, readOnly: true} + volumes: + - name: config + configMap: + name: {{ .Release.Name }}-config + - name: credentials + secret: + secretName: {{ required "registry.pullSecret is required" .Values.registry.pullSecret }} + defaultMode: 0440 diff --git a/deploy/registry-connectivity/chart/values.yaml b/deploy/registry-connectivity/chart/values.yaml new file mode 100644 index 00000000..77ea6ffc --- /dev/null +++ b/deploy/registry-connectivity/chart/values.yaml @@ -0,0 +1,52 @@ +image: "" # Required immutable checker image (repository@sha256:...). +imagePullSecrets: [] +nodeSelector: {} +tolerations: [] +registry: + deployment: registry + podSelector: + app: managed-registry + url: "" + allowedAuthOrigins: [] + ports: [5000, 5001] + # CA bundle paths INSIDE each seed, not paths on the checker/client. + caFile: "" + authCAFile: "" + # Optional private object-storage CA for cross-origin blob redirects. + redirectCAFile: "" + # A retained, single-platform Nydus manifest (not an image index). + probeRepository: "" + probeDigest: "" + # Existing Secret with credentials.json: {"username": ..., "password": ...}. + # Use a pull-only identity restricted to the probe repository. + pullSecret: "" +dragonfly: + namespace: dragonfly-system + podSelector: + app: dragonfly + component: seed-client + container: seed-client + seedNames: + - dragonfly-seed-client-0 + - dragonfly-seed-client-1 + - dragonfly-seed-client-2 + - dragonfly-seed-client-3 + - dragonfly-seed-client-4 + proxy: http://127.0.0.1:4001 + proxyCheckEnabled: true + # Enable only after disabling/redacting request dumps and testing a dummy + # credential against all proxy log sinks (console, files and collectors). + proxyLoggingVerified: false + # HTTPS CONNECT is terminated by Dragonfly: curl needs this signing CA. + proxyCAFile: "" +networkPolicy: + enabled: false + # Verify existing policies preserve BuildKit/direct-client ingress first. + existingClientsReviewed: false + name: registry-dragonfly-seed-ingress +schedule: '* * * * *' +suspend: false +recheckSeconds: 600 +resources: + requests: {cpu: 50m, memory: 96Mi} + limits: {cpu: '1', memory: 256Mi} diff --git a/deploy/registry-connectivity/check.py b/deploy/registry-connectivity/check.py new file mode 100644 index 00000000..35d12167 --- /dev/null +++ b/deploy/registry-connectivity/check.py @@ -0,0 +1,337 @@ +"""Verify Registry authentication and Nydus reads from every Dragonfly seed. + +Credentials are supplied to remote curl over exec stdin, never command arguments. +Only digests, sizes, timings and fixed diagnostic messages are recorded. +""" +from __future__ import annotations + +import base64 +import concurrent.futures +import hashlib +import gzip +import io +import json +import os +import re +import tarfile +import time +from datetime import datetime, timezone +from pathlib import Path +from urllib.parse import urlencode, urlsplit, urljoin + +# A dedicated retained probe is tiny. Bound both transport and decompressed +# content instead of accepting arbitrary task-image bootstraps in this Job. +LIMIT = 1024 * 1024 +CONCURRENCY = 2 +MAX_REDIRECTS = 5 +MARKER = b"\n__AKERNEL_HTTP_STATUS__" +EXIT_MARKER = b"\n__AKERNEL_CURL_EXIT__" +BOOTSTRAP = "containerd.io/snapshot/nydus-bootstrap" + + +class CheckFailed(RuntimeError): + """A credential-free diagnostic safe for the Job log.""" + + +def origin(url): + parsed = urlsplit(url) + if parsed.scheme not in ("http", "https") or not parsed.hostname or parsed.username or parsed.password: + raise CheckFailed("invalid endpoint") + return parsed.scheme, parsed.hostname, parsed.port or (443 if parsed.scheme == "https" else 80) + + +def challenge(headers, allowed_origins): + value = headers.get("www-authenticate", "") + if not value.lower().startswith("bearer "): + raise CheckFailed("Registry did not return a Bearer challenge") + fields = dict(re.findall(r'(\w+)="([^"\r\n]*)"', value)) + if not fields.get("service") or origin(fields.get("realm", "")) not in {origin(u) for u in allowed_origins}: + raise CheckFailed("token realm is not explicitly trusted") + if urlsplit(fields["realm"]).query or urlsplit(fields["realm"]).fragment: + raise CheckFailed("token realm must not contain query or fragment") + return fields["realm"], fields["service"] + + +def decode_response(encoded): + if len(encoded) > 2 * LIMIT: + raise CheckFailed("seed response exceeds limit") + try: + raw = base64.b64decode(encoded, validate=False) + response, exit_code = raw.rsplit(EXIT_MARKER, 1) + if int(exit_code.strip()) != 0: + raise CheckFailed("seed curl transfer failed") + response, code = response.rsplit(MARKER, 1) + status = int(code.strip()) + # curl may include proxy CONNECT / HTTP interim header blocks. + while True: + head, body = response.split(b"\r\n\r\n", 1) + lines = head.decode("iso-8859-1").split("\r\n") + if not lines[0].startswith("HTTP/"): + raise ValueError("missing status") + headers = {k.lower(): v.strip() for k, v in (line.split(":", 1) for line in lines[1:] if ":" in line)} + if body.startswith(b"HTTP/") and ("connection established" in lines[0].lower() or lines[0].split()[1] == "100"): + response = body + continue + break + if status == 0 or len(body) > LIMIT or len(head) > 64 * 1024: + raise ValueError("failed or too large") + return status, headers, body + except (ValueError, IndexError) as error: + raise CheckFailed("seed HTTP request failed or returned an invalid response") from error + + +def curl_command(length): + return ["/bin/sh", "-c", 'head -c "$1" | { curl --config - --silent --show-error --suppress-connect-headers --include --write-out "\\n__AKERNEL_HTTP_STATUS__%{http_code}\\n"; result=$?; printf "\\n__AKERNEL_CURL_EXIT__%s\\n" "$result"; } | base64', "registry-check", str(length)] + + +def digest_bytes(data, expected): + if not re.fullmatch(r"sha256:[0-9a-f]{64}", expected or ""): + raise CheckFailed("an immutable sha256 digest is required") + if "sha256:" + hashlib.sha256(data).hexdigest() != expected: + raise CheckFailed("content digest mismatch") + + +def bootstrap_descriptor(manifest): + layers = manifest.get("layers", []) + if not layers or BOOTSTRAP not in layers[-1].get("annotations", {}): + raise CheckFailed("probe is not a Nydus manifest with a bootstrap layer") + layer = layers[-1] + if not isinstance(layer.get("size"), int) or not 0 < layer["size"] <= LIMIT: + raise CheckFailed("bootstrap layer size is invalid") + return layer + + +def bootstrap_contents(layer, descriptor): + digest_bytes(layer, descriptor["digest"]) + if len(layer) != descriptor["size"]: + raise CheckFailed("bootstrap layer size mismatch") + try: + # Limit decompression before tar parsing, including skipped members. + if layer.startswith(b"\x1f\x8b"): + with gzip.GzipFile(fileobj=io.BytesIO(layer)) as compressed: + archive_data = compressed.read(2 * LIMIT + 1) + else: + archive_data = layer + if len(archive_data) > 2 * LIMIT: + raise CheckFailed("bootstrap archive exceeds decompression limit") + with tarfile.open(fileobj=io.BytesIO(archive_data), mode="r:") as archive: + member = archive.getmember("image/image.boot") + if not member.isfile() or not 0 < member.size <= LIMIT: + raise CheckFailed("bootstrap is not a bounded regular file") + data = archive.extractfile(member).read(LIMIT + 1) + if len(data) != member.size: + raise CheckFailed("bootstrap file is truncated") + return {"sha256": hashlib.sha256(data).hexdigest(), "bytes": len(data)} + except (tarfile.TarError, KeyError, OSError, EOFError) as error: + raise CheckFailed("cannot read image/image.boot from bootstrap layer") from error + + +class SeedHTTP: + def __init__(self, api, namespace, pod, container, timeout=30): + self.api, self.namespace, self.pod, self.container, self.timeout = api, namespace, pod, container, timeout + + def get(self, url, headers=None, proxy=None, ca_file=None): + from kubernetes import client + from kubernetes.stream import stream + + # Fixed-length stdin gives curl EOF without depending on a particular + # Kubernetes exec websocket protocol's half-close implementation. + options = {"url": url, "connect-timeout": "5", "max-time": str(self.timeout), "max-filesize": str(LIMIT), "noproxy": "" if proxy else "*", "proxy": proxy or ""} + if proxy and urlsplit(proxy).scheme != "http": + raise CheckFailed("Dragonfly probe requires an HTTP CONNECT proxy") + if ca_file: + options["cacert"] = ca_file + config = "".join(f"{key} = {json.dumps(value)}\n" for key, value in options.items()) + config += "".join(f"header = {json.dumps(key + ': ' + value)}\n" for key, value in (headers or {}).items()) + payload = config.encode() + command = curl_command(len(payload)) + # Kubernetes stream temporarily replaces ApiClient.request. A client + # shared by concurrent seed checks would corrupt HTTP/exec dispatch. + isolated = client.ApiClient(configuration=self.api.api_client.configuration) + connection = None + output = [] + size = 0 + deadline = time.monotonic() + self.timeout + 15 + try: + connection = stream(client.CoreV1Api(isolated).connect_get_namespaced_pod_exec, self.pod, self.namespace, container=self.container, command=command, stdin=True, stdout=True, stderr=True, tty=False, _preload_content=False, _request_timeout=10) + connection.write_stdin(config) + while connection.is_open(): + if time.monotonic() >= deadline: + raise CheckFailed("seed exec deadline exceeded") + connection.update(timeout=1) + while connection.peek_stdout(): + chunk = connection.read_stdout() + size += len(chunk) + if size > LIMIT * 2: + raise CheckFailed("seed response exceeds limit") + output.append(chunk) + if connection.peek_stderr(): + connection.read_stderr() # Never expose upstream text/credentials. + return decode_response("".join(output)) + finally: + if connection is not None: + connection.close() + isolated.close() + + +def validate_probe_security(cfg): + if cfg.get("proxyCheckEnabled", True): + if cfg.get("proxyLoggingVerified") is not True: + raise CheckFailed("proxy checks require verified credential-safe Dragonfly logging") + if urlsplit(cfg["registry"]).scheme == "https" and not cfg.get("proxyCAFile"): + raise CheckFailed("HTTPS proxy checks require the Dragonfly signing CA bundle path in each seed") + + +def read_bootstrap(http, cfg, url, headers, proxy=None): + """Follow bounded blob redirects without forwarding cross-origin secrets. + + Each hop is a separate curl request: intermediate headers/bodies cannot be + mistaken for the final layer. Credentials are never restored after leaving + the initial origin, even if a later redirect returns to it. + """ + initial_origin = origin(url) + forwarded = dict(headers) + visited = set() + for hop in range(MAX_REDIRECTS + 1): + if url in visited: + raise CheckFailed("bootstrap redirect loop detected") + visited.add(url) + if proxy: + ca_file = cfg.get("proxyCAFile") + if urlsplit(url).scheme == "https" and not ca_file: + raise CheckFailed("HTTPS proxy redirect requires the Dragonfly signing CA bundle") + else: + ca_file = cfg.get("registryCAFile") if origin(url) == initial_origin else cfg.get("redirectCAFile") + status, response_headers, body = http.get(url, forwarded, proxy=proxy, ca_file=ca_file) + if status == 200: + return body + if status not in (301, 302, 303, 307, 308) or not response_headers.get("location"): + raise CheckFailed("authenticated bootstrap read failed") + target = urljoin(url, response_headers["location"]) + target_origin = origin(target) # HTTP(S) only, with no URL credentials. + if urlsplit(url).scheme == "https" and urlsplit(target).scheme != "https": + raise CheckFailed("bootstrap redirect would downgrade HTTPS") + if origin(url) != target_origin: + forwarded = {key: value for key, value in forwarded.items() if key.lower() not in ("authorization", "proxy-authorization", "cookie")} + url = target + raise CheckFailed("bootstrap redirect limit exceeded") + + +def verify_seed(http, cfg, credential): + validate_probe_security(cfg) + start = time.monotonic() + registry = cfg["registry"].rstrip("/") + status, headers, _ = http.get(registry + "/v2/", ca_file=cfg.get("registryCAFile")) + if status != 401: + raise CheckFailed("anonymous Registry access did not require authentication") + realm, service = challenge(headers, cfg["allowedAuthOrigins"]) + scope = "repository:" + cfg["repository"] + ":pull" + basic = base64.b64encode((credential["username"] + ":" + credential["password"]).encode()).decode() + status, _, body = http.get(realm + "?" + urlencode({"service": service, "scope": scope}), {"Authorization": "Basic " + basic}, ca_file=cfg.get("authCAFile")) + if status != 200: + raise CheckFailed("pull token acquisition failed") + try: + token_data = json.loads(body) + token = token_data.get("token") or token_data.get("access_token") + if not isinstance(token, str) or not token: + raise ValueError("missing token") + except (ValueError, TypeError) as error: + raise CheckFailed("invalid pull token response") from error + auth = {"Authorization": "Bearer " + token, "Accept": "application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json"} + root = registry + "/v2/" + cfg["repository"] + status, _, body = http.get(root + "/manifests/" + cfg["digest"], auth, ca_file=cfg.get("registryCAFile")) + if status != 200: + raise CheckFailed("authenticated manifest read failed") + digest_bytes(body, cfg["digest"]) + try: + descriptor = bootstrap_descriptor(json.loads(body)) + except (ValueError, TypeError) as error: + raise CheckFailed("invalid manifest JSON") from error + url = root + "/blobs/" + descriptor["digest"] + body = read_bootstrap(http, cfg, url, auth) + result = bootstrap_contents(body, descriptor) + proxy_checked = cfg.get("proxyCheckEnabled", True) + if proxy_checked: + body = read_bootstrap(http, cfg, url, auth, proxy=cfg.get("seedProxy", "http://127.0.0.1:4001")) + if bootstrap_contents(body, descriptor) != result: + raise CheckFailed("Dragonfly proxy bootstrap differs from Registry") + return {"seed": http.pod, "passed": True, "proxy_checked": proxy_checked, "bootstrap": result, "layer_digest": descriptor["digest"], "seconds": round(time.monotonic() - start, 3)} + + +def ready(pod): + return not pod.metadata.deletion_timestamp and any(c.type == "Ready" and c.status == "True" for c in (pod.status.conditions or [])) + + +def fingerprint(api, apps, cfg): + deployment = apps.read_namespaced_deployment(cfg["deployment"], cfg["namespace"], _request_timeout=20) + spec, status = deployment.spec, deployment.status + count = spec.replicas or 0 + if count < 1 or (status.observed_generation or 0) < deployment.metadata.generation or (status.updated_replicas or 0) != count or (status.available_replicas or 0) != count or (status.replicas or 0) != count: + raise CheckFailed("Registry rollout is not ready") + selector = ",".join(f"{k}={v}" for k, v in sorted(cfg["registrySelector"].items())) + pods = [p for p in api.list_namespaced_pod(cfg["namespace"], label_selector=selector, _request_timeout=20).items if not p.metadata.deletion_timestamp] + if len(pods) != count or not all(ready(p) for p in pods): + raise CheckFailed("Registry Pods are not ready") + selector = ",".join(f"{k}={v}" for k, v in sorted(cfg["seedSelector"].items())) + seeds = api.list_namespaced_pod(cfg["seedNamespace"], label_selector=selector, _request_timeout=20).items + selected = {p.metadata.name: p for p in seeds if p.metadata.name in cfg["seedNames"]} + names = cfg["seedNames"] + if not names or len(set(names)) != len(names) or set(selected) != set(names) or not all(ready(p) for p in selected.values()): + raise CheckFailed("all configured Dragonfly seeds must be distinct, present and Ready") + identities = {"deployment": [deployment.metadata.uid, deployment.metadata.generation], "registry": sorted((p.metadata.uid, [(s.name, s.image_id, s.restart_count) for s in (p.status.container_statuses or [])]) for p in pods), "seeds": sorted((name, p.metadata.uid, [(s.name, s.image_id, s.restart_count) for s in (p.status.container_statuses or [])]) for name, p in selected.items()), "config": cfg} + return hashlib.sha256(json.dumps(identities, sort_keys=True).encode()).hexdigest() + + +def execute(api, apps, cfg, credential, state, force=False, now=None, http_factory=SeedHTTP): + validate_probe_security(cfg) + now = time.time() if now is None else now + before = fingerprint(api, apps, cfg) + if not force and state.get("fingerprint") == before and now - state.get("checked_epoch", 0) < cfg.get("recheckSeconds", 600): + return None + results = [] + def one(name): + try: + return verify_seed(http_factory(api, cfg["seedNamespace"], name, cfg["seedContainer"]), cfg, credential) + except CheckFailed as error: + return {"seed": name, "passed": False, "error": str(error)} + except Exception: + return {"seed": name, "passed": False, "error": "seed exec or HTTP transport failed"} + with concurrent.futures.ThreadPoolExecutor(max_workers=CONCURRENCY) as pool: + results = list(pool.map(one, cfg["seedNames"])) + if not all(r["passed"] for r in results): + print(json.dumps({"event": "verification_failed", "seeds": results}), flush=True) + raise CheckFailed("one or more Dragonfly seeds failed; success state was not advanced") + if len({r["bootstrap"]["sha256"] for r in results}) != 1: + raise CheckFailed("seed bootstrap digests disagree") + if fingerprint(api, apps, cfg) != before: + raise CheckFailed("Registry or seed identity changed during validation; retry required") + return {"fingerprint": before, "checked_epoch": now, "checked_at": datetime.fromtimestamp(now, timezone.utc).isoformat(), "manifest_digest": cfg["digest"], "seeds": results} + + +def main(): + from kubernetes import client, config + config.load_incluster_config() + api, apps = client.CoreV1Api(), client.AppsV1Api() + cfg = json.loads(Path(os.environ.get("CONFIG_FILE", "/config/config.json")).read_text()) + credential = json.loads(Path("/credentials/credentials.json").read_text()) + cm = api.read_namespaced_config_map(cfg["stateConfigMap"], cfg["namespace"], _request_timeout=20) + state = json.loads((cm.data or {}).get("state.json", "{}")) + result = execute(api, apps, cfg, credential, state, force=os.environ.get("FORCE_CHECK") == "1") + if result is None: + print(json.dumps({"event": "unchanged", "last_success": state.get("checked_at")}), flush=True) + return + # resourceVersion prevents overlapping manual/hook Jobs overwriting state. + api.patch_namespaced_config_map(cfg["stateConfigMap"], cfg["namespace"], {"metadata": {"resourceVersion": cm.metadata.resource_version}, "data": {"state.json": json.dumps(result, sort_keys=True)}}, field_manager="registry-connectivity-check", _request_timeout=20) + print(json.dumps({"event": "verified", **result}), flush=True) + + +if __name__ == "__main__": + try: + main() + except CheckFailed as error: + print(json.dumps({"event": "failed", "error": str(error)}), flush=True) + raise SystemExit(1) + except Exception: + # Kubernetes/HTTP exceptions may include response bodies and credentials. + print(json.dumps({"event": "failed", "error": "verification API or configuration error"}), flush=True) + raise SystemExit(1) diff --git a/deploy/registry-connectivity/requirements.txt b/deploy/registry-connectivity/requirements.txt new file mode 100644 index 00000000..720ed308 --- /dev/null +++ b/deploy/registry-connectivity/requirements.txt @@ -0,0 +1,15 @@ +# Pin the complete checker dependency set for reproducible operator images. +certifi==2026.10.23 +charset-normalizer==3.5.2 +durationpy==0.11 +idna==3.20 +kubernetes==35.0.0 +oauthlib==4.0.1 +python-dateutil==4.0.0 +PyYAML==6.0.3 +requests==2.34.2 +requests-oauthlib==2.0.0 +six==1.17.0 +tzdata==2026.6 +urllib3==2.8.0 +websocket-client==1.9.2 diff --git a/deploy/registry-connectivity/tests/test_chart.py b/deploy/registry-connectivity/tests/test_chart.py new file mode 100644 index 00000000..0c58217a --- /dev/null +++ b/deploy/registry-connectivity/tests/test_chart.py @@ -0,0 +1,98 @@ +"""Render and check security boundaries without a cluster.""" +import json +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + +import yaml + +CHART = Path(__file__).resolve().parents[1] / "chart" +HELM = shutil.which("helm") or str(Path.home() / ".local/bin/helm") + + +class ChartTests(unittest.TestCase): + def values(self): + return {"image": "registry.example/check@sha256:" + "a" * 64, "registry": {"url": "https://registry.example", "allowedAuthOrigins": ["https://auth.example"], "probeRepository": "operations/nydus", "probeDigest": "sha256:" + "b" * 64, "pullSecret": "probe-pull"}, "networkPolicy": {"enabled": True, "existingClientsReviewed": True}, "dragonfly": {"proxyLoggingVerified": True, "proxyCAFile": "/trust/dragonfly-ca.pem"}} + + def render(self, values): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "values.json" + path.write_text(json.dumps(values)) + return subprocess.run([HELM, "template", "guard", str(CHART), "--namespace", "registry", "-f", str(path)], text=True, capture_output=True) + + def objects(self): + result = self.render(self.values()) + self.assertEqual(result.returncode, 0, result.stderr) + return list(yaml.safe_load_all(result.stdout)) + + def test_namespace_and_pod_labels_share_one_peer_and_only_data_auth_ports(self): + policy = next(o for o in self.objects() if o["kind"] == "NetworkPolicy") + rule = policy["spec"]["ingress"][0] + self.assertEqual(len(rule["from"]), 1) + self.assertEqual(rule["from"][0], {"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": "dragonfly-system"}}, "podSelector": {"matchLabels": {"app": "dragonfly", "component": "seed-client"}}}) + self.assertEqual(rule["ports"], [{"protocol": "TCP", "port": 5000}, {"protocol": "TCP", "port": 5001}]) + + def test_policy_is_opt_in_and_requires_existing_client_review(self): + values = self.values() + values.pop("networkPolicy") + result = self.render(values) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(any(o and o["kind"] == "NetworkPolicy" for o in yaml.safe_load_all(result.stdout))) + values["networkPolicy"] = {"enabled": True} + self.assertNotEqual(self.render(values).returncode, 0) + + def test_empty_or_invalid_port_lists_are_rejected(self): + for ports in [[], None, [0], [65536], ["all"], [-1]]: + values = self.values() + values["registry"]["ports"] = ports + self.assertNotEqual(self.render(values).returncode, 0, repr(ports)) + + def test_state_configmap_does_not_own_job_written_data(self): + state = next(o for o in self.objects() if o["kind"] == "ConfigMap" and o["metadata"]["name"] == "guard-state") + self.assertNotIn("data", state) + + def test_proxy_credential_logging_and_https_ca_are_explicit(self): + values = self.values() + values["dragonfly"]["proxyLoggingVerified"] = False + self.assertNotEqual(self.render(values).returncode, 0) + values["dragonfly"]["proxyLoggingVerified"] = True + values["dragonfly"]["proxyCAFile"] = "" + self.assertNotEqual(self.render(values).returncode, 0) + values["dragonfly"]["proxyCheckEnabled"] = False + self.assertEqual(self.render(values).returncode, 0) + + def test_configured_three_and_five_seed_topologies_and_bad_lists(self): + for count in [3, 5]: + values = self.values() + names = [f"seed-{i}" for i in range(count)] + values["dragonfly"]["seedNames"] = names + result = self.render(values) + self.assertEqual(result.returncode, 0, result.stderr) + roles = [o for o in yaml.safe_load_all(result.stdout) if o["kind"] == "Role"] + rules = [r for role in roles for r in role["rules"]] + self.assertFalse(any("secrets" in r["resources"] for r in rules)) + rule = next(r for r in rules if "pods/exec" in r["resources"]) + self.assertEqual(rule["resourceNames"], names) + for names in [[], ["seed-0", "seed-0"]]: + values = self.values() + values["dragonfly"]["seedNames"] = names + self.assertNotEqual(self.render(values).returncode, 0) + + def test_chart_does_not_take_over_registry_or_storage(self): + kinds = {o["kind"] for o in self.objects()} + self.assertFalse(kinds.intersection({"Deployment", "PersistentVolumeClaim", "Service", "Secret"})) + + def test_empty_selectors_or_auth_origins_and_unpinned_image_fail_closed(self): + for field in ["podSelector", "allowedAuthOrigins"]: + values = self.values() + values["registry"][field] = None + self.assertNotEqual(self.render(values).returncode, 0) + values = self.values() + values["image"] = "registry.example/check:latest" + self.assertNotEqual(self.render(values).returncode, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/deploy/registry-connectivity/tests/test_check.py b/deploy/registry-connectivity/tests/test_check.py new file mode 100644 index 00000000..a30d93f7 --- /dev/null +++ b/deploy/registry-connectivity/tests/test_check.py @@ -0,0 +1,135 @@ +import contextlib +import hashlib +import gzip +import importlib.util +import io +import json +from pathlib import Path +from types import SimpleNamespace as NS +import tarfile +import unittest +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location("registry_check", Path(__file__).resolve().parents[1] / "check.py") +check = importlib.util.module_from_spec(spec) +spec.loader.exec_module(check) + + +def sha(data): + return "sha256:" + hashlib.sha256(data).hexdigest() + + +def fixture(): + output = io.BytesIO() + with gzip.GzipFile(fileobj=output, mode="wb", mtime=0) as compressed: + with tarfile.open(fileobj=compressed, mode="w:") as archive: + data = b"Nydus-bootstrap-probe" + member = tarfile.TarInfo("image/image.boot") + member.size = len(data) + archive.addfile(member, io.BytesIO(data)) + layer = output.getvalue() + manifest = json.dumps({"layers": [{"digest": sha(layer), "size": len(layer), "annotations": {check.BOOTSTRAP: "true"}}]}).encode() + return layer, manifest + + +def pod(name, uid=None, is_ready=True): + return NS(metadata=NS(name=name, uid=uid or name, deletion_timestamp=None), status=NS(conditions=[NS(type="Ready", status="True" if is_ready else "False")], container_statuses=[NS(name="test", image_id="sha256:1", restart_count=0)])) + + +def configuration(): + return {"namespace": "registry", "deployment": "registry", "registrySelector": {"app": "registry"}, "seedNamespace": "dragonfly", "seedSelector": {"app": "dragonfly", "component": "seed-client"}, "seedContainer": "seed-client", "seedNames": [f"seed-{i}" for i in range(5)], "registry": "http://registry.test:5000", "allowedAuthOrigins": ["http://auth.test:5001"], "proxyLoggingVerified": True, "repository": "health/probe", "digest": sha(fixture()[1])} + + +class API: + def __init__(self): + self.registry = [pod("registry-0")] + self.seeds = [pod(f"seed-{i}") for i in range(5)] + self.generation = 1 + + def read_namespaced_deployment(self, *args, **kwargs): + return NS(metadata=NS(uid="deployment", generation=self.generation), spec=NS(replicas=1), status=NS(observed_generation=self.generation, updated_replicas=1, available_replicas=1, replicas=1)) + + def list_namespaced_pod(self, namespace, **kwargs): + return NS(items=self.registry if namespace == "registry" else self.seeds) + + +class CheckerTests(unittest.TestCase): + def test_requires_trusted_token_origin_before_sending_credentials(self): + with self.assertRaisesRegex(check.CheckFailed, "not explicitly trusted"): + check.challenge({"www-authenticate": 'Bearer realm="http://other.test/auth",service="registry"'}, ["http://auth.test:5001"]) + + def test_content_is_verified_and_real_bootstrap_is_read(self): + layer, manifest = fixture() + result = check.bootstrap_contents(layer, check.bootstrap_descriptor(json.loads(manifest))) + self.assertEqual(result["sha256"], hashlib.sha256(b"Nydus-bootstrap-probe").hexdigest()) + with self.assertRaisesRegex(check.CheckFailed, "digest mismatch"): + check.bootstrap_contents(layer + b"x", check.bootstrap_descriptor(json.loads(manifest))) + + def test_oci_without_nydus_bootstrap_is_rejected(self): + with self.assertRaises(check.CheckFailed): + check.bootstrap_descriptor({"layers": [{"size": 1}]}) + + def test_migration_rollout_and_seed_replacement_change_fingerprint(self): + api, cfg = API(), configuration() + first = check.fingerprint(api, api, cfg) + api.registry[0].metadata.uid = "migrated" + self.assertNotEqual(check.fingerprint(api, api, cfg), first) + second = check.fingerprint(api, api, cfg) + api.generation += 1 + self.assertNotEqual(check.fingerprint(api, api, cfg), second) + second = check.fingerprint(api, api, cfg) + api.seeds[0].metadata.uid = "replacement" + self.assertNotEqual(check.fingerprint(api, api, cfg), second) + + def test_configured_three_and_five_seeds_all_pass_and_fail_closed(self): + for count in [3, 5]: + api, cfg = API(), configuration() + api.seeds = api.seeds[:count] + cfg["seedNames"] = cfg["seedNames"][:count] + check.fingerprint(api, api, cfg) + with patch.object(check, "verify_seed", return_value={"passed": True, "bootstrap": {"sha256": "one"}}): + result = check.execute(api, api, cfg, {}, {}) + self.assertEqual(len(result["seeds"]), count) + self.assertEqual(result["fingerprint"], check.fingerprint(api, api, cfg)) + api.seeds[-1].status.conditions[0].status = "False" + with self.assertRaises(check.CheckFailed): + check.fingerprint(api, api, cfg) + api.seeds.pop() + with self.assertRaises(check.CheckFailed): + check.fingerprint(api, api, cfg) + api, cfg = API(), configuration() + cfg["seedNames"] = ["seed-0", "seed-0"] + with self.assertRaises(check.CheckFailed): + check.fingerprint(api, api, cfg) + cfg["seedNames"] = [] + with self.assertRaises(check.CheckFailed): + check.fingerprint(api, api, cfg) + + def test_unchanged_state_skips_until_periodic_recheck(self): + api, cfg = API(), configuration() + state = {"fingerprint": check.fingerprint(api, api, cfg), "checked_epoch": 100} + with patch.object(check, "verify_seed", side_effect=AssertionError("unexpected network call")): + self.assertIsNone(check.execute(api, api, cfg, {}, state, now=101)) + + def test_every_seed_must_pass_and_errors_are_redacted(self): + api, cfg = API(), configuration() + def verify(http, *args): + if http.pod == "seed-4": + raise RuntimeError("secret-password") + return {"seed": http.pod, "passed": True, "bootstrap": {"sha256": "one"}} + factory = lambda api, namespace, name, container: NS(pod=name) + output = io.StringIO() + with patch.object(check, "verify_seed", side_effect=verify), contextlib.redirect_stdout(output): + with self.assertRaises(check.CheckFailed): + check.execute(api, api, cfg, {}, {}, http_factory=factory) + self.assertNotIn("secret-password", output.getvalue()) + self.assertIn('"passed": false', output.getvalue()) + + def test_rollout_during_check_never_advances_success_state(self): + api, cfg = API(), configuration() + with patch.object(check, "fingerprint", side_effect=["before", "after"]), patch.object(check, "verify_seed", return_value={"passed": True, "bootstrap": {"sha256": "one"}}): + with self.assertRaisesRegex(check.CheckFailed, "identity changed"): + check.execute(api, api, cfg, {}, {}) + +if __name__ == "__main__": + unittest.main() diff --git a/deploy/registry-connectivity/tests/test_redirects.py b/deploy/registry-connectivity/tests/test_redirects.py new file mode 100644 index 00000000..bd5f8630 --- /dev/null +++ b/deploy/registry-connectivity/tests/test_redirects.py @@ -0,0 +1,187 @@ +"""Registry redirects through the real curl/SeedHTTP path; only exec is local.""" +import contextlib +import http.client +import http.server +import json +import subprocess +import threading +from types import SimpleNamespace as NS +from urllib.parse import urlsplit +import unittest +from unittest.mock import patch + +from test_check import check, configuration, fixture + + +@contextlib.contextmanager +def server(handler): + instance = http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler) + thread = threading.Thread(target=instance.serve_forever, daemon=True) + thread.start() + try: + yield instance + finally: + instance.shutdown() + instance.server_close() + thread.join() + + +@contextlib.contextmanager +def local_exec(): + class WS: + opened, unread = True, True + def __init__(self, command): + self.command = command + def write_stdin(self, config): + for line in config.splitlines(): + if line.startswith("header ="): + header = json.loads(line.split("=", 1)[1]) + if header.lower().startswith("authorization:"): + assert header not in " ".join(self.command) + self.result = subprocess.run(self.command, input=config.encode(), capture_output=True, timeout=10) + def is_open(self): + return self.opened + def update(self, **kwargs): + self.opened = False + def peek_stdout(self): + return self.unread + def read_stdout(self): + self.unread = False + return self.result.stdout.decode() + def peek_stderr(self): + return False + def close(self): + self.opened = False + api = NS(ApiClient=lambda **kwargs: NS(close=lambda: None), CoreV1Api=lambda client: NS(connect_get_namespaced_pod_exec=None)) + modules = {"kubernetes": NS(client=api), "kubernetes.stream": NS(stream=lambda method, *args, **kwargs: WS(kwargs["command"]))} + with patch.dict("sys.modules", modules): + yield check.SeedHTTP(NS(api_client=NS(configuration=None)), "test", "seed", "proxy") + + +class RedirectTests(unittest.TestCase): + def verify(self, proxied, cross_origin=False): + layer, manifest = fixture() + events = [] + dummy = "dummy-redirect-pull-token" + class Objects(http.server.BaseHTTPRequestHandler): + def do_GET(self): + events.append(("object", self.path, self.headers.get("Authorization"))) + self.send_response(200) + self.send_header("Content-Length", str(len(layer))) + self.end_headers() + self.wfile.write(layer) + def log_message(self, *args): + pass + with server(Objects) as objects: + class Registry(http.server.BaseHTTPRequestHandler): + def do_GET(self): + events.append(("registry", self.path, self.headers.get("Authorization"))) + if self.path == "/v2/": + self.send_response(401) + self.send_header("WWW-Authenticate", f'Bearer realm="http://127.0.0.1:{self.server.server_port}/auth",service="registry"') + self.end_headers() + return + if self.path.startswith("/auth?"): + body = json.dumps({"token": dummy}).encode() + elif "/manifests/" in self.path: + body = manifest + elif self.path.startswith("/objects/"): + body = layer + else: + target = f"http://127.0.0.1:{objects.server_port}/objects/blob?signature=DUMMY-SIGNED-QUERY" if cross_origin else "/objects/blob" + self.send_response(307) + self.send_header("Location", target) + self.send_header("Content-Length", "0") + self.end_headers() + return + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + def log_message(self, *args): + pass + class Proxy(http.server.BaseHTTPRequestHandler): + def do_GET(self): + target = urlsplit(self.path) + with contextlib.closing(http.client.HTTPConnection(target.hostname, target.port, timeout=3)) as upstream: + upstream.request("GET", target.path + ("?" + target.query if target.query else ""), headers=dict(self.headers)) + response = upstream.getresponse() + body = response.read() + self.send_response(response.status) + if response.getheader("Location"): + self.send_header("Location", response.getheader("Location")) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + def log_message(self, *args): + pass + with server(Registry) as registry, server(Proxy) as proxy, local_exec() as seed_http: + cfg = configuration() + cfg.update(registry=f"http://127.0.0.1:{registry.server_port}", allowedAuthOrigins=[f"http://127.0.0.1:{registry.server_port}"], seedProxy=f"http://127.0.0.1:{proxy.server_port}", proxyCheckEnabled=proxied) + result = check.verify_seed(seed_http, cfg, {"username": "dummy", "password": "dummy"}) + self.assertTrue(result["passed"]) + self.assertEqual(result["proxy_checked"], proxied) + self.assertNotIn(dummy, json.dumps(result)) + auth_requests = [e for e in events if e[1].startswith("/auth?")] + self.assertTrue(auth_requests) + self.assertTrue(all(e[2].startswith("Basic ") for e in auth_requests)) + objects_seen = [e for e in events if e[0] == "object"] + if cross_origin: + self.assertEqual(len(objects_seen), 2 if proxied else 1) + self.assertTrue(all(e[2] is None for e in objects_seen)) + self.assertTrue(all("signature=DUMMY-SIGNED-QUERY" in e[1] for e in objects_seen)) + else: + completed = [e for e in events if e[1] == "/objects/blob"] + self.assertTrue(completed) + self.assertTrue(all(e[2] == "Bearer " + dummy for e in completed)) + + def test_same_origin_direct_and_proxy_redirects(self): + for proxy in [False, True]: + with self.subTest(proxy=proxy): + self.verify(proxy) + + def test_cross_origin_strips_authorization_direct_and_proxy(self): + for proxy in [False, True]: + with self.subTest(proxy=proxy): + self.verify(proxy, cross_origin=True) + + def test_redirect_loop_and_limit_fail(self): + for loop in [True, False]: + class HTTP: + calls = 0 + def get(self, url, *args, **kwargs): + self.calls += 1 + return 307, {"location": url if loop else f"/next/{self.calls}"}, b"" + http = HTTP() + with self.assertRaisesRegex(check.CheckFailed, "redirect (loop|limit)"): + check.read_bootstrap(http, {}, "http://registry.test/blob", {}) + self.assertLessEqual(http.calls, check.MAX_REDIRECTS + 1) + + def test_cross_origin_secrets_never_return_and_ca_is_per_origin(self): + calls = [] + responses = [(307, {"location": "https://objects.test/blob?sig=dummy"}, b""), (302, {"location": "https://registry.test/final"}, b""), (200, {}, b"ok")] + class HTTP: + def get(self, url, headers, **kwargs): + calls.append((url, dict(headers), kwargs.get("ca_file"))) + return responses.pop(0) + result = check.read_bootstrap(HTTP(), {"registryCAFile": "/trust/registry.pem", "redirectCAFile": "/trust/objects.pem"}, "https://registry.test/blob", {"Authorization": "Bearer dummy", "Cookie": "dummy", "Accept": "blob"}) + self.assertEqual(result, b"ok") + self.assertEqual([c[2] for c in calls], ["/trust/registry.pem", "/trust/objects.pem", "/trust/registry.pem"]) + self.assertEqual(calls[1][1], {"Accept": "blob"}) + self.assertEqual(calls[2][1], {"Accept": "blob"}) + + def test_https_downgrade_and_non_http_redirects_fail_before_next_request(self): + for location in ["http://objects.test/blob", "file:///etc/passwd", "https://user:password@objects.test/blob"]: + class HTTP: + calls = 0 + def get(self, *args, **kwargs): + self.calls += 1 + return 307, {"location": location}, b"" + http = HTTP() + with self.assertRaises(check.CheckFailed): + check.read_bootstrap(http, {}, "https://registry.test/blob", {"Authorization": "Bearer dummy"}) + self.assertEqual(http.calls, 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/deploy/registry-connectivity/tests/test_transport.py b/deploy/registry-connectivity/tests/test_transport.py new file mode 100644 index 00000000..1ce40512 --- /dev/null +++ b/deploy/registry-connectivity/tests/test_transport.py @@ -0,0 +1,135 @@ +"""Exercise the actual seed shell command against local HTTP servers.""" +import base64 +import http.server +import json +import subprocess +import threading +import unittest + +from test_check import check, sha + + +class TransportTests(unittest.TestCase): + def test_real_curl_rejects_200_with_truncated_content_length(self): + body = b"expected-bytes-with-the-correct-content-digest" + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header("Content-Length", str(len(body) + 10)) + self.end_headers() + self.wfile.write(body) + self.close_connection = True + def log_message(self, *args): + pass + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + config = 'url = "http://127.0.0.1:%s/probe"\nmax-time = "3"\nnoproxy = "*"\n' % server.server_port + result = subprocess.run(check.curl_command(len(config.encode())), input=config.encode(), capture_output=True, timeout=5) + self.assertEqual(result.returncode, 0) # base64 itself succeeds. + raw = base64.b64decode(result.stdout) + self.assertIn(body, raw) + self.assertIn(check.MARKER + b"200", raw) + self.assertIn(check.EXIT_MARKER + b"18", raw) + check.digest_bytes(body, sha(body)) # Content digest alone is insufficient. + with self.assertRaisesRegex(check.CheckFailed, "curl transfer failed"): + check.decode_response(result.stdout) + finally: + server.shutdown() + server.server_close() + thread.join() + + def test_https_connect_response_headers_do_not_become_bootstrap_bytes(self): + import select + import socket + import socketserver + import ssl + import tempfile + from pathlib import Path + with tempfile.TemporaryDirectory() as tmp: + cert, key = Path(tmp) / "ca.pem", Path(tmp) / "key.pem" + subprocess.run(["openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes", "-keyout", str(key), "-out", str(cert), "-days", "1", "-subj", "/CN=localhost", "-addext", "subjectAltName=DNS:localhost"], capture_output=True, check=True, timeout=10) + body = b"actual-bootstrap-content" + class Registry(http.server.BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + def log_message(self, *args): + pass + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Registry) + tls = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + tls.load_cert_chain(str(cert), str(key)) + server.socket = tls.wrap_socket(server.socket, server_side=True) + class Proxy(socketserver.StreamRequestHandler): + def handle(self): + request = self.rfile.readline() + if not request.startswith(b"CONNECT "): + return + while self.rfile.readline().strip(): + pass + with socket.create_connection(("127.0.0.1", server.server_port), timeout=3) as upstream: + # Dragonfly can return OK rather than 'Connection established'. + self.wfile.write(b"HTTP/1.1 200 OK\r\n\r\n") + self.wfile.flush() + while True: + readable, _, _ = select.select([self.connection, upstream], [], [], 3) + if not readable: + return + for stream in readable: + data = stream.recv(65536) + if not data: + return + (upstream if stream is self.connection else self.connection).sendall(data) + proxy = socketserver.ThreadingTCPServer(("127.0.0.1", 0), Proxy) + proxy.daemon_threads = True + workers = [threading.Thread(target=s.serve_forever, daemon=True) for s in [server, proxy]] + for worker in workers: + worker.start() + try: + config = f'url = "https://localhost:{server.server_port}/blob"\nproxy = "http://127.0.0.1:{proxy.server_address[1]}"\nnoproxy = ""\nmax-time = "3"\n' + # No insecure flag: a missing CA must produce a curl failure. + failed = subprocess.run(check.curl_command(len(config.encode())), input=config.encode(), capture_output=True, timeout=5) + with self.assertRaisesRegex(check.CheckFailed, "curl transfer failed"): + check.decode_response(failed.stdout) + config += "cacert = " + json.dumps(str(cert)) + "\n" + result = subprocess.run(check.curl_command(len(config.encode())), input=config.encode(), capture_output=True, timeout=5, check=True) + status, _, received = check.decode_response(result.stdout) + self.assertEqual((status, received), (200, body)) + finally: + for instance in [server, proxy]: + instance.shutdown() + instance.server_close() + for worker in workers: + worker.join() + + def test_missing_exit_marker_never_counts_as_success(self): + with self.assertRaises(check.CheckFailed): + check.decode_response(base64.b64encode(b"HTTP/1.1 200 OK\r\n\r\nbody" + check.MARKER + b"200\n")) + + def test_oversized_compressed_archive_fails_before_tar_parsing(self): + import gzip + data = gzip.compress(b"x" * (2 * check.LIMIT + 1)) + descriptor = {"digest": sha(data), "size": len(data)} + with self.assertRaisesRegex(check.CheckFailed, "decompression limit"): + check.bootstrap_contents(data, descriptor) + + def test_proxy_logging_and_https_trust_are_required_before_network(self): + from test_check import configuration + class HTTP: + def get(self, *args, **kwargs): + raise AssertionError("must not send credentials") + for overrides in [{"proxyLoggingVerified": False}, {"registry": "https://registry.test", "registryCAFile": "/trust/registry.pem"}]: + with self.subTest(config=overrides), self.assertRaises(check.CheckFailed): + check.verify_seed(HTTP(), {**configuration(), **overrides}, {}) + + def test_oversized_response_is_rejected(self): + raw = b"HTTP/1.1 200 OK\r\n\r\n" + b"x" * (check.LIMIT + 1) + raw += check.MARKER + b"200\n" + check.EXIT_MARKER + b"0\n" + with self.assertRaises(check.CheckFailed): + check.decode_response(base64.b64encode(raw)) + +if __name__ == "__main__": + unittest.main()