diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 4584d725f9b..c8bda4fbeac 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -456,7 +456,10 @@ REPOSITORY_LOCATIONS = dict( org_libc_musl = dict( sha256 = "7d5b0b6062521e4627e099e4c9dc8248d32a30285e959b7eecaa780cf8cfd4a4", strip_prefix = "musl-1.2.3", - urls = ["http://musl.libc.org/releases/musl-1.2.3.tar.gz"], + urls = [ + "https://sources.openwrt.org/musl-1.2.3.tar.gz", + "http://musl.libc.org/releases/musl-1.2.3.tar.gz", + ], manual_license_name = "libc/musl", ), rules_cc = dict( diff --git a/ci/artifact_utils.sh b/ci/artifact_utils.sh index a1eec1a7760..e6d7c0dca26 100644 --- a/ci/artifact_utils.sh +++ b/ci/artifact_utils.sh @@ -107,7 +107,15 @@ create_manifest_update() { tag_name="release/${component}/v${version}" # actions/checkout doesn't get the tag annotation properly. git fetch origin tag "${tag_name}" -f - timestamp="$(git tag -l --format "%(taggerdate:raw)" "${tag_name}" | awk '{print $1}' | jq '. | todate')" + # taggerdate is empty for a LIGHTWEIGHT tag → produces `timestamp: ,` → jq syntax + # error → release-metadata step fails even though the image built fine. Fall back to + # the tagged commit's committer date so the manifest is well-formed regardless of how + # the release tag was cut (annotated vs lightweight). + raw_ts="$(git tag -l --format "%(taggerdate:raw)" "${tag_name}" | awk '{print $1}')" + if [ -z "${raw_ts}" ]; then + raw_ts="$(git log -1 --format="%ct" "${tag_name}")" + fi + timestamp="$(printf '%s' "${raw_ts}" | jq '. | todate')" jq -s \ "[{name: \"${component}\", artifact: [{timestamp: ${timestamp}, commitHash: \"${commit_hash}\", versionStr: \"${version}\", availableArtifactMirrors: .}]}]" \ diff --git a/k8s/vizier/BUILD.bazel b/k8s/vizier/BUILD.bazel index 69a66ce180b..81341f7f77e 100644 --- a/k8s/vizier/BUILD.bazel +++ b/k8s/vizier/BUILD.bazel @@ -23,7 +23,6 @@ load("//bazel:kustomize.bzl", "kustomize_build") package(default_visibility = ["//visibility:public"]) VIZIER_IMAGE_TO_LABEL = { - "$(IMAGE_PREFIX)/vizier-adaptive_export_image:$(BUNDLE_VERSION)": "//src/vizier/services/adaptive_export:adaptive_export_image", "$(IMAGE_PREFIX)/vizier-cert_provisioner_image:$(BUNDLE_VERSION)": "//src/utils/cert_provisioner:cert_provisioner_image", "$(IMAGE_PREFIX)/vizier-cloud_connector_server_image:$(BUNDLE_VERSION)": "//src/vizier/services/cloud_connector:cloud_connector_server_image", "$(IMAGE_PREFIX)/vizier-kelvin_image:$(BUNDLE_VERSION)": "//src/vizier/services/agent/kelvin:kelvin_image", diff --git a/k8s/vizier/adaptive_export/README.md b/k8s/vizier/adaptive_export/README.md new file mode 100644 index 00000000000..26e448dc109 --- /dev/null +++ b/k8s/vizier/adaptive_export/README.md @@ -0,0 +1,58 @@ +# The export path: what is defined where + +Two fork-only components move query results off a cluster: `adaptive-export` in the `pl` +namespace, and `dx-daemon` in the `honey` namespace. This file is the map, so nothing has +to be remembered twice. + +## One installer + +``` +skaffold deploy -f skaffold/skaffold_export.yaml +``` + +Two things have to exist first: a running Vizier, and the ClickHouse that holds +`forensic_db`. adaptive-export applies its own schema at startup and exits rather than run +against tables it cannot verify, so without ClickHouse it installs correctly and then +crashloops on `schema apply failed; refusing to proceed with possibly drifted tables`. That +is adaptive-export being careful, not a broken install. Install ClickHouse, then this. + +That is the whole install, on an empty cluster or on top of a running one. It is +idempotent: a second run with no change touches nothing and restarts nothing. A pod is +replaced only when an image pin changes. The file holds two Skaffold configs, +`adaptive-export` and `dx-daemon`, and `dx-daemon` declares `requires: adaptive-export`, +so the order is declared rather than remembered. Select one with `-m `. + +## Where each thing lives + +| Thing | Defined in | +| --- | --- | +| adaptive-export DaemonSet and Service | `k8s/vizier/adaptive_export/daemonset.yaml` | +| adaptive-export ServiceAccount, Role, ClusterRole and bindings | `k8s/vizier/adaptive_export/rbac.yaml` | +| adaptive-export image pin | `images:` in `k8s/vizier/adaptive_export/kustomization.yaml` | +| dx-daemon DaemonSet | `k8s/vizier/dx/dx-daemon.yaml` | +| dx-daemon image pin | `images:` in `k8s/vizier/dx/kustomization.yaml` | +| every secret and ConfigMap fix-up | the `before` hooks in `skaffold/skaffold_export.yaml` | + +Each component is self-contained. The AE kustomization does not reach into `../bootstrap`, +which is what used to render the same two files twice under two different label sets and +leave `adaptive-export-control` with no endpoints. + +## Not part of a vizier release + +The release bundle is built by the `kustomize_build` targets in `k8s/vizier/BUILD.bazel`, +which glob `base/`, `bootstrap/`, `pem/` and the two metadata overlays. Neither +`adaptive_export/` nor `dx/` is in any of those globs, so a vizier release installs +neither, and this path is rolled on its own with the command above. + +## Inputs the installer reads + +| Variable | Used for | If unset | +| --- | --- | --- | +| `PIXIE_API_KEY`, or `PX_API_KEY` | the `pixie-api-key` in `pl-adaptive-export-secrets` | a live secret is left alone; on a cluster with no secret the install stops | +| `AE_CH_DSN` | the `clickhouse-dsn` in the same secret | the in-cluster forensic database default | +| `AE_PULL_CONFIG`, or `DOCKER_CONFIG` | the `adaptive-export-pull` image pull secret | an existing pull secret is reused; otherwise the install stops | +| `DX_PULL_CONFIG`, or `DOCKER_CONFIG` | the `dx-pull` image pull secret | a default config location is tried, and the install stops if nothing there is readable | +| `DX_CH_HTTP_URL` | dx's database endpoint | the in-cluster default | + +Secrets are only ever seeded, never overwritten with a placeholder. A key is written only +when one is supplied. diff --git a/k8s/vizier/adaptive_export/daemonset.yaml b/k8s/vizier/adaptive_export/daemonset.yaml new file mode 100644 index 00000000000..d3f42fc6aa6 --- /dev/null +++ b/k8s/vizier/adaptive_export/daemonset.yaml @@ -0,0 +1,109 @@ +--- +# adaptive-export: node-local forensic capture operator. DaemonSet so each pod +# queries its own node's vizier-pem (pem-direct). Secret seeded per-cluster. +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: adaptive-export + labels: {name: adaptive-export, plane: control} +spec: + selector: + matchLabels: {name: adaptive-export} + template: + metadata: + labels: {name: adaptive-export, plane: control} + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - {key: kubernetes.io/os, operator: In, values: [linux]} + serviceAccountName: pl-adaptive-export-service-account + imagePullSecrets: + - name: adaptive-export-pull + containers: + - name: adaptive-export + image: vizier-adaptive_export_image:latest + ports: + - {name: control, containerPort: 9100, hostPort: 9100} + resources: + requests: {cpu: 100m, memory: 128Mi} + limits: {cpu: "1", memory: 1Gi} + env: + - name: HOST_IP + valueFrom: {fieldRef: {fieldPath: status.hostIP}} + # Profiling listener, bound to loopback: unreachable from the cluster, + # reachable through `kubectl port-forward` for a heap or CPU sample. + - {name: AE_PPROF_ADDR, value: "127.0.0.1:6061"} + - name: ADAPTIVE_VIZIER_DIRECT_ADDR + value: "$(HOST_IP):50305" + - name: PL_JWT_SIGNING_KEY + valueFrom: {secretKeyRef: {name: pl-cluster-secrets, key: jwt-signing-key}} + - name: PX_DISABLE_TLS + value: "1" + - name: PL_NAMESPACE + valueFrom: {fieldRef: {fieldPath: metadata.namespace}} + - name: NODE_NAME + valueFrom: {fieldRef: {fieldPath: spec.nodeName}} + - name: PIXIE_API_KEY + valueFrom: {secretKeyRef: {name: pl-adaptive-export-secrets, key: pixie-api-key}} + - name: CLICKHOUSE_DSN + valueFrom: {secretKeyRef: {name: pl-adaptive-export-secrets, key: clickhouse-dsn}} + - name: KUBESCAPE_TABLE + value: "kubescape_logs" + - name: EXPORT_MODE + value: "never" + # Control surface is secure-by-default (#96): TLS + bearer-JWT auth are ON + # out of the box. The service-tls-certs keypair mounted at /certs below is + # used for TLS (else AE self-generates an ephemeral in-memory cert), and + # PL_JWT_SIGNING_KEY above turns on auth. CONTROL_TLS / CONTROL_REQUIRE_AUTH + # are deprecated no-ops; set CONTROL_INSECURE=true only to opt out (dev). + - name: CONTROL_ADDR + value: ":9100" + - name: ADAPTIVE_PUSH_PIXIE_ROWS + value: "true" + # Capture is driven by the control surface only; no kubescape_logs poll. + - name: ADAPTIVE_TRIGGER_POLL + value: "false" + # Convert any evidence table still on a plain MergeTree. Leader-only, + # and a no-op once every table has been converted. + - name: ADAPTIVE_MIGRATE_ENGINES + value: "true" + - name: ADAPTIVE_RECONCILE + value: "true" + - name: DEPLOY_TRACEPOINTS + value: "true" + - name: INSTALL_PRESET_SCRIPTS + value: "false" + - name: ADAPTIVE_MAX_INFLIGHT_QUERIES_GLOBAL + value: "4" + - name: ADAPTIVE_ORDER_CHUNK_SEC + value: "600" + - name: VERBOSE + value: "true" + volumeMounts: + - {name: certs, mountPath: /certs, readOnly: true} + securityContext: + allowPrivilegeEscalation: false + capabilities: {drop: [ALL]} + seccompProfile: {type: RuntimeDefault} + volumes: + - name: certs + secret: {secretName: service-tls-certs} + securityContext: + runAsUser: 10100 + runAsGroup: 10100 + fsGroup: 10100 + runAsNonRoot: true + seccompProfile: {type: RuntimeDefault} +--- +apiVersion: v1 +kind: Service +metadata: + name: adaptive-export-control +spec: + selector: {name: adaptive-export} + internalTrafficPolicy: Local # dx reaches its co-located (same-node) AE + ports: + - {name: control, port: 9100, targetPort: 9100} diff --git a/k8s/vizier/adaptive_export/kustomization.yaml b/k8s/vizier/adaptive_export/kustomization.yaml new file mode 100644 index 00000000000..3920e5ddaaa --- /dev/null +++ b/k8s/vizier/adaptive_export/kustomization.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: pl +# Everything adaptive-export needs lives in this directory. It used to reach back into +# ../bootstrap, which meant the same two files were rendered by two kustomizations: this +# one and the vizier bootstrap. The two applied different commonLabels, so the Service +# selector written by one did not match the pods written by the other, and a deploy on top +# of a vizier-installed AE left adaptive-export-control with no endpoints at all — dx's +# hand-off silently going nowhere. One definition, one label set, no divergence possible. +# The pl-adaptive-export-secrets secret is NOT a resource here. It is created by the +# before-deploy hook in skaffold/skaffold_export.yaml, which seeds it only when a key is +# supplied and otherwise leaves a live secret untouched. There used to be a second +# definition of the same secret in this directory (secrets.example.yaml); whenever it was +# applied it clobbered the real pixie-api-key with its placeholder, which is the recurring +# "AE unauthenticated / writes 0" bug. One definition, in the installer. See README.md. +resources: +- rbac.yaml +- daemonset.yaml +images: +- name: vizier-adaptive_export_image + newName: docker.io/tanzeee/adaptive-export + newTag: 0.16.50-6dc818c diff --git a/k8s/vizier/bootstrap/adaptive_export_role.yaml b/k8s/vizier/adaptive_export/rbac.yaml similarity index 52% rename from k8s/vizier/bootstrap/adaptive_export_role.yaml rename to k8s/vizier/adaptive_export/rbac.yaml index 33887150f37..eeb292ff801 100644 --- a/k8s/vizier/bootstrap/adaptive_export_role.yaml +++ b/k8s/vizier/adaptive_export/rbac.yaml @@ -37,6 +37,22 @@ subjects: name: pl-adaptive-export-service-account namespace: pl --- +# The ONE cluster-scoped grant adaptive-export still needs, and why. +# +# config.findVizierNamespace() lists pods across the cluster to locate the vizier +# namespace, then reads pl-cluster-secrets there for cluster-id and cluster-name. Without +# it AE logs "unable to find vizier namespace: pods is forbidden ... at the cluster scope", +# falls back to environment variables, and fatals on a missing PIXIE_CLUSTER_ID. Verified +# by removing it: the pods CrashLoop. main.go's leader election lists pods too, but only in +# its own namespace, so this covers both and the Role above does not repeat it. +# +# What this role no longer grants, because it was dead: read on the kubescape +# containerprofiles and rogueartifacts CRDs and on the node-agent-bundle-policy ConfigMap. +# Those belonged to kssync, which has moved out of adaptive-export and carries its own +# RBAC where it now lives. +# +# To retire this ClusterRole entirely, AE would have to take the vizier namespace from +# PL_NAMESPACE — already injected into its pod spec — instead of discovering it. apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: diff --git a/k8s/vizier/bootstrap/adaptive_export_deployment.yaml b/k8s/vizier/bootstrap/adaptive_export_deployment.yaml deleted file mode 100644 index 2db195ff408..00000000000 --- a/k8s/vizier/bootstrap/adaptive_export_deployment.yaml +++ /dev/null @@ -1,115 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: adaptive-export -spec: - replicas: 0 - selector: - matchLabels: - name: adaptive-export - template: - metadata: - labels: - name: adaptive-export - plane: control - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - # The beta.kubernetes.io/os label has been deprecated since - # k8s v1.14; every modern kubelet sets kubernetes.io/os. The - # single term below is enough — kept both ORed terms in the - # past for pre-1.14 compatibility. - - matchExpressions: - - key: kubernetes.io/os - operator: In - values: - - linux - serviceAccountName: pl-adaptive-export-service-account - containers: - - name: adaptive-export - image: vizier-adaptive_export_image:latest - # Bounded so AE can never memory-pressure a node (measured: AE uses - # only ~16-38Mi steady; passthrough with the raised 1M-row cap can - # spike, so 1Gi caps the worst case). CPU was pinned at the old 300m - # limit under concurrent passthrough → raised to 1 core. - resources: - requests: - cpu: 200m - memory: 128Mi - limits: - cpu: "1" - memory: 1Gi - env: - - name: PL_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: PIXIE_API_KEY - valueFrom: - secretKeyRef: - name: pl-adaptive-export-secrets - key: pixie-api-key - - name: CLICKHOUSE_DSN - valueFrom: - secretKeyRef: - name: pl-adaptive-export-secrets - key: clickhouse-dsn - - name: VERBOSE - value: "true" - - name: DETECTION_INTERVAL_SEC - value: "10" - - name: DETECTION_LOOKBACK_SEC - value: "30" - # EXPORT_MODE controls the reconcile behaviour: - # auto - detection drives on/off (default) - # always - plugin always enabled (bypass detection) - # never - plugin always disabled and ch-* scripts purged - - name: EXPORT_MODE - value: "auto" - # Number of consecutive empty detection ticks before auto-disable fires. - - name: EXPORT_QUIET_TICKS - value: "6" - # Optional overrides for the ClickHouse PxL scripts. When unset they are - # parsed from CLICKHOUSE_DSN. Individual fields win over the parsed DSN. - # Defaults below match soc/tree/clickhouse-lab (forensic-soc-db CHI, - # ingest_writer user, forensic_db database). - - name: KUBESCAPE_TABLE - value: "kubescape_logs" - # - name: CLICKHOUSE_HOST - # value: "clickhouse-forensic-soc-db.clickhouse.svc.cluster.local" - # - name: CLICKHOUSE_PORT - # value: "9000" - # - name: CLICKHOUSE_USER - # value: "ingest_writer" - # - name: CLICKHOUSE_PASSWORD - # value: "changeme-ingest" - # - name: CLICKHOUSE_DATABASE - # value: "forensic_db" - # TLS for the control surface (CONTROL_TLS=true). server.crt/key from the - # same service-tls-certs secret the broker/PEM use; without this the dx - # bearer JWT crosses the CNI in cleartext. Harmless when control is off. - volumeMounts: - - name: certs - mountPath: /certs - readOnly: true - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - seccompProfile: - type: RuntimeDefault - volumes: - - name: certs - secret: - secretName: service-tls-certs - securityContext: - runAsUser: 10100 - runAsGroup: 10100 - fsGroup: 10100 - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault diff --git a/k8s/vizier/bootstrap/adaptive_export_secrets.yaml b/k8s/vizier/bootstrap/adaptive_export_secrets.yaml deleted file mode 100644 index 9250676dca4..00000000000 --- a/k8s/vizier/bootstrap/adaptive_export_secrets.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# SEED-ONLY template — NOT in kustomization.yaml (separation of concerns). -# Real credentials are written by `make ae-auth` (pixie-api-key from keys.env, -# clickhouse-dsn = the fixed forensic-CH constant). Do NOT add this back to the -# bundle: a re-apply would clobber the real pixie-api-key with the placeholder -# (the recurring "AE unauthenticated / writes 0" bug). Apply this by hand ONLY -# to seed a brand-new cluster so the AE pod's secretKeyRef resolves before -# ae-auth runs. ---- -apiVersion: v1 -kind: Secret -metadata: - name: pl-adaptive-export-secrets -type: Opaque -stringData: - # Replace with your actual Pixie API key from https://work.withpixie.ai - pixie-api-key: "PIXIE_API_KEY_PLACEHOLDER" - # ClickHouse DSN matches soc/tree/clickhouse-lab (CHI "forensic-soc-db", - # ingest_writer user with INSERT rights into the forensic_db database). - # Format: user:password@host:port/database - clickhouse-dsn: >- - ingest_writer:changeme-ingest@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:9000/forensic_db diff --git a/k8s/vizier/bootstrap/kustomization.yaml b/k8s/vizier/bootstrap/kustomization.yaml index e2afd14af16..714f5676426 100644 --- a/k8s/vizier/bootstrap/kustomization.yaml +++ b/k8s/vizier/bootstrap/kustomization.yaml @@ -15,11 +15,3 @@ resources: - cert_provisioner_role.yaml - cert_provisioner_job.yaml - vizier_crd_role.yaml -- adaptive_export_role.yaml -# adaptive_export_secrets.yaml is intentionally NOT bundled here: it holds real -# credentials (pixie-api-key, clickhouse-dsn) owned by `make ae-auth`. Bundling -# it meant every infra re-apply clobbered the real key with the placeholder. -# Separation of concerns: infra (role+deployment) re-appliable; secret is -# created ONCE by ae-auth and never touched by this kustomization. ponytail: -# apply adaptive_export_secrets.yaml manually only to seed a fresh cluster. -- adaptive_export_deployment.yaml diff --git a/k8s/vizier/dx/dx-daemon.yaml b/k8s/vizier/dx/dx-daemon.yaml new file mode 100644 index 00000000000..de1189dc5e2 --- /dev/null +++ b/k8s/vizier/dx/dx-daemon.yaml @@ -0,0 +1,144 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: {name: dx-daemon, namespace: honey} +--- +# B1: every dx instance reads kubescape's state (container profiles, rogue +# artifacts) from the api-server for its own shadow manager; the trust policy +# is a configmap in this namespace. Read-only. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: {name: dx-daemon-kubescape-reader} +rules: +- apiGroups: [spdx.softwarecomposition.kubescape.io] + resources: [containerprofiles, rogueartifacts] + verbs: [get, list] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: {name: dx-daemon-kubescape-reader} +roleRef: {apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: dx-daemon-kubescape-reader} +subjects: +- {kind: ServiceAccount, name: dx-daemon, namespace: honey} +--- +# One dx instance hands the mirror rows on to AE; the others only read. The +# sender is whoever holds this Lease, so a node that stops renewing is +# replaced without a restart. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: {name: dx-daemon-kssync, namespace: honey} +rules: +- apiGroups: [coordination.k8s.io] + resources: [leases] + resourceNames: [dx-kssync] + verbs: [get, update] +- apiGroups: [coordination.k8s.io] + resources: [leases] + verbs: [create] +- apiGroups: [""] + resources: [configmaps] + resourceNames: [node-agent-bundle-policy] + verbs: [get] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: {name: dx-daemon-kssync, namespace: honey} +roleRef: {apiGroup: rbac.authorization.k8s.io, kind: Role, name: dx-daemon-kssync} +subjects: +- {kind: ServiceAccount, name: dx-daemon, namespace: honey} +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: dx-daemon + namespace: honey + labels: {app: dx-daemon} +spec: + selector: {matchLabels: {app: dx-daemon}} + template: + metadata: + labels: {app: dx-daemon} + spec: + serviceAccountName: dx-daemon + tolerations: [{operator: Exists}] + terminationGracePeriodSeconds: 35 + # dx-pull is built by the skaffold_export.yaml before-hook from the + # operator's local registry config (DX_PULL_CONFIG); without it the pod + # goes ImagePullBackOff. + imagePullSecrets: + - name: dx-pull + containers: + - name: dx-daemon + # Build note: -literals without -tiny; -tiny's pclntab stripping + # SIGSEGV'd under load. Survives the kill-chain (restarts=0, 4 rounds). + # Carries the evidence-manifest + DX_FOREST_PUSHDOWN code. + image: vizier-dx_daemon_image:latest + ports: + - {name: findings, containerPort: 9099, hostPort: 9099} + env: + - {name: NODE_NAME, valueFrom: {fieldRef: {fieldPath: spec.nodeName}}} + - {name: HOST_IP, valueFrom: {fieldRef: {fieldPath: status.hostIP}}} + - {name: POD_NAMESPACE, valueFrom: {fieldRef: {fieldPath: metadata.namespace}}} + # #164: the capture policy is the experiment's one variable. adaptive_base is + # the shipped behaviour; a static-arm lab sets "static" here and dx pulls the + # node's trailing window every DX_STATIC_WINDOW_S instead of following referrals. + - {name: DX_CAPTURE_POLICY, value: "adaptive_base"} + - {name: DX_RECEIVER_TLS, value: "1"} + # AE control surface is TLS-by-default (#96); the dx client TLS-skip-verifies + # the in-cluster (self-signed/shared) cert and attaches its bearer JWT. + - {name: AE_CONTROL_ADDR, value: "https://adaptive-export-control.pl.svc.cluster.local:9100"} + - {name: PX_API_KEY, valueFrom: {secretKeyRef: {name: dx-pixie-auth, key: api-key, optional: true}}} + - {name: PX_CLUSTER_ID, valueFrom: {secretKeyRef: {name: dx-pixie-auth, key: cluster-id, optional: true}}} + - {name: PX_CLOUD_ADDR, valueFrom: {secretKeyRef: {name: dx-pixie-auth, key: cloud-addr, optional: true}}} + - {name: DX_BENCH, value: "pemdirect"} + - name: PL_JWT_SIGNING_KEY + valueFrom: {secretKeyRef: {name: dx-vizier-direct, key: jwt-signing-key, optional: true}} + - {name: DX_VIZIER_DIRECT_ADDR, value: "vizier-query-broker-svc.pl.svc.cluster.local:50300"} + - {name: PX_DISABLE_TLS, value: "1"} + - name: DX_CLUSTER_MALIGNANT_HTTP + valueFrom: {secretKeyRef: {name: dx-metastasis-ch, key: http-url, optional: true}} + - {name: DX_PX_TIMEOUT_S, value: "90"} + - {name: DX_TELEMETRY_CACHE, value: "1"} + - {name: DX_WORKERS, value: "4"} + # evidence-graph: forest-scope the evidence, write the per-anomaly edge set, + # sink it straight to forensic_db.dx_evidence_graph (soc ingest_writer). + - {name: DX_FOREST_SCOPE, value: "1"} + # FOREST_PUSHDOWN: push the dc_snoop ppid-lineage filter + # INTO the PxL so dx pulls only the alert pod's subtree, not the whole node — + # frees the node-local PEM so AE can export dc_snoop under load (validated: 0→1777). + - {name: DX_PRECORRELATE_GRAPH, value: "1"} + - {name: DX_NARROW_WINDOW_BEFORE_MS, value: "150"} + - {name: DX_NARROW_WINDOW_AFTER_MS, value: "150"} # server-side protocol capture lands just after the anomaly + - name: DX_EVIDENCE_GRAPH_CH + value: >- + http://ingest_writer:changeme-ingest@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:8123/forensic_db + # Direct alert intake key (ES256 private). Optional: without the secret + # dx logs "direct alerts OFF" and /findings stays the only intake. + volumeMounts: + - {name: alert-key, mountPath: /etc/dx/node-agent-direct-jwt, readOnly: true} + readinessProbe: + httpGet: {path: /healthz, port: 9099, scheme: HTTPS} + initialDelaySeconds: 3 + periodSeconds: 10 + resources: + # memory: the precorrelate/full-evidence workup (DX_PRECORRELATE_GRAPH) + pemdirect + # gRPC result streams pull the per-anomaly evidence set into memory; at 1Gi dx is + # OOM-killed mid-workup (exit 137) BEFORE it writes the graph → crash-loop, empty + # graph. Measured peak ~1.3GB/round under the redis kill-chain + # fault 1); 3Gi clears it reliably on an 8GiB node (validated: restarts=0 over 6+ rounds). + requests: {cpu: 50m, memory: 1Gi} + limits: {cpu: "2", memory: 3Gi} + volumes: + - name: alert-key + secret: {secretName: node-agent-direct-jwt, optional: true, items: [{key: key.pem, path: key.pem}]} +--- +apiVersion: v1 +kind: Service +metadata: + name: dx-daemon + namespace: honey +spec: + selector: {app: dx-daemon} + internalTrafficPolicy: Local + ports: + - {name: findings, port: 9099, targetPort: 9099} diff --git a/k8s/vizier/dx/kustomization.yaml b/k8s/vizier/dx/kustomization.yaml new file mode 100644 index 00000000000..f772cb22773 --- /dev/null +++ b/k8s/vizier/dx/kustomization.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: honey +resources: +- dx-daemon.yaml +images: +- name: vizier-dx_daemon_image + newName: docker.io/tanzeee/dx-daemon + newTag: 0.6.0-shadow-rc102 + diff --git a/private/cockpit/cloud_ingress.yaml b/private/cockpit/cloud_ingress.yaml index 2a89929844c..14651d50680 100644 --- a/private/cockpit/cloud_ingress.yaml +++ b/private/cockpit/cloud_ingress.yaml @@ -6,13 +6,15 @@ metadata: namespace: plc annotations: external-dns.alpha.kubernetes.io/hostname: >- - test.austrianopencloudcommunity.org,work.test.austrianopencloudcommunity.org + test.austrianopencloudcommunity.org,work.test.austrianopencloudcommunity.org,soc.k8sstormcenter.com,work.soc.k8sstormcenter.com cert-manager.io/cluster-issuer: "letsencrypt-prod" spec: tls: - hosts: - test.austrianopencloudcommunity.org - work.test.austrianopencloudcommunity.org + - soc.k8sstormcenter.com + - work.soc.k8sstormcenter.com secretName: cloud-proxy-tls-certs rules: - host: test.austrianopencloudcommunity.org @@ -77,3 +79,65 @@ spec: name: cloud-proxy-service port: number: 443 + - host: soc.k8sstormcenter.com + http: + paths: + - path: /px.services + pathType: Prefix + backend: + service: + name: vzconn-service + port: + number: 51600 + - path: /px.cloudapi + pathType: Prefix + backend: + service: + name: api-service + port: + number: 51200 + - path: /px.api + pathType: Prefix + backend: + service: + name: cloud-proxy-service + port: + number: 4444 + - path: / + pathType: Prefix + backend: + service: + name: cloud-proxy-service + port: + number: 443 + - host: work.soc.k8sstormcenter.com + http: + paths: + - path: /px.services + pathType: Prefix + backend: + service: + name: vzconn-service + port: + number: 51600 + - path: /px.cloudapi + pathType: Prefix + backend: + service: + name: api-service + port: + number: 51200 + - path: /px.api + pathType: Prefix + backend: + service: + name: cloud-proxy-service + port: + number: 4444 + - path: / + pathType: Prefix + backend: + service: + name: cloud-proxy-service + port: + number: 443 diff --git a/skaffold/skaffold_export.yaml b/skaffold/skaffold_export.yaml new file mode 100644 index 00000000000..fcb415648f9 --- /dev/null +++ b/skaffold/skaffold_export.yaml @@ -0,0 +1,239 @@ +--- +# The evidence export path: adaptive-export and dx-daemon, defined once each, installed +# together. +# +# skaffold deploy -f skaffold/skaffold_export.yaml # both, in order +# skaffold deploy -f skaffold/skaffold_export.yaml -m dx-daemon # just one +# +# Ordering is declared, not remembered: dx reads pl-adaptive-export-secrets and hands every +# row to adaptive-export-control, so adaptive-export has to exist first. +# +# Both halves are idempotent. Deploying an unchanged tree changes nothing and restarts no +# pods; a new image tag rolls, which is the only thing that should. Both pin their image in +# their kustomization, so that is the one place a version changes. +apiVersion: skaffold/v4beta11 +kind: Config +metadata: + name: adaptive-export +# skaffold stamps a fresh skaffold.dev/run-id into whatever it deploys. In a pod template +# that is a template change, so an unchanged tree still rolled every pod on every deploy. +# Denying it on that one path leaves the image path alone: a new tag still rolls, a +# repeated deploy does not. +resourceSelector: + deny: + # Skaffold stamps a fresh run-id label on everything it deploys. On a pod template that + # rewrites the DaemonSet spec and rolls every pod on every run; on a Service it rewrites + # the object for nothing. Deny both, so a run with no image change changes nothing. + - groupKind: DaemonSet.apps + labels: [".spec.template.metadata.labels", ".metadata.labels"] + - groupKind: Service + labels: [".metadata.labels"] +manifests: + kustomize: + paths: + - k8s/vizier/adaptive_export +deploy: + kubectl: + # No defaultNamespace here. Every object's namespace comes from the kustomization + # (pl), so setting it again only produced a "multiple defaultNamespace values set" + # warning on every run and a second place to keep in sync. + hooks: + before: + - host: + command: + - bash + - -c + - | + set -e + # The adaptive-export-control Service routes dx's row hand-off to the pods. + # On a cluster first bootstrapped by px-deploy, a Service of this name already + # exists with the generic vizier selector, which matches no adaptive-export pod. + # kubectl apply will not repair it: the object carries no last-applied + # annotation, so the selector map is merged into rather than replaced, and the + # endpoints stay empty. AE listens, nothing reaches it, and dx logs a hand-off + # failure per batch with no retry. On k3s-1 that discarded every row for hours. + # Assert the selector this manifest declares. A JSON patch replaces the map; a + # merge patch would not. + WANT='{"name":"adaptive-export"}' + HAVE=$(kubectl -n pl get svc adaptive-export-control -o jsonpath='{.spec.selector}' 2>/dev/null || true) + if [ -n "$HAVE" ] && [ "$HAVE" != "$WANT" ]; then + echo "adaptive-export-control: selector is $HAVE, replacing with $WANT" + kubectl -n pl patch svc adaptive-export-control --type=json \ + -p='[{"op":"replace","path":"/spec/selector","value":{"name":"adaptive-export"}}]' >/dev/null + fi + - host: + command: + - bash + - -c + - | + set -e + # Create the adaptive-export pull secret in pl from the operator's + # own local registry config. Nothing is read from or written to this + # repository. Point AE_PULL_CONFIG at that config. + # Resolve the registry config the way the deploy step is invoked: + # explicit AE_PULL_CONFIG, else the DOCKER_CONFIG the caller already exports, + # else the default location. No registry is named here. + CFG="${AE_PULL_CONFIG:-${DOCKER_CONFIG:+$DOCKER_CONFIG/config.json}}" + CFG="${CFG:-$HOME/.registry-pull/config.json}" + if [ -f "$CFG" ]; then + kubectl -n pl create secret generic adaptive-export-pull \ + --type=kubernetes.io/dockerconfigjson \ + --from-file=.dockerconfigjson="$CFG" \ + --dry-run=client -o yaml | kubectl apply -f - >/dev/null + echo "adaptive-export-pull applied in pl (from $CFG)" + elif kubectl -n pl get secret adaptive-export-pull >/dev/null 2>&1; then + echo "adaptive-export-pull already in the cluster; no local config needed" + else + echo "ERROR: no registry config at $CFG and no adaptive-export-pull secret in pl." >&2 + echo " The image is private, so the pods would sit in ImagePullBackOff." >&2 + echo " Set AE_PULL_CONFIG or DOCKER_CONFIG, or create the secret once by hand." >&2 + exit 1 + fi + - host: + command: + - bash + - -c + - | + set -e + # PL_CLOUD_ADDR must carry an explicit :443 or the AE cloud client crashloops. + CA=$(kubectl -n pl get cm pl-cloud-config -o jsonpath='{.data.PL_CLOUD_ADDR}' 2>/dev/null || true) + case "$CA" in + ""|*:*) ;; + *) kubectl -n pl patch cm pl-cloud-config --type merge -p "{\"data\":{\"PL_CLOUD_ADDR\":\"$CA:443\"}}" ;; + esac + # Seed pl-adaptive-export-secrets only when a key is supplied; never clobber a good secret. + API="${PIXIE_API_KEY:-${PX_API_KEY:-}}" + CH_HOST=clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:9000 + CH_DSN="${AE_CH_DSN:-ingest_writer:changeme-ingest@$CH_HOST/forensic_db}" + if [ -n "$API" ]; then + kubectl -n pl create secret generic pl-adaptive-export-secrets \ + --from-literal=pixie-api-key="$API" \ + --from-literal=clickhouse-dsn="$CH_DSN" \ + --dry-run=client -o yaml | kubectl apply -f - + elif ! kubectl -n pl get secret pl-adaptive-export-secrets >/dev/null 2>&1; then + echo "ERROR: set PIXIE_API_KEY (or source keys.env) to seed pl-adaptive-export-secrets" >&2 + exit 1 + else + echo "pl-adaptive-export-secrets exists; PIXIE_API_KEY unset -> leaving it untouched" + fi + os: [linux, darwin] + + after: + - host: + command: + - bash + - -c + - | + set -e + # Fail loudly if the Service resolves to nothing. An empty endpoint list here + # means every dx row hand-off is dropped, and the only symptom is a log line on + # the other side, which is how this went unnoticed for hours. + # + # Wait for the rollout FIRST. This check used to run straight after apply and + # read the endpoints of the pods being replaced, so it reported success at the + # exact moment the new pods were still Terminating — a green gate over a broken + # cluster. Nothing after the rollout is stale. + kubectl -n pl rollout status ds/adaptive-export --timeout=120s + for i in 1 2 3 4 5 6 7 8 9 10; do + N=$(kubectl -n pl get endpointslice -l kubernetes.io/service-name=adaptive-export-control \ + -o jsonpath='{range .items[*]}{range .endpoints[*]}{.addresses[0]}{"\n"}{end}{end}' \ + 2>/dev/null | grep -c . || true) + [ "${N:-0}" -gt 0 ] && break + sleep 3 + done + if [ "${N:-0}" -gt 0 ]; then + echo "adaptive-export-control: $N endpoint(s) — dx can reach AE" + else + echo "FATAL: adaptive-export-control has NO endpoints; every dx row hand-off will be dropped" >&2 + kubectl -n pl get svc adaptive-export-control -o jsonpath='{.spec.selector}' >&2; echo >&2 + kubectl -n pl get pods -l name=adaptive-export --show-labels >&2 + exit 1 + fi +--- +apiVersion: skaffold/v4beta11 +kind: Config +metadata: + name: dx-daemon +requires: + - configs: [adaptive-export] +# Same reason as adaptive-export: skaffold stamps a fresh skaffold.dev/run-id into the pod +# template on every deploy, which is a template change, so an unchanged tree rolled every +# pod. Denied on that path only — a new image tag still rolls. +resourceSelector: + deny: + # Skaffold stamps a fresh run-id label on everything it deploys. On a pod template that + # rewrites the DaemonSet spec and rolls every pod on every run; on a Service it rewrites + # the object for nothing. Deny both, so a run with no image change changes nothing. + - groupKind: DaemonSet.apps + labels: [".spec.template.metadata.labels", ".metadata.labels"] + - groupKind: Service + labels: [".metadata.labels"] +manifests: + kustomize: + paths: + - k8s/vizier/dx +deploy: + kubectl: + # No defaultNamespace here. Every object's namespace comes from the kustomization + # (honey), so setting it again only produced a "multiple defaultNamespace values set" + # warning on every run and a second place to keep in sync. + hooks: + before: + - host: + command: + - bash + - -c + - | + set -e + kubectl create namespace honey --dry-run=client -o yaml | kubectl apply -f - + JWT=$(kubectl -n pl get secret pl-cluster-secrets -o jsonpath='{.data.jwt-signing-key}' | base64 -d) + CID=$(kubectl -n pl get secret pl-cluster-secrets -o jsonpath='{.data.cluster-id}' | base64 -d) + CA=$(kubectl -n pl get cm pl-cloud-config -o jsonpath='{.data.PL_CLOUD_ADDR}') + API=$(kubectl -n pl get secret pl-adaptive-export-secrets \ + -o jsonpath='{.data.pixie-api-key}' 2>/dev/null | base64 -d) + CH_HOST=clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:8123 + CH_URL="${DX_CH_HTTP_URL:-http://ingest_writer:changeme-ingest@$CH_HOST/?database=forensic_db}" + kubectl -n honey create secret generic dx-vizier-direct \ + --from-literal=jwt-signing-key="$JWT" \ + --dry-run=client -o yaml | kubectl apply -f - + kubectl -n honey create secret generic dx-pixie-auth \ + --from-literal=api-key="$API" \ + --from-literal=cluster-id="$CID" \ + --from-literal=cloud-addr="$CA" \ + --dry-run=client -o yaml | kubectl apply -f - + kubectl -n honey create secret generic dx-metastasis-ch \ + --from-literal=http-url="$CH_URL" \ + --dry-run=client -o yaml | kubectl apply -f - + # Resolve the registry config the way the deploy step is invoked: + # explicit DX_PULL_CONFIG, else the DOCKER_CONFIG the caller already exports, + # else the default location. No registry is named here. + PULL="${DX_PULL_CONFIG:-${DOCKER_CONFIG:+$DOCKER_CONFIG/config.json}}" + PULL="${PULL:-$HOME/.registry-pull/config.json}" + [ -r "$PULL" ] || { echo "dx-pull: registry config $PULL not readable (set DX_PULL_CONFIG)" >&2; exit 1; } + kubectl -n honey create secret generic dx-pull \ + --type=kubernetes.io/dockerconfigjson \ + --from-file=.dockerconfigjson="$PULL" \ + --dry-run=client -o yaml | kubectl apply -f - + os: [linux, darwin] + after: + - host: + command: + - bash + - -c + - | + set -e + # Wait for the rollout before judging anything, or this reads the state of the + # pods being replaced and reports success while they terminate. + kubectl -n honey rollout status ds/dx-daemon --timeout=120s + # dx hands every row to adaptive-export. If that Service has no endpoints the + # hand-off is dropped with nothing but a line in dx's own log to show for it. + N=$(kubectl -n pl get endpointslice -l kubernetes.io/service-name=adaptive-export-control \ + -o jsonpath='{range .items[*]}{range .endpoints[*]}{.addresses[0]}{"\n"}{end}{end}' \ + 2>/dev/null | grep -c . || true) + if [ "${N:-0}" -gt 0 ]; then + echo "dx-daemon: up, and adaptive-export-control has $N endpoint(s)" + else + echo "FATAL: adaptive-export-control has NO endpoints; dx will drop every hand-off" >&2 + exit 1 + fi + os: [linux, darwin] diff --git a/skaffold/skaffold_vizier.yaml b/skaffold/skaffold_vizier.yaml index 58b6bba70af..7f44c668916 100644 --- a/skaffold/skaffold_vizier.yaml +++ b/skaffold/skaffold_vizier.yaml @@ -45,13 +45,6 @@ build: args: - --config=x86_64_sysroot - --compilation_mode=opt - - image: vizier-adaptive_export_image - context: . - bazel: - target: //src/vizier/services/adaptive_export:adaptive_export_image.tar - args: - - --config=x86_64_sysroot - - --compilation_mode=opt tagPolicy: dateTime: {} local: diff --git a/src/pxl_scripts/Makefile b/src/pxl_scripts/Makefile index 1cca03f4dc5..4e8a3562658 100644 --- a/src/pxl_scripts/Makefile +++ b/src/pxl_scripts/Makefile @@ -15,7 +15,7 @@ # SPDX-License-Identifier: Apache-2.0 # Update dir name here if you want to add a new directory. -dirs := bpftrace px pxbeta sotw +dirs := bpftrace dx px pxbeta sotw script_files := $(foreach dir,$(dirs),$(wildcard $(dir)/**/*)) EXECUTABLES ?= px diff --git a/src/pxl_scripts/README.md b/src/pxl_scripts/README.md index ff919d732c7..9f71d1f4052 100644 --- a/src/pxl_scripts/README.md +++ b/src/pxl_scripts/README.md @@ -15,6 +15,10 @@ To update this README, run `make update_readme` in this directory. - bpftrace/[sync_snoop](https://github.com/pixie-io/pixie/tree/main/src/pxl_scripts/bpftrace/sync_snoop): Tracing file system sync events. - bpftrace/[tcp_drops](https://github.com/pixie-io/pixie/tree/main/src/pxl_scripts/bpftrace/tcp_drops): Shows TCP drop counts in the cluster. - bpftrace/[tcp_retransmits](https://github.com/pixie-io/pixie/tree/main/src/pxl_scripts/bpftrace/tcp_retransmits): Shows TCP retransmission counts in the cluster. +- dx/[breakout](https://github.com/pixie-io/pixie/tree/main/src/pxl_scripts/dx/breakout): Presentation +- dx/[dns_resolve](https://github.com/pixie-io/pixie/tree/main/src/pxl_scripts/dx/dns_resolve): SOC DNS Resolution +- dx/[evidence_graph](https://github.com/pixie-io/pixie/tree/main/src/pxl_scripts/dx/evidence_graph): SOC Evidence Graph +- dx/[fullchain](https://github.com/pixie-io/pixie/tree/main/src/pxl_scripts/dx/fullchain): SOC Full-Chain - px/[agent_status](https://github.com/pixie-io/pixie/tree/main/src/pxl_scripts/px/agent_status): This script gets the status of all the pixie agents (PEMs/Collectors) running. - px/[agent_status_diagnostics](https://github.com/pixie-io/pixie/tree/main/src/pxl_scripts/px/agent_status_diagnostics): This script performs diagnostics on the agents' (PEMs/Collectors) status - px/[amqp_data](https://github.com/pixie-io/pixie/tree/main/src/pxl_scripts/px/amqp_data): Shows a sample of amqp messages in the cluster. @@ -27,7 +31,6 @@ To update this README, run `make update_readme` in this directory. - px/[dns_data](https://github.com/pixie-io/pixie/tree/main/src/pxl_scripts/px/dns_data): Show a sample of DNS traffic in the cluster. - px/[dns_flow_graph](https://github.com/pixie-io/pixie/tree/main/src/pxl_scripts/px/dns_flow_graph): Overview of DNS requests in the cluster, with latency stats. - px/[dns_query_summary](https://github.com/pixie-io/pixie/tree/main/src/pxl_scripts/px/dns_query_summary): Overview of DNS queries from pods in a namespace, grouped by the name being resolved and the rates of success. -- px/[dx_evidence_graph](https://github.com/pixie-io/pixie/tree/main/src/pxl_scripts/px/dx_evidence_graph): Severity-weighted, all-protocol pod-to-pod graph for one investigation. Edge records emitted by dx with weight (sum of CRS evidence severity) on the edges and max_severity colouring the heat. - px/[funcs](https://github.com/pixie-io/pixie/tree/main/src/pxl_scripts/px/funcs): Gets a list all of the funcs available in Pixie. - px/[http_data](https://github.com/pixie-io/pixie/tree/main/src/pxl_scripts/px/http_data): Shows most recent HTTP messages in the cluster. - px/[http_data_filtered](https://github.com/pixie-io/pixie/tree/main/src/pxl_scripts/px/http_data_filtered): Show a sample of HTTP requests in the Cluster filtered by service, pod, request path & response status code. diff --git a/src/pxl_scripts/dx/breakout/breakout.pxl b/src/pxl_scripts/dx/breakout/breakout.pxl new file mode 100644 index 00000000000..78ba895622a --- /dev/null +++ b/src/pxl_scripts/dx/breakout/breakout.pxl @@ -0,0 +1,59 @@ +# Copyright 2018- The Pixie Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +import px + + +def _host(start_time: str, clickhouse_dsn: str, host: str): + df = px.DataFrame('dx_src__kubescape_mitre', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + df = df[px.contains(df.pod, host)] + df.podns = px.pluck_array(px.split(df.pod, '/'), 0) + df = df[df.podns != 'clickhouse'] + df.symptom = df.alert + df.sign = px.pluck_array(px.split(df.alert, ':'), 0) + df.site = px.pluck_array(px.split(px.pluck_array(px.split(df.alert, ':'), 1), ' '), 1) + return df + + +def course(start_time: str, clickhouse_dsn: str, host: str): + df = _host(start_time, clickhouse_dsn, host) + return df[['ts', 'symptom', 'comm', 'pid', 'ppid', 'cmdline']] + + +def signs(start_time: str, clickhouse_dsn: str, host: str): + df = _host(start_time, clickhouse_dsn, host) + agg = df.groupby(['sign']).agg(count=('uniqueID', px.count)) + return agg[['sign', 'count']] + + +def presentation(start_time: str, clickhouse_dsn: str, host: str): + df = _host(start_time, clickhouse_dsn, host) + agg = df.groupby(['pod', 'site', 'sign']).agg( + count=('uniqueID', px.count), symptom=('symptom', px.any)) + agg.from_entity = px.Pod(agg.pod) + agg.to_entity = agg.site + return agg[['from_entity', 'to_entity', 'sign', 'symptom', 'count']] + + +def discharge(start_time: str, clickhouse_dsn: str): + df = px.DataFrame('dx_ord__http_events', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + df = df[df.req_method == 'POST'] + df.podns = px.pluck_array(px.split(df.pod, '/'), 0) + df = df[df.podns != 'clickhouse'] + df.host = df.pod + df.recipient = df.remote_addr + df.specimen = df.req_body + return df[['ts', 'order_id', 'host', 'recipient', 'req_path', 'resp_status', 'specimen']] diff --git a/src/pxl_scripts/dx/breakout/manifest.yaml b/src/pxl_scripts/dx/breakout/manifest.yaml new file mode 100644 index 00000000000..406d0dffed8 --- /dev/null +++ b/src/pxl_scripts/dx/breakout/manifest.yaml @@ -0,0 +1,3 @@ +--- +short: Presentation +long: Presentation diff --git a/src/pxl_scripts/dx/breakout/vis.json b/src/pxl_scripts/dx/breakout/vis.json new file mode 100644 index 00000000000..14d7debe7e6 --- /dev/null +++ b/src/pxl_scripts/dx/breakout/vis.json @@ -0,0 +1,171 @@ +{ + "variables": [ + { + "name": "start_time", + "type": "PX_STRING", + "description": "", + "defaultValue": "-2h" + }, + { + "name": "clickhouse_dsn", + "type": "PX_STRING", + "description": "", + "defaultValue": "forensic_analyst:changeme-analyst@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:9000/forensic_db" + }, + { + "name": "host", + "type": "PX_STRING", + "description": "", + "defaultValue": "opsboard" + } + ], + "globalFuncs": [ + { + "outputName": "g_presentation", + "func": { + "name": "presentation", + "args": [ + { + "name": "start_time", + "variable": "start_time" + }, + { + "name": "clickhouse_dsn", + "variable": "clickhouse_dsn" + }, + { + "name": "host", + "variable": "host" + } + ] + } + }, + { + "outputName": "g_signs", + "func": { + "name": "signs", + "args": [ + { + "name": "start_time", + "variable": "start_time" + }, + { + "name": "clickhouse_dsn", + "variable": "clickhouse_dsn" + }, + { + "name": "host", + "variable": "host" + } + ] + } + }, + { + "outputName": "g_discharge", + "func": { + "name": "discharge", + "args": [ + { + "name": "start_time", + "variable": "start_time" + }, + { + "name": "clickhouse_dsn", + "variable": "clickhouse_dsn" + } + ] + } + }, + { + "outputName": "g_course", + "func": { + "name": "course", + "args": [ + { + "name": "start_time", + "variable": "start_time" + }, + { + "name": "clickhouse_dsn", + "variable": "clickhouse_dsn" + }, + { + "name": "host", + "variable": "host" + } + ] + } + } + ], + "widgets": [ + { + "name": "Presentation", + "position": { + "x": 0, + "y": 0, + "w": 12, + "h": 4 + }, + "globalFuncOutputName": "g_presentation", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Graph", + "adjacencyList": { + "fromColumn": "from_entity", + "toColumn": "to_entity" + }, + "edgeWeightColumn": "count", + "edgeColorColumn": "count", + "edgeLabelColumn": "sign", + "edgeThresholds": { + "mediumThreshold": 3, + "highThreshold": 10 + }, + "edgeHoverInfo": [ + "sign", + "symptom", + "count" + ], + "edgeLength": 400 + } + }, + { + "name": "Discharge", + "position": { + "x": 0, + "y": 4, + "w": 12, + "h": 3 + }, + "globalFuncOutputName": "g_discharge", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Table" + } + }, + { + "name": "Presenting signs", + "position": { + "x": 0, + "y": 7, + "w": 4, + "h": 4 + }, + "globalFuncOutputName": "g_signs", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Table" + } + }, + { + "name": "Clinical course", + "position": { + "x": 4, + "y": 7, + "w": 8, + "h": 4 + }, + "globalFuncOutputName": "g_course", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Table" + } + } + ] +} diff --git a/src/pxl_scripts/dx/dns_resolve/dns_resolve.pxl b/src/pxl_scripts/dx/dns_resolve/dns_resolve.pxl new file mode 100644 index 00000000000..d7e18627e3b --- /dev/null +++ b/src/pxl_scripts/dx/dns_resolve/dns_resolve.pxl @@ -0,0 +1,40 @@ +# Copyright 2018- The Pixie Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +import px + + +def dns_graph(start_time: str, clickhouse_dsn: str, order_id: str): + win = px.DataFrame('dx_orders_win', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + win = win[win.order_id == order_id] + win = win[['lo', 'hi']] + win.k = 1 + d = px.DataFrame('dx_dns_resolve', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + d.k = 1 + d = d.merge(win, how='inner', left_on=['k'], right_on=['k'], suffixes=['', '_w']) + d = d[d.event_time >= d.lo] + d = d[d.event_time <= d.hi] + + d.resolver_ip = px.replace(':53', d.to_node, '') + d.r_svc = px.service_id_to_service_name(px.ip_to_service_id(d.resolver_ip)) + d.r_nsl = px.nslookup(d.resolver_ip) + d.resolver = px.select(d.r_svc != '', d.r_svc, d.r_nsl) + d.q_svc = px.pod_name_to_service_name(d.from_node) + d.querier = px.select(d.q_svc != '', d.q_svc, d.from_node) + + d.from_entity = px.select(d.kind == 'query', d.querier, d.from_node) + d.to_entity = px.select(d.kind == 'query', d.resolver, d.to_node) + return d[['from_entity', 'to_entity', 'edge_label', 'kind', 'ts']] diff --git a/src/pxl_scripts/dx/dns_resolve/manifest.yaml b/src/pxl_scripts/dx/dns_resolve/manifest.yaml new file mode 100644 index 00000000000..cc25c6d788f --- /dev/null +++ b/src/pxl_scripts/dx/dns_resolve/manifest.yaml @@ -0,0 +1,3 @@ +--- +short: SOC DNS Resolution +long: SOC DNS Resolution diff --git a/src/pxl_scripts/dx/dns_resolve/vis.json b/src/pxl_scripts/dx/dns_resolve/vis.json new file mode 100644 index 00000000000..ac7ca816e02 --- /dev/null +++ b/src/pxl_scripts/dx/dns_resolve/vis.json @@ -0,0 +1,83 @@ +{ + "variables": [ + { + "name": "start_time", + "type": "PX_STRING", + "description": "", + "defaultValue": "-6h" + }, + { + "name": "clickhouse_dsn", + "type": "PX_STRING", + "description": "", + "defaultValue": "forensic_analyst:changeme-analyst@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:9000/forensic_db" + }, + { + "name": "order_id", + "type": "PX_STRING", + "description": "", + "defaultValue": "" + } + ], + "globalFuncs": [ + { + "outputName": "g_dns", + "func": { + "name": "dns_graph", + "args": [ + { + "name": "start_time", + "variable": "start_time" + }, + { + "name": "clickhouse_dsn", + "variable": "clickhouse_dsn" + }, + { + "name": "order_id", + "variable": "order_id" + } + ] + } + } + ], + "widgets": [ + { + "name": "DNS resolution", + "position": { + "x": 0, + "y": 0, + "w": 12, + "h": 5 + }, + "globalFuncOutputName": "g_dns", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Graph", + "adjacencyList": { + "fromColumn": "from_entity", + "toColumn": "to_entity" + }, + "edgeLabelColumn": "edge_label", + "edgeHoverInfo": [ + "edge_label", + "kind", + "ts" + ], + "edgeLength": 400 + } + }, + { + "name": "DNS edges", + "position": { + "x": 0, + "y": 5, + "w": 12, + "h": 4 + }, + "globalFuncOutputName": "g_dns", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Table" + } + } + ] +} diff --git a/src/pxl_scripts/dx/evidence_graph/evidence_graph.pxl b/src/pxl_scripts/dx/evidence_graph/evidence_graph.pxl new file mode 100644 index 00000000000..6198340d326 --- /dev/null +++ b/src/pxl_scripts/dx/evidence_graph/evidence_graph.pxl @@ -0,0 +1,203 @@ +# Copyright 2018- The Pixie Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +import px + + +def _ord(start_time: str, clickhouse_dsn: str, view: str, order_id: str): + df = px.DataFrame(view, clickhouse_dsn=clickhouse_dsn, start_time=start_time) + df = df[df.order_id == order_id] + return df.drop(['order_id', 'row_time', 'event_time']) + + +def evidence_graph(start_time: str, clickhouse_dsn: str, table: str, case_id: str): + orders = px.DataFrame('dx_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + orders = orders[['order_id', 'kubescape_uid', 'rule_id', 'pod', 'culprit_key']] + orders = orders[px.contains(orders.culprit_key, case_id)] + anom = px.DataFrame(table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) + df = orders.merge(anom, how='inner', left_on=['kubescape_uid', 'rule_id'], + right_on=['uniqueID', 'rule'], suffixes=['', '_a']) + df.subject_ns = px.pluck_array(px.split(df.subject_pod, '/'), 0) + df = df[df.subject_ns != 'pl'] + df = df[df.subject_ns != 'px-operator'] + df.order_link = px.script_reference(df.order_id, 'dx/evidence_graph', { + 'start_time': start_time, + 'clickhouse_dsn': clickhouse_dsn, + 'graph_table': table, + 'order_id': df.order_id, + 'case_id': df.culprit_key, + }) + df.from_entity = px.Pod(df.subject_pod) + df.to_entity = df.target + return df[['from_entity', 'to_entity', 'order_link', 'rule_mitre', 'mitre_tactic', + 'mitre_technique', 'order_id', 'rule', 'process', 'target', 'target_kind', + 'severity', 'alert', 'subject_pod']] + + +def cases(start_time: str, clickhouse_dsn: str, case_id: str): + df = px.DataFrame('dx_cases', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + df = df[px.contains(df.culprit_key, case_id)] + df.subject_ns = px.pluck_array(px.split(df.subject_pod, '/'), 0) + df = df[df.subject_ns != 'pl'] + df = df[df.subject_ns != 'px-operator'] + df.from_entity = df.culprit_key + df.to_entity = df.subject_pod + ':' + df.rule_id + df.case_link = px.script_reference(df.culprit_key, 'dx/evidence_graph', { + 'start_time': start_time, + 'clickhouse_dsn': clickhouse_dsn, + 'graph_table': 'dx_kubescape_mitre', + 'order_id': df.order_id, + 'case_id': df.culprit_key, + }) + df.order_link = df.case_link + return df[['from_entity', 'to_entity', 'culprit_key', 'rule_id', 'mitre_tactic', + 'mitre_technique', 'severity', 'alert', 'order_id', 'order_link']] + + +def orders(start_time: str, clickhouse_dsn: str, graph_table: str, case_id: str): + df = px.DataFrame('dx_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + df = df[px.contains(df.culprit_key, case_id)] + ks = px.DataFrame('dx_src__kubescape_mitre', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + ks = ks[['uniqueID', 'infected_pid', 'offender', 'profile']] + df = df.merge(ks, how='left', left_on=['kubescape_uid'], right_on=['uniqueID'], suffixes=['', '_ks']) + df = df.drop(['uniqueID']) + alerts = px.DataFrame(graph_table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) + alerts = alerts[['uniqueID', 'rule', 'mitre_tactic', 'mitre_technique', 'ts']] + df = df.merge(alerts, how='left', left_on=['kubescape_uid', 'rule_id'], + right_on=['uniqueID', 'rule'], suffixes=['', '_k']) + df.pod_ns = px.pluck_array(px.split(df.pod, '/'), 0) + df = df[df.pod_ns != 'pl'] + df = df[df.pod_ns != 'px-operator'] + df.order = px.script_reference(df.order_id, 'dx/evidence_graph', { + 'start_time': start_time, + 'clickhouse_dsn': clickhouse_dsn, + 'graph_table': graph_table, + 'order_id': df.order_id, + 'case_id': df.culprit_key, + }) + df.Alert = df.disc + return df[['order', 'ts', 'rule_id', 'Alert', 'offender', 'infected_pid', 'profile', + 'mitre_tactic', 'mitre_technique', 'pod', 'culprit_key']] + + +def kubescape(start_time: str, clickhouse_dsn: str, order_id: str): + orders = px.DataFrame('dx_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + orders = orders[orders.order_id == order_id] + orders = orders[['kubescape_uid', 'rule_id']] + k = px.DataFrame('dx_src__kubescape_mitre', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + j = k.merge(orders, how='inner', left_on=['uniqueID', 'RuleID'], + right_on=['kubescape_uid', 'rule_id'], suffixes=['', '_ord']) + return j.drop(['row_time', 'event_time', 'kubescape_uid', 'rule_id', 'uniqueID']) + + +def capability(start_time: str, clickhouse_dsn: str, order_id: str): + orders = px.DataFrame('dx_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + orders = orders[orders.order_id == order_id] + orders = orders[['kubescape_uid']] + k = px.DataFrame('dx_src__kubescape_mitre', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + seed = k.merge(orders, how='inner', left_on=['uniqueID'], right_on=['kubescape_uid'], suffixes=['', '_o']) + seed = seed.groupby(['infected_pid', 'pod']).agg(hits=('RuleID', px.count)) + seed = seed[['infected_pid', 'pod']] + allk = px.DataFrame('dx_src__kubescape_mitre', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + acts = allk.merge(seed, how='inner', left_on=['infected_pid', 'pod'], + right_on=['infected_pid', 'pod'], suffixes=['', '_s']) + return acts[['ts', 'RuleID', 'mitre_tactic', 'mitre_technique', 'alert', + 'offender', 'infected_pid', 'profile']] + + +def conn(start_time: str, clickhouse_dsn: str, order_id: str): + return _ord(start_time, clickhouse_dsn, 'dx_ord__conn_stats', order_id) + + +def redis(start_time: str, clickhouse_dsn: str, order_id: str): + return _ord(start_time, clickhouse_dsn, 'dx_ord__redis_events', order_id) + + +def http(start_time: str, clickhouse_dsn: str, order_id: str): + return _ord(start_time, clickhouse_dsn, 'dx_ord__http_events', order_id).drop(['namespace']) + + +def dns(start_time: str, clickhouse_dsn: str, order_id: str): + return _ord(start_time, clickhouse_dsn, 'dx_ord__dns_events', order_id).drop(['namespace']) + + +def pgsql(start_time: str, clickhouse_dsn: str, order_id: str): + return _ord(start_time, clickhouse_dsn, 'dx_ord__pgsql_events', order_id).drop(['namespace']) + + +def mysql(start_time: str, clickhouse_dsn: str, order_id: str): + return _ord(start_time, clickhouse_dsn, 'dx_ord__mysql_events', order_id).drop(['namespace']) + + +def dc_snoop(start_time: str, clickhouse_dsn: str, order_id: str): + return _ord(start_time, clickhouse_dsn, 'dx_ord__dc_snoop', order_id).drop(['namespace']) + + +def creds_change(start_time: str, clickhouse_dsn: str, order_id: str): + df = _ord(start_time, clickhouse_dsn, 'dx_ord__creds_change', order_id).drop(['namespace']) + return df[df.old_uid != df.new_uid] + + +def cql(start_time: str, clickhouse_dsn: str, order_id: str): + return _ord(start_time, clickhouse_dsn, 'dx_ord__cql_events', order_id).drop(['namespace']) + + +def mongodb(start_time: str, clickhouse_dsn: str, order_id: str): + return _ord(start_time, clickhouse_dsn, 'dx_ord__mongodb_events', order_id).drop(['namespace']) + + +def stack_trace(start_time: str, clickhouse_dsn: str, order_id: str): + w = px.DataFrame('dx_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + w = w[w.order_id == order_id] + w = w[['pod']] + st = px.DataFrame(table='stack_traces.beta', start_time=start_time) + st.pod = st.ctx['pod'] + st = st.merge(w, how='inner', left_on=['pod'], right_on=['pod'], suffixes=['', '_w']) + st = st.groupby(['pod', 'stack_trace']).agg(count=('count', px.sum)) + return st[['pod', 'stack_trace', 'count']] + + +def stack_diff(start_time: str, clickhouse_dsn: str, order_id: str): + orders = px.DataFrame('dx_orders_win', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + orders = orders[orders.order_id == order_id] + orders = orders[['pod', 'lo', 'hi']] + orders.alo = orders.lo + 270000000000 + orders.ahi = orders.lo + 330000000000 + orders.blo = orders.lo + 210000000000 + st = px.DataFrame(table='stack_traces.beta', start_time=start_time) + st.pod = st.ctx['pod'] + st.row_time = px.time_to_int64(st.time_) + st = st.merge(orders, how='inner', left_on=['pod'], right_on=['pod'], suffixes=['', '_o']) + + base = st[st.row_time >= st.blo] + base = base[base.row_time < base.alo] + base = base.groupby(['pod', 'stack_trace']).agg(count=('count', px.sum)) + + atk = st[st.row_time >= st.alo] + atk = atk[atk.row_time <= atk.ahi] + atk = atk.groupby(['pod', 'stack_trace']).agg(count=('count', px.sum)) + + diff = base.merge(atk, how='right', left_on=['stack_trace'], right_on=['stack_trace'], + suffixes=['_base', '_atk']) + diff.pod = diff.pod_atk + diff.stack_trace = px.replace(' ', diff.stack_trace_atk, '') + diff.count = diff.count_atk + diff.delta = diff.count_atk - diff.count_base + + total = atk.groupby(['pod']).agg(total=('count', px.sum)) + merged = diff.merge(total, how='inner', left_on=['pod'], right_on=['pod'], suffixes=['', '_t']) + merged.percent = 100 * merged.count / merged.total + return merged[['stack_trace', 'count', 'delta', 'percent', 'pod']] diff --git a/src/pxl_scripts/dx/evidence_graph/manifest.yaml b/src/pxl_scripts/dx/evidence_graph/manifest.yaml new file mode 100644 index 00000000000..32d0b78b669 --- /dev/null +++ b/src/pxl_scripts/dx/evidence_graph/manifest.yaml @@ -0,0 +1,3 @@ +--- +short: SOC Evidence Graph +long: SOC Evidence Graph diff --git a/src/pxl_scripts/dx/evidence_graph/vis.json b/src/pxl_scripts/dx/evidence_graph/vis.json new file mode 100644 index 00000000000..db0c52adf37 --- /dev/null +++ b/src/pxl_scripts/dx/evidence_graph/vis.json @@ -0,0 +1,657 @@ +{ + "variables": [ + { + "name": "start_time", + "type": "PX_STRING", + "description": "", + "defaultValue": "-7d" + }, + { + "name": "clickhouse_dsn", + "type": "PX_STRING", + "description": "", + "defaultValue": "forensic_analyst:changeme-analyst@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:9000/forensic_db" + }, + { + "name": "graph_table", + "type": "PX_STRING", + "description": "", + "defaultValue": "dx_kubescape_mitre" + }, + { + "name": "order_id", + "type": "PX_STRING", + "description": "", + "defaultValue": "" + }, + { + "name": "case_id", + "type": "PX_STRING", + "description": "", + "defaultValue": "" + } + ], + "globalFuncs": [ + { + "outputName": "g_graph", + "func": { + "name": "evidence_graph", + "args": [ + { + "name": "start_time", + "variable": "start_time" + }, + { + "name": "clickhouse_dsn", + "variable": "clickhouse_dsn" + }, + { + "name": "table", + "variable": "graph_table" + }, + { + "name": "case_id", + "variable": "case_id" + } + ] + } + }, + { + "outputName": "g_cases", + "func": { + "name": "cases", + "args": [ + { + "name": "start_time", + "variable": "start_time" + }, + { + "name": "clickhouse_dsn", + "variable": "clickhouse_dsn" + }, + { + "name": "case_id", + "variable": "case_id" + } + ] + } + }, + { + "outputName": "g_orders", + "func": { + "name": "orders", + "args": [ + { + "name": "start_time", + "variable": "start_time" + }, + { + "name": "clickhouse_dsn", + "variable": "clickhouse_dsn" + }, + { + "name": "graph_table", + "variable": "graph_table" + }, + { + "name": "case_id", + "variable": "case_id" + } + ] + } + }, + { + "outputName": "g_kube", + "func": { + "name": "kubescape", + "args": [ + { + "name": "start_time", + "variable": "start_time" + }, + { + "name": "clickhouse_dsn", + "variable": "clickhouse_dsn" + }, + { + "name": "order_id", + "variable": "order_id" + } + ] + } + }, + { + "outputName": "g_conn", + "func": { + "name": "conn", + "args": [ + { + "name": "start_time", + "variable": "start_time" + }, + { + "name": "clickhouse_dsn", + "variable": "clickhouse_dsn" + }, + { + "name": "order_id", + "variable": "order_id" + } + ] + } + }, + { + "outputName": "g_redis", + "func": { + "name": "redis", + "args": [ + { + "name": "start_time", + "variable": "start_time" + }, + { + "name": "clickhouse_dsn", + "variable": "clickhouse_dsn" + }, + { + "name": "order_id", + "variable": "order_id" + } + ] + } + }, + { + "outputName": "g_http", + "func": { + "name": "http", + "args": [ + { + "name": "start_time", + "variable": "start_time" + }, + { + "name": "clickhouse_dsn", + "variable": "clickhouse_dsn" + }, + { + "name": "order_id", + "variable": "order_id" + } + ] + } + }, + { + "outputName": "g_dns", + "func": { + "name": "dns", + "args": [ + { + "name": "start_time", + "variable": "start_time" + }, + { + "name": "clickhouse_dsn", + "variable": "clickhouse_dsn" + }, + { + "name": "order_id", + "variable": "order_id" + } + ] + } + }, + { + "outputName": "g_pgsql", + "func": { + "name": "pgsql", + "args": [ + { + "name": "start_time", + "variable": "start_time" + }, + { + "name": "clickhouse_dsn", + "variable": "clickhouse_dsn" + }, + { + "name": "order_id", + "variable": "order_id" + } + ] + } + }, + { + "outputName": "g_mysql", + "func": { + "name": "mysql", + "args": [ + { + "name": "start_time", + "variable": "start_time" + }, + { + "name": "clickhouse_dsn", + "variable": "clickhouse_dsn" + }, + { + "name": "order_id", + "variable": "order_id" + } + ] + } + }, + { + "outputName": "g_dcsnoop", + "func": { + "name": "dc_snoop", + "args": [ + { + "name": "start_time", + "variable": "start_time" + }, + { + "name": "clickhouse_dsn", + "variable": "clickhouse_dsn" + }, + { + "name": "order_id", + "variable": "order_id" + } + ] + } + }, + { + "outputName": "g_stack", + "func": { + "name": "stack_trace", + "args": [ + { + "name": "start_time", + "variable": "start_time" + }, + { + "name": "clickhouse_dsn", + "variable": "clickhouse_dsn" + }, + { + "name": "order_id", + "variable": "order_id" + } + ] + } + }, + { + "outputName": "g_stackdiff", + "func": { + "name": "stack_diff", + "args": [ + { + "name": "start_time", + "variable": "start_time" + }, + { + "name": "clickhouse_dsn", + "variable": "clickhouse_dsn" + }, + { + "name": "order_id", + "variable": "order_id" + } + ] + } + }, + { + "outputName": "g_capability", + "func": { + "name": "capability", + "args": [ + { + "name": "start_time", + "variable": "start_time" + }, + { + "name": "clickhouse_dsn", + "variable": "clickhouse_dsn" + }, + { + "name": "order_id", + "variable": "order_id" + } + ] + } + }, + { + "outputName": "g_credschange", + "func": { + "name": "creds_change", + "args": [ + { + "name": "start_time", + "variable": "start_time" + }, + { + "name": "clickhouse_dsn", + "variable": "clickhouse_dsn" + }, + { + "name": "order_id", + "variable": "order_id" + } + ] + } + }, + { + "outputName": "g_cql", + "func": { + "name": "cql", + "args": [ + { + "name": "start_time", + "variable": "start_time" + }, + { + "name": "clickhouse_dsn", + "variable": "clickhouse_dsn" + }, + { + "name": "order_id", + "variable": "order_id" + } + ] + } + }, + { + "outputName": "g_mongodb", + "func": { + "name": "mongodb", + "args": [ + { + "name": "start_time", + "variable": "start_time" + }, + { + "name": "clickhouse_dsn", + "variable": "clickhouse_dsn" + }, + { + "name": "order_id", + "variable": "order_id" + } + ] + } + } + ], + "widgets": [ + { + "name": "CASES", + "position": { + "x": 0, + "y": 0, + "w": 12, + "h": 4 + }, + "globalFuncOutputName": "g_cases", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Graph", + "adjacencyList": { + "fromColumn": "from_entity", + "toColumn": "to_entity" + }, + "edgeWeightColumn": "severity", + "edgeColorColumn": "severity", + "edgeLabelColumn": "mitre_technique", + "edgeThresholds": { + "mediumThreshold": 5, + "highThreshold": 8 + }, + "edgeHoverInfo": [ + "order_link", + "culprit_key", + "rule_id", + "mitre_tactic", + "mitre_technique", + "alert", + "severity", + "order_id" + ], + "edgeLength": 500 + } + }, + { + "name": "Evidence graph", + "position": { + "x": 0, + "y": 8, + "w": 12, + "h": 4 + }, + "globalFuncOutputName": "g_graph", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Graph", + "adjacencyList": { + "fromColumn": "from_entity", + "toColumn": "to_entity" + }, + "edgeWeightColumn": "severity", + "edgeColorColumn": "severity", + "edgeLabelColumn": "rule_mitre", + "edgeThresholds": { + "mediumThreshold": 5, + "highThreshold": 8 + }, + "edgeHoverInfo": [ + "order_link", + "rule", + "mitre_tactic", + "mitre_technique", + "alert", + "process", + "target", + "severity", + "order_id" + ], + "edgeLength": 500 + } + }, + { + "name": "ORDERS", + "position": { + "x": 0, + "y": 8, + "w": 12, + "h": 3 + }, + "globalFuncOutputName": "g_orders", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Table" + } + }, + { + "name": "kubescape_logs for order", + "position": { + "x": 0, + "y": 11, + "w": 12, + "h": 4 + }, + "globalFuncOutputName": "g_kube", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Table" + } + }, + { + "name": "conn_stats", + "position": { + "x": 0, + "y": 15, + "w": 6, + "h": 4 + }, + "globalFuncOutputName": "g_conn", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Table" + } + }, + { + "name": "redis_events", + "position": { + "x": 6, + "y": 15, + "w": 6, + "h": 4 + }, + "globalFuncOutputName": "g_redis", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Table" + } + }, + { + "name": "http_events", + "position": { + "x": 0, + "y": 19, + "w": 6, + "h": 4 + }, + "globalFuncOutputName": "g_http", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Table" + } + }, + { + "name": "dns_events", + "position": { + "x": 6, + "y": 19, + "w": 6, + "h": 4 + }, + "globalFuncOutputName": "g_dns", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Table" + } + }, + { + "name": "pgsql_events", + "position": { + "x": 0, + "y": 23, + "w": 6, + "h": 4 + }, + "globalFuncOutputName": "g_pgsql", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Table" + } + }, + { + "name": "mysql_events", + "position": { + "x": 6, + "y": 23, + "w": 6, + "h": 4 + }, + "globalFuncOutputName": "g_mysql", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Table" + } + }, + { + "name": "dc_snoop", + "position": { + "x": 0, + "y": 27, + "w": 6, + "h": 4 + }, + "globalFuncOutputName": "g_dcsnoop", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Table" + } + }, + { + "name": "stack_trace", + "position": { + "x": 6, + "y": 27, + "w": 6, + "h": 4 + }, + "globalFuncOutputName": "g_stack", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Table" + } + }, + { + "name": "Differential stack trace", + "position": { + "x": 0, + "y": 31, + "w": 12, + "h": 4 + }, + "globalFuncOutputName": "g_stackdiff", + "displaySpec": { + "@type": "types.px.dev/px.vispb.StackTraceFlameGraph", + "stacktraceColumn": "stack_trace", + "countColumn": "count", + "percentageColumn": "percent", + "podColumn": "pod", + "differenceColumn": "delta" + } + }, + { + "name": "Capability result for order", + "position": { + "x": 0, + "y": 35, + "w": 12, + "h": 4 + }, + "globalFuncOutputName": "g_capability", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Table" + } + }, + { + "name": "creds_change", + "position": { + "x": 0, + "y": 39, + "w": 6, + "h": 4 + }, + "globalFuncOutputName": "g_credschange", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Table" + } + }, + { + "name": "cql", + "position": { + "x": 0, + "y": 43, + "w": 6, + "h": 4 + }, + "globalFuncOutputName": "g_cql", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Table" + } + }, + { + "name": "mongodb", + "position": { + "x": 0, + "y": 47, + "w": 6, + "h": 4 + }, + "globalFuncOutputName": "g_mongodb", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Table" + } + } + ] +} diff --git a/src/pxl_scripts/dx/fullchain/fullchain.pxl b/src/pxl_scripts/dx/fullchain/fullchain.pxl new file mode 100644 index 00000000000..12146d9320b --- /dev/null +++ b/src/pxl_scripts/dx/fullchain/fullchain.pxl @@ -0,0 +1,40 @@ +# Copyright 2018- The Pixie Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +import px + + +def _seed_pods(start_time: str, clickhouse_dsn: str, min_sev: int): + e = px.DataFrame('dx_fullchain_edges', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + s = e[e.sev >= min_sev] + s = s.groupby(['pod']).agg(n=('sev', px.count)) + return s[['pod']] + + +def fullchain(start_time: str, clickhouse_dsn: str, min_sev: int): + e = px.DataFrame('dx_fullchain_edges', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + seeds = _seed_pods(start_time, clickhouse_dsn, min_sev) + g = e.merge(seeds, how='inner', left_on=['pod'], right_on=['pod'], suffixes=['', '_s']) + g.from_entity = g.from_node + g.to_entity = g.to_node + return g[['from_entity', 'to_entity', 'edge_label', 'kind', 'pod']] + + +def group(start_time: str, clickhouse_dsn: str, min_sev: int): + e = px.DataFrame('dx_fullchain_edges', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + s = e[e.sev >= min_sev] + s = s.groupby(['pod', 'from_node', 'edge_label']).agg(alerts=('sev', px.count), sev=('sev', px.max)) + return s[['pod', 'edge_label', 'sev', 'alerts']] diff --git a/src/pxl_scripts/dx/fullchain/manifest.yaml b/src/pxl_scripts/dx/fullchain/manifest.yaml new file mode 100644 index 00000000000..70ade68a356 --- /dev/null +++ b/src/pxl_scripts/dx/fullchain/manifest.yaml @@ -0,0 +1,3 @@ +--- +short: SOC Full-Chain +long: SOC Full-Chain diff --git a/src/pxl_scripts/dx/fullchain/vis.json b/src/pxl_scripts/dx/fullchain/vis.json new file mode 100644 index 00000000000..60b22aa246c --- /dev/null +++ b/src/pxl_scripts/dx/fullchain/vis.json @@ -0,0 +1,103 @@ +{ + "variables": [ + { + "name": "start_time", + "type": "PX_STRING", + "description": "", + "defaultValue": "-90m" + }, + { + "name": "clickhouse_dsn", + "type": "PX_STRING", + "description": "", + "defaultValue": "forensic_analyst:changeme-analyst@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:9000/forensic_db" + }, + { + "name": "min_sev", + "type": "PX_INT64", + "description": "", + "defaultValue": "5" + } + ], + "globalFuncs": [ + { + "outputName": "g_fc", + "func": { + "name": "fullchain", + "args": [ + { + "name": "start_time", + "variable": "start_time" + }, + { + "name": "clickhouse_dsn", + "variable": "clickhouse_dsn" + }, + { + "name": "min_sev", + "variable": "min_sev" + } + ] + } + }, + { + "outputName": "g_grp", + "func": { + "name": "group", + "args": [ + { + "name": "start_time", + "variable": "start_time" + }, + { + "name": "clickhouse_dsn", + "variable": "clickhouse_dsn" + }, + { + "name": "min_sev", + "variable": "min_sev" + } + ] + } + } + ], + "widgets": [ + { + "name": "Full-chain incident graph", + "position": { + "x": 0, + "y": 0, + "w": 12, + "h": 6 + }, + "globalFuncOutputName": "g_fc", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Graph", + "adjacencyList": { + "fromColumn": "from_entity", + "toColumn": "to_entity" + }, + "edgeLabelColumn": "edge_label", + "edgeHoverInfo": [ + "kind", + "edge_label", + "pod" + ], + "edgeLength": 500 + } + }, + { + "name": "Incident group", + "position": { + "x": 0, + "y": 6, + "w": 12, + "h": 3 + }, + "globalFuncOutputName": "g_grp", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Table" + } + } + ] +} diff --git a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl deleted file mode 100644 index c5860a94a2c..00000000000 --- a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2018- The Pixie Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 - -import px - - -def dx_evidence_graph(start_time: str, clickhouse_dsn: str, table: str): - df = px.DataFrame(table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) - df.requestor = px.select(df.requestor_pod == '', - px.select(df.requestor_service == '', df.requestor_ip, df.requestor_service), - df.requestor_pod) - df.responder = px.select(df.responder_pod == '', - px.select(df.responder_service == '', df.responder_ip, df.responder_service), - df.responder_pod) - return df[['requestor', 'responder', - 'requestor_pod', 'responder_pod', - 'requestor_service', 'responder_service', - 'requestor_ip', 'responder_ip', - 'weight', 'max_severity', 'confidence', - 'edge_kind', 'condition', 'criteria', 'num_findings', - 'investigation_id']] diff --git a/src/pxl_scripts/px/dx_evidence_graph/manifest.yaml b/src/pxl_scripts/px/dx_evidence_graph/manifest.yaml deleted file mode 100644 index 35ec4c613e4..00000000000 --- a/src/pxl_scripts/px/dx_evidence_graph/manifest.yaml +++ /dev/null @@ -1,6 +0,0 @@ ---- -short: DX Evidence Graph -long: > - Severity-weighted, all-protocol pod-to-pod graph for one investigation. Edge records emitted by - dx with weight (sum of CRS evidence severity) on the edges and - max_severity colouring the heat. diff --git a/src/pxl_scripts/px/dx_evidence_graph/vis.json b/src/pxl_scripts/px/dx_evidence_graph/vis.json deleted file mode 100644 index 90befd97383..00000000000 --- a/src/pxl_scripts/px/dx_evidence_graph/vis.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "variables": [ - { - "name": "start_time", - "type": "PX_STRING", - "description": "Start time of the window.", - "defaultValue": "-15m" - }, - { - "name": "clickhouse_dsn", - "type": "PX_STRING", - "description": "ClickHouse DSN: user:pass@host:port/db.", - "defaultValue": "forensic_analyst:changeme-analyst@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:9000/forensic_db" - }, - { - "name": "table", - "type": "PX_STRING", - "description": "dx_evidence_graph", - "defaultValue": "dx_evidence_graph_malignant" - } - ], - "globalFuncs": [ - { - "outputName": "dx_graph", - "func": { - "name": "dx_evidence_graph", - "args": [ - {"name": "start_time", "variable": "start_time"}, - {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, - {"name": "table", "variable": "table"} - ] - } - } - ], - "widgets": [ - { - "name": "DX Evidence Graph", - "position": {"x": 0, "y": 0, "w": 12, "h": 5}, - "globalFuncOutputName": "dx_graph", - "displaySpec": { - "@type": "types.px.dev/px.vispb.Graph", - "adjacencyList": { - "fromColumn": "requestor", - "toColumn": "responder" - }, - "edgeWeightColumn": "weight", - "edgeColorColumn": "max_severity", - "edgeLabelColumn": "edge_kind", - "edgeThresholds": { - "mediumThreshold": 3, - "highThreshold": 4 - }, - "edgeHoverInfo": [ - "edge_kind", - "condition", - "criteria", - "weight", - "max_severity", - "confidence", - "num_findings", - "investigation_id" - ], - "edgeLength": 500 - } - }, - { - "name": "Edges", - "position": {"x": 0, "y": 5, "w": 12, "h": 4}, - "globalFuncOutputName": "dx_graph", - "displaySpec": { - "@type": "types.px.dev/px.vispb.Table" - } - } - ] -} diff --git a/src/ui/src/containers/live-widgets/graph/graph.tsx b/src/ui/src/containers/live-widgets/graph/graph.tsx index a8a642eec84..7e4c7e1c14b 100644 --- a/src/ui/src/containers/live-widgets/graph/graph.tsx +++ b/src/ui/src/containers/live-widgets/graph/graph.tsx @@ -46,6 +46,7 @@ import { } from './graph-utils'; import { formatByDataType, formatBySemType } from '../../format-data/format-data'; import { deepLinkURLFromSemanticType } from '../utils/live-view-params'; +import { ScriptReference } from '../utils/script-reference'; interface AdjacencyList { toColumn: string; @@ -157,8 +158,11 @@ export const Graph = React.memo(({ const [graph, setGraph] = React.useState(null); const [pinned, setPinned] = React.useState>([]); const pinSeq = React.useRef(0); + const [edgeScriptRefs, setEdgeScriptRefs] = React.useState< + Map>(() => new Map()); const [edgeLabels, setEdgeLabels] = React.useState>(() => new Map()); const [labelOffsets, setLabelOffsets] = React.useState>(() => new Map()); @@ -198,6 +202,7 @@ export const Graph = React.memo(({ const nodes = new visData.DataSet(); const idToSemType = {}; const labelMap = new Map(); + const scriptRefMap = new Map(); const selfLoopCounts = new Map(); const selfLoopRank = new Map(); @@ -257,8 +262,17 @@ export const Graph = React.memo(({ if (edgeHoverInfo && edgeHoverInfo.length > 0) { let edgeInfo = ''; - edgeHoverInfo.forEach((info, i) => { + edgeHoverInfo.forEach((info) => { if (info != null) { + // Script-reference columns become the deep link in the pinned popup, + // not a line in the hover text (a hover tooltip can't be clicked). + if (info.semType === SemanticType.ST_SCRIPT_REFERENCE) { + const scriptRef = d[info.name]; + if (scriptRef && scriptRef.script) { + scriptRefMap.set(edgeId, { label: scriptRef.label, script: scriptRef.script, args: scriptRef.args }); + } + return; + } let val: string; if (info.semType === SemanticType.ST_NONE || info.semType === SemanticType.ST_UNSPECIFIED) { val = formatByDataType(info.type, d[info.name]); @@ -266,7 +280,7 @@ export const Graph = React.memo(({ const valWithUnits = formatBySemType(info.semType, d[info.name]); val = `${valWithUnits.val} ${valWithUnits.units}`; } - edgeInfo = `${edgeInfo}${i === 0 ? '' : '
'} ${info.name}: ${val}`; + edgeInfo = `${edgeInfo}${edgeInfo === '' ? '' : '
'} ${info.name}: ${val}`; } }); edge.title = edgeInfo; @@ -279,6 +293,7 @@ export const Graph = React.memo(({ nodes, edges, idToSemType, }); setEdgeLabels(labelMap); + setEdgeScriptRefs(scriptRefMap); setLabelOffsets((prev) => { const next = new Map(); selfLoopRank.forEach((rank, edgeId) => { @@ -331,6 +346,7 @@ export const Graph = React.memo(({ title: String(edgeData?.title ?? ''), x: rect.left + params.pointer.DOM.x, y: rect.top + params.pointer.DOM.y, + scriptRef: edgeScriptRefs.get(edgeId), }]); } }); @@ -401,7 +417,7 @@ export const Graph = React.memo(({ const onPinPointerDown = React.useCallback((key: number, initialX: number, initialY: number) => (e: React.PointerEvent) => { - if ((e.target as HTMLElement).closest('[data-pin-close]')) return; + if ((e.target as HTMLElement).closest('a, [data-pin-close]')) return; e.stopPropagation(); const startX = e.clientX; const startY = e.clientY; @@ -490,6 +506,17 @@ export const Graph = React.memo(({ touchAction: 'none', }} > + {p.scriptRef && ( +
+ +
+ )}
RUNNING and the old one + // RUNNING->TERMINATED. That cycle drops kprobe:lookup_fast on a running + // PEM — bpftrace warns "could not attach probe ... skipping" on the PEM's + // stderr and carries on — so dc_snoop silently degrades to M-only until + // the PEM restarts. Re-upserting on every AE boot therefore UNDOES the + // deployed program's R probe on each rollout. + if !force { + if _, err := adapter.Query(ctx, verify); err == nil { + log.WithFields(log.Fields{"tracepoint": tp.Name, "table": tp.Table}). + Info("tracepoint already deployed — skipping upsert (a re-upsert cycles it and drops probes; set ADAPTIVE_TRACEPOINT_REDEPLOY=1 after changing a preset)") + continue + } + } + // Fire the deploy mutation. pxapi's result collector cannot decode the // mutation-info response the vizier returns for a pxtrace deploy // ("stream: unimplemented type"), so a Query error here is NOT a // deployment failure — the UpsertTracepoint applies server-side - // regardless. Success is confirmed below by the tracepoint's output - // table becoming queryable (PENDING_STATE -> RUNNING_STATE). + // regardless. Success is confirmed below by the output table becoming + // queryable. if _, err := adapter.Query(ctx, tp.Script); err != nil { log.WithError(err).WithField("tracepoint", tp.Name). Debug("deploy mutation returned a stream error (expected for pxtrace mutations) — confirming via table") } - // Confirm the tracepoint reached RUNNING by polling its output table. A - // plain DataFrame query on a not-yet-deployed table fails PxL compilation - // ("Table '' not found"); once the tracepoint is RUNNING the query - // compiles and returns (0 rows is fine — RUNNING, just no captures yet). - // Re-fire the deploy every few attempts in case the first didn't take. - verify := "import px\npx.display(px.DataFrame(table='" + tp.Table + "', start_time='-5s').head(1))\n" running := false for attempt := 1; attempt <= 12; attempt++ { if _, err := adapter.Query(ctx, verify); err == nil { @@ -998,8 +1144,12 @@ func deployDesiredTracepoints(ctx context.Context, adapter *pixieapi.Adapter) { time.Sleep(5 * time.Second) } if running { + // A queryable table means the tracepoint deployed, NOT that every + // probe in the program attached. A multi-probe program can report + // RUNNING with one probe silently skipped, and lookup_fast cannot be + // re-attached inside a running PEM — recovery needs a fresh PEM. log.WithFields(log.Fields{"tracepoint": tp.Name, "table": tp.Table}). - Info("bpftrace tracepoint deployed + RUNNING (permanent, idempotent upsert)") + Info("bpftrace tracepoint deployed + table queryable (per-probe attachment NOT verified here)") } else { log.WithField("tracepoint", tp.Name). Warn("bpftrace tracepoint not confirmed RUNNING after deploy — its dark table stays empty until it deploys") diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/BUILD.bazel b/src/vizier/services/adaptive_export/internal/clickhouse/BUILD.bazel index 4826adf14e4..402aa3015ac 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/BUILD.bazel +++ b/src/vizier/services/adaptive_export/internal/clickhouse/BUILD.bazel @@ -40,6 +40,7 @@ pl_go_test( "ddl_test.go", "insert_test.go", "metadata_invariants_test.go", + "schema_registration_test.go", "timestamp_invariants_test.go", ], embed = [":clickhouse"], diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/apply.go b/src/vizier/services/adaptive_export/internal/clickhouse/apply.go index 411b79f443d..3522f35e585 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/apply.go +++ b/src/vizier/services/adaptive_export/internal/clickhouse/apply.go @@ -21,6 +21,7 @@ import ( "context" "encoding/json" "fmt" + "log" "strings" "px.dev/pixie/src/vizier/services/adaptive_export/internal/chhttp" @@ -59,12 +60,6 @@ var OperatorOwnedTables = []string{ "dc_snoop", "creds_change", "stack_trace", - "dx_vfs_events", - "dx_unlink", - "dx_dlookup", - "dx_mprotect", - "dx_bpf", - "dx_ptrace", // operator's write targets. "adaptive_attribution", "trigger_watermark", @@ -77,12 +72,52 @@ var OperatorOwnedTables = []string{ // globally-registered table to read. dx emits edges, AE persists. // Not a pixie socket_tracer table → not in PixieTables(). "dx_evidence_graph", - // rule-ins-only VIEW over dx_evidence_graph; created AFTER it (depends on it). - "dx_evidence_graph_malignant", // dx §9 completeness manifest — one row per verdict naming the evidence rows // dx consulted. Created on boot so POST /dx/evidence_manifest has a target. // Independent of dx_evidence_graph. Not a pixie table → not in PixieTables(). "dx_evidence_manifest", + // dx per-referral order seeds (#136 evidence-loss fix) — created on boot so + // dx's direct INSERT has a target. Not a pixie table → not in PixieTables(). + "dx_order_seeds", + // consulted-records bridge (#136 stamping) — created on boot so dx's INSERT has + // a target. Not a pixie table → not in PixieTables(). + "dx_order_records", + // NEW identity-model tables (added alongside; dx INSERTs). Not pixie tables. + "dx_orders", + "dx_order_edges", + // order-UUID pre-correlation VIEWS (#136) — created LAST, after every base + // table above exists (kubescape_logs, the socket_tracer tables, dc_snoop, + // dx_order_seeds). Read by the px/dx_evidence_graph dashboard. Not pixie tables. + "dx_anomaly_orders", + "dx_kubescape_anomalies", + "dx_src__kubescape_logs", + "dx_src__stack_trace", + // NEW identity-model join views (after their base table + dx_order_edges exist). + "dx_ord__conn_stats", + "dx_ord__redis_events", + "dx_ord__http_events", + "dx_ord__dns_events", + "dx_ord__pgsql_events", + "dx_ord__mysql_events", + "dx_ord__dc_snoop", + "dx_ord__stack_trace", + // MITRE ATT&CK enrichment (over kubescape_logs) + per-order window (over + // dx_orders) — VIEWS, created after their base tables. px/dx_evidence_graph reads them. + "dx_kubescape_mitre", + "dx_src__kubescape_mitre", + "dx_orders_win", + // DNS resolution reconstruction view (dx/dns_resolve UI) — over dns_events. + "dx_dns_resolve", + // cql/mongodb/creds_change order-bridge views — created after their base tables. + "dx_ord__cql_events", + "dx_ord__mongodb_events", + "dx_ord__creds_change", + "dx_cases", + "dx_case_links", + // generic alert flatten (dx/breakout PxL) + story edge views (dx/breakout, dx/fullchain). + "dx_alerts", + "dx_breakout_story", + "dx_fullchain_edges", } // Applier applies operator-owned DDL to a ClickHouse cluster over the @@ -115,6 +150,14 @@ func (a *Applier) Apply(ctx context.Context) error { return fmt.Errorf("apply: get DDL for %s: %w", table, err) } if err := a.execute(ctx, ddl); err != nil { + // Views are DERIVED + best-effort: a not-yet-ready base (e.g. the + // soc-owned kubescape_logs before its installer has run) must NOT fatal + // the boot. Log + continue; the next boot retries once the base exists. + // Tables stay boot-critical (fatal) — a missing operator table is real. + if strings.Contains(ddl, "CREATE VIEW") { + log.Printf("[ae] deferred view %s (base not ready?): %v", table, err) + continue + } return fmt.Errorf("apply: create %s: %w", table, err) } } diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/apply_test.go b/src/vizier/services/adaptive_export/internal/clickhouse/apply_test.go index 3c834d11a80..e98ff939253 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/apply_test.go +++ b/src/vizier/services/adaptive_export/internal/clickhouse/apply_test.go @@ -67,10 +67,16 @@ func TestApply_ExecutesEveryOperatorOwnedTable(t *testing.T) { if !strings.Contains(bodies[len(bodies)-1], "forensic_db."+lastTable) { t.Fatalf("last DDL not for %s; got: %s", lastTable, bodies[len(bodies)-1]) } - // And ensure no kubescape DDL leaked through. + // And ensure no kubescape DDL leaked through. Match the CREATE TABLE form, + // not a bare mention: the order-UUID views (dx_kubescape_anomalies, + // dx_src__kubescape_logs) legitimately SELECT FROM forensic_db.kubescape_logs, + // so a substring check on the table name alone flags reading it as creating it. + // Ownership is about who issues the CREATE TABLE, which is still never AE. for _, b := range bodies { - if strings.Contains(b, "forensic_db.alerts") || strings.Contains(b, "forensic_db.kubescape_logs") { - t.Fatalf("operator's Apply must not create kubescape tables; got:\n%s", b) + for _, ks := range []string{"alerts", "kubescape_logs"} { + if strings.Contains(b, "CREATE TABLE IF NOT EXISTS forensic_db."+ks) { + t.Fatalf("operator's Apply must not create kubescape tables; got:\n%s", b) + } } } } @@ -228,7 +234,10 @@ func TestOperatorOwnedTables_DoesNotIncludeKubescape(t *testing.T) { // plugin can auto-DDL them with the wrong schema), then the operator's // own write targets in declared order. func TestOperatorOwnedTables_TrailingOperatorTables(t *testing.T) { - want := []string{"adaptive_attribution", "trigger_watermark", "ae_reconcile", "dx_evidence_graph", "dx_evidence_graph_malignant", "dx_evidence_manifest"} + want := []string{ + "adaptive_attribution", "trigger_watermark", "ae_reconcile", "dx_evidence_graph", "dx_evidence_manifest", "dx_order_seeds", "dx_order_records", "dx_orders", "dx_order_edges", + "dx_anomaly_orders", "dx_kubescape_anomalies", "dx_src__kubescape_logs", "dx_src__stack_trace", "dx_ord__conn_stats", "dx_ord__redis_events", "dx_ord__http_events", "dx_ord__dns_events", "dx_ord__pgsql_events", "dx_ord__mysql_events", "dx_ord__dc_snoop", "dx_ord__stack_trace", "dx_kubescape_mitre", "dx_src__kubescape_mitre", "dx_orders_win", "dx_dns_resolve", "dx_ord__cql_events", "dx_ord__mongodb_events", "dx_ord__creds_change", "dx_cases", "dx_case_links", "dx_alerts", "dx_breakout_story", "dx_fullchain_edges", + } got := OperatorOwnedTables[len(OperatorOwnedTables)-len(want):] for i, w := range want { if got[i] != w { diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/columns_test.go b/src/vizier/services/adaptive_export/internal/clickhouse/columns_test.go index 2e3a94bfb73..4ef3dde5f7d 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/columns_test.go +++ b/src/vizier/services/adaptive_export/internal/clickhouse/columns_test.go @@ -37,7 +37,7 @@ func TestColumns_http_events_ExactList(t *testing.T) { "content_type", "req_headers", "req_method", "req_path", "req_body", "req_body_size", "resp_headers", "resp_status", "resp_message", "resp_body", "resp_body_size", "latency", - "hostname", "event_time", + "hostname", "event_time", "unique_id", } if !reflect.DeepEqual(got, want) { t.Fatalf("Columns(http_events) mismatch:\n got=%v\nwant=%v", got, want) @@ -56,7 +56,7 @@ func TestColumns_conn_stats_ExactList(t *testing.T) { "time_", "upid", "namespace", "pod", "remote_addr", "remote_port", "trace_role", "addr_family", "protocol", "ssl", "conn_open", "conn_close", "conn_active", - "bytes_sent", "bytes_recv", "hostname", "event_time", + "bytes_sent", "bytes_recv", "hostname", "event_time", "remote_pod", "unique_id", } if !reflect.DeepEqual(got, want) { t.Fatalf("Columns(conn_stats) mismatch:\n got=%v\nwant=%v", got, want) diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/ddl.go b/src/vizier/services/adaptive_export/internal/clickhouse/ddl.go index 0eb315882e9..fbebddc01fa 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/ddl.go +++ b/src/vizier/services/adaptive_export/internal/clickhouse/ddl.go @@ -60,12 +60,6 @@ var KnownTables = []string{ "dc_snoop", "creds_change", "stack_trace", - "dx_vfs_events", - "dx_unlink", - "dx_dlookup", - "dx_mprotect", - "dx_bpf", - "dx_ptrace", // operator-owned attribution table "adaptive_attribution", // operator-owned persistent trigger cursor @@ -76,14 +70,53 @@ var KnownTables = []string{ // operator-owned dx evidence-graph edge list (read by the Pixie // dx_evidence_graph UI via clickhouse_dsn). NOT a pixie table. "dx_evidence_graph", - // rule-ins-only VIEW over dx_evidence_graph (condition != ''); the - // dx_evidence_graph UI reads this by default so benign rows are filtered - // in ClickHouse, not pulled. Must follow dx_evidence_graph (depends on it). - "dx_evidence_graph_malignant", // operator-owned dx §9 completeness manifest — one row per verdict naming // the evidence rows dx consulted (POST /dx/evidence_manifest). NOT a pixie // table. Independent of dx_evidence_graph. "dx_evidence_manifest", + // operator-owned dx per-referral order seeds (#136 evidence-loss fix). dx + // INSERTs one row per anomaly so dx_anomaly_orders can window every uniqueID. + // NOT a pixie table. + "dx_order_seeds", + // order-UUID consulted-records bridge (#136 stamping): the records dx consulted + // per primary kubescape log, stamped with its order_id. dx INSERTs. NOT a pixie + // table. + "dx_order_records", + // NEW identity-model tables (added alongside dx_order_seeds/records). NOT pixie tables. + "dx_orders", + "dx_order_edges", + // order-UUID pre-correlation views (#136) read by the px/dx_evidence_graph + // dashboard. VIEWS, created after their base tables (kubescape_logs ensured + // first). NOT pixie tables. Order matches schema.sql (appended at the end). + "dx_anomaly_orders", + "dx_kubescape_anomalies", + "dx_src__kubescape_logs", + "dx_src__stack_trace", + // NEW identity-model join view. + "dx_ord__conn_stats", + "dx_ord__redis_events", + "dx_ord__http_events", + "dx_ord__dns_events", + "dx_ord__pgsql_events", + "dx_ord__mysql_events", + "dx_ord__dc_snoop", + "dx_ord__stack_trace", + // MITRE ATT&CK enrichment + per-order window views (px/dx_evidence_graph). + "dx_kubescape_mitre", + "dx_src__kubescape_mitre", + "dx_orders_win", + // DNS resolution reconstruction (dx/dns_resolve UI). + "dx_dns_resolve", + // cql/mongodb/creds_change order-bridge views. + "dx_ord__cql_events", + "dx_ord__mongodb_events", + "dx_ord__creds_change", + "dx_cases", + "dx_case_links", + // generic alert flatten + breakout/fullchain story edge views. + "dx_alerts", + "dx_breakout_story", + "dx_fullchain_edges", } // ErrUnknownTable is returned by DDL / Columns when asked for a table @@ -103,7 +136,7 @@ func DDL(table string) (string, error) { identifier = "`" + table + "`" } start := -1 - for _, kw := range []string{"CREATE TABLE IF NOT EXISTS forensic_db.", "CREATE VIEW IF NOT EXISTS forensic_db."} { + for _, kw := range []string{"CREATE TABLE IF NOT EXISTS forensic_db.", "CREATE OR REPLACE VIEW forensic_db."} { if start = strings.Index(canonicalSchema, kw+identifier); start >= 0 { break } @@ -140,12 +173,6 @@ func PixieTables() []string { "dc_snoop", "creds_change", "stack_trace", - "dx_vfs_events", - "dx_unlink", - "dx_dlookup", - "dx_mprotect", - "dx_bpf", - "dx_ptrace", } } diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/ddl_test.go b/src/vizier/services/adaptive_export/internal/clickhouse/ddl_test.go index 0da8c706d3d..576bc1c6721 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/ddl_test.go +++ b/src/vizier/services/adaptive_export/internal/clickhouse/ddl_test.go @@ -32,7 +32,7 @@ func TestDDL_ReturnsCanonicalForKnownTables(t *testing.T) { t.Fatalf("DDL(%q): %v", name, err) } if !strings.HasPrefix(ddl, "CREATE TABLE IF NOT EXISTS forensic_db.") && - !strings.HasPrefix(ddl, "CREATE VIEW IF NOT EXISTS forensic_db.") { + !strings.HasPrefix(ddl, "CREATE OR REPLACE VIEW forensic_db.") { t.Fatalf("DDL(%q) wrong prefix: %q", name, ddl[:minInt(70, len(ddl))]) } if !strings.HasSuffix(ddl, ";") { diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/metadata_invariants_test.go b/src/vizier/services/adaptive_export/internal/clickhouse/metadata_invariants_test.go index 33ba3d5f106..b922fc60286 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/metadata_invariants_test.go +++ b/src/vizier/services/adaptive_export/internal/clickhouse/metadata_invariants_test.go @@ -25,7 +25,6 @@ import "testing" // guards that this list stays a subset of the operator-owned pixie tables. var darkVectorTables = []string{ "dc_snoop", "creds_change", "stack_trace", - "dx_vfs_events", "dx_unlink", "dx_dlookup", "dx_mprotect", "dx_bpf", "dx_ptrace", } // TestDarkVectorTablesHaveFullMetadata enforces attribution CONSISTENCY: every @@ -33,8 +32,8 @@ var darkVectorTables = []string{ // (namespace, pod, container, hostname), so any dark-vector event is attributable // to its workload uniformly. A table missing one of these is an inconsistency // that dx projection + forensic joins would silently drop — this was the concrete -// gap in dx_vfs_events/dx_unlink (no comm) and all six dx_ tables (no container) -// before they were reconciled to the canonical dc_snoop/creds_change shape. +// gap in the dx_* tracepoint tables before they were removed (no tracepoint was +// ever deployed for them, so they could only ever return zero rows). func TestDarkVectorTablesHaveFullMetadata(t *testing.T) { required := []string{"namespace", "pod", "container", "hostname"} for _, tbl := range darkVectorTables { diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql b/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql index 6fa8ed7f002..9d343c415b7 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql +++ b/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql @@ -107,10 +107,11 @@ CREATE TABLE IF NOT EXISTS forensic_db.http_events ( resp_body_size Int64, latency Int64, hostname String, - event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9) + event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9), + unique_id String DEFAULT '' ) ENGINE = ReplacingMergeTree() PARTITION BY toYYYYMM(event_time) - ORDER BY (hostname, event_time, time_, upid, trace_role, remote_port, local_port, latency, req_method, req_path); + ORDER BY (hostname, event_time, time_, upid, trace_role, remote_port, local_port, latency, req_method, req_path, req_body, resp_status, resp_body); -- http2_messages.beta — http2_messages_table.h CREATE TABLE IF NOT EXISTS forensic_db.`http2_messages.beta` ( @@ -152,10 +153,11 @@ CREATE TABLE IF NOT EXISTS forensic_db.dns_events ( resp_body String, latency Int64, hostname String, - event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9) + event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9), + unique_id String DEFAULT '' ) ENGINE = ReplacingMergeTree() PARTITION BY toYYYYMM(event_time) - ORDER BY (hostname, event_time, time_, upid, trace_role, remote_port, local_port, latency, req_body); + ORDER BY (hostname, event_time, time_, upid, trace_role, remote_port, local_port, latency, req_body, resp_body); -- redis_events — redis_table.h CREATE TABLE IF NOT EXISTS forensic_db.redis_events ( @@ -174,10 +176,11 @@ CREATE TABLE IF NOT EXISTS forensic_db.redis_events ( resp String, latency Int64, hostname String, - event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9) + event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9), + unique_id String DEFAULT '' ) ENGINE = ReplacingMergeTree() PARTITION BY toYYYYMM(event_time) - ORDER BY (hostname, event_time, time_, upid, trace_role, remote_port, local_port, latency, req_cmd); + ORDER BY (hostname, event_time, time_, upid, trace_role, remote_port, local_port, latency, req_cmd, req_args, resp); -- mysql_events — mysql_table.h CREATE TABLE IF NOT EXISTS forensic_db.mysql_events ( @@ -197,10 +200,11 @@ CREATE TABLE IF NOT EXISTS forensic_db.mysql_events ( resp_body String, latency Int64, hostname String, - event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9) + event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9), + unique_id String DEFAULT '' ) ENGINE = MergeTree() PARTITION BY toYYYYMM(event_time) - ORDER BY (hostname, event_time); + ORDER BY (hostname, event_time, time_, upid, remote_addr, remote_port, latency, req_cmd, req_body, resp_status, resp_body); -- pgsql_events — pgsql_table.h CREATE TABLE IF NOT EXISTS forensic_db.pgsql_events ( @@ -218,10 +222,11 @@ CREATE TABLE IF NOT EXISTS forensic_db.pgsql_events ( resp String, latency Int64, hostname String, - event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9) + event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9), + unique_id String DEFAULT '' ) ENGINE = MergeTree() PARTITION BY toYYYYMM(event_time) - ORDER BY (hostname, event_time); + ORDER BY (hostname, event_time, time_, upid, remote_addr, remote_port, latency, req, resp); -- cql_events — cass_table.h CREATE TABLE IF NOT EXISTS forensic_db.cql_events ( @@ -241,10 +246,11 @@ CREATE TABLE IF NOT EXISTS forensic_db.cql_events ( resp_body String, latency Int64, hostname String, - event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9) + event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9), + unique_id String DEFAULT '' ) ENGINE = MergeTree() PARTITION BY toYYYYMM(event_time) - ORDER BY (hostname, event_time); + ORDER BY (hostname, event_time, time_, upid, remote_addr, remote_port, latency, req_op, req_body, resp_op, resp_body); -- mongodb_events — mongodb_table.h CREATE TABLE IF NOT EXISTS forensic_db.mongodb_events ( @@ -264,10 +270,11 @@ CREATE TABLE IF NOT EXISTS forensic_db.mongodb_events ( resp_body String, latency Int64, hostname String, - event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9) + event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9), + unique_id String DEFAULT '' ) ENGINE = MergeTree() PARTITION BY toYYYYMM(event_time) - ORDER BY (hostname, event_time); + ORDER BY (hostname, event_time, time_, upid, remote_addr, remote_port, latency, req_cmd, req_body, resp_status, resp_body); -- kafka_events.beta — kafka_table.h CREATE TABLE IF NOT EXISTS forensic_db.`kafka_events.beta` ( @@ -383,7 +390,9 @@ CREATE TABLE IF NOT EXISTS forensic_db.conn_stats ( bytes_sent Int64, bytes_recv Int64, hostname String, - event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9) + event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9), + remote_pod String DEFAULT '', + unique_id String DEFAULT '' ) ENGINE = ReplacingMergeTree() PARTITION BY toYYYYMM(event_time) ORDER BY (hostname, event_time, time_, upid, remote_addr, remote_port, trace_role); @@ -531,11 +540,6 @@ CREATE TABLE IF NOT EXISTS forensic_db.dx_evidence_graph ( TTL toDateTime(fromUnixTimestamp64Nano(event_time)) + INTERVAL 30 DAY DELETE SETTINGS index_granularity = 8192; --- dx_evidence_graph_malignant — rule-ins-only view (condition != '') the --- dx_evidence_graph UI reads by default so benign rows stay in ClickHouse. -CREATE VIEW IF NOT EXISTS forensic_db.dx_evidence_graph_malignant AS - SELECT * FROM forensic_db.dx_evidence_graph WHERE `condition` != ''; - -- dx_evidence_manifest — the §9 completeness contract: one row per verdict -- (ruled_in | metastasis), naming the evidence rows dx consulted so the -- validator can join them against what AE persisted (write⊇read, checkable). @@ -567,86 +571,120 @@ CREATE TABLE IF NOT EXISTS forensic_db.dx_evidence_manifest ( TTL toDateTime(fromUnixTimestamp64Nano(event_time)) + INTERVAL 30 DAY DELETE SETTINGS index_granularity = 8192; +-- dx_order_seeds — one row per ORDER dx opens (entlein/dx#136 evidence-loss fix). +-- dx owns the order identity: it computes order_id and decides the dedup +-- granularity (1:1 with uniqueID today; finer — per rule/target/event — later). +-- AE only stores and surfaces exactly what dx emits, so the key is order_id and +-- NOTHING here assumes how many orders map to a uniqueID. dx INSERTs (POST-less, +-- direct CH); AE owns the DDL. ReplacingMergeTree ORDER BY (order_id) dedups +-- re-fires of the same order. NOT a pixie table. +CREATE TABLE IF NOT EXISTS forensic_db.dx_order_seeds ( + order_id String, + unique_id String, + rule_id String, + pod String, + event_time UInt64, + hostname String, + case_key String +) ENGINE = ReplacingMergeTree() + ORDER BY (order_id) + PARTITION BY toYYYYMM(fromUnixTimestamp64Nano(event_time)) + TTL toDateTime(fromUnixTimestamp64Nano(event_time)) + INTERVAL 30 DAY DELETE + SETTINGS index_granularity = 8192; + +-- dx_order_records — the STAMPED consulted set (entlein/dx#136 stamping model). dx +-- writes one row per (order_id, finding): each record it consulted during the workup +-- for a primary kubescape log, stamped with that log's order_id. The panels read THIS +-- (the exact consulted set) instead of a ±300s time window. event_time is derived from +-- time_ so px can read it (UInt64 + hostname, no Bool cols). AE owns the DDL; dx +-- INSERTs. ReplacingMergeTree collapses re-stamps of the same (order_id,row). +CREATE TABLE IF NOT EXISTS forensic_db.dx_order_records ( + order_id String, + unique_id String, + src_table String, + vector String, + source String, + time_ Int64, + pod String, + remote_addr String, + path String, + comm String, + dns_name String, + hostname String, + event_time UInt64 DEFAULT toUInt64(time_) +) ENGINE = ReplacingMergeTree() + ORDER BY (order_id, src_table, time_, pod, remote_addr, path, comm, dns_name) + PARTITION BY toYYYYMM(fromUnixTimestamp64Nano(event_time)) + TTL toDateTime(fromUnixTimestamp64Nano(event_time)) + INTERVAL 30 DAY DELETE + SETTINGS index_granularity = 8192; + +-- ── NEW identity model (added ALONGSIDE dx_order_seeds/records, which stay) ─── +-- dx_orders — one row per kubescape detection INSTANT. order_id is TRULY unique = +-- hash(uniqueID|Disc|event_time_ns). kubescape_uid/disc are provenance only, NEVER +-- keys. dx INSERTs; AE owns the DDL. +CREATE TABLE IF NOT EXISTS forensic_db.dx_orders ( + order_id String, + kubescape_uid String, + rule_id String, + disc String, + pod String, + event_time UInt64, + hostname String, + culprit_key String DEFAULT '' +) ENGINE = ReplacingMergeTree() + ORDER BY (order_id) + PARTITION BY toYYYYMM(fromUnixTimestamp64Nano(event_time)) + TTL toDateTime(fromUnixTimestamp64Nano(event_time)) + INTERVAL 30 DAY DELETE + SETTINGS index_granularity = 8192; + +-- dx_order_edges — the identity bridge. One row per (order, consulted pixie row): +-- links order_id to a base-table row via unique_id = the dx-computed content hash +-- of the row's fields (FNV-1a 64, lowercase hex String), the SAME value dx stamps +-- onto that base row's unique_id column — so they match by construction, no +-- CH-side hashing. String (not UInt64): a 64-bit integer does not survive a JSON +-- decode through float64. Many-to-many: a row consulted by N orders → N edges; +-- re-stamps collapse. dx INSERTs. +CREATE TABLE IF NOT EXISTS forensic_db.dx_order_edges ( + order_id String, + src_table String, + unique_id String, + hostname String, + event_time UInt64 DEFAULT 0 +) ENGINE = ReplacingMergeTree() + ORDER BY (order_id, src_table, unique_id) + SETTINGS index_granularity = 8192; + +-- dx_ord__conn_stats — join view: conn_stats rows consulted for an order, via the +-- bridge (edge.unique_id = conn_stats.unique_id). Panel filters by order_id. +CREATE OR REPLACE VIEW forensic_db.dx_ord__conn_stats AS +SELECT + e.order_id AS order_id, + toString(c.time_) AS ts, + toInt64(toUnixTimestamp64Nano(c.time_)) AS row_time, + fromUnixTimestamp64Nano(toInt64(e.event_time)) AS event_time, + c.namespace AS namespace, + c.pod AS pod, + c.remote_addr AS remote_addr, + c.remote_port AS remote_port, + c.trace_role AS trace_role, + c.protocol AS protocol, + c.conn_open AS conn_open, + c.conn_close AS conn_close, + c.conn_active AS conn_active, + c.bytes_sent AS bytes_sent, + e.hostname AS hostname +FROM forensic_db.dx_order_edges AS e +INNER JOIN forensic_db.conn_stats AS c ON c.unique_id = e.unique_id +WHERE e.src_table = 'conn_stats'; + -- ── dx dark-vector tracepoint tables (entlein/dx#126) ──────────────────────── -- Fed by AE-owned bpftrace UpsertTracepoint probes (constantly enabled, no TTL). -- Emit raw kernel pid+comm (NOT upid); namespace/pod enriched at pull time via a -- process_stats join on pid. One column per line (schema-verify parser is line-oriented). -- (dx_dcsnoop superseded by forensic_db.dc_snoop — canonical DateTime64(9)/Int64 -- schema with full k8s metadata; see the dark-vector section above.) -CREATE TABLE IF NOT EXISTS forensic_db.dx_vfs_events ( - time_ DateTime64(9, 'UTC'), - pid Int64, - comm String, - op String, - file String, - namespace String, - pod String, - container String, - hostname String, - event_time DateTime64(9, 'UTC') -) ENGINE = MergeTree ORDER BY (event_time, pod); - -CREATE TABLE IF NOT EXISTS forensic_db.dx_unlink ( - time_ DateTime64(9, 'UTC'), - pid Int64, - comm String, - op String, - file String, - namespace String, - pod String, - container String, - hostname String, - event_time DateTime64(9, 'UTC') -) ENGINE = MergeTree ORDER BY (event_time, pod); - -CREATE TABLE IF NOT EXISTS forensic_db.dx_dlookup ( - time_ DateTime64(9, 'UTC'), - pid Int64, - comm String, - file String, - namespace String, - pod String, - container String, - hostname String, - event_time DateTime64(9, 'UTC') -) ENGINE = MergeTree ORDER BY (event_time, pod); - -CREATE TABLE IF NOT EXISTS forensic_db.dx_mprotect ( - time_ DateTime64(9, 'UTC'), - pid Int64, - comm String, - prot UInt64, - namespace String, - pod String, - container String, - hostname String, - event_time DateTime64(9, 'UTC') -) ENGINE = MergeTree ORDER BY (event_time, pod); - -- (dx_creds superseded by forensic_db.creds_change — canonical schema with -- old_uid/new_uid + full k8s metadata.) -CREATE TABLE IF NOT EXISTS forensic_db.dx_bpf ( - time_ DateTime64(9, 'UTC'), - pid Int64, - comm String, - namespace String, - pod String, - container String, - hostname String, - event_time DateTime64(9, 'UTC') -) ENGINE = MergeTree ORDER BY (event_time, pod); - -CREATE TABLE IF NOT EXISTS forensic_db.dx_ptrace ( - time_ DateTime64(9, 'UTC'), - pid Int64, - comm String, - namespace String, - pod String, - container String, - hostname String, - event_time DateTime64(9, 'UTC') -) ENGINE = MergeTree ORDER BY (event_time, pod); - -- dc_snoop (dentry cache, V1/V2 process+file) — exported via the OTel/ClickHouse -- retention plugin (px.export). pid-keyed; t = R (reference) / M (miss). -- One column per line (schema-verify parser is line-oriented). @@ -660,7 +698,8 @@ CREATE TABLE IF NOT EXISTS forensic_db.dc_snoop ( pod String, container String, hostname String, - event_time DateTime64(9, 'UTC') + event_time DateTime64(9, 'UTC'), + unique_id String DEFAULT '' ) ENGINE = ReplacingMergeTree ORDER BY (time_, pid, comm, t, file, pod); -- stack_trace (native continuous profiler stack_traces.beta, V9) — OTel export. @@ -674,7 +713,8 @@ CREATE TABLE IF NOT EXISTS forensic_db.stack_trace ( stack_trace_id Int64, stack_trace String, count Int64, - event_time DateTime64(9, 'UTC') + event_time DateTime64(9, 'UTC'), + unique_id String DEFAULT '' ) ENGINE = ReplacingMergeTree ORDER BY (time_, upid, stack_trace_id, pod); -- creds_change (commit_creds privilege-escalation to root, V7) — OTel export. @@ -688,5 +728,458 @@ CREATE TABLE IF NOT EXISTS forensic_db.creds_change ( pod String, container String, hostname String, - event_time DateTime64(9, 'UTC') + event_time DateTime64(9, 'UTC'), + unique_id String DEFAULT '' ) ENGINE = ReplacingMergeTree ORDER BY (time_, pid, comm, old_uid, new_uid, pod); + +-- ── Order-UUID pre-correlation views (entlein/dx#136) ──────────────────────── +-- The px/dx_evidence_graph multi-panel dashboard reads these. Each is created on +-- boot AFTER its base table (Apply is fatal on a missing base): all bases are +-- OperatorOwned, and kubescape_logs is ensured in OperatorOwnedTables just before +-- these views. px read contract: expose event_time UInt64 + hostname + NO Bool cols; +-- ts=toString(time_) readable, row_time Int64 ns for the PxL interval-join. Views +-- are not pixie socket_tracer tables → absent from PixieTables(). + +-- dx_anomaly_orders: ONE row per order dx opened. order_id is dx-assigned and +-- dx owns its granularity (hash(uniqueID) = 1:1 with the log today; finer later), +-- so the view dedups on order_id and makes NO assumption about orders-per-uniqueID. +-- lo/hi are kept for reference (the ±300s span); the CONSULTED records for the +-- order live in dx_order_records, stamped with this order_id. +CREATE OR REPLACE VIEW forensic_db.dx_anomaly_orders AS +SELECT unique_id AS uniqueID, rule_id AS rule, pod, + toInt64(event_time) - 300000000000 AS lo, + toInt64(event_time) + 300000000000 AS hi, + order_id, + hostname, event_time +FROM forensic_db.dx_order_seeds +LIMIT 1 BY order_id; + +-- dx_kubescape_anomalies: L1 kill-chain graph (subject_pod -> target), deduped by uniqueID. +CREATE OR REPLACE VIEW forensic_db.dx_kubescape_anomalies AS +SELECT JSONExtractString(BaseRuntimeMetadata, 'uniqueID') AS uniqueID, + concat(JSONExtractString(RuntimeK8sDetails, 'podNamespace'), '/', JSONExtractString(RuntimeK8sDetails, 'podName')) AS subject_pod, + RuleID AS rule, + JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'process'), 'name') AS process, + multiIf(JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'dns'), 'domain') != '', JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'dns'), 'domain'), JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'network'), 'dstIP') != '', JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'network'), 'dstIP'), JSONExtractString(JSONExtractRaw(BaseRuntimeMetadata, 'arguments'), 'path') != '', JSONExtractString(JSONExtractRaw(BaseRuntimeMetadata, 'arguments'), 'path'), JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'file'), 'name') != '', concat(JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'file'), 'directory'), '/', JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'file'), 'name')), 'unknown') AS target, + multiIf(JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'dns'), 'domain') != '', 'domain', JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'network'), 'dstIP') != '', 'endpoint', (JSONExtractString(JSONExtractRaw(BaseRuntimeMetadata, 'arguments'), 'path') != '') OR (JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'file'), 'name') != ''), 'file', 'other') AS target_kind, + toInt8OrZero(JSONExtractString(BaseRuntimeMetadata, 'severity')) AS severity, + message AS alert, hostname, event_time +FROM forensic_db.kubescape_logs +WHERE RuleID != '' AND JSONExtractString(BaseRuntimeMetadata, 'uniqueID') != '' +LIMIT 1 BY uniqueID; + +-- dx_src__kubescape_logs: anomaly detail (process tree comm/cmdline/pcomm) per panel. +CREATE OR REPLACE VIEW forensic_db.dx_src__kubescape_logs AS +SELECT toString(fromUnixTimestamp64Nano(toInt64(event_time))) AS ts, toInt64(event_time) AS row_time, event_time, + RuleID, JSONExtractString(BaseRuntimeMetadata, 'uniqueID') AS uniqueID, + JSONExtractString(JSONExtractRaw(RuntimeProcessDetails, 'processTree'), 'comm') AS comm, + JSONExtractString(JSONExtractRaw(RuntimeProcessDetails, 'processTree'), 'pcomm') AS parent, + JSONExtractString(JSONExtractRaw(RuntimeProcessDetails, 'processTree'), 'cmdline') AS cmdline, + message AS alert, + concat(JSONExtractString(RuntimeK8sDetails, 'podNamespace'), '/', JSONExtractString(RuntimeK8sDetails, 'podName')) AS pod, hostname +FROM forensic_db.kubescape_logs WHERE RuleID != ''; + +-- dx_src__stack_trace: original schema + ts/row_time/event_time. +CREATE OR REPLACE VIEW forensic_db.dx_src__stack_trace AS +SELECT toString(time_) AS ts, toInt64(toUnixTimestamp64Nano(time_)) AS row_time, toUInt64(toUnixTimestamp64Nano(event_time)) AS event_time, + namespace, pod, container, stack_trace_id, stack_trace, count, hostname +FROM forensic_db.stack_trace; + +CREATE OR REPLACE VIEW forensic_db.dx_ord__redis_events AS +SELECT + e.order_id AS order_id, + toString(c.time_) AS ts, + toInt64(toUnixTimestamp64Nano(c.time_)) AS row_time, + fromUnixTimestamp64Nano(toInt64(e.event_time)) AS event_time, + c.namespace AS namespace, + c.pod AS pod, + c.remote_addr AS remote_addr, + c.remote_port AS remote_port, + c.trace_role AS trace_role, + c.req_cmd AS req_cmd, + c.req_args AS req_args, + c.resp AS resp, + c.latency AS latency, + e.hostname AS hostname +FROM forensic_db.dx_order_edges AS e +INNER JOIN forensic_db.redis_events AS c ON c.unique_id = e.unique_id +WHERE e.src_table = 'redis_events'; + +CREATE OR REPLACE VIEW forensic_db.dx_ord__http_events AS +SELECT + e.order_id AS order_id, + toString(c.time_) AS ts, + toInt64(toUnixTimestamp64Nano(c.time_)) AS row_time, + fromUnixTimestamp64Nano(toInt64(e.event_time)) AS event_time, + c.namespace AS namespace, + c.pod AS pod, + c.remote_addr AS remote_addr, + c.remote_port AS remote_port, + c.req_method AS req_method, + c.req_path AS req_path, + c.req_body AS req_body, + c.resp_status AS resp_status, + c.resp_body AS resp_body, + c.latency AS latency, + e.hostname AS hostname +FROM forensic_db.dx_order_edges AS e +INNER JOIN forensic_db.http_events AS c ON c.unique_id = e.unique_id +WHERE e.src_table = 'http_events'; + +CREATE OR REPLACE VIEW forensic_db.dx_ord__dns_events AS +SELECT + e.order_id AS order_id, + toString(c.time_) AS ts, + toInt64(toUnixTimestamp64Nano(c.time_)) AS row_time, + fromUnixTimestamp64Nano(toInt64(e.event_time)) AS event_time, + c.namespace AS namespace, + c.pod AS pod, + c.remote_addr AS remote_addr, + c.remote_port AS remote_port, + c.req_body AS req_body, + c.resp_body AS resp_body, + c.latency AS latency, + e.hostname AS hostname +FROM forensic_db.dx_order_edges AS e +INNER JOIN forensic_db.dns_events AS c ON c.unique_id = e.unique_id +WHERE e.src_table = 'dns_events'; + +CREATE OR REPLACE VIEW forensic_db.dx_ord__pgsql_events AS +SELECT + e.order_id AS order_id, + toString(c.time_) AS ts, + toInt64(toUnixTimestamp64Nano(c.time_)) AS row_time, + fromUnixTimestamp64Nano(toInt64(e.event_time)) AS event_time, + c.namespace AS namespace, + c.pod AS pod, + c.remote_addr AS remote_addr, + c.remote_port AS remote_port, + c.req AS req, + c.resp AS resp, + c.latency AS latency, + e.hostname AS hostname +FROM forensic_db.dx_order_edges AS e +INNER JOIN forensic_db.pgsql_events AS c ON c.unique_id = e.unique_id +WHERE e.src_table = 'pgsql_events'; + +CREATE OR REPLACE VIEW forensic_db.dx_ord__mysql_events AS +SELECT + e.order_id AS order_id, + toString(c.time_) AS ts, + toInt64(toUnixTimestamp64Nano(c.time_)) AS row_time, + fromUnixTimestamp64Nano(toInt64(e.event_time)) AS event_time, + c.namespace AS namespace, + c.pod AS pod, + c.remote_addr AS remote_addr, + c.remote_port AS remote_port, + c.req_cmd AS req_cmd, + c.req_body AS req_body, + c.resp_status AS resp_status, + c.resp_body AS resp_body, + c.latency AS latency, + e.hostname AS hostname +FROM forensic_db.dx_order_edges AS e +INNER JOIN forensic_db.mysql_events AS c ON c.unique_id = e.unique_id +WHERE e.src_table = 'mysql_events'; + +-- dx_ord__cql_events / dx_ord__mongodb_events / dx_ord__creds_change — bridge +-- views: exactly the rows dx consulted for an order, joined on the dx-stamped +-- unique_id. Same shape as the other dx_ord__ views; the panel filters order_id. +CREATE OR REPLACE VIEW forensic_db.dx_ord__cql_events AS +SELECT + e.order_id AS order_id, + toString(c.time_) AS ts, + toInt64(toUnixTimestamp64Nano(c.time_)) AS row_time, + fromUnixTimestamp64Nano(toInt64(e.event_time)) AS event_time, + c.namespace AS namespace, + c.pod AS pod, + c.remote_addr AS remote_addr, + c.remote_port AS remote_port, + c.req_op AS req_op, + c.req_body AS req_body, + c.resp_op AS resp_op, + c.resp_body AS resp_body, + c.latency AS latency, + e.hostname AS hostname +FROM forensic_db.dx_order_edges AS e +INNER JOIN forensic_db.cql_events AS c ON c.unique_id = e.unique_id +WHERE e.src_table = 'cql_events'; + +CREATE OR REPLACE VIEW forensic_db.dx_ord__mongodb_events AS +SELECT + e.order_id AS order_id, + toString(c.time_) AS ts, + toInt64(toUnixTimestamp64Nano(c.time_)) AS row_time, + fromUnixTimestamp64Nano(toInt64(e.event_time)) AS event_time, + c.namespace AS namespace, + c.pod AS pod, + c.remote_addr AS remote_addr, + c.remote_port AS remote_port, + c.req_cmd AS req_cmd, + c.req_body AS req_body, + c.resp_status AS resp_status, + c.resp_body AS resp_body, + c.latency AS latency, + e.hostname AS hostname +FROM forensic_db.dx_order_edges AS e +INNER JOIN forensic_db.mongodb_events AS c ON c.unique_id = e.unique_id +WHERE e.src_table = 'mongodb_events'; + +CREATE OR REPLACE VIEW forensic_db.dx_ord__creds_change AS +SELECT + e.order_id AS order_id, + toString(c.time_) AS ts, + toInt64(toUnixTimestamp64Nano(c.time_)) AS row_time, + fromUnixTimestamp64Nano(toInt64(e.event_time)) AS event_time, + c.namespace AS namespace, + c.pod AS pod, + c.pid AS pid, + c.comm AS comm, + c.old_uid AS old_uid, + c.new_uid AS new_uid, + c.container AS container, + e.hostname AS hostname +FROM forensic_db.dx_order_edges AS e +INNER JOIN forensic_db.creds_change AS c ON c.unique_id = e.unique_id +WHERE e.src_table = 'creds_change'; + +-- dx_cases — the meta-grouping above orders: every order carries culprit_key +-- (ns/pod/RootPID). Orders sharing a culprit_key are one actor's steps (read + +-- exfil + spawns). Joins the kubescape mitre/severity so a case node can be +-- coloured, and counts distinct evidence protocols the culprit touched. +CREATE OR REPLACE VIEW forensic_db.dx_cases AS +SELECT + o.culprit_key AS culprit_key, + o.order_id AS order_id, + o.rule_id AS rule_id, + o.pod AS subject_pod, + o.disc AS alert, + m.mitre_tactic AS mitre_tactic, + m.mitre_technique AS mitre_technique, + m.severity AS severity, + toInt64(toUnixTimestamp64Nano(fromUnixTimestamp64Nano(o.event_time))) AS event_time, + o.hostname AS hostname +FROM forensic_db.dx_orders AS o +LEFT JOIN forensic_db.dx_kubescape_mitre AS m + ON m.uniqueID = o.kubescape_uid AND m.rule = o.rule_id +WHERE o.culprit_key != ''; + +-- dx_case_links — the cross-pod bridge over cases: an order's conn_stats +-- evidence resolves remote_pod (the peer pod), and a culprit lives on that pod. +-- Links the sink's exfil-receipt culprit back to the attacker's culprit that +-- opened the connection. Directed: from = this order's culprit, to = peer culprit. +CREATE OR REPLACE VIEW forensic_db.dx_case_links AS +SELECT DISTINCT + a.culprit_key AS from_culprit, + c.remote_pod AS peer_pod, + b.culprit_key AS to_culprit, + a.hostname AS hostname +FROM forensic_db.dx_orders AS a +INNER JOIN forensic_db.dx_order_edges AS e + ON e.order_id = a.order_id AND e.src_table = 'conn_stats' +INNER JOIN forensic_db.conn_stats AS c + ON c.unique_id = e.unique_id AND c.remote_pod != '' +INNER JOIN forensic_db.dx_orders AS b + ON b.pod = c.remote_pod +WHERE a.culprit_key != '' AND b.culprit_key != '' AND a.culprit_key != b.culprit_key; + +CREATE OR REPLACE VIEW forensic_db.dx_ord__dc_snoop AS +SELECT + e.order_id AS order_id, + toString(c.time_) AS ts, + toInt64(toUnixTimestamp64Nano(c.time_)) AS row_time, + fromUnixTimestamp64Nano(toInt64(e.event_time)) AS event_time, + c.pid AS pid, + c.comm AS comm, + c.t AS t, + c.file AS file, + c.namespace AS namespace, + c.pod AS pod, + c.container AS container, + e.hostname AS hostname +FROM forensic_db.dx_order_edges AS e +INNER JOIN forensic_db.dc_snoop AS c ON c.unique_id = e.unique_id +WHERE e.src_table = 'dc_snoop'; + +CREATE OR REPLACE VIEW forensic_db.dx_ord__stack_trace AS +SELECT + e.order_id AS order_id, + toString(c.time_) AS ts, + toInt64(toUnixTimestamp64Nano(c.time_)) AS row_time, + fromUnixTimestamp64Nano(toInt64(e.event_time)) AS event_time, + c.namespace AS namespace, + c.pod AS pod, + c.container AS container, + c.stack_trace_id AS stack_trace_id, + c.stack_trace AS stack_trace, + c.count AS count, + e.hostname AS hostname +FROM forensic_db.dx_order_edges AS e +INNER JOIN forensic_db.stack_trace AS c ON c.unique_id = e.unique_id +WHERE e.src_table = 'stack_trace'; + +-- ── MITRE ATT&CK enrichment over kubescape_logs (px/dx_evidence_graph) ──────── +-- dx_kubescape_mitre: L1 graph source — one row per (uniqueID, rule) so orders +-- keyed on (uniqueID, RuleID) all join; MITRE + resolved target from BaseRuntimeMetadata. +CREATE OR REPLACE VIEW forensic_db.dx_kubescape_mitre AS +SELECT JSONExtractString(BaseRuntimeMetadata, 'uniqueID') AS uniqueID, + concat(JSONExtractString(RuntimeK8sDetails, 'podNamespace'), '/', JSONExtractString(RuntimeK8sDetails, 'podName')) AS subject_pod, + RuleID AS rule, + JSONExtractString(BaseRuntimeMetadata, 'mitreTactic') AS mitre_tactic, + JSONExtractString(BaseRuntimeMetadata, 'mitreTechnique') AS mitre_technique, + concat(RuleID, ' · ', JSONExtractString(BaseRuntimeMetadata, 'mitreTechnique')) AS rule_mitre, + JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'process'), 'name') AS process, + multiIf(JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'dns'), 'domain') != '', +JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'dns'), 'domain'), +JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'network'), 'dstIP') != '', +JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'network'), 'dstIP'), JSONExtractString(JSONExtractRaw(BaseRuntimeMetadata, +'arguments'), 'path') != '', JSONExtractString(JSONExtractRaw(BaseRuntimeMetadata, 'arguments'), 'path'), +JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'file'), 'name') != '', +concat(JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'file'), 'directory'), '/', +JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'file'), 'name')), 'unknown') AS target, + multiIf(JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'dns'), 'domain') != '', 'domain', +JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'network'), 'dstIP') != '', 'endpoint', +(JSONExtractString(JSONExtractRaw(BaseRuntimeMetadata, 'arguments'), 'path') != '') OR (JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, +'identifiers'), 'file'), 'name') != ''), 'file', 'other') AS target_kind, + toInt8OrZero(JSONExtractString(BaseRuntimeMetadata, 'severity')) AS severity, + message AS alert, hostname, event_time, + toString(fromUnixTimestamp64Nano(toInt64(event_time))) AS ts +FROM forensic_db.kubescape_logs +WHERE RuleID != '' AND JSONExtractString(BaseRuntimeMetadata, 'uniqueID') != '' +LIMIT 1 BY uniqueID, rule; + +-- dx_src__kubescape_mitre: kubescape detail panel — MITRE cols after RuleID, plus +-- process tree (comm/pcomm/cmdline). ts/row_time/event_time px-connector convention. +-- pid is the processTree ROOT (the entrypoint, e.g. dumb-init). The process that +-- actually tripped the rule is infectedPID, exposed as infected_pid with its comm +-- (offender) and violated profile. Keying evidence on pid shows the wrong process, +-- which makes a true positive read as a false positive. +-- NOTE: statements here are extracted by scanning to the FIRST ';' after the CREATE +-- (ddl.go statementFor), so a ';' anywhere inside a statement — including inside a +-- comment — truncates it and the apply fails. Keep prose above the CREATE. +CREATE OR REPLACE VIEW forensic_db.dx_src__kubescape_mitre AS +SELECT toString(fromUnixTimestamp64Nano(toInt64(event_time))) AS ts, toInt64(event_time) AS row_time, event_time, + RuleID, + JSONExtractString(BaseRuntimeMetadata, 'mitreTactic') AS mitre_tactic, + JSONExtractString(BaseRuntimeMetadata, 'mitreTechnique') AS mitre_technique, + JSONExtractString(BaseRuntimeMetadata, 'uniqueID') AS uniqueID, + JSONExtractString(JSONExtractRaw(RuntimeProcessDetails, 'processTree'), 'comm') AS comm, + JSONExtractInt(JSONExtractRaw(RuntimeProcessDetails, 'processTree'), 'pid') AS pid, + JSONExtractString(JSONExtractRaw(RuntimeProcessDetails, 'processTree'), 'pcomm') AS parent, + JSONExtractInt(JSONExtractRaw(RuntimeProcessDetails, 'processTree'), 'ppid') AS ppid, + JSONExtractString(JSONExtractRaw(RuntimeProcessDetails, 'processTree'), 'cmdline') AS cmdline, + JSONExtractInt(BaseRuntimeMetadata, 'infectedPID') AS infected_pid, + JSONExtractString(BaseRuntimeMetadata, 'identifiers', 'process', 'name') AS offender, + JSONExtractString(BaseRuntimeMetadata, 'profileMetadata', 'name') AS profile, + message AS alert, + concat(JSONExtractString(RuntimeK8sDetails, 'podNamespace'), '/', JSONExtractString(RuntimeK8sDetails, 'podName')) AS pod, hostname +FROM forensic_db.kubescape_logs WHERE RuleID != ''; + +-- dx_orders_win: per-order ±300s baseline/attack window for the differential +-- flamegraph (stack_diff). hostname carried so the px node-shard resolves. +CREATE OR REPLACE VIEW forensic_db.dx_orders_win AS +SELECT order_id, pod, + toInt64(event_time) - 300000000000 AS lo, + toInt64(event_time) + 300000000000 AS hi, + event_time, hostname +FROM forensic_db.dx_orders; + +-- dx_dns_resolve: DNS resolution edges exploded from dns_events resp_body +-- (querier->resolver, then the answer tree name->CNAME / name->A). Read by the +-- dx/dns_resolve UI, time-windowed via dx_orders_win. Column named event_time +-- (int64 ns) because the px ClickHouse connector defaults its cursor there. +CREATE OR REPLACE VIEW forensic_db.dx_dns_resolve AS +SELECT toInt64(toUnixTimestamp64Nano(dns_events.event_time)) AS event_time, + toString(dns_events.event_time) AS ts, hostname, + if(pod != '', pod, if(local_addr != '', concat('client:', local_addr), 'client')) AS from_node, + concat(remote_addr, ':', toString(remote_port)) AS to_node, + JSONExtractString(JSONExtractArrayRaw(req_body, 'queries')[1], 'name') AS edge_label, + 'query' AS kind +FROM forensic_db.dns_events +WHERE resp_body != '' AND resp_body != '{}' +UNION ALL +SELECT toInt64(toUnixTimestamp64Nano(dns_events.event_time)) AS event_time, + toString(dns_events.event_time) AS ts, hostname, + JSONExtractString(ans, 'name') AS from_node, + concat(JSONExtractString(ans, 'cname'), JSONExtractString(ans, 'addr')) AS to_node, + JSONExtractString(ans, 'type') AS edge_label, + lower(JSONExtractString(ans, 'type')) AS kind +FROM forensic_db.dns_events +ARRAY JOIN JSONExtractArrayRaw(resp_body, 'answers') AS ans +WHERE resp_body != '' AND JSONExtractString(ans, 'type') != '' + AND concat(JSONExtractString(ans, 'cname'), JSONExtractString(ans, 'addr')) NOT IN ('', '-'); + +-- dx_alerts: GENERIC thin flatten of kubescape_logs (pod/namespace/rule/message/ +-- sev). Reusable seed for any narrative; story logic (target/kind) is derived in +-- the PxL (dx/breakout), never here — so the DDL is portable and stays stable. +CREATE OR REPLACE VIEW forensic_db.dx_alerts AS +SELECT + fromUnixTimestamp64Nano(toInt64(event_time)) AS event_time, + hostname AS hostname, + JSONExtractString(RuntimeK8sDetails,'namespace') AS namespace, + JSONExtractString(RuntimeK8sDetails,'podName') AS pod, + RuleID AS rule, + message AS message, + JSONExtractInt(BaseRuntimeMetadata,'severity') AS sev +FROM forensic_db.kubescape_logs +WHERE JSONExtractString(RuntimeK8sDetails,'namespace') NOT IN + ('honey','pl','clickhouse','kube-system','kube-public','kube-node-lease', + 'local-path-storage','px-operator','olm','cert-manager',''); + +-- dx_breakout_story: runtime-breakout edges (pod -> off-profile target, kind). +-- Story-specific SQL kept for the shipped dashboard; newer dx/breakout PxL derives +-- target/kind from dx_alerts in PxL instead. +CREATE OR REPLACE VIEW forensic_db.dx_breakout_story AS +SELECT + fromUnixTimestamp64Nano(toInt64(event_time)) AS event_time, + hostname AS hostname, + JSONExtractString(RuntimeK8sDetails,'namespace') AS namespace, + JSONExtractString(RuntimeK8sDetails,'podName') AS pod, + JSONExtractString(RuntimeK8sDetails,'podName') AS from_node, + multiIf(RuleID='R0002' AND position(message,'serviceaccount')>0 AND position(message,'token')>0,'serviceaccount/token (SA cred)', + RuleID='R0002', extractGroups(message,' to (.+)$')[1], + RuleID='R0001', concat('proc:',coalesce(nullIf(extractGroups(message,'([^ /]+)$')[1],''),'?')), + RuleID='R0012', concat('ingress<-',coalesce(nullIf(extractGroups(message,'from: ([^ ]+)')[1],''),'peer')), + RuleID='R0011', concat('egress->',coalesce(nullIf(extractGroups(message,'to: ([^ ]+)')[1],''),'peer')), + RuleID='R0005', concat('dns:',coalesce(nullIf(extractGroups(message,'([^ ]+)$')[1],''),'?')), + RuleID) AS to_node, + RuleID AS rule, + multiIf(RuleID='R0001','process', + RuleID='R0002' AND position(message,'serviceaccount')>0 AND position(message,'token')>0,'token-read', + RuleID='R0002' AND match(message,'\\.so'),'libload', + RuleID='R0002' AND position(message,'/proc/')>0,'proc-read', + RuleID='R0002' AND (position(message,'/etc/')>0 OR position(message,'/runc')>0 OR position(message,'/root')>0),'sensitive', + RuleID='R0002' AND position(message,'/tmp')>0,'tmp-write', + RuleID='R0002','file-access', + RuleID='R0012','ingress', RuleID='R0011','egress', RuleID='R0005','dns', + RuleID='R0004','capability', RuleID='R0003','syscall', + RuleID IN ('R0006','R0007','R0008'),'cred-access','alert') AS kind, + JSONExtractInt(BaseRuntimeMetadata,'severity') AS sev +FROM forensic_db.kubescape_logs +WHERE JSONExtractString(RuntimeK8sDetails,'namespace') NOT IN + ('honey','pl','clickhouse','kube-system','kube-public','kube-node-lease', + 'local-path-storage','px-operator','olm','cert-manager','gmp-system', + 'gmp-public','storm','lightening','chain-loadgen',''); + +-- dx_fullchain_edges: cross-sensor exfil edges (kubescape seed + conn egress + +-- dns + pgsql) normalized to one edge shape. Feeds dx/fullchain. The UNION must +-- live in SQL (PxL cannot union DataFrames); labels stay source-based, not parsed. +CREATE OR REPLACE VIEW forensic_db.dx_fullchain_edges AS +SELECT toDateTime64(fromUnixTimestamp64Nano(toInt64(event_time)),9) AS event_time, hostname AS hostname, + JSONExtractString(RuntimeK8sDetails,'podName') AS pod, concat('alert:',RuleID) AS from_node, + JSONExtractString(RuntimeK8sDetails,'podName') AS to_node, RuleID AS edge_label, 'flagged' AS kind, + toInt32(JSONExtractInt(BaseRuntimeMetadata,'severity')) AS sev +FROM forensic_db.kubescape_logs +WHERE JSONExtractString(RuntimeK8sDetails,'namespace') NOT IN + ('honey','pl','clickhouse','kube-system','kube-public','kube-node-lease', + 'local-path-storage','px-operator','olm','cert-manager','') +UNION ALL +SELECT toDateTime64(time_,9), hostname, pod, pod, remote_addr, + concat('conn/',toString(protocol)), 'connects', toInt32(5) +FROM forensic_db.conn_stats WHERE trace_role=1 AND remote_addr!='' +UNION ALL +SELECT toDateTime64(time_,9), hostname, pod, pod, req_body, 'dns', 'resolves', toInt32(5) +FROM forensic_db.dns_events WHERE req_body!='' +UNION ALL +SELECT toDateTime64(time_,9), hostname, pod, pod, substring(req,1,60), 'sql', 'sql', toInt32(5) +FROM forensic_db.pgsql_events WHERE req!=''; diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/schema_registration_test.go b/src/vizier/services/adaptive_export/internal/clickhouse/schema_registration_test.go new file mode 100644 index 00000000000..b2e1051c0fb --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/clickhouse/schema_registration_test.go @@ -0,0 +1,104 @@ +/* + * Copyright 2018- The Pixie Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package clickhouse + +import ( + "regexp" + "testing" +) + +// Drift guard between schema.sql and the two registration lists. +// +// schema.sql is the source of truth for DDL, but nothing is created unless the +// object is also listed: Apply iterates OperatorOwnedTables, and DDL/Columns +// resolve through KnownTables. An object that exists in schema.sql and in +// neither list is inert — and inert *silently*: no error at boot, no log line, +// just a view that is never created and panels that return nothing. That has +// happened twice (dx_base__dc_snoop, then dx_ord__{cql,mongodb,creds_change}), +// both caught on a rig rather than in CI. +// +// The reverse drift is cheaper to hit but also worth pinning: a name listed +// with no DDL behind it fails only when DDL() is called for it. + +// createStmt matches the object name in `CREATE TABLE IF NOT EXISTS forensic_db. +// ` (tables) or `CREATE OR REPLACE VIEW forensic_db.` (views — replaced +// on every Apply so a changed definition never needs a manual DROP), with or +// without the backticks used for dotted names (e.g. `http2_messages.beta`). +var createStmt = regexp.MustCompile("(?i)CREATE\\s+(?:TABLE\\s+IF\\s+NOT\\s+EXISTS|OR\\s+REPLACE\\s+VIEW)\\s+forensic_db\\.`?([A-Za-z0-9_.]+)`?") + +// socOwnedTables are declared in schema.sql so the operator can verify and read +// them, but are created by the soc/clickhouse-lab installer — AE must never +// issue their CREATE TABLE. See TestOperatorOwnedTables_DoesNotIncludeKubescape. +var socOwnedTables = map[string]bool{ + "alerts": true, + "kubescape_logs": true, +} + +func schemaObjects(t *testing.T) map[string]bool { + t.Helper() + objs := map[string]bool{} + for _, m := range createStmt.FindAllStringSubmatch(canonicalSchema, -1) { + objs[m[1]] = true + } + if len(objs) == 0 { + t.Fatal("parsed no CREATE statements out of schema.sql — the regex or the file shape changed") + } + return objs +} + +func TestEverySchemaObjectIsRegistered(t *testing.T) { + known := map[string]bool{} + for _, n := range KnownTables { + known[n] = true + } + owned := map[string]bool{} + for _, n := range OperatorOwnedTables { + owned[n] = true + } + + for name := range schemaObjects(t) { + if !known[name] { + t.Errorf("%q is in schema.sql but missing from KnownTables — DDL(%q) will not resolve it", name, name) + } + if socOwnedTables[name] { + if owned[name] { + t.Errorf("%q is soc-owned; AE must not create it", name) + } + continue + } + if !owned[name] { + t.Errorf("%q is in schema.sql but missing from OperatorOwnedTables — Apply will never create it, "+ + "and nothing will report that at boot", name) + } + } +} + +func TestEveryRegisteredNameHasDDL(t *testing.T) { + objs := schemaObjects(t) + for _, name := range KnownTables { + if !objs[name] { + t.Errorf("%q is in KnownTables but has no CREATE statement in schema.sql", name) + } + } + for _, name := range OperatorOwnedTables { + if !objs[name] { + t.Errorf("%q is in OperatorOwnedTables but has no CREATE statement in schema.sql", name) + } + } +} diff --git a/src/vizier/services/adaptive_export/internal/config/config.go b/src/vizier/services/adaptive_export/internal/config/config.go index 86e60250e34..87644654112 100644 --- a/src/vizier/services/adaptive_export/internal/config/config.go +++ b/src/vizier/services/adaptive_export/internal/config/config.go @@ -19,6 +19,7 @@ package config import ( "context" "fmt" + "net/url" "os" "strconv" "strings" @@ -178,9 +179,9 @@ func setUpConfig() error { log.SetLevel(log.DebugLevel) } - log.Debugf("Config from environment - ClickHouse DSN: %s", clickhouseDSN) + log.Debugf("Config from environment - ClickHouse DSN: %s", redactDSN(clickhouseDSN)) log.Debugf("Config from environment - Pixie Cluster ID: %s", pixieClusterID) - log.Debugf("Config from environment - Pixie API Key: %s", pixieAPIKey) + log.Debugf("Config from environment - Pixie API Key: %s", redactSecret(pixieAPIKey)) log.Debugf("Config from environment - Cluster Name: %s", clusterName) log.Debugf("Config from environment - Pixie Host: %s", pixieHost) @@ -209,10 +210,10 @@ func setUpConfig() error { } log.Debugf("Final config - Pixie Cluster ID: %s", pixieClusterID) - log.Debugf("Final config - Pixie API Key: %s", pixieAPIKey) + log.Debugf("Final config - Pixie API Key: %s", redactSecret(pixieAPIKey)) log.Debugf("Final config - Cluster Name: %s", clusterName) log.Debugf("Final config - Pixie Host: %s", pixieHost) - log.Debugf("Final config - ClickHouse DSN: %s", clickhouseDSN) + log.Debugf("Final config - ClickHouse DSN: %s", redactDSN(clickhouseDSN)) collectInterval, err := getIntEnvWithDefault(envCollectInterval, defCollectInterval) if err != nil { @@ -548,3 +549,30 @@ func (a *worker) DetectionInterval() int64 { return a.detectionInterval } func (a *worker) DetectionLookback() int64 { return a.detectionLookback } func (a *worker) ExportMode() string { return a.exportMode } func (a *worker) ExportQuietTicks() int64 { return a.exportQuietTicks } + +// Debug logs land in pod logs, readable by anyone with cluster read access, so +// secrets must never be printed verbatim. Length is kept because "set but wrong +// length" distinguishes a real key from a PLACEHOLDER without disclosing it. +func redactSecret(s string) string { + if s == "" { + return "" + } + return fmt.Sprintf("", len(s)) +} + +// The DSN carries a password in its userinfo, but the host/database are the +// first thing to check when AE comes up Running and writes nothing — so mask +// only the credentials and keep the rest legible. +func redactDSN(dsn string) string { + if dsn == "" { + return "" + } + u, err := url.Parse(dsn) + if err != nil || u.User == nil { + return redactSecret(dsn) + } + if _, hasPW := u.User.Password(); hasPW { + u.User = url.UserPassword(u.User.Username(), "***") + } + return u.String() +} diff --git a/src/vizier/services/adaptive_export/internal/control/BUILD.bazel b/src/vizier/services/adaptive_export/internal/control/BUILD.bazel index c22b1b8ba71..6253c6a54d5 100644 --- a/src/vizier/services/adaptive_export/internal/control/BUILD.bazel +++ b/src/vizier/services/adaptive_export/internal/control/BUILD.bazel @@ -19,19 +19,26 @@ load("//bazel:pl_build_system.bzl", "pl_go_test") go_library( name = "control", - srcs = ["server.go"], + srcs = [ + "server.go", + "tls.go", + ], importpath = "px.dev/pixie/src/vizier/services/adaptive_export/internal/control", visibility = ["//src/vizier/services/adaptive_export:__subpackages__"], deps = [ "//src/shared/services/utils", "//src/vizier/services/adaptive_export/internal/activeset", "//src/vizier/services/adaptive_export/internal/anomaly", + "@com_github_sirupsen_logrus//:logrus", ], ) pl_go_test( name = "control_test", - srcs = ["server_test.go"], + srcs = [ + "server_test.go", + "tls_test.go", + ], embed = [":control"], deps = [ "//src/shared/services/utils", diff --git a/src/vizier/services/adaptive_export/internal/control/server.go b/src/vizier/services/adaptive_export/internal/control/server.go index 96292715fcb..a98b000136b 100644 --- a/src/vizier/services/adaptive_export/internal/control/server.go +++ b/src/vizier/services/adaptive_export/internal/control/server.go @@ -31,9 +31,13 @@ import ( "context" "encoding/json" "net/http" + "os" + "strconv" "strings" "time" + log "github.com/sirupsen/logrus" + jwtutils "px.dev/pixie/src/shared/services/utils" "px.dev/pixie/src/vizier/services/adaptive_export/internal/activeset" "px.dev/pixie/src/vizier/services/adaptive_export/internal/anomaly" @@ -67,6 +71,26 @@ type exportAller interface { // anomaly is comfortably inside the pulled slice. const controlExportLookback = 600 * time.Second +// A DEGENERATE /query window (hi <= lo) is widened to controlExportLookback: a +// zero-width window keyed on one finding's timestamp matches no pixie rows. +// +// A deliberately TIGHT window is NOT degenerate and must be honoured. kubescape's +// alert timestamp is the kernel event time (bpf_ktime_get_boot_ns, converted to +// wall clock once, never re-stamped) and it reaches AE as nanos end-to-end, so a +// ±50ms span around an anomaly is meaningful — it is how a chatty protocol +// (pgsql/mysql) stays readable instead of returning tens of thousands of rows. +// This floor previously widened ANY sub-5s window to 600s, silently inflating an +// intentional ±50ms request by 6000× and defeating caller-side narrowing. +// The floor drops to 1ms: still wide enough to catch a sub-microsecond point +// window (which matches no rows), far below any intentional millisecond span. +// ADAPTIVE_MIN_QUERY_WINDOW_MS overrides it. +func minControlQueryWindow() time.Duration { + if v, err := strconv.Atoi(os.Getenv("ADAPTIVE_MIN_QUERY_WINDOW_MS")); err == nil && v > 0 { + return time.Duration(v) * time.Millisecond + } + return time.Millisecond +} + // The control API carries timestamps in the pipeline's ONE unit: unix // NANOSECONDS — the same unit as forensic_db.*.event_time and dx's referral // windows. Read them with time.Unix(0, ns). (This spot previously did @@ -86,12 +110,21 @@ type manifestWriter interface { WriteEvidenceManifest(ctx context.Context, jsonEachRow []byte) error } +// rowsWriter persists dx-handed pixie base rows (loop 1: conn_stats with a +// pre-stamped unique_id) into forensic_db. through the SAME sink the +// controller capture path uses (sink.ClickHouseHTTP.WritePixieRows). +// nil → /dx/rows 501s. +type rowsWriter interface { + WritePixieRows(ctx context.Context, table string, rows []map[string]any) error +} + // Server is the control HTTP surface. type Server struct { set exporter runner queryRunner // may be nil; /query then returns 501 graph graphWriter // may be nil; /dx/evidence_graph then returns 501 manifest manifestWriter // may be nil; /dx/evidence_manifest then returns 501 + rows rowsWriter // may be nil; /dx/rows then returns 501 mux *http.ServeMux verify func(bearer string) error // nil → auth disabled; set via SetAuth } @@ -106,6 +139,7 @@ func New(set exporter, runner queryRunner) *Server { s.mux.HandleFunc("/query", s.handleQuery) s.mux.HandleFunc("/dx/evidence_graph", s.handleDXEvidenceGraph) s.mux.HandleFunc("/dx/evidence_manifest", s.handleDXEvidenceManifest) + s.mux.HandleFunc("/dx/rows", s.handleDXRows) return s } @@ -115,6 +149,9 @@ func (s *Server) SetGraphWriter(g graphWriter) { s.graph = g } // SetManifestWriter wires the dx_evidence_manifest sink. func (s *Server) SetManifestWriter(m manifestWriter) { s.manifest = m } +// SetRowsWriter wires the /dx/rows base-row sink (loop 1). +func (s *Server) SetRowsWriter(rw rowsWriter) { s.rows = rw } + // SetAuth turns on bearer-JWT auth for the control surface, verified with the // SAME shared lib + signing key the vizier broker/PEM use (px.dev/pixie/src/ // shared/services/utils). dx already mints a service JWT (GenerateJWTForService, @@ -179,6 +216,63 @@ func (s *Server) handleDXEvidenceGraph(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusAccepted) } +// dxRowsAllowedTables guards /dx/rows against arbitrary-table writes: only the +// bridged tables dx hands base rows for (each carries a pre-stamped unique_id and +// a dx_ord__ view) are accepted. Mirrors evidencegraph.UIDColsByTable on the dx side. +var dxRowsAllowedTables = map[string]bool{ + "conn_stats": true, + "redis_events": true, + "http_events": true, + "dns_events": true, + "pgsql_events": true, + "mysql_events": true, + "cql_events": true, + "mongodb_events": true, + "dc_snoop": true, + "stack_trace": true, + // creds_change is dx-bridged too (dark tracepoint, keyed on its own columns). + "creds_change": true, +} + +// dxRowsReq is the /dx/rows wire body: dx-handed base rows for one table. +type dxRowsReq struct { + Table string `json:"table"` + Rows []map[string]any `json:"rows"` +} + +// handleDXRows ingests dx-handed base rows (loop 1: conn_stats carrying a +// pre-stamped content-hash unique_id, a hex String) and writes them to +// forensic_db.
via the same sink path the controller capture uses. +// decodeNumber (UseNumber) keeps large integer columns as json.Number so the +// fast encoder emits exact decimal text; the shared decode() would cast them to +// float64, and the sink's appendFloat renders large values in scientific +// notation, which ClickHouse rejects for Int64/UInt64 columns (whole batch 502). +func (s *Server) handleDXRows(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + if s.rows == nil { + w.WriteHeader(http.StatusNotImplemented) + return + } + var req dxRowsReq + if !decodeNumber(w, r, &req) || !dxRowsAllowedTables[req.Table] { + w.WriteHeader(http.StatusBadRequest) + return + } + if len(req.Rows) == 0 { + w.WriteHeader(http.StatusAccepted) + return + } + if err := s.rows.WritePixieRows(r.Context(), req.Table, req.Rows); err != nil { + log.WithField("table", req.Table).WithField("rows", len(req.Rows)).WithError(err).Error("dx/rows: WritePixieRows failed") + w.WriteHeader(http.StatusBadGateway) + return + } + w.WriteHeader(http.StatusAccepted) +} + // dxManifest mirrors the wire shape of dx's manifest.Manifest (internal/manifest). // Scalars map to typed forensic_db.dx_evidence_manifest columns; the nested // collections are held as raw JSON and persisted as JSON text in String columns @@ -295,6 +389,14 @@ func decode(w http.ResponseWriter, r *http.Request, v any) bool { return json.NewDecoder(r.Body).Decode(v) == nil } +func decodeNumber(w http.ResponseWriter, r *http.Request, v any) bool { + defer r.Body.Close() + r.Body = http.MaxBytesReader(w, r.Body, maxControlBodyBytes) + dec := json.NewDecoder(r.Body) + dec.UseNumber() + return dec.Decode(v) == nil +} + // ── handlers ────────────────────────────────────────────────────────── func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) @@ -353,8 +455,12 @@ func (s *Server) handleQuery(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusBadRequest) return } - err := s.runner.OrderQuery(req.target(), req.Table, - time.Unix(0, req.Window[0]).UTC(), time.Unix(0, req.Window[1]).UTC(), req.QueryID) + hi := time.Unix(0, req.Window[1]).UTC() + lo := time.Unix(0, req.Window[0]).UTC() + if d := hi.Sub(lo); d <= 0 || d < minControlQueryWindow() { + lo = hi.Add(-controlExportLookback) // widen a degenerate (or floored) window + } + err := s.runner.OrderQuery(req.target(), req.Table, lo, hi, req.QueryID) if err != nil { w.WriteHeader(http.StatusBadGateway) return diff --git a/src/vizier/services/adaptive_export/internal/control/server_test.go b/src/vizier/services/adaptive_export/internal/control/server_test.go index 15b3122ad0e..ae57dbe6eb4 100644 --- a/src/vizier/services/adaptive_export/internal/control/server_test.go +++ b/src/vizier/services/adaptive_export/internal/control/server_test.go @@ -21,6 +21,7 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "strconv" "strings" "testing" "time" @@ -45,12 +46,14 @@ func (f *fakeExporter) Remove(k activeset.Key) { f.removes = append(f.removes, k // fakeRunner records OrderQuery calls; err controls the failure path. type fakeRunner struct { - calls []string // "table|ns/pod|queryID" - err error + calls []string // "table|ns/pod|queryID" + lastStart, lastEnd time.Time + err error } func (f *fakeRunner) OrderQuery(t anomaly.Target, table string, start, end time.Time, qid string) error { f.calls = append(f.calls, table+"|"+t.Namespace+"/"+t.Pod+"|"+qid) + f.lastStart, f.lastEnd = start, end return f.err } @@ -311,3 +314,72 @@ func TestEvidenceManifest(t *testing.T) { t.Fatalf("writer error: got %d, want 502", r.StatusCode) } } + +// TestQueryWidensNarrowWindow — a control client that sends a near-zero window +// (lo≈hi, e.g. dx keying on a single finding's event_time) would capture nothing; +// the handler widens it to controlExportLookback ending at hi so the evidence +// leading up to the referral is still captured. +func TestQueryWidensNarrowWindow(t *testing.T) { + rn := &fakeRunner{} + srv := New(&fakeExporter{}, rn) + // hi = 10_000_000_000 ns, lo = hi - 512ns → a 512ns window (passes lo= %v; got %v", minControlQueryWindow(), got) + } + // hi must be preserved (we widen the lower bound only). + if rn.lastEnd.UnixNano() != hi { + t.Errorf("hi must be preserved; want %d got %d", hi, rn.lastEnd.UnixNano()) + } +} + +// TestQueryHonoursIntentionalTightWindow — a deliberate +/-50ms span around an +// anomaly must reach the runner UNCHANGED. kubescape's alert timestamp is the +// kernel event time and travels as nanos end-to-end, so a millisecond window is +// meaningful; widening it to 600s is what made a chatty protocol (pgsql/mysql) +// return tens of thousands of rows per referral. +func TestQueryHonoursIntentionalTightWindow(t *testing.T) { + rn := &fakeRunner{} + srv := New(&fakeExporter{}, rn) + hi := int64(10_000_000_000) + lo := hi - int64(100*time.Millisecond) // +/-50ms around the anomaly + resp := do(t, srv, http.MethodPost, "/query", + `{"pod":"p","namespace":"postgres-oss","table":"pgsql_events","query_id":"q1","window":[`+ + itoa(lo)+`,`+itoa(hi)+`]}`) + if resp.StatusCode != http.StatusAccepted { + t.Fatalf("want 202, got %d", resp.StatusCode) + } + if got := rn.lastEnd.Sub(rn.lastStart); got != 100*time.Millisecond { + t.Fatalf("tight window must pass through unchanged; want 100ms got %v", got) + } + if rn.lastStart.UnixNano() != lo { + t.Errorf("lo must be preserved; want %d got %d", lo, rn.lastStart.UnixNano()) + } +} + +// A comfortably-wide window is passed through unchanged (no over-widening). +func TestQueryWideWindowUnchanged(t *testing.T) { + rn := &fakeRunner{} + srv := New(&fakeExporter{}, rn) + hi := int64(1_000_000_000_000) // 1000s in ns, so lo stays positive + lo := hi - int64(120*time.Second) + resp := do(t, srv, http.MethodPost, "/query", + `{"pod":"p","namespace":"redis-demo","table":"dc_snoop","query_id":"q2","window":[`+ + itoa(lo)+`,`+itoa(hi)+`]}`) + if resp.StatusCode != http.StatusAccepted { + t.Fatalf("want 202, got %d", resp.StatusCode) + } + if got := rn.lastEnd.Sub(rn.lastStart); got != 120*time.Second { + t.Errorf("wide window must pass through unchanged; want 120s got %v", got) + } +} + +func itoa(n int64) string { return strconv.FormatInt(n, 10) } diff --git a/src/vizier/services/adaptive_export/internal/control/tls.go b/src/vizier/services/adaptive_export/internal/control/tls.go new file mode 100644 index 00000000000..cc59c8455c1 --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/control/tls.go @@ -0,0 +1,128 @@ +// Copyright 2018- The Pixie Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package control + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "net" + "os" + "time" +) + +// TLSConfig builds the server-side *tls.Config for the control surface. +// +// If BOTH certFile and keyFile exist and load, the mounted keypair is used +// (the shared service-tls-certs the broker/PEM already carry). Otherwise an +// ephemeral in-memory self-signed cert is generated so TLS works with zero +// extra secrets — dx skip-verifies the in-cluster cert, so a self-signed cert +// is sufficient to stop the bearer JWT crossing the CNI in cleartext. +// +// The bool return reports whether the cert was self-generated (true) vs +// loaded from disk (false), for the caller's boot log. +func TLSConfig(certFile, keyFile string, hostnames ...string) (*tls.Config, bool, error) { + if certFile != "" && keyFile != "" && fileExists(certFile) && fileExists(keyFile) { + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, false, fmt.Errorf("load mounted keypair %s/%s: %w", certFile, keyFile, err) + } + return &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS12}, false, nil + } + cert, err := selfSignedCert(hostnames...) + if err != nil { + return nil, false, err + } + return &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS12}, true, nil +} + +func fileExists(p string) bool { + fi, err := os.Stat(p) + return err == nil && !fi.IsDir() +} + +// selfSignedCert mints an ephemeral in-memory self-signed certificate: +// ECDSA P-256, 1y validity, SAN covering localhost + 127.0.0.1 + ::1 and any +// extra hostnames (the pod/node name). Nothing is written to disk; the key +// lives only in the returned tls.Certificate. +func selfSignedCert(hostnames ...string) (tls.Certificate, error) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return tls.Certificate{}, fmt.Errorf("generate ecdsa key: %w", err) + } + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return tls.Certificate{}, fmt.Errorf("generate serial: %w", err) + } + now := time.Now() + tmpl := x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{CommonName: "adaptive-export-control"}, + NotBefore: now.Add(-time.Hour), + NotAfter: now.AddDate(1, 0, 0), // 1y validity + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + DNSNames: []string{"localhost"}, + IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback}, + } + for _, h := range hostnames { + if h == "" { + continue + } + if ip := net.ParseIP(h); ip != nil { + tmpl.IPAddresses = append(tmpl.IPAddresses, ip) + } else { + tmpl.DNSNames = append(tmpl.DNSNames, h) + } + } + der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &key.PublicKey, key) + if err != nil { + return tls.Certificate{}, fmt.Errorf("create certificate: %w", err) + } + return tls.Certificate{ + Certificate: [][]byte{der}, + PrivateKey: key, + Leaf: &tmpl, + }, nil +} + +// certToPEM renders a tls.Certificate (as produced by selfSignedCert, holding a +// single DER cert + an *ecdsa.PrivateKey) as PEM cert + PEM key bytes — the +// on-disk shape of a mounted /certs/server.{crt,key} keypair. +func certToPEM(cert tls.Certificate) ([]byte, []byte, error) { + if len(cert.Certificate) == 0 { + return nil, nil, fmt.Errorf("certToPEM: empty certificate chain") + } + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert.Certificate[0]}) + ec, ok := cert.PrivateKey.(*ecdsa.PrivateKey) + if !ok { + return nil, nil, fmt.Errorf("certToPEM: private key is not *ecdsa.PrivateKey") + } + der, err := x509.MarshalECPrivateKey(ec) + if err != nil { + return nil, nil, fmt.Errorf("marshal ec private key: %w", err) + } + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: der}) + return certPEM, keyPEM, nil +} diff --git a/src/vizier/services/adaptive_export/internal/control/tls_test.go b/src/vizier/services/adaptive_export/internal/control/tls_test.go new file mode 100644 index 00000000000..e6f53680c83 --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/control/tls_test.go @@ -0,0 +1,194 @@ +// Copyright 2018- The Pixie Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package control + +import ( + "crypto/tls" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" + + jwtutils "px.dev/pixie/src/shared/services/utils" +) + +// serveTLS starts the control server over TLS on 127.0.0.1:0 using the given +// *tls.Config and returns the base https URL + a shutdown func. +func serveTLS(t *testing.T, cfg *tls.Config, srv *Server) (string, func()) { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + httpSrv := &http.Server{Handler: srv.Handler(), TLSConfig: cfg} + go func() { _ = httpSrv.ServeTLS(ln, "", "") }() + return "https://" + ln.Addr().String(), func() { _ = httpSrv.Close() } +} + +func skipVerifyClient() *http.Client { + return &http.Client{ + Timeout: 3 * time.Second, + Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}, //nolint:gosec // test: dx skip-verifies the in-cluster self-signed cert + } +} + +// TestTLSConfigSelfSigned: with no mounted cert files, TLSConfig self-generates +// an in-memory cert (the default secure path when /certs is absent). +func TestTLSConfigSelfSigned(t *testing.T) { + cfg, selfSigned, err := TLSConfig("/no/such/cert.crt", "/no/such/key.key", "some-pod") + if err != nil { + t.Fatalf("TLSConfig self-gen: %v", err) + } + if !selfSigned { + t.Fatal("expected selfSigned=true when cert files are absent") + } + if cfg == nil || len(cfg.Certificates) != 1 { + t.Fatalf("expected exactly one in-memory certificate, got %+v", cfg) + } +} + +// TestTLSServesHealthz: the server serves TLS by default (self-gen path) and a +// TLS client can reach /healthz. This is T1's "no cleartext by default". +func TestTLSServesHealthz(t *testing.T) { + cfg, selfSigned, err := TLSConfig("", "", "localhost") + if err != nil { + t.Fatalf("TLSConfig: %v", err) + } + if !selfSigned { + t.Fatal("expected self-signed cert") + } + base, stop := serveTLS(t, cfg, New(&fakeExporter{}, nil)) + defer stop() + + resp, err := skipVerifyClient().Get(base + "/healthz") + if err != nil { + t.Fatalf("TLS GET /healthz: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("healthz over TLS = %d, want 200", resp.StatusCode) + } +} + +// TestTLSRejectsUnauthenticated: over TLS with a signing key configured, an +// unauthenticated control request is rejected (401). This is T1's "requires +// the bearer JWT when a signing key is present" — verified end-to-end on the +// real TLS listener, not just the handler. +func TestTLSRejectsUnauthenticated(t *testing.T) { + const key = "0123456789abcdef0123456789abcdef" + srv := New(&fakeExporter{}, nil) + srv.SetAuth(key, "vizier") + + cfg, _, err := TLSConfig("", "", "localhost") + if err != nil { + t.Fatalf("TLSConfig: %v", err) + } + base, stop := serveTLS(t, cfg, srv) + defer stop() + client := skipVerifyClient() + + // No bearer → 401. + resp, err := client.Post(base+"/export/start", "application/json", strings.NewReader(`{"pod":"p","t_end":1}`)) + if err != nil { + t.Fatalf("TLS POST: %v", err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("unauthenticated over TLS = %d, want 401", resp.StatusCode) + } + + // Valid bearer → not 401. + good, err := jwtutils.SignJWTClaims(jwtutils.GenerateJWTForService("dx", "vizier"), key) + if err != nil { + t.Fatalf("mint token: %v", err) + } + req, _ := http.NewRequest(http.MethodPost, base+"/export/start", strings.NewReader(`{"namespace":"n","pod":"p","t_end":1}`)) + req.Header.Set("Authorization", "Bearer "+good) + resp2, err := client.Do(req) + if err != nil { + t.Fatalf("TLS POST authed: %v", err) + } + resp2.Body.Close() + if resp2.StatusCode == http.StatusUnauthorized { + t.Fatal("valid bearer wrongly rejected over TLS") + } +} + +// TestTLSConfigMountedCert: when cert+key files exist, TLSConfig loads them +// (selfSigned=false) — the /certs/server.{crt,key} shared-cert path. +func TestTLSConfigMountedCert(t *testing.T) { + dir := t.TempDir() + certPath := filepath.Join(dir, "server.crt") + keyPath := filepath.Join(dir, "server.key") + writePEMKeypair(t, certPath, keyPath) + + cfg, selfSigned, err := TLSConfig(certPath, keyPath, "localhost") + if err != nil { + t.Fatalf("TLSConfig mounted: %v", err) + } + if selfSigned { + t.Fatal("expected selfSigned=false when cert files exist") + } + if cfg == nil || len(cfg.Certificates) != 1 { + t.Fatalf("expected one loaded certificate, got %+v", cfg) + } +} + +// TestPlaintextPathServes: the CONTROL_INSECURE opt-out serves plain HTTP. This +// mirrors main.go's insecure branch (httpSrv.ListenAndServe with the same +// handler) — a plaintext client reaches /healthz. +func TestPlaintextPathServes(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + httpSrv := &http.Server{Handler: New(&fakeExporter{}, nil).Handler()} + go func() { _ = httpSrv.Serve(ln) }() + defer httpSrv.Close() + + resp, err := (&http.Client{Timeout: 3 * time.Second}).Get("http://" + ln.Addr().String() + "/healthz") + if err != nil { + t.Fatalf("plaintext GET /healthz: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("plaintext healthz = %d, want 200", resp.StatusCode) + } +} + +// writePEMKeypair mints a self-signed cert via the same helper and writes it as +// PEM cert+key files, so the mounted-cert load path can be exercised. +func writePEMKeypair(t *testing.T, certPath, keyPath string) { + t.Helper() + cert, err := selfSignedCert("localhost") + if err != nil { + t.Fatalf("selfSignedCert: %v", err) + } + certPEM, keyPEM, err := certToPEM(cert) + if err != nil { + t.Fatalf("certToPEM: %v", err) + } + if err := os.WriteFile(certPath, certPEM, 0o600); err != nil { + t.Fatalf("write cert: %v", err) + } + if err := os.WriteFile(keyPath, keyPEM, 0o600); err != nil { + t.Fatalf("write key: %v", err) + } +} diff --git a/src/vizier/services/adaptive_export/internal/controller/BUILD.bazel b/src/vizier/services/adaptive_export/internal/controller/BUILD.bazel index 6024b7ce5c7..dc4bdca23b1 100644 --- a/src/vizier/services/adaptive_export/internal/controller/BUILD.bazel +++ b/src/vizier/services/adaptive_export/internal/controller/BUILD.bazel @@ -36,12 +36,14 @@ pl_go_test( name = "controller_test", srcs = [ "controller_test.go", + "order_chunk_test.go", "order_query_test.go", ], embed = [":controller"], deps = [ "//src/vizier/services/adaptive_export/internal/anomaly", "//src/vizier/services/adaptive_export/internal/kubescape", + "//src/vizier/services/adaptive_export/internal/reconcile", "//src/vizier/services/adaptive_export/internal/sink", ], ) diff --git a/src/vizier/services/adaptive_export/internal/controller/controller.go b/src/vizier/services/adaptive_export/internal/controller/controller.go index 5a4b44ec1c4..01e317359c0 100644 --- a/src/vizier/services/adaptive_export/internal/controller/controller.go +++ b/src/vizier/services/adaptive_export/internal/controller/controller.go @@ -33,7 +33,9 @@ import ( "context" "errors" "fmt" + "strings" "sync" + "sync/atomic" "time" log "github.com/sirupsen/logrus" @@ -135,6 +137,10 @@ type Config struct { // captures over overlapping windows. Defaulted to 30s in defaulted(). ExportAllFloor time.Duration + // OrderChunk is the sub-window the ordered path walks the capture window in. + // Defaulted in defaulted(); env ADAPTIVE_ORDER_CHUNK_SEC overrides. + OrderChunk time.Duration + // === Throughput-protection knobs === // // At high anomaly rates (many concurrent active hashes), the default @@ -207,9 +213,20 @@ func (c *Config) defaulted() Config { if out.ExportAllFloor == 0 { out.ExportAllFloor = 30 * time.Second } + if out.OrderChunk == 0 { + out.OrderChunk = defaultOrderChunk + } return out } +const ( + // defaultOrderChunk = the full control lookback: one query per table, subdividing + // only on timeout (pre-chunking every table 10x-amplifies queries on one PEM). + defaultOrderChunk = 600 * time.Second + // orderMinChunk is the adaptive-subdivision floor; a span this small that still fails is surfaced. + orderMinChunk = 1 * time.Second +) + // Controller is the live orchestrator. One instance per operator process. type Controller struct { trig Trigger @@ -243,8 +260,17 @@ type Controller struct { exportAllMu sync.Mutex exportAllAt map[string]time.Time // per-target floor for OrderExportAll (steer-all) + + // Consecutive transient failures on the ordered path; any success resets it. Above + // orderBreakerTrip captureSpan stops subdividing so a saturated PEM isn't flooded. + orderTimeoutStreak atomic.Int32 } +const ( + maxOrderSplitDepth = 3 // cap captureSpan recursion (≤2^depth leaves/chunk) + orderBreakerTrip = 8 // consecutive timeouts above which subdivision stops +) + // New wires a Controller. nil clock falls through to RealClock. // nil querier disables the rev-1 push path (controller will only // write attribution rows; expects cloud's retention plugin to write @@ -295,23 +321,78 @@ func (c *Controller) OrderQuery(target anomaly.Target, table string, start, end return errors.New("controller: no pixie querier (operator-side push disabled)") } now := c.clock.Now() - q, err := pxl.QueryFor(table, target, start, end, now) - if err != nil { - return err + chunk := c.cfg.OrderChunk + if chunk <= 0 { + chunk = defaultOrderChunk + } + // Walk the window in OrderChunk sub-windows; captureSpan subdivides any that time out. + var readTotal, wroteTotal int + var firstErr error + for s := start; s.Before(end); s = s.Add(chunk) { + e := s.Add(chunk) + if e.After(end) { + e = end + } + qid := fmt.Sprintf("%s:%d-%d", queryID, s.Unix(), e.Unix()) + r, w, err := c.captureSpan(target, table, s, e, qid, 0) + readTotal += r + wroteTotal += w + if err != nil && firstErr == nil { + firstErr = err + } + } + recErr := "" + if firstErr != nil { + recErr = firstErr.Error() + } + // One reconcile row per table, aggregating every chunk. + c.cfg.Rec.Record(context.Background(), reconcile.Row{ + TS: now, Mode: "ordered", Table: table, + Namespace: target.Namespace, Pod: target.Pod, + WinStart: start, WinEnd: end, + ReadCount: int64(readTotal), WroteCount: int64(wroteTotal), + WriteErr: recErr, Hostname: c.cfg.Hostname, + }) + return firstErr +} + +// captureSpan captures [start,end) for one table, subdividing a transient failure into +// half-spans down to orderMinChunk. Bounded by maxOrderSplitDepth + the breaker so a +// saturated PEM isn't stormed; overlapping retries dedupe in the ReplacingMergeTree tables. +func (c *Controller) captureSpan(target anomaly.Target, table string, start, end time.Time, queryID string, depth int) (int, int, error) { + r, w, e := c.orderQuerySlice(target, table, start, end, queryID) + if e == nil || !isRetriableSpanErr(e) || end.Sub(start) <= orderMinChunk { + return r, w, e + } + if depth >= maxOrderSplitDepth { + return r, w, e // depth-capped: don't amplify a persistently-failing span + } + if c.orderTimeoutStreak.Load() > orderBreakerTrip { + // PEM saturated (sustained timeouts) — splitting would only add load. + log.WithFields(log.Fields{"table": table, "pod": target.Pod}). + Warn("ordered capture: circuit-breaker open (PEM saturated), not subdividing") + return r, w, e + } + log.WithError(e).WithFields(log.Fields{ + "table": table, "pod": target.Pod, "span": end.Sub(start).String(), "depth": depth, + }).Warn("ordered capture: transient failure, subdividing span") + mid := start.Add(end.Sub(start) / 2) + r1, w1, e1 := c.captureSpan(target, table, start, mid, queryID+".l", depth+1) + r2, w2, e2 := c.captureSpan(target, table, mid, end, queryID+".r", depth+1) + if e1 != nil { + return r1 + r2, w1 + w2, e1 + } + return r1 + r2, w1 + w2, e2 +} + +// orderQuerySlice runs one (target, table, [start,end)) capture and writes the rows. +// It records no reconcile row — the OrderQuery driver aggregates and records once. +func (c *Controller) orderQuerySlice(target anomaly.Target, table string, start, end time.Time, queryID string) (int, int, error) { + now := c.clock.Now() + q, qerr := pxl.QueryFor(table, target, start, end, now) + if qerr != nil { + return 0, 0, qerr } - // Background ctx with per-op timeouts mirroring pushPixieRows: a control-ordered - // capture must complete independently of any anomaly window's lifecycle. - var readCount, wroteCount int - var recErr string - defer func() { - c.cfg.Rec.Record(context.Background(), reconcile.Row{ - TS: now, Mode: "ordered", Table: table, - Namespace: target.Namespace, Pod: target.Pod, - WinStart: start, WinEnd: end, - ReadCount: int64(readCount), WroteCount: int64(wroteCount), - WriteErr: recErr, Hostname: c.cfg.Hostname, - }) - }() if c.globalSem != nil { c.globalSem <- struct{}{} defer func() { <-c.globalSem }() @@ -320,25 +401,46 @@ func (c *Controller) OrderQuery(target anomaly.Target, table string, start, end rows, qerr := c.querier.Query(qctx, q) cancel() if qerr != nil { - recErr = qerr.Error() - return qerr + if isRetriableSpanErr(qerr) { + c.orderTimeoutStreak.Add(1) // feed the saturation breaker + } + return 0, 0, qerr } - readCount = len(rows) + c.orderTimeoutStreak.Store(0) // a completed query clears the breaker if len(rows) == 0 { - return nil // nothing to persist; the read/0-wrote reconcile row still records it + return 0, 0, nil } wctx, wcancel := context.WithTimeout(context.Background(), 60*time.Second) werr := c.sink.WritePixieRows(wctx, table, rows) wcancel() if werr != nil { - recErr = werr.Error() - return werr + return len(rows), 0, werr } - wroteCount = len(rows) log.WithFields(log.Fields{ "table": table, "rows": len(rows), "pod": target.Pod, "query_id": queryID, }).Info("ordered pixie rows written to forensic_db (dx→AE /query)") - return nil + return len(rows), len(rows), nil +} + +// isRetriableSpanErr reports whether an error is a transient timeout/overload (vs. a +// structural error like a missing table) — covers ctx deadlines and gRPC status strings. +func isRetriableSpanErr(err error) bool { + if err == nil { + return false + } + if errors.Is(err, context.DeadlineExceeded) { + return true + } + s := strings.ToLower(err.Error()) + for _, m := range []string{ + "deadline", "timeout", "exceeded", "resourceexhausted", + "resource exhausted", "unavailable", "context canceled", "context cancelled", + } { + if strings.Contains(s, m) { + return true + } + } + return false } // OrderExportAll runs a one-shot OrderQuery for EVERY configured pixie table for diff --git a/src/vizier/services/adaptive_export/internal/controller/order_chunk_test.go b/src/vizier/services/adaptive_export/internal/controller/order_chunk_test.go new file mode 100644 index 00000000000..10522afd941 --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/controller/order_chunk_test.go @@ -0,0 +1,226 @@ +/* + * Copyright 2018- The Pixie Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package controller + +// Tests for the CHUNKED + adaptively-subdividing ordered capture path — the +// durable fix for heavy tables (dc_snoop) losing the per-query deadline race under +// the OrderExportAll fan-out. Each chunk is a both-sides bounded pixie query; a +// chunk that still times out under contention is halved down to orderMinChunk. + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "px.dev/pixie/src/vizier/services/adaptive_export/internal/reconcile" +) + +// countingQuerier counts Query calls and can fail the first failN calls with a +// configurable error (to exercise adaptive subdivision) or fail every call. +type countingQuerier struct { + mu sync.Mutex + calls int + rows []map[string]any + failN int // fail the first failN calls, then succeed + failAll bool // fail every call + failErr error // error to return on a failed call +} + +func (q *countingQuerier) Query(context.Context, string) ([]map[string]any, error) { + q.mu.Lock() + defer q.mu.Unlock() + q.calls++ + if q.failAll || q.calls <= q.failN { + return nil, q.failErr + } + return q.rows, nil +} + +func (q *countingQuerier) callCount() int { + q.mu.Lock() + defer q.mu.Unlock() + return q.calls +} + +// recordingRec captures every reconcile Row so a test can assert the ordered path +// records exactly ONE aggregated row per table (not one per chunk). +type recordingRec struct { + mu sync.Mutex + rows []reconcile.Row +} + +func (r *recordingRec) Record(_ context.Context, row reconcile.Row) { + r.mu.Lock() + defer r.mu.Unlock() + r.rows = append(r.rows, row) +} + +func chunkCtl(snk Sink, q PixieQuerier, rec reconcile.Recorder, chunk time.Duration) *Controller { + cfg := defaultCfg() + cfg.OrderChunk = chunk + cfg.Rec = rec + c := New(newFakeTrigger(), snk, cfg, &fakeClock{t: canonicalEventTime}) + if q != nil { + c = c.WithPixieQuerier(q) + } + return c +} + +var errDeadline = errors.New("rpc error: code = DeadlineExceeded desc = context deadline exceeded") + +// A wide window is walked in OrderChunk-sized slices: one pixie query per chunk, +// each writing its rows. 180s window / 60s chunk = 3 bounded queries. +func TestOrderQueryChunksWideWindow(t *testing.T) { + snk := newRecordingSink() + q := &countingQuerier{rows: []map[string]any{{"comm": "whoami"}}} + end := canonicalEventTime + start := end.Add(-180 * time.Second) + if err := chunkCtl(snk, q, reconcile.Nop{}, 60*time.Second). + OrderQuery(oqTarget, "dc_snoop", start, end, "qid-w"); err != nil { + t.Fatalf("OrderQuery: %v", err) + } + if got := q.callCount(); got != 3 { + t.Errorf("want 3 chunk queries for a 180s/60s window, got %d", got) + } + if got := snk.count("dc_snoop"); got != 3 { + t.Errorf("want 3 rows written (one per chunk), got %d", got) + } +} + +// The ordered path records exactly ONE reconcile row per table, aggregating the +// per-chunk read/wrote counts — a forensic dump reads per-table, not per-chunk. +func TestOrderQuerySingleReconcileRowPerTable(t *testing.T) { + snk := newRecordingSink() + rec := &recordingRec{} + q := &countingQuerier{rows: []map[string]any{{"comm": "cat"}}} + end := canonicalEventTime + start := end.Add(-120 * time.Second) // 2 chunks + if err := chunkCtl(snk, q, rec, 60*time.Second). + OrderQuery(oqTarget, "dc_snoop", start, end, "qid-r"); err != nil { + t.Fatalf("OrderQuery: %v", err) + } + if len(rec.rows) != 1 { + t.Fatalf("want 1 aggregated reconcile row, got %d", len(rec.rows)) + } + if rec.rows[0].ReadCount != 2 || rec.rows[0].WroteCount != 2 { + t.Errorf("want aggregated read=2 wrote=2 across chunks, got read=%d wrote=%d", + rec.rows[0].ReadCount, rec.rows[0].WroteCount) + } + if rec.rows[0].WriteErr != "" { + t.Errorf("clean capture must record no error, got %q", rec.rows[0].WriteErr) + } +} + +// A chunk that fails with a TRANSIENT (deadline) error is retried as narrower +// half-spans and recovers — the flaky-capture fix. The querier fails only its first +// call, so the initial full-chunk query subdivides and the halves succeed. +func TestCaptureSpanSubdividesOnTransientError(t *testing.T) { + snk := newRecordingSink() + q := &countingQuerier{rows: []map[string]any{{"comm": "getent"}}, failN: 1, failErr: errDeadline} + end := canonicalEventTime + start := end.Add(-8 * time.Second) // single 60s chunk covers it → one initial query + if err := chunkCtl(snk, q, reconcile.Nop{}, 60*time.Second). + OrderQuery(oqTarget, "dc_snoop", start, end, "qid-t"); err != nil { + t.Fatalf("transient failure must recover via subdivision, got %v", err) + } + // call 1 (8s span) fails → split into two 4s halves (calls 2 & 3), both succeed. + if got := q.callCount(); got != 3 { + t.Errorf("want 3 calls (1 failed + 2 half-span retries), got %d", got) + } + if got := snk.count("dc_snoop"); got != 2 { + t.Errorf("want 2 half-span writes after subdivision, got %d", got) + } +} + +// A NON-transient error (e.g. a missing table) surfaces immediately — +// no wasteful subdivision. Exactly one query per chunk, error returned. +func TestCaptureSpanDoesNotSplitNonTransient(t *testing.T) { + snk := newRecordingSink() + q := &countingQuerier{failAll: true, failErr: errors.New("table 'no_such_table' not found")} + end := canonicalEventTime + start := end.Add(-30 * time.Second) // < one chunk → single chunk + // Uses a KNOWN table with a non-transient failure: the point is the error + // class, not the table. (Previously keyed on dx_bpf, which no longer exists — + // OrderQuery now rejects it before the querier runs, so nothing subdivided.) + err := chunkCtl(snk, q, reconcile.Nop{}, 60*time.Second). + OrderQuery(oqTarget, "dc_snoop", start, end, "qid-n") + if err == nil { + t.Fatal("non-transient error must surface") + } + if got := q.callCount(); got != 1 { + t.Errorf("non-transient error must NOT subdivide; want 1 call, got %d", got) + } +} + +// A persistently-timing-out span subdivides down to orderMinChunk and then surfaces +// the error instead of looping forever — the recursion terminates at the floor. +func TestCaptureSpanTerminatesAtMinChunk(t *testing.T) { + snk := newRecordingSink() + q := &countingQuerier{failAll: true, failErr: errDeadline} + end := canonicalEventTime + start := end.Add(-4 * time.Second) // 4s → 2s → 1s (floor), bounded call count + err := chunkCtl(snk, q, reconcile.Nop{}, 60*time.Second). + OrderQuery(oqTarget, "dc_snoop", start, end, "qid-f") + if err == nil { + t.Fatal("a span that never succeeds must ultimately surface the error") + } + // 4s→(2s,2s)→each (1s,1s): calls = 1 + 2 + 4 = 7, finite. Assert it stayed bounded. + if got := q.callCount(); got == 0 || got > 15 { + t.Errorf("subdivision must terminate at orderMinChunk with a bounded call count, got %d", got) + } +} + +// A persistently-timing-out multi-chunk window must NOT explode into a query storm. +// Without guards, 10 chunks each subdividing 60s→1s ≈ 10×64 = 640 queries against a +// saturated PEM. The depth cap (≤2^3 leaves/chunk) + circuit-breaker (stop +// subdividing after orderBreakerTrip consecutive timeouts) bound it hard. +func TestOrderQueryCircuitBreakerBoundsStorm(t *testing.T) { + snk := newRecordingSink() + q := &countingQuerier{failAll: true, failErr: errDeadline} + end := canonicalEventTime + start := end.Add(-600 * time.Second) // 10 chunks @ 60s, all time out + _ = chunkCtl(snk, q, reconcile.Nop{}, 60*time.Second). + OrderQuery(oqTarget, "dc_snoop", start, end, "qid-storm") + got := q.callCount() + if got > 60 { + t.Errorf("depth-cap + circuit-breaker must bound the storm; got %d calls (want <=60, ungrafted would be ~640)", got) + } + if got < 10 { + t.Errorf("must still attempt each of the 10 chunks at least once; got %d", got) + } +} + +// A healthy PEM (queries succeed) must NOT trip the breaker — subdivision stays +// available for genuinely-oversized windows. A querier that fails ONCE then succeeds +// still subdivides and recovers (breaker reset by the success). +func TestCircuitBreakerResetsOnSuccess(t *testing.T) { + snk := newRecordingSink() + q := &countingQuerier{rows: []map[string]any{{"comm": "cat"}}, failN: 1, failErr: errDeadline} + end := canonicalEventTime + start := end.Add(-8 * time.Second) + if err := chunkCtl(snk, q, reconcile.Nop{}, 60*time.Second). + OrderQuery(oqTarget, "dc_snoop", start, end, "qid-reset"); err != nil { + t.Fatalf("single transient failure must recover (breaker must not latch); got %v", err) + } + if snk.count("dc_snoop") < 1 { + t.Errorf("recovered subdivision must write rows; got %d", snk.count("dc_snoop")) + } +} diff --git a/src/vizier/services/adaptive_export/internal/pxl/compile.go b/src/vizier/services/adaptive_export/internal/pxl/compile.go index cdd21c5313c..82d312c36b9 100644 --- a/src/vizier/services/adaptive_export/internal/pxl/compile.go +++ b/src/vizier/services/adaptive_export/internal/pxl/compile.go @@ -71,8 +71,6 @@ func CompilePassthrough(table string, window time.Duration) (string, error) { // PodEnrichPxL. var darkVectorTables = map[string]bool{ "dc_snoop": true, "creds_change": true, - "dx_vfs_events": true, "dx_unlink": true, "dx_dlookup": true, - "dx_mprotect": true, "dx_bpf": true, "dx_ptrace": true, } // IsDarkVector reports whether table is a pid-keyed dx tracepoint table. @@ -95,8 +93,18 @@ const darkProcStatsWindow = "-2m" // aggregator/kelvin asid, not the per-PEM asid of the data, so pid+asid never // matches. df.pod here is the BARE pod name (proc.ctx['pod']). Best-effort: blank // for host/transient pids (correct — no pod). +// NodeHostname is the AE pod's k8s node (NODE_NAME), stamped as a literal onto the +// pid-keyed dark tables. The process_stats-merge node resolution misses transient +// attack pids (most dc_snoop rows blank), leaving the table un-px-readable; AE is +// node-local, so its node is the correct shard key for every row it captures. +var NodeHostname string + func PodEnrichPxL(table string) string { if darkVectorTables[table] { + hostLine := "df.hostname = df.node\n" + if NodeHostname != "" { + hostLine = "df.hostname = '" + escapePxL(NodeHostname) + "'\n" + } // process_stats is the COST of the dark-table merge: a busy node samples // every live pid every ~10-30s, so a wide window is a huge scan that // competes with the fast native-table queries for the shared query-slot @@ -108,12 +116,23 @@ func PodEnrichPxL(table string) string { return "proc = px.DataFrame(table='process_stats', start_time='" + darkProcStatsWindow + "')\n" + "proc.pod = proc.ctx['pod']\n" + "proc.namespace = proc.ctx['namespace']\n" + + // node resolved from the SAME process_stats upid the pod/ns come from, so + // dark-vector rows (dc_snoop et al.) carry hostname and become px-readable + // (#136). Transient attack pids that miss process_stats resolve blank — + // the same accepted limitation as pod/ns above. + "proc.node = px.upid_to_node_name(proc.upid)\n" + "proc.pid = px.upid_to_pid(proc.upid)\n" + - "proc = proc.groupby(['pod', 'namespace', 'pid']).agg()\n" + - "df = df.merge(proc, how='left', left_on=['pid'], right_on=['pid'], suffixes=['', '_x'])\n" + "proc = proc.groupby(['pod', 'namespace', 'node', 'pid']).agg()\n" + + "df = df.merge(proc, how='left', left_on=['pid'], right_on=['pid'], suffixes=['', '_x'])\n" + + hostLine } return "df.namespace = px.upid_to_namespace(df.upid)\n" + - "df.pod = px.upid_to_pod_name(df.upid)\n" + "df.pod = px.upid_to_pod_name(df.upid)\n" + + // hostname = the capture node — the leading ORDER BY column on every + // socket_tracer table. AE left it empty (only stack_trace stamped it), so + // px reads of these tables (and the #136 order-UUID views) could not filter + // by hostname and the pushdown prefix (hostname,event_time) was unusable. + "df.hostname = px.upid_to_node_name(df.upid)\n" } // Render fills a CompilePassthrough template with the precise [sliceStart, diff --git a/src/vizier/services/adaptive_export/internal/pxl/compile_test.go b/src/vizier/services/adaptive_export/internal/pxl/compile_test.go index e5a16cc3e56..8d0065aab54 100644 --- a/src/vizier/services/adaptive_export/internal/pxl/compile_test.go +++ b/src/vizier/services/adaptive_export/internal/pxl/compile_test.go @@ -95,7 +95,7 @@ func TestPodEnrichPxL_DarkVsNative(t *testing.T) { if !strings.Contains(native, "px.upid_to_pod_name(df.upid)") { t.Errorf("native table must resolve pod via upid: %q", native) } - for _, tbl := range []string{"dc_snoop", "creds_change", "dx_bpf", "dx_ptrace"} { + for _, tbl := range []string{"dc_snoop", "creds_change"} { q := PodEnrichPxL(tbl) if strings.Contains(q, "df.upid") { t.Errorf("%s (pid-keyed) must NOT reference df.upid: %q", tbl, q) diff --git a/src/vizier/services/adaptive_export/internal/pxl/queryfor.go b/src/vizier/services/adaptive_export/internal/pxl/queryfor.go index 4f9d8d6d37c..097d2fbc3e7 100644 --- a/src/vizier/services/adaptive_export/internal/pxl/queryfor.go +++ b/src/vizier/services/adaptive_export/internal/pxl/queryfor.go @@ -63,21 +63,17 @@ func QueryFor(table string, t anomaly.Target, sliceStart, sliceEnd, now time.Tim var b strings.Builder b.WriteString(pxSetMaxRows) b.WriteString("import px\n") - b.WriteString("df = px.DataFrame(table='" + pixieSourceFor(table) + "', start_time='" + relStart + "')\n") + // Bound the source scan on both sides; the df.time_ < sliceEnd filter trims the exact upper bound. + dfArgs := "table='" + pixieSourceFor(table) + "', start_time='" + relStart + "'" + if relEnd := relEndBound(now, sliceEnd); relEnd != "" { + dfArgs += ", end_time='" + relEnd + "'" + } + b.WriteString("df = px.DataFrame(" + dfArgs + ")\n") b.WriteString("df = df[df.time_ >= px.int64_to_time(" + strconv.FormatInt(sliceStart.UnixNano(), 10) + ")]\n") b.WriteString("df = df[df.time_ < px.int64_to_time(" + strconv.FormatInt(sliceEnd.UnixNano(), 10) + ")]\n") - // Native tables: px.upid_to_pod_name returns "/" (carnot: - // metadata_ops.h UPIDToPodNameUDF::Exec → absl::Substitute("$0/$1", ns, name)), - // not the bare pod name. Dark-vector tracepoint tables (pid-keyed) resolve pod - // via a process_stats pid-merge instead and yield a BARE pod name (dx#126). + // px.upid_to_pod_name yields "/"; dark-vector tables resolve pod via a pid-merge (bare name). if table == "stack_trace" { - // stack_trace is the CANONICAL native continuous profiler (stack_traces.beta, - // upid-keyed — NOT a pid tracepoint, so NOT a dark-vector pid-merge). Resolve - // pod/namespace/container/hostname exactly like the export preset - // (script/presets/stack_trace.pxl) and stamp event_time = time_ so the CH - // stack_trace row is complete. df.ctx['pod'] is the NAMESPACED "/" - // key (verified live), so the pod filter is namespaced — same as the native - // upid_to_pod_name path below. + // Native profiler (stack_traces.beta): resolve pod/ns/container from ctx, stamp event_time. b.WriteString("df.namespace = df.ctx['namespace']\n") b.WriteString("df.pod = df.ctx['pod']\n") b.WriteString("df.container = df.ctx['container']\n") @@ -94,22 +90,11 @@ func QueryFor(table string, t anomaly.Target, sliceStart, sliceEnd, now time.Tim } } } else if IsDarkVector(table) { - // Dark-vector tracepoints emit a RAW kernel pid. The malignant transient - // pids an incident actually produces — an attack's whoami/cat/getent - // children — are too short-lived to land in process_stats, so their - // pod/namespace resolves BLANK; a pod (or even namespace) filter drops - // exactly the evidence, which is why the dark tables came back empty. - // The AE is node-local (pem-direct → the node's own PEM), so the query is - // already scoped to the alert's node. - // - // ORDER MATTERS: drop the infra/self comms FIRST (env-driven, no recompile), - // THEN do the process_stats pid-merge. The node's dark stream is huge - // (Formatter/vector/runc/... thousands of rows per window); merging every - // one against process_stats is the query that timed out and silently - // dropped dc_snoop. Filtering comm first shrinks the merge to the handful - // of workload rows (bash/redis/whoami/cat), so the dark capture completes. + // Node-scoped (transient attack pids resolve blank ns, so no pod filter). Drop + // own-stack comms before the pid-merge to keep it cheap, then drop infra namespaces. b.WriteString(darkCommExclusion(table)) b.WriteString(PodEnrichPxL(table)) + b.WriteString(darkNamespaceExclusion()) } else { b.WriteString(PodEnrichPxL(table)) if t.Namespace != "" { @@ -130,11 +115,16 @@ func QueryFor(table string, t anomaly.Target, sliceStart, sliceEnd, now time.Tim return b.String(), nil } -// pixieSourceFor returns the Pixie table a builtin is sourced FROM when it -// differs from the ClickHouse table it is written TO. stack_trace is written to -// CH as 'stack_trace' but sourced from the CANONICAL native continuous profiler -// 'stack_traces.beta' — the always-on Pixie profiler, NOT an AE-invented table. -// (Dotted-name DataFrames compile fine in a direct query; verified live.) +// relEndBound returns a relative end_time ("-s"), or "" when sliceEnd is at/after now. +func relEndBound(now, sliceEnd time.Time) string { + gap := now.Sub(sliceEnd) + if gap < time.Second { + return "" // at/after now → default end_time (scan to now) + } + return "-" + strconv.FormatInt(int64(gap/time.Second), 10) + "s" +} + +// pixieSourceFor maps a CH table to the pixie table it's read from (stack_trace ← stack_traces.beta). func pixieSourceFor(table string) string { if table == "stack_trace" { return "stack_traces.beta" @@ -142,18 +132,13 @@ func pixieSourceFor(table string) string { return table } -// darkVectorHasComm lists the dark-vector tables that carry a `comm` column, so -// the infra-comm exclusion only emits for those (stack_trace is upid-only). +// Dark-vector tables carrying a comm column (so the comm exclusion applies). var darkVectorHasComm = map[string]bool{ - "dc_snoop": true, "creds_change": true, "dx_vfs_events": true, - "dx_unlink": true, "dx_dlookup": true, "dx_mprotect": true, - "dx_bpf": true, "dx_ptrace": true, + "dc_snoop": true, "creds_change": true, } -// darkExcludeCommsDefault is the node's own infra/self comms dropped from the -// node-scoped dark capture so the workload's activity stands out. Overridable at -// runtime via DC_SNOOP_EXCLUDE_COMMS (csv) — a process can be added without a -// recompile. Kept in sync with script.presets defaultExcludeComms. +// Own-stack + node/system comms dropped from the node-scoped dark capture; workload +// comms (redis-*, etc.) are never listed. Override via DC_SNOOP_EXCLUDE_COMMS (csv). var darkExcludeCommsDefault = []string{ "pem", "kelvin", "containerd", "containerd-shim", "runc", "node-agent", "runc:[2:INIT]", "runc:[1:CHILD]", @@ -165,10 +150,26 @@ var darkExcludeCommsDefault = []string{ "ConfigReloader", "clickhouse-oper", "Formatter", "(setup.sh)", "cmd", "vector-worker", "metrics-server", "local-path-prov", "portmap", "(udev-worker)", "systemd-resolve", "systemd-timesyn", + "systemd-udevd", "systemd-sysctl", "host-local", "bridge", "flannel", + "loopback", "bandwidth", "dbus-daemon", "mount", "umount", "tailscaled", + "grpc_health_pro", "kubevuln", "opm", "(spawn)", "kube-proxy", + "pause", "systemd-logind", +} + +// Kernel-thread families whose names carry a variable suffix (kworker/u8:3) that +// exact match misses; dropped via px.contains. +var darkExcludeCommSubstrings = []string{ + "kworker", "ksoftirqd", "migration", "rcu_", "kthreadd", "kdevtmpfs", + "kcompactd", "khugepaged", "kswapd", "watchdog", "cpuhp", "ksmd", "irq/", +} + +// Infra namespaces dropped from the node-scoped dark capture. Blank-namespace rows +// (transient attack children) survive. Override via DC_SNOOP_EXCLUDE_NAMESPACES. +var darkExcludeNamespacesDefault = []string{ + "pl", "honey", "px-operator", "olm", "clickhouse", "socdemo", "socdemo-ch", + "kube-system", "kube-public", "kube-node-lease", "local-path-storage", } -// darkCommExclusion builds the infra-comm drop filter for a dark-vector table -// that has a comm column. Returns "" for comm-less tables (stack_trace). func darkCommExclusion(table string) string { if !darkVectorHasComm[table] { return "" @@ -186,6 +187,26 @@ func darkCommExclusion(table string) string { for _, c := range comms { b.WriteString("df = df[df.comm != '" + escapePxL(c) + "']\n") } + for _, s := range darkExcludeCommSubstrings { + b.WriteString("df = df[px.logicalNot(px.contains(df.comm, '" + escapePxL(s) + "'))]\n") + } + return b.String() +} + +func darkNamespaceExclusion() string { + nss := darkExcludeNamespacesDefault + if v := strings.TrimSpace(os.Getenv("DC_SNOOP_EXCLUDE_NAMESPACES")); v != "" { + nss = nil + for _, s := range strings.Split(v, ",") { + if s = strings.TrimSpace(s); s != "" { + nss = append(nss, s) + } + } + } + var b strings.Builder + for _, ns := range nss { + b.WriteString("df = df[df.namespace != '" + escapePxL(ns) + "']\n") + } return b.String() } diff --git a/src/vizier/services/adaptive_export/internal/pxl/queryfor_test.go b/src/vizier/services/adaptive_export/internal/pxl/queryfor_test.go index 562ea794cc0..6ca0023fd61 100644 --- a/src/vizier/services/adaptive_export/internal/pxl/queryfor_test.go +++ b/src/vizier/services/adaptive_export/internal/pxl/queryfor_test.go @@ -18,6 +18,7 @@ package pxl import ( "errors" + "strconv" "strings" "testing" "time" @@ -257,14 +258,15 @@ func TestEscapePxL_TableDriven(t *testing.T) { // df = df[df.time_ < ...] 1 // df.namespace = px.upid_to_namespace(...) 1 // df.pod = px.upid_to_pod_name(...) 1 +// df.hostname = px.upid_to_node_name(...) 1 // df = df[df.namespace == '...'] 1 // df = df[df.pod == '...'] 1 // px.display(df, '...') 1 -// (trailing newline → empty 11th split) 1 +// (trailing newline → empty 12th split) 1 // -// Total: 10 statements + trailing empty == strings.Split == 11 entries. +// Total: 11 statements + trailing empty == strings.Split == 12 entries. func TestQueryFor_RejectsInjectionInTargetFields(t *testing.T) { - const wantLines = 11 + const wantLines = 12 cases := []struct { name string @@ -336,7 +338,105 @@ func TestQueryFor_PodOnlyRegexEscapesQuoteMetaInjection(t *testing.T) { if err != nil { t.Fatalf("QueryFor: %v", err) } - if strings.Contains(q, "exec(") || strings.Count(q, "\n") > 9 { + if strings.Contains(q, "exec(") || strings.Count(q, "\n") > 10 { t.Fatalf("pod-only path injection succeeded:\n%s", q) } } + +// TestQueryFor_EndTimeBoundsPastWindow — a window whose upper bound is in the past +// must emit a relative end_time so the PEM scan is bounded on BOTH sides (not +// [sliceStart, now]). The precise upper bound is still enforced by the df.time_ < +// nanos post-filter. +func TestQueryFor_EndTimeBoundsPastWindow(t *testing.T) { + // sliceEnd 2 minutes before now → end_time must appear. + end := fixedNow.Add(-2 * time.Minute) + start := fixedNow.Add(-7 * time.Minute) + q, err := QueryFor("dc_snoop", target, start, end, fixedNow) + if err != nil { + t.Fatalf("QueryFor: %v", err) + } + if !strings.Contains(q, "end_time='-120s'") { + t.Fatalf("past-window query must bound the source scan with end_time='-120s'; got:\n%s", q) + } + // exact upper bound still trimmed precisely in nanos. + if !strings.Contains(q, "df = df[df.time_ < px.int64_to_time("+ + strconv.FormatInt(end.UnixNano(), 10)+")]") { + t.Fatalf("precise nanos upper-bound filter must remain; got:\n%s", q) + } +} + +// TestQueryFor_NoEndTimeAtLiveEdge — a window that reaches now must NOT emit +// end_time (scan to the live edge), preserving the pre-chunking behavior for the +// most-recent slice. +func TestQueryFor_NoEndTimeAtLiveEdge(t *testing.T) { + q, err := QueryFor("dc_snoop", target, fixedNow.Add(-1*time.Minute), fixedNow, fixedNow) + if err != nil { + t.Fatalf("QueryFor: %v", err) + } + if strings.Contains(q, "end_time=") { + t.Fatalf("live-edge window must not bound end_time; got:\n%s", q) + } +} + +// TestQueryFor_DarkNamespaceExclusion — the node-scoped dark capture (dc_snoop) +// must drop infra namespaces (pl, kube-system, …) while KEEPING blank-namespace +// transient rows (the attack's short-lived children). Mirrors the shipped preset. +func TestQueryFor_DarkNamespaceExclusion(t *testing.T) { + q, err := QueryFor("dc_snoop", target, fixedStart, fixedEnd, fixedNow) + if err != nil { + t.Fatalf("QueryFor: %v", err) + } + // namespace drops present for infra + for _, ns := range []string{"pl", "kube-system", "clickhouse"} { + if !strings.Contains(q, "df = df[df.namespace != '"+ns+"']") { + t.Errorf("dark capture must drop infra namespace %q; got:\n%s", ns, q) + } + } + // must NOT pin to the alert pod's namespace (node-scoped keeps blank + other workloads) + if strings.Contains(q, "df = df[df.namespace == '") { + t.Errorf("dark capture must not pin df.namespace ==; got:\n%s", q) + } + // host/CNI comm drops present + for _, c := range []string{"host-local", "systemd-udevd", "tailscaled", "kubevuln"} { + if !strings.Contains(q, "df = df[df.comm != '"+c+"']") { + t.Errorf("dark capture must drop host/CNI comm %q; got:\n%s", c, q) + } + } +} + +// TestQueryFor_DarkNamespaceExclusion_EnvOverride — DC_SNOOP_EXCLUDE_NAMESPACES +// replaces the default list. +func TestQueryFor_DarkNamespaceExclusion_EnvOverride(t *testing.T) { + t.Setenv("DC_SNOOP_EXCLUDE_NAMESPACES", "foo,bar") + q, err := QueryFor("dc_snoop", target, fixedStart, fixedEnd, fixedNow) + if err != nil { + t.Fatalf("QueryFor: %v", err) + } + if !strings.Contains(q, "df = df[df.namespace != 'foo']") || !strings.Contains(q, "df = df[df.namespace != 'bar']") { + t.Errorf("env override must emit foo/bar drops; got:\n%s", q) + } + if strings.Contains(q, "df = df[df.namespace != 'pl']") { + t.Errorf("env override must REPLACE the default (no 'pl'); got:\n%s", q) + } +} + +// dc_snoop drops kernel-thread families (variable suffix) via px.logicalNot(px.contains), +// keeps workload comms (redis-server), and doesn't pin the alert pod's namespace. +func TestQueryFor_DarkCommSubstringExclusion(t *testing.T) { + q, err := QueryFor("dc_snoop", target, fixedStart, fixedEnd, fixedNow) + if err != nil { + t.Fatalf("QueryFor: %v", err) + } + for _, sub := range []string{"kworker", "ksoftirqd", "rcu_"} { + want := "df = df[px.logicalNot(px.contains(df.comm, '" + sub + "'))]" + if !strings.Contains(q, want) { + t.Errorf("want kernel-thread drop %q; got:\n%s", want, q) + } + } + if strings.Contains(q, "df.comm != 'redis-server'") || strings.Contains(q, "df.comm, 'redis") { + t.Errorf("workload comm redis-* must NOT be excluded; got:\n%s", q) + } + if !strings.Contains(q, "df = df[df.comm != 'pause']") { + t.Errorf("want exact drop of 'pause'; got:\n%s", q) + } +} diff --git a/src/vizier/services/adaptive_export/internal/pxl/tables.go b/src/vizier/services/adaptive_export/internal/pxl/tables.go index d04f107fa2f..6bad717007d 100644 --- a/src/vizier/services/adaptive_export/internal/pxl/tables.go +++ b/src/vizier/services/adaptive_export/internal/pxl/tables.go @@ -81,12 +81,6 @@ var builtinTables = []TableSpec{ {Name: "dc_snoop", Protocol: "tracepoint (dentry lookup, V1/V2)"}, {Name: "creds_change", Protocol: "tracepoint (commit_creds priv-esc, V7)"}, {Name: "stack_trace", Protocol: "profiler (stack_traces.beta, V9)"}, - {Name: "dx_vfs_events", Protocol: "tracepoint"}, - {Name: "dx_unlink", Protocol: "tracepoint"}, - {Name: "dx_dlookup", Protocol: "tracepoint"}, - {Name: "dx_mprotect", Protocol: "tracepoint"}, - {Name: "dx_bpf", Protocol: "tracepoint"}, - {Name: "dx_ptrace", Protocol: "tracepoint"}, } // Registry is the extension surface for users to register their own diff --git a/src/vizier/services/adaptive_export/internal/pxl/tables_test.go b/src/vizier/services/adaptive_export/internal/pxl/tables_test.go index d0d8a38ed54..e410c61149c 100644 --- a/src/vizier/services/adaptive_export/internal/pxl/tables_test.go +++ b/src/vizier/services/adaptive_export/internal/pxl/tables_test.go @@ -27,11 +27,12 @@ import ( // kafka_events.beta, amqp_events, mux_events, tls_events, conn_stats). // Update this guard if the spec adds / removes a table. // 13 socket_tracer tables + 9 dark-vector tables: dc_snoop (V1/V2), creds_change -// (V7), stack_trace (V9) — canonical schemas, active — plus dx_vfs_events, -// dx_unlink, dx_dlookup, dx_mprotect, dx_bpf, dx_ptrace reserved for the bpftraces -// still to be written (entlein/dx#126). Update this guard if the spec changes. +// (V7), stack_trace (V9) — canonical schemas, active. The six reserved dx_* +// tracepoint tables (dx_vfs_events/dx_unlink/dx_dlookup/dx_mprotect/dx_bpf/ +// dx_ptrace) were removed: no bpftrace was ever deployed for them, so every +// steer-all query on them could only fail PxL compile. Update if the spec changes. func TestBuiltinTables_Count(t *testing.T) { - const want = 22 + const want = 16 if got := len(builtinTables); got != want { t.Fatalf("builtinTables = %d entries, want %d", got, want) } diff --git a/src/vizier/services/adaptive_export/internal/script/presets/dc_snoop_deploy.pxl b/src/vizier/services/adaptive_export/internal/script/presets/dc_snoop_deploy.pxl index bf8333f0a4b..bfe0d4f4f3c 100644 --- a/src/vizier/services/adaptive_export/internal/script/presets/dc_snoop_deploy.pxl +++ b/src/vizier/services/adaptive_export/internal/script/presets/dc_snoop_deploy.pxl @@ -23,8 +23,28 @@ import pxtrace +# HEADER MODE ONLY — no BTF-only type here. +# PROVEN: mixing BTF-generated types with kernel includes in ONE program breaks +# it — the BTF struct nameidata layout is not the one assumed below, so the +# lookup_fast read yields garbage while the probe reports healthy. +# SUSPECTED, not proven: a BTF-mode program running earlier in the SAME PEM +# process writes __btf_generated_header.h and makes later header-mode compiles +# fail on kernel_cap_struct/__be64 redefinitions. Best explanation for compiles +# that fail only after other tracepoints have run, but not established. +# Includes can also clash with EACH OTHER (qspinlock redefinition seen when +# extending this set), so keep the include set minimal and re-test any change +# on a PEM that has compiled nothing else. +# +# A skipped probe does NOT fail the deploy: bpftrace warns "could not attach +# probe ... skipping" on the PEM's stderr and continues, so the tracepoint +# reports RUNNING with half the program dead. lookup_fast also cannot be +# re-attached inside a running PEM, so this cannot be repaired by re-upserting +# — confirm capture per probe type (t=R and t=M), and recover with a fresh PEM. program = """ #include +#include +#include +#include // from fs/namei.c: struct nameidata { @@ -33,21 +53,32 @@ struct nameidata { // [...] }; -// comment out this block to avoid showing hits: +// 4026531836 = PROC_PID_INIT_INO, the init pid namespace (constant across +// kernels). Guarding in kprobe context drops host processes before the printf +// and before the map write, so they never reach the perf ring at all. Without +// it the ring overflows and takes the rows we WANT down with the noise. kprobe:lookup_fast, kprobe:lookup_fast.constprop.* { - $nd = (struct nameidata *)arg0; - printf("time_:%llu pid:%d comm:%s t:%s file:%s", - nsecs, pid, comm, "R", str($nd->last.name)); + $t = (struct task_struct *)curtask; + if ($t->nsproxy->pid_ns_for_children->ns.inum != 4026531836) { + $nd = (struct nameidata *)arg0; + printf("time_:%llu pid:%d comm:%s t:%s file:%s", + nsecs, pid, comm, "R", str($nd->last.name)); + } } kprobe:d_lookup { - $name = (struct qstr *)arg1; - @fname[tid] = $name->name; + $t = (struct task_struct *)curtask; + if ($t->nsproxy->pid_ns_for_children->ns.inum != 4026531836) { + $name = (struct qstr *)arg1; + @fname[tid] = $name->name; + } } +// Deliberately unguarded: the map is only ever populated by the guarded kprobe +// above, so a host process can never reach this block. kretprobe:d_lookup /@fname[tid]/ { diff --git a/src/vizier/services/adaptive_export/internal/sink/clickhouse.go b/src/vizier/services/adaptive_export/internal/sink/clickhouse.go index 11b0aed7202..cb108b2dde0 100644 --- a/src/vizier/services/adaptive_export/internal/sink/clickhouse.go +++ b/src/vizier/services/adaptive_export/internal/sink/clickhouse.go @@ -289,7 +289,9 @@ func (s *ClickHouseHTTP) Write(ctx context.Context, rows []AttributionRow) error return fmt.Errorf("sink: encode %d attribution rows: %w", len(rows), err) } if _, err := s.c.Insert(ctx, - fmt.Sprintf("INSERT INTO %s.adaptive_attribution FORMAT JSONEachRow", s.cfg.Database), + // One row per anomaly snapshot -> one part per INSERT; merges cannot keep up + // (34k parts for 15 KiB observed). wait_for_async_insert keeps it synchronous. + fmt.Sprintf("INSERT INTO %s.adaptive_attribution SETTINGS async_insert=1, wait_for_async_insert=1 FORMAT JSONEachRow", s.cfg.Database), body, chhttp.InsertOptions{FailLoud: true}); err != nil { return fmt.Errorf("sink: POST: %w", err) } diff --git a/src/vizier/services/adaptive_export/internal/sink/clickhouse_test.go b/src/vizier/services/adaptive_export/internal/sink/clickhouse_test.go index 0eb42adcc76..1d9d2c03ff0 100644 --- a/src/vizier/services/adaptive_export/internal/sink/clickhouse_test.go +++ b/src/vizier/services/adaptive_export/internal/sink/clickhouse_test.go @@ -70,7 +70,7 @@ func TestSink_Write_PostsCorrectQueryAndBody(t *testing.T) { if err := s.Write(context.Background(), []AttributionRow{row}); err != nil { t.Fatalf("Write: %v", err) } - want := "INSERT INTO forensic_db.adaptive_attribution FORMAT JSONEachRow" + want := "INSERT INTO forensic_db.adaptive_attribution SETTINGS async_insert=1, wait_for_async_insert=1 FORMAT JSONEachRow" if gotQuery != want { t.Fatalf("query = %q, want %q", gotQuery, want) } diff --git a/src/vizier/services/adaptive_export/internal/streaming/BUILD.bazel b/src/vizier/services/adaptive_export/internal/streaming/BUILD.bazel index 94823988493..73b8a81b55d 100644 --- a/src/vizier/services/adaptive_export/internal/streaming/BUILD.bazel +++ b/src/vizier/services/adaptive_export/internal/streaming/BUILD.bazel @@ -24,6 +24,7 @@ go_library( visibility = ["//src/vizier/services/adaptive_export:__subpackages__"], deps = [ "//src/vizier/services/adaptive_export/internal/activeset", + "//src/vizier/services/adaptive_export/internal/pxl", "//src/vizier/services/adaptive_export/internal/reconcile", "@com_github_sirupsen_logrus//:logrus", ], diff --git a/src/vizier/services/adaptive_export/internal/streaming/scanner.go b/src/vizier/services/adaptive_export/internal/streaming/scanner.go index d77941e886e..251415133f8 100644 --- a/src/vizier/services/adaptive_export/internal/streaming/scanner.go +++ b/src/vizier/services/adaptive_export/internal/streaming/scanner.go @@ -27,6 +27,7 @@ import ( log "github.com/sirupsen/logrus" "px.dev/pixie/src/vizier/services/adaptive_export/internal/activeset" + "px.dev/pixie/src/vizier/services/adaptive_export/internal/pxl" "px.dev/pixie/src/vizier/services/adaptive_export/internal/reconcile" ) @@ -275,9 +276,20 @@ func (s *TableScanner) buildPxL(f Filter) string { b.WriteString(pxSetMaxRows) b.WriteString("import px\n") b.WriteString("df = px.DataFrame(table='" + s.cfg.Table + "', start_time='" + relStart + "')\n") - b.WriteString("df.namespace = px.upid_to_namespace(df.upid)\n") - b.WriteString("df.pod = px.upid_to_pod_name(df.upid)\n") - if f.Mode == FilterModeAllowlist && len(f.Pods) > 0 { + // Pod/ns enrichment via the SINGLE dark-aware builder (pxl.PodEnrichPxL): native + // tables carry upid → direct px.upid_to_* resolution; dark-vector tracepoint + // tables (dc_snoop/creds_change/dx_*) carry a raw kernel pid with NO upid, so pod + // is resolved by the process_stats pid-merge. #89 restored dark-vector export but + // only in the retention builder (pxl/queryfor.go); this streaming scanner — the + // path dx steers EXCLUSIVELY — kept hardcoding px.upid_to_*(df.upid) and threw + // "Column 'upid' not found" for every dark table. Reuse the one builder here. + b.WriteString(pxl.PodEnrichPxL(s.cfg.Table)) + // The pod-allowlist filter is NATIVE-tables only. Dark-vector rows are + // node-scoped: a transient attack pid (sh→whoami) resolves a BLANK pod, so a + // pod-equality filter would delete exactly the malignant evidence (entlein/dx#129). + // Capture the dark tables node-wide and let dx's process forest scope by lineage + // (mirrors pxl/queryfor.go's dark-vector node-scoped branch). + if !pxl.IsDarkVector(s.cfg.Table) && f.Mode == FilterModeAllowlist && len(f.Pods) > 0 { // Allowlist clause. PxL syntax exploration (2026-05-17): // - `or` between equalities → "Expected two arguments to 'or'" // - `|` between equalities → "Operator '|' not handled" diff --git a/src/vizier/services/adaptive_export/internal/streaming/scanner_test.go b/src/vizier/services/adaptive_export/internal/streaming/scanner_test.go index 0e5a6b9ac1f..4828b7f5f9b 100644 --- a/src/vizier/services/adaptive_export/internal/streaming/scanner_test.go +++ b/src/vizier/services/adaptive_export/internal/streaming/scanner_test.go @@ -240,3 +240,26 @@ func TestScanner_QueriesOnNonEmptyFilter(t *testing.T) { t.Fatalf("writer received no rows; expected at least 1") } } + +// TestScanner_DarkVectorPidMergeNodeScoped locks the #89-completion fix (the +// dx-steered streaming path): a dark-vector tracepoint table (dc_snoop) carries a +// raw kernel pid with NO upid, so buildPxL must resolve pod via the process_stats +// pid-merge (pxl.PodEnrichPxL) — NOT px.upid_to_pod_name(df.upid) which threw +// "Column 'upid' not found" — and must be NODE-SCOPED (no pod allowlist: transient +// attack pids resolve a blank pod, so a pod filter would drop the malignant +// evidence; dx's process forest scopes by lineage). +func TestScanner_DarkVectorPidMergeNodeScoped(t *testing.T) { + cfg := ScannerConfig{Table: "dc_snoop"}.defaulted() + s := &TableScanner{cfg: cfg} + f := Filter{Mode: FilterModeAllowlist, Pods: []activeset.Key{{Namespace: "redis", Pod: "redis-0"}}} + pxl := s.buildPxL(f) + if strings.Contains(pxl, "px.upid_to_pod_name(df.upid)") || strings.Contains(pxl, "px.upid_to_namespace(df.upid)") { + t.Fatalf("dark table must NOT resolve pod via df.upid (no upid column):\n%s", pxl) + } + if !strings.Contains(pxl, "table='process_stats'") || !strings.Contains(pxl, "left_on=['pid']") { + t.Fatalf("dark table must pid-merge process_stats for pod:\n%s", pxl) + } + if strings.Contains(pxl, "px.regex_match") { + t.Fatalf("dark table must be node-scoped (no pod allowlist filter):\n%s", pxl) + } +} diff --git a/src/vizier/services/adaptive_export/internal/trigger/BUILD.bazel b/src/vizier/services/adaptive_export/internal/trigger/BUILD.bazel index 0445d9211f4..8ffeb29b8e4 100644 --- a/src/vizier/services/adaptive_export/internal/trigger/BUILD.bazel +++ b/src/vizier/services/adaptive_export/internal/trigger/BUILD.bazel @@ -21,6 +21,8 @@ go_library( name = "trigger", srcs = [ "clickhouse.go", + "dedup.go", + "metrics.go", "watermark.go", ], importpath = "px.dev/pixie/src/vizier/services/adaptive_export/internal/trigger", @@ -28,6 +30,8 @@ go_library( deps = [ "//src/vizier/services/adaptive_export/internal/chhttp", "//src/vizier/services/adaptive_export/internal/kubescape", + "@com_github_prometheus_client_golang//prometheus", + "@com_github_prometheus_client_golang//prometheus/promauto", "@com_github_sirupsen_logrus//:logrus", ], ) @@ -37,10 +41,15 @@ pl_go_test( srcs = [ "clickhouse_internal_test.go", "clickhouse_test.go", + "dedup_test.go", "fingerprint_bench_test.go", + "lookback_test.go", "oracle_test.go", "watermark_test.go", ], embed = [":trigger"], - deps = ["//src/vizier/services/adaptive_export/internal/kubescape"], + deps = [ + "//src/vizier/services/adaptive_export/internal/kubescape", + "@com_github_prometheus_client_golang//prometheus/testutil", + ], ) diff --git a/src/vizier/services/adaptive_export/internal/trigger/clickhouse.go b/src/vizier/services/adaptive_export/internal/trigger/clickhouse.go index 80e03f3b942..1f548e37cc2 100644 --- a/src/vizier/services/adaptive_export/internal/trigger/clickhouse.go +++ b/src/vizier/services/adaptive_export/internal/trigger/clickhouse.go @@ -84,13 +84,48 @@ type Config struct { // hardcoded to 5s, which under any backlog caused every poll to // time out mid-stream → watermark never advanced. HTTPTimeout time.Duration + + // Lookback (#97 / F8 / AE-9): when > 0, each poll re-scans + // [watermark-Lookback, ∞) instead of the strict [watermark, ∞) and + // dedupes re-seen rows by content fingerprint, so an out-of-order / + // clock-skewed / restart-buried row that lands within the window is + // still processed EXACTLY ONCE (no drop, no duplicate). Rows below + // watermark-Lookback stay dropped — the documented bound. 0 keeps + // the legacy strict high-water-mark behavior (anything below the + // watermark is dropped forever). Production default is 300s via + // ADAPTIVE_TRIGGER_LOOKBACK_SEC in cmd/main.go; the zero value here + // is legacy so existing callers/tests are unchanged. + Lookback time.Duration + + // MaxSkew is the wall-clock poison clamp (#97): a row whose + // NORMALIZED event_time is more than MaxSkew past now is still + // emitted once, but never advances the watermark, so a single + // corrupted/oversized timestamp (the 1.78e18 leftover of loadtest + // E8) cannot jump the cursor past all real data and silently halt + // the trigger. Also applied to the persisted watermark at load, so + // an ALREADY-poisoned cursor self-recovers on restart without the + // manual `ALTER TABLE trigger_watermark DELETE`. <=0 → 1h. + MaxSkew time.Duration + + // DedupMaxEntries caps the lookback dedup set (memory bound). An + // in-window fingerprint evicted by capacity may re-emit once, so + // size it >= the max rows expected per lookback window. + // <=0 → 4*PollLimit. + DedupMaxEntries int } +// defaultMaxSkew is the default wall-clock poison-clamp bound (#97): +// an event_time more than this far in the future is implausible. +const defaultMaxSkew = time.Hour + // ClickHouseHTTP polls forensic_db.
over the ClickHouse HTTP // interface, scoped to a single node. type ClickHouseHTTP struct { cfg Config client *http.Client + // now is the wall clock used by the poison clamp (#97). + // Injectable for deterministic tests; time.Now in production. + now func() time.Time } // New validates Config and returns a ready trigger. @@ -143,9 +178,19 @@ func New(cfg Config) (*ClickHouseHTTP, error) { if cfg.HTTPTimeout <= 0 { cfg.HTTPTimeout = 30 * time.Second } + if cfg.Lookback < 0 { + return nil, fmt.Errorf("trigger: Lookback must be >= 0 (got %v)", cfg.Lookback) + } + if cfg.MaxSkew <= 0 { + cfg.MaxSkew = defaultMaxSkew + } + if cfg.DedupMaxEntries <= 0 { + cfg.DedupMaxEntries = 4 * cfg.PollLimit + } return &ClickHouseHTTP{ cfg: cfg, client: &http.Client{Timeout: cfg.HTTPTimeout}, + now: time.Now, }, nil } @@ -169,11 +214,15 @@ func (t *ClickHouseHTTP) Subscribe(ctx context.Context) (<-chan kubescape.Event, func (t *ClickHouseHTTP) run(ctx context.Context, out chan<- kubescape.Event) { defer close(out) // Watermark uses event_time as the cursor PLUS a set of row - // fingerprints already pushed at that exact event_time. This - // closes the race where two kubescape rows share the same - // event_time but the second arrives after our previous poll: the - // query is `event_time >= watermark` (inclusive) and we skip rows - // whose fingerprint we have already seen at the boundary. + // fingerprints already pushed. In legacy strict mode (Lookback==0) + // the query is `event_time >= watermark` (inclusive) and the + // fingerprint set covers only the exact boundary event_time — + // closing the race where two kubescape rows share the same + // event_time but the second arrives after our previous poll. With + // a bounded lookback (#97, the F8/AE-9 fix) the query starts at + // max(0, watermark-Lookback) and the fingerprint set is a bounded + // LRU over the whole re-scanned window, so out-of-order / skewed / + // restart-buried rows inside the window are captured exactly once. // // Cold-start order: persistent store > InitialWatermark > 0. // The persistent store is the production answer to "operator @@ -203,7 +252,39 @@ func (t *ClickHouseHTTP) run(ctx context.Context, out chan<- kubescape.Event) { // pre-fix persisted seconds watermark (or a non-seconds InitialWatermark) // is interpreted on the same scale as chNormEventTimeNanos in the SQL. watermark = normalizeEventTimeNanos(watermark) + maxSkewNS := uint64(t.cfg.MaxSkew.Nanoseconds()) + lookbackNS := uint64(t.cfg.Lookback.Nanoseconds()) + // Self-recovery from an ALREADY-poisoned persisted cursor (#97 T1): + // a pre-fix deployment could have persisted a far-future watermark + // (loadtest E8's leftover 1.78e18-style value). Clamp it to + // wall-clock so fresh rows flow again on restart WITHOUT the manual + // `ALTER TABLE trigger_watermark DELETE WHERE 1=1` + redeploy. + if nowNS := uint64(t.now().UnixNano()); watermark > nowNS+maxSkewNS { + log.WithFields(log.Fields{"watermark": watermark, "clamped_to": nowNS}). + Warn("trigger: persisted watermark is implausibly far in the future — clamping to wall-clock (poison recovery, #97)") + watermark = nowNS + } + wmGauge := metricWatermarkNS.WithLabelValues(t.cfg.Table, t.cfg.Hostname) + wmGauge.Set(float64(watermark)) + // Dedup state. Strict mode (Lookback==0) keeps the legacy exact + // boundary set; lookback mode dedupes the whole re-scanned window + // with a bounded LRU (#97). rejectedSeen exists only in strict mode: + // a clamp-rejected row never falls below the cursor, so without a + // fingerprint record it would re-emit on every poll. seenAtBoundary := map[string]bool{} + var seenInWindow *dedupLRU + var rejectedSeen *dedupLRU + if lookbackNS > 0 { + seenInWindow = newDedupLRU(t.cfg.DedupMaxEntries) + } else { + rejectedSeen = newDedupLRU(t.cfg.DedupMaxEntries) + } + // catchup lifts a poll's lower bound above the sliding lookback + // floor while an in-window backlog is wider than PollLimit: without + // it every poll would re-fetch the same fully-deduped first + // PollLimit rows and never reach deeper into the window. Cleared as + // soon as a poll returns under capacity (back to full-window scans). + var catchup uint64 ticker := time.NewTicker(t.cfg.PollInterval) defer ticker.Stop() @@ -253,7 +334,21 @@ func (t *ClickHouseHTTP) run(ctx context.Context, out chan<- kubescape.Event) { }() pollOnce := func() { - rows, maxSeen, err := t.fetchSince(ctx, watermark) + // Bounded lookback (#97): scan from max(0, watermark-Lookback) + // so rows that landed BELOW the cursor (out-of-order, clock + // skew, restart burial) are still fetched; the dedup LRU makes + // re-seen rows exactly-once. Lookback==0 → legacy strict HWM. + queryFrom := watermark + if lookbackNS > 0 { + queryFrom = 0 + if watermark > lookbackNS { + queryFrom = watermark - lookbackNS + } + if catchup > queryFrom { + queryFrom = catchup + } + } + rows, maxFetched, err := t.fetchSince(ctx, queryFrom) // Partial-read tolerance: when the body read is cut short by // HTTP timeout / connection reset, fetchSince returns the rows // it managed to parse + err. We still process those rows so @@ -267,6 +362,19 @@ func (t *ClickHouseHTTP) run(ctx context.Context, out chan<- kubescape.Event) { log.WithError(err).WithField("partial_rows", len(rows)). Warn("trigger: poll partial — advancing on what parsed") } + // Wall-clock poison clamp (#97): any normalized event_time past + // now+MaxSkew must never advance the cursor. acceptedMax is the + // advancement target — the max normalized event_time among rows + // that PASS the clamp. With no poison rows it equals maxFetched, + // so the monotonic happy path is byte-identical to before. + skewLimit := uint64(t.now().UnixNano()) + maxSkewNS + acceptedMax := uint64(0) + for _, row := range rows { + if evn := normalizeEventTimeNanos(row.EventTime); evn <= skewLimit && evn > acceptedMax { + acceptedMax = evn + } + } + wmAtPollStart := watermark nextSeen := map[string]bool{} // Periodic in-loop save: when pollOnce is draining a large // initial backlog, the watermark advances long before the @@ -276,45 +384,122 @@ func (t *ClickHouseHTTP) run(ctx context.Context, out chan<- kubescape.Event) { // with the time-based throttle inside flushWatermark, this // produces at most one persistent INSERT per WatermarkSaveInterval. const saveEveryN = 256 - skippedAtBoundary := 0 + skippedSeen := 0 + emitted := 0 for i, row := range rows { fp := rowFingerprint(row) // Cursor comparisons are in NORMALIZED nanos (F8): the raw // event_time unit is not enforced, so compare on the same scale - // as the SQL filter (chNormEventTimeNanos) and maxSeen. + // as the SQL filter (chNormEventTimeNanos) and acceptedMax. evn := normalizeEventTimeNanos(row.EventTime) - if evn == watermark && seenAtBoundary[fp] { - skippedAtBoundary++ - continue // already pushed in a prior poll at this exact boundary + if lookbackNS > 0 { + if seenInWindow.Contains(fp) { + skippedSeen++ + continue // already pushed in a prior scan of this window + } + } else { + if evn == watermark && seenAtBoundary[fp] { + skippedSeen++ + continue // already pushed in a prior poll at this exact boundary + } + if rejectedSeen.Contains(fp) { + continue // clamp-rejected row re-fetched (it never sinks below the cursor) + } } - ev, err := kubescape.Extract(row) - if err != nil { - log.WithError(err).Debug("trigger: skip incomplete row") + poison := evn > skewLimit + ev, exErr := kubescape.Extract(row) + if exErr != nil { + log.WithError(exErr).Debug("trigger: skip incomplete row") + // Register the fingerprint anyway (lookback / poison): + // the row can never become extractable, and without a + // record it would be re-fetched + re-logged every poll + // for as long as it stays above the scan floor. + if lookbackNS > 0 { + seenInWindow.Add(fp, evn) + } else if poison { + rejectedSeen.Add(fp, evn) + } continue } - // Promote the per-row (normalized) event_time into the watermark - // immediately so flushWatermark below can persist mid-drain. - if evn > watermark { - watermark = evn - dirty = true + if poison { + // Emit the row once (it may be a real anomaly with a + // mangled timestamp) but do NOT let it advance the + // cursor: one 1.78e18 row must not jump the watermark + // past all real seconds rows (F8 halt). + metricEventTimeRejected.Inc() + log.WithFields(log.Fields{ + "event_time": row.EventTime, + "normalized": evn, + "skew_limit": skewLimit, + }).Warn("trigger: event_time beyond wall-clock skew bound — processing row WITHOUT advancing watermark (poison clamp, #97)") + } else { + if evn < wmAtPollStart { + // A row the legacy strict HWM would have dropped — + // captured via the lookback (T2). Observable proof + // the fix is doing work (T3). + metricBelowWatermark.Inc() + } + // Promote the per-row (normalized) event_time into the watermark + // immediately so flushWatermark below can persist mid-drain. + if evn > watermark { + watermark = evn + dirty = true + wmGauge.Set(float64(watermark)) + } + } + if lookbackNS > 0 { + seenInWindow.Add(fp, evn) + } else if poison { + rejectedSeen.Add(fp, evn) } select { case out <- ev: case <-ctx.Done(): return } - if evn == maxSeen { + emitted++ + if !poison && evn == acceptedMax { nextSeen[fp] = true } if i > 0 && i%saveEveryN == 0 { flushWatermark() } } - if maxSeen > watermark { - watermark = maxSeen + if lookbackNS > 0 { + if acceptedMax > watermark { + watermark = acceptedMax + dirty = true + wmGauge.Set(float64(watermark)) + } + // Paging within the window: a saturated response means the + // window holds more rows than PollLimit — lift the floor so + // the next poll pages FORWARD instead of re-fetching the + // same deduped prefix forever. + if len(rows) >= t.cfg.PollLimit { + if emitted == 0 && skippedSeen == len(rows) { + // Every row in the saturated page was already seen — + // step past the page entirely (lookback analog of the + // legacy 1ns boundary escape). + catchup = maxFetched + 1 + } else if acceptedMax > catchup { + catchup = acceptedMax + } + } else { + catchup = 0 + } + // Entries below the sliding floor can never be re-fetched; + // evict them so the LRU stays at ~window size. + floor := uint64(0) + if watermark > lookbackNS { + floor = watermark - lookbackNS + } + seenInWindow.EvictBelow(floor) + } else if acceptedMax > watermark { + watermark = acceptedMax seenAtBoundary = nextSeen dirty = true - } else if maxSeen == watermark { + wmGauge.Set(float64(watermark)) + } else if acceptedMax == watermark { // no progress this tick — preserve boundary set, optionally extend for fp := range nextSeen { seenAtBoundary[fp] = true @@ -329,10 +514,11 @@ func (t *ClickHouseHTTP) run(ctx context.Context, out chan<- kubescape.Event) { // the next poll, which is acceptable: the fingerprint dedup already // tolerates boundary overlap, and we prefer forward progress over // an infinite loop. - if skippedAtBoundary > 0 && len(nextSeen) == 0 && len(rows) >= t.cfg.PollLimit { + if skippedSeen > 0 && len(nextSeen) == 0 && len(rows) >= t.cfg.PollLimit { watermark++ seenAtBoundary = map[string]bool{} dirty = true + wmGauge.Set(float64(watermark)) log.WithField("watermark", watermark). Warn("trigger: boundary paging escape — advanced watermark by 1ns to unblock poll") } diff --git a/src/vizier/services/adaptive_export/internal/trigger/dedup.go b/src/vizier/services/adaptive_export/internal/trigger/dedup.go new file mode 100644 index 00000000000..ca1c5dda9a0 --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/trigger/dedup.go @@ -0,0 +1,98 @@ +// Copyright 2018- The Pixie Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package trigger + +import "container/list" + +// dedupLRU is a bounded, insertion-ordered set of row fingerprints, +// each tagged with the row's normalized event_time (nanos). It is the +// #97 (F8/AE-9) extension of the old single-boundary `seenAtBoundary` +// map: with a bounded lookback the trigger re-fetches every row in +// [watermark-Lookback, watermark] on each poll, so dedup must cover the +// whole window, not just the exact watermark boundary. +// +// Eviction is two-fold: +// - EvictBelow(floor): entries whose event_time has slid below the +// lookback floor can never be returned by the SELECT again, so they +// are dropped eagerly to keep the set at ~window size. +// - capacity: Add evicts the OLDEST INSERTION when over max, bounding +// memory even if the window holds more rows than expected. An +// in-window entry evicted by capacity may cause one duplicate emit — +// the documented trade-off for bounded memory (size it >= the max +// rows per window; default 4*PollLimit). +// +// Not goroutine-safe; owned by the single poll loop. +type dedupLRU struct { + max int + ll *list.List // front = oldest insertion + items map[string]*list.Element +} + +type dedupEntry struct { + fp string + evn uint64 // normalized event_time (nanos) +} + +func newDedupLRU(capacity int) *dedupLRU { + if capacity <= 0 { + capacity = 1 + } + return &dedupLRU{max: capacity, ll: list.New(), items: map[string]*list.Element{}} +} + +// Contains reports whether fp was Added and not yet evicted. +func (d *dedupLRU) Contains(fp string) bool { + _, ok := d.items[fp] + return ok +} + +// Add records fp with its normalized event_time. No-op if already +// present. Evicts oldest insertions while over capacity. +func (d *dedupLRU) Add(fp string, evn uint64) { + if _, ok := d.items[fp]; ok { + return + } + d.items[fp] = d.ll.PushBack(dedupEntry{fp: fp, evn: evn}) + for d.ll.Len() > d.max { + d.removeElement(d.ll.Front()) + } +} + +// EvictBelow drops entries with evn < floor, popping from the oldest +// insertion. Insertion order tracks the poll's ORDER BY event_time, so +// in the common case this removes exactly the expired prefix. A late +// arrival (low evn inserted after a higher one) may survive behind a +// newer entry until capacity eviction — harmless: Contains on an +// expired fp only suppresses a row the SELECT can no longer return. +func (d *dedupLRU) EvictBelow(floor uint64) { + for e := d.ll.Front(); e != nil; { + if e.Value.(dedupEntry).evn >= floor { + return + } + next := e.Next() + d.removeElement(e) + e = next + } +} + +// Len returns the number of live entries. +func (d *dedupLRU) Len() int { return d.ll.Len() } + +func (d *dedupLRU) removeElement(e *list.Element) { + delete(d.items, e.Value.(dedupEntry).fp) + d.ll.Remove(e) +} diff --git a/src/vizier/services/adaptive_export/internal/trigger/dedup_test.go b/src/vizier/services/adaptive_export/internal/trigger/dedup_test.go new file mode 100644 index 00000000000..c139b21831c --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/trigger/dedup_test.go @@ -0,0 +1,92 @@ +// Copyright 2018- The Pixie Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package trigger + +import "testing" + +func TestDedupLRU_AddContains(t *testing.T) { + d := newDedupLRU(8) + if d.Contains("a") { + t.Fatalf("empty LRU claims to contain a") + } + d.Add("a", 100) + d.Add("b", 200) + if !d.Contains("a") || !d.Contains("b") { + t.Fatalf("added fingerprints not found") + } + if d.Len() != 2 { + t.Fatalf("Len = %d, want 2", d.Len()) + } + // Duplicate Add is a no-op (no double-entry, no reorder). + d.Add("a", 100) + if d.Len() != 2 { + t.Fatalf("duplicate Add changed Len to %d", d.Len()) + } +} + +func TestDedupLRU_CapacityEvictsOldestInsertion(t *testing.T) { + d := newDedupLRU(3) + d.Add("a", 1) + d.Add("b", 2) + d.Add("c", 3) + d.Add("d", 4) // over capacity → "a" (oldest insertion) evicted + if d.Contains("a") { + t.Fatalf("oldest entry not evicted at capacity") + } + for _, fp := range []string{"b", "c", "d"} { + if !d.Contains(fp) { + t.Fatalf("entry %q evicted unexpectedly", fp) + } + } + if d.Len() != 3 { + t.Fatalf("Len = %d, want 3", d.Len()) + } +} + +func TestDedupLRU_EvictBelow(t *testing.T) { + d := newDedupLRU(8) + d.Add("a", 100) + d.Add("b", 200) + d.Add("c", 300) + d.EvictBelow(250) + if d.Contains("a") || d.Contains("b") { + t.Fatalf("entries below floor survived EvictBelow") + } + if !d.Contains("c") { + t.Fatalf("entry at/above floor was evicted") + } + // EvictBelow stops at the first entry >= floor (prefix semantics): + // a late arrival (low evn inserted AFTER a higher one) survives — + // documented as harmless. + d.Add("late", 50) + d.EvictBelow(250) + if !d.Contains("late") { + t.Fatalf("late-arrival entry behind a newer one should survive prefix eviction") + } +} + +func TestDedupLRU_ZeroCapacityIsSafe(t *testing.T) { + d := newDedupLRU(0) // clamped to 1 + d.Add("a", 1) + if !d.Contains("a") { + t.Fatalf("single entry not retained") + } + d.Add("b", 2) + if d.Contains("a") || !d.Contains("b") { + t.Fatalf("capacity-1 eviction wrong: a=%v b=%v", d.Contains("a"), d.Contains("b")) + } +} diff --git a/src/vizier/services/adaptive_export/internal/trigger/lookback_test.go b/src/vizier/services/adaptive_export/internal/trigger/lookback_test.go new file mode 100644 index 00000000000..8fa153001fa --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/trigger/lookback_test.go @@ -0,0 +1,321 @@ +// Copyright 2018- The Pixie Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +// Bounded-lookback + wall-clock poison-clamp tests (#97 / F8 / AE-9). +// No live ClickHouse: a stub HTTP server implements the trigger's +// JSONEachRow contract INCLUDING the `>= ` watermark predicate, +// so re-poll semantics (the essence of lookback) are exercised for real. + +package trigger + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "regexp" + "sort" + "strconv" + "sync" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus/testutil" +) + +// fakeCH is a stub ClickHouse HTTP endpoint that stores rows and, like +// the real server, only returns rows whose NORMALIZED event_time is >= +// the bound parsed out of the trigger's SELECT. +type fakeCH struct { + mu sync.Mutex + rows []fakeRow + srv *httptest.Server +} + +type fakeRow struct { + eventTime uint64 // raw, unit-ambiguous — exactly like production + ruleID string + pid int +} + +var boundRE = regexp.MustCompile(`>= (\d+) ORDER`) + +func newFakeCH(t *testing.T) *fakeCH { + f := &fakeCH{} + f.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query().Get("query") + m := boundRE.FindStringSubmatch(q) + if m == nil { + t.Errorf("query without >= bound: %q", q) + w.WriteHeader(400) + return + } + bound, err := strconv.ParseUint(m[1], 10, 64) + if err != nil { + t.Errorf("unparseable bound in query %q: %v", q, err) + w.WriteHeader(400) + return + } + f.mu.Lock() + var out []fakeRow + for _, row := range f.rows { + if normalizeEventTimeNanos(row.eventTime) >= bound { + out = append(out, row) + } + } + f.mu.Unlock() + sort.Slice(out, func(i, j int) bool { + return normalizeEventTimeNanos(out[i].eventTime) < normalizeEventTimeNanos(out[j].eventTime) + }) + for _, row := range out { + fmt.Fprintf(w, + `{"RuleID":%q,"RuntimeK8sDetails":"{\"podName\":\"p-1\",\"podNamespace\":\"ns\"}","RuntimeProcessDetails":"{\"processTree\":{\"pid\":%d,\"comm\":\"c\"}}","event_time":"%d","hostname":"node-1"}`+"\n", + row.ruleID, row.pid, row.eventTime) + } + })) + return f +} + +func (f *fakeCH) add(r fakeRow) { + f.mu.Lock() + f.rows = append(f.rows, r) + f.mu.Unlock() +} + +func (f *fakeCH) close() { f.srv.Close() } + +// testBase is a fixed "now" for deterministic clamp behavior: +// 2026-05-29T… ≈ 1.7805e9 seconds. +const testBase = uint64(1_780_500_000) + +func fixedNow() time.Time { return time.Unix(int64(testBase), 0) } + +// newLookbackTrigger builds a trigger against the fake server with the +// #97 config (300s lookback) and a pinned wall clock. +func newLookbackTrigger(t *testing.T, f *fakeCH, hostname string, lookback time.Duration) *ClickHouseHTTP { + t.Helper() + tr, err := New(Config{ + Endpoint: f.srv.URL, + Hostname: hostname, + PollInterval: 20 * time.Millisecond, + Lookback: lookback, + }) + if err != nil { + t.Fatalf("New: %v", err) + } + tr.now = fixedNow // deterministic poison clamp + return tr +} + +// TestTrigger_LookbackCapturesLateArrivalExactlyOnce — T2: a row that +// lands BELOW the watermark but inside the lookback window is processed +// exactly once (no drop, no duplicate over many re-polls), and a row +// below watermark-lookback stays dropped (the documented bound). Also +// asserts ae_trigger_below_watermark_total increments (T3). +func TestTrigger_LookbackCapturesLateArrivalExactlyOnce(t *testing.T) { + f := newFakeCH(t) + defer f.close() + f.add(fakeRow{eventTime: testBase, ruleID: "R1", pid: 111}) // head row → watermark = testBase + + belowBefore := testutil.ToFloat64(metricBelowWatermark) + + tr := newLookbackTrigger(t, f, "node-lb", 300*time.Second) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ch, _ := tr.Subscribe(ctx) + + // Wait for the head row so the watermark is at testBase. + select { + case ev := <-ch: + if ev.Target.PID != 111 { + t.Fatalf("first event PID = %d, want 111", ev.Target.PID) + } + case <-time.After(500 * time.Millisecond): + t.Fatalf("timeout waiting for head row") + } + + // Late arrival 60s below the watermark (inside the 300s window) and + // one 400s below (outside the window). + f.add(fakeRow{eventTime: testBase - 60, ruleID: "R2", pid: 222}) + f.add(fakeRow{eventTime: testBase - 400, ruleID: "R3", pid: 333}) + + got := map[uint64]int{} + deadline := time.Now().Add(400 * time.Millisecond) // ~20 re-polls of the same window + for time.Now().Before(deadline) { + select { + case ev := <-ch: + got[ev.Target.PID]++ + case <-time.After(20 * time.Millisecond): + } + } + if got[222] != 1 { + t.Errorf("late-arrival row emitted %d times, want exactly 1 (T2)", got[222]) + } + if got[333] != 0 { + t.Errorf("row below watermark-lookback emitted %d times, want 0 (documented bound)", got[333]) + } + if got[111] != 0 { + t.Errorf("head row re-emitted %d times after initial delivery (window dedup failed)", got[111]) + } + if delta := testutil.ToFloat64(metricBelowWatermark) - belowBefore; delta < 1 { + t.Errorf("ae_trigger_below_watermark_total delta = %v, want >= 1", delta) + } +} + +// TestTrigger_PoisonRowDoesNotHalt — T1 (the F8 non-halt guarantee): a +// row carrying the real E8 poison timestamp (1.78e18-style far-future +// vs the pinned clock) is clamp-rejected from advancing the watermark, +// the reject metric increments, the watermark gauge stays wall-clock- +// bounded, and SUBSEQUENT seconds rows are still processed — no manual +// watermark reset needed. +func TestTrigger_PoisonRowDoesNotHalt(t *testing.T) { + // The exact leftover value from loadtest E8's poisoned watermark. + // Normalized it stays 1.781559e18 ns ≈ 12 days past the pinned + // clock (1.7805e9 s) — beyond the 1h MaxSkew. + const poisonET = uint64(1781559619170395824) + + f := newFakeCH(t) + defer f.close() + f.add(fakeRow{eventTime: testBase - 10, ruleID: "R1", pid: 111}) + + rejBefore := testutil.ToFloat64(metricEventTimeRejected) + + tr := newLookbackTrigger(t, f, "node-poison", 300*time.Second) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ch, _ := tr.Subscribe(ctx) + + select { + case ev := <-ch: + if ev.Target.PID != 111 { + t.Fatalf("first event PID = %d, want 111", ev.Target.PID) + } + case <-time.After(500 * time.Millisecond): + t.Fatalf("timeout waiting for head row") + } + + // Inject the poison row; it is emitted once (real anomaly, mangled + // timestamp) but must not advance the cursor. + f.add(fakeRow{eventTime: poisonET, ruleID: "RPOISON", pid: 666}) + select { + case ev := <-ch: + if ev.Target.PID != 666 { + t.Fatalf("expected poison row emission, got PID %d", ev.Target.PID) + } + case <-time.After(500 * time.Millisecond): + t.Fatalf("poison row was dropped entirely; want emitted-once-without-advance") + } + if delta := testutil.ToFloat64(metricEventTimeRejected) - rejBefore; delta < 1 { + t.Errorf("ae_trigger_event_time_rejected_total delta = %v, want >= 1", delta) + } + + // THE F8 guarantee: a fresh seconds row AFTER the poison must flow. + // Under the old strict HWM the cursor sat at 1.78e18 and this row + // was below it forever (25/25 ticks at n_anomalies=0 in E8). + f.add(fakeRow{eventTime: testBase + 5, ruleID: "R2", pid: 222}) + var got222 int + deadline := time.Now().Add(600 * time.Millisecond) + for time.Now().Before(deadline) && got222 == 0 { + select { + case ev := <-ch: + if ev.Target.PID == 222 { + got222++ + } + case <-time.After(20 * time.Millisecond): + } + } + if got222 != 1 { + t.Fatalf("post-poison seconds row emitted %d times, want 1 (T1 non-halt)", got222) + } + + // Watermark gauge stays wall-clock-bounded: it advanced to the real + // row (testBase+5 s), NOT to the poison value. + wantWM := float64(normalizeEventTimeNanos(testBase + 5)) + if got := testutil.ToFloat64(metricWatermarkNS.WithLabelValues("kubescape_logs", "node-poison")); got != wantWM { + t.Errorf("ae_trigger_watermark_ns = %v, want %v (wall-clock-bounded, not poison)", got, wantWM) + } +} + +// TestTrigger_PoisonPersistedWatermarkSelfRecovers — the E8 recovery +// scenario without the manual ALTER TABLE … DELETE: a pre-fix deployment +// left a far-future watermark behind; on start the trigger clamps it to +// wall-clock and fresh rows flow again. +func TestTrigger_PoisonPersistedWatermarkSelfRecovers(t *testing.T) { + const poisonWM = uint64(1781559619170395824) + + f := newFakeCH(t) + defer f.close() + f.add(fakeRow{eventTime: testBase, ruleID: "R1", pid: 111}) + + tr := newLookbackTrigger(t, f, "node-recover", 300*time.Second) + tr.cfg.InitialWatermark = poisonWM // simulates the poisoned persisted cursor + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ch, _ := tr.Subscribe(ctx) + + select { + case ev := <-ch: + if ev.Target.PID != 111 { + t.Fatalf("recovered event PID = %d, want 111", ev.Target.PID) + } + case <-time.After(500 * time.Millisecond): + t.Fatalf("fresh row not delivered — poisoned persisted watermark was not clamped (still halted)") + } +} + +// TestTrigger_LookbackZeroIsStrictHWM — T4: LOOKBACK=0 preserves the +// legacy strict high-water-mark exactly — the poll bound IS the +// watermark (no window subtraction) and a below-watermark row stays +// dropped. (The monotonic happy path itself is pinned by the existing +// clickhouse_test.go suite, which runs with the zero-value Lookback.) +func TestTrigger_LookbackZeroIsStrictHWM(t *testing.T) { + f := newFakeCH(t) + defer f.close() + f.add(fakeRow{eventTime: testBase, ruleID: "R1", pid: 111}) + + tr := newLookbackTrigger(t, f, "node-strict", 0) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ch, _ := tr.Subscribe(ctx) + + select { + case ev := <-ch: + if ev.Target.PID != 111 { + t.Fatalf("first event PID = %d, want 111", ev.Target.PID) + } + case <-time.After(500 * time.Millisecond): + t.Fatalf("timeout waiting for head row") + } + + // A late arrival below the watermark: with strict HWM the SELECT + // bound equals the watermark, so it is never fetched again → dropped. + f.add(fakeRow{eventTime: testBase - 60, ruleID: "R2", pid: 222}) + got := map[uint64]int{} + deadline := time.Now().Add(300 * time.Millisecond) + for time.Now().Before(deadline) { + select { + case ev := <-ch: + got[ev.Target.PID]++ + case <-time.After(20 * time.Millisecond): + } + } + if got[222] != 0 { + t.Errorf("strict mode emitted a below-watermark row %d times; want 0 (legacy behavior)", got[222]) + } + if got[111] != 0 { + t.Errorf("strict mode re-emitted the boundary row %d times; want 0", got[111]) + } +} diff --git a/src/vizier/services/adaptive_export/internal/trigger/metrics.go b/src/vizier/services/adaptive_export/internal/trigger/metrics.go new file mode 100644 index 00000000000..4fc9dee9eda --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/trigger/metrics.go @@ -0,0 +1,57 @@ +// Copyright 2018- The Pixie Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package trigger + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +// Watermark observability (#97 / F8 / AE-9). Registered on the DEFAULT +// prometheus registry via promauto — the same pattern the rest of pixie +// uses (e.g. query_broker's queryExec* summaries) — and served by the +// shared services/metrics /metrics handler wired up in cmd/main.go. +// Before these existed a watermark halt was completely invisible: writes +// stopped, no error, no signal (loadtest E8). +var ( + // metricWatermarkNS tracks the trigger's current cursor in + // normalized unix NANOS, per (table, hostname). A flat gauge while + // kubescape rows keep arriving is the F8 silent-halt signature — + // alert on it. + metricWatermarkNS = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "ae_trigger_watermark_ns", + Help: "Current trigger high-water-mark cursor in normalized unix nanoseconds, per (table, hostname).", + }, []string{"table", "hostname"}) + + // metricBelowWatermark counts rows processed with a normalized + // event_time BELOW the poll-start watermark — i.e. out-of-order / + // clock-skewed / restart-buried rows the legacy strict HWM silently + // dropped and the bounded lookback now captures. + metricBelowWatermark = promauto.NewCounter(prometheus.CounterOpts{ + Name: "ae_trigger_below_watermark_total", + Help: "Rows seen with event_time below the prior watermark that the bounded lookback captured (strict HWM would have dropped them).", + }) + + // metricEventTimeRejected counts poison clamps: rows whose + // normalized event_time was implausibly far in the future + // (> now + MaxSkew) and were therefore barred from advancing the + // watermark. + metricEventTimeRejected = promauto.NewCounter(prometheus.CounterOpts{ + Name: "ae_trigger_event_time_rejected_total", + Help: "Rows whose normalized event_time exceeded now+max-skew and were rejected from advancing the watermark (poison clamp).", + }) +) diff --git a/tools/private/copybara/copy.bara.sky b/tools/private/copybara/copy.bara.sky index 9a54dddb6bb..91aa223b644 100644 --- a/tools/private/copybara/copy.bara.sky +++ b/tools/private/copybara/copy.bara.sky @@ -40,7 +40,7 @@ ignored_dirs = [ # stay pinned; fork-only k8s additions are protected via fork_only_files below. "k8s/cloud/dev/plugin_db_updater_job.yaml", # fork PL_PLUGIN_REPO override "k8s/cloud/overlays/plugin_job/plugin_job.yaml", # fork PL_PLUGIN_REPO override - "k8s/vizier/BUILD.bazel", # fork adaptive_export image entry + "k8s/vizier/BUILD.bazel", # fork kustomize_build glob divergence "k8s/cloud/overlays/exposed_services_traefik/**", # To be upstreamed "scripts/create_cloud_secrets.sh", # cert-manager support (upstream) "skaffold/**", @@ -144,8 +144,9 @@ fork_only_files = [ ".sops.yaml", # fork SOPS encryption config "PLATFORM.md", "ci/private/**", - "k8s/vizier/bootstrap/adaptive_export_*", + "k8s/vizier/adaptive_export/**", # fork export path: AE manifests + RBAC "k8s/vizier/bootstrap/kustomization.yaml", + "k8s/vizier/dx/**", # fork export path: dx-daemon manifests "private/**", # fork cloud deploy config (cockpit/, skaffold_cloud.yaml) "src/carnot/planner/ir/clickhouse_*", "src/e2e_test/adaptive_export_loadtest/**", # fork AE load-test harness