meter.yaml

# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# telemetry-meter: an OTLP tee that counts the telemetry that substrate sends,
# for each service that sends it.
#
# See benchmarking/telemetry/README.md. Send substrate to
#   http://telemetry-meter.benchmarking.svc.cluster.local:4317
# for the duration of a measurement, then scrape :8889 for
#
#   substrate_spans_total{service_name="..."}       spans from each service
#   substrate_datapoints_total{service_name="..."}  datapoints from each service
#
# Scrape :8888 for the otelcol_* receiver counters of the meter.
#
# The meter is a tee. It counts the telemetry and then sends it on to the
# managed collector, thus the resource values of the managed collector stay
# correct during a measurement. It is not a replacement for that collector.

apiVersion: v1
kind: Namespace
metadata:
  name: benchmarking
  labels:
    # Admits the benchmark clients to the Substrate API through
    # manifests/ate-install/ate-system-networkpolicies.yaml.
    ate.dev/api-access: "true"
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: telemetry-meter-config
  namespace: benchmarking
data:
  config.yaml: |
    receivers:
      otlp:
        protocols:
          grpc:
            endpoint: 0.0.0.0:4317
          http:
            endpoint: 0.0.0.0:4318

    processors:
      # Refuse data instead of growing the heap until the kernel stops the
      # container. otelcol_receiver_refused_*, already in the pass criteria
      # of a run, counts each refusal. The percentages are of the container
      # memory limit below.
      memory_limiter:
        check_interval: 1s
        limit_percentage: 80
        spike_limit_percentage: 15

      # The count connector sends deltas. Each batch has a new start timestamp,
      # and the Prometheus exporter reads a new start timestamp as a reset of
      # the counter. Without deltatocumulative the counters do not accumulate,
      # and each scrape shows only the last batch.
      #
      # max_streams bounds the memory. A stream is for each process and
      # count, and not for each service, because serverboot gives each
      # process its own service.instance.id. 24 processes held 51 streams, at
      # approximately 5 KiB each. Above the limit new streams drop with no log
      # line and the totals become too low; README.md holds the pass criteria
      # that catch it.
      deltatocumulative:
        max_streams: 20000

    connectors:
      count:
        spans:
          substrate_spans:
            description: Spans received, by emitting service.
            attributes:
              # The connector reads each key from the attributes of the item,
              # then the scope, then the resource, thus service.name resolves
              # from the resource and no transform in front of it is needed.
              # The Prometheus exporter writes the key as service_name.
              #
              # Without a default value, the connector removes each item that
              # has no service.name from the count. The item does not show as
              # unknown, and the totals are too low with no error message.
              - key: service.name
                default_value: unknown
        datapoints:
          substrate_datapoints:
            description: Metric datapoints received, by emitting service.
            attributes:
              - key: service.name
                default_value: unknown

    exporters:
      prometheus:
        endpoint: 0.0.0.0:8889

      # The meter is a tee, not a terminal sink. It counts the telemetry and
      # then sends it on to the collector that the cluster uses. Thus the
      # managed collector keeps the load of the run, and its CPU, memory, and
      # replica count stay correct while you measure the volume.
      #
      # To make the meter terminal, delete the two forward pipelines below.
      # The counts continue to work. A terminal meter keeps the telemetry of a
      # long run out of Cloud Monitoring, but it makes the resource values of
      # the managed collector those of an idle collector.
      otlp/managed:
        endpoint: opentelemetry-collector.gke-managed-otel.svc.cluster.local:4317
        tls:
          insecure: true

    service:
      pipelines:
        # The count pipelines and the forward pipelines read the same receiver.
        # The count pipelines only count. The connector declares MutatesData
        # false, thus it changes nothing that goes to the managed collector.
        #
        # memory_limiter is first in each pipeline that reads the receiver.
        # metrics/count-out has no limiter: a refusal there would remove the
        # measurement of the run.
        traces/count:
          receivers: [otlp]
          processors: [memory_limiter]
          exporters: [count]
        traces/forward:
          receivers: [otlp]
          processors: [memory_limiter]
          exporters: [otlp/managed]
        metrics/count:
          receivers: [otlp]
          processors: [memory_limiter]
          exporters: [count]
        metrics/forward:
          receivers: [otlp]
          processors: [memory_limiter]
          exporters: [otlp/managed]

        # The counts that the connector makes, out to the Prometheus exporter.
        metrics/count-out:
          receivers: [count]
          processors: [deltatocumulative]
          exporters: [prometheus]

        # Forward raw OTLP metrics directly to the Prometheus exporter on :8889.
        # (monitoring.yaml auto-scrapes telemetry-meter:8889 via pod annotations).
        metrics/raw:
          receivers: [otlp]
          processors: [memory_limiter]
          exporters: [prometheus]
      telemetry:
        metrics:
          level: detailed
          readers:
            - pull:
                exporter:
                  prometheus:
                    host: 0.0.0.0
                    port: 8888
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: telemetry-meter
  namespace: benchmarking
spec:
  replicas: 1
  selector:
    matchLabels:
      app: telemetry-meter
  template:
    metadata:
      labels:
        app: telemetry-meter
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "8889"
    spec:
      containers:
      - name: collector
        image: otel/opentelemetry-collector-contrib:0.152.0
        args: ["--config=/conf/config.yaml"]
        env:
        # The Go runtime does not read the cgroup limit; without this it sizes
        # the heap from the memory of the node. Keep it under the hard limit
        # of memory_limiter, so the runtime collects before the meter refuses.
        - name: GOMEMLIMIT
          value: 3000MiB
        ports:
        - {name: otlp-grpc, containerPort: 4317}
        - {name: otlp-http, containerPort: 4318}
        - {name: self, containerPort: 8888}
        - {name: counts, containerPort: 8889}
        resources:
          requests:
            cpu: "1"
            memory: 1Gi
          limits:
            memory: 4Gi
        volumeMounts:
        - name: config
          mountPath: /conf
      volumes:
      - name: config
        configMap:
          name: telemetry-meter-config
---
apiVersion: v1
kind: Service
metadata:
  name: telemetry-meter
  namespace: benchmarking
spec:
  selector:
    app: telemetry-meter
  ports:
  - {name: otlp-grpc, port: 4317, targetPort: 4317}
  - {name: otlp-http, port: 4318, targetPort: 4318}
  - {name: self, port: 8888, targetPort: 8888}
  - {name: counts, port: 8889, targetPort: 8889}