diff --git a/.github/workflows/runtime-metrics-conventions.yml b/.github/workflows/runtime-metrics-conventions.yml new file mode 100644 index 000000000..034c1def9 --- /dev/null +++ b/.github/workflows/runtime-metrics-conventions.yml @@ -0,0 +1,61 @@ +name: Runtime Metrics Conventions + +on: + push: + branches: + - master + - release/** + pull_request: + paths: + - lib/sentry/metrics/** + - lib/sentry/metric.ex + - test/sentry/metrics/runtime_conventions_test.exs + - test/fixtures/sentry_conventions/** + - scripts/check_metrics_conventions.sh + - .github/workflows/runtime-metrics-conventions.yml + schedule: + - cron: "0 6 * * 1" + workflow_dispatch: + inputs: + conventions_ref: + description: sentry-conventions branch, tag or commit to check against + required: false + +env: + MIX_ENV: test + CONVENTIONS_REF: ${{ inputs.conventions_ref || 'feat/metric-model' }} + +jobs: + conventions: + name: Runtime metrics match sentry-conventions + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - name: Check out this repository + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + - name: Setup Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: "24" + + - name: Setup Elixir and Erlang + uses: erlef/setup-beam@fc68ffb90438ef2936bbb3251622353b3dcb2f93 # v1.24.0 + with: + elixir-version: "1.18" + otp-version: "27.2" + + - name: Cache Elixir dependencies + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 + with: + path: | + deps + _build + key: ${{ runner.os }}-elixir-1.18-otp-27.2-mix-${{ hashFiles('**/mix*.lock') }} + + - name: Download Mix dependencies + run: mix deps.get --check-locked + + - name: Check runtime metrics against sentry-conventions + run: mix test.conventions diff --git a/lib/sentry/config.ex b/lib/sentry/config.ex index 4f8fb677c..0d1af7f5a 100644 --- a/lib/sentry/config.ex +++ b/lib/sentry/config.ex @@ -584,8 +584,11 @@ defmodule Sentry.Config do type: :boolean, default: false, doc: """ - Whether to attach `elixir_version` and `otp_release` attributes to every - reported measurement, so metrics can be grouped by runtime version. Off by + Whether to attach runtime version attributes to every reported + measurement, so metrics can be grouped by runtime version: + `process.runtime.name` (`"elixir"`), `process.runtime.version` (the + Elixir version), `process.runtime.engine.name` (`"BEAM"`) and + `process.runtime.engine.version` (the OTP release, such as `"27"`). Off by default: the versions change only on upgrade, and attaching them to every point starts a fresh series for each metric on every rolling deploy. *Available since 14.0.0*. diff --git a/lib/sentry/metrics.ex b/lib/sentry/metrics.ex index 24f84814d..f7494e709 100644 --- a/lib/sentry/metrics.ex +++ b/lib/sentry/metrics.ex @@ -17,7 +17,7 @@ defmodule Sentry.Metrics do # Record a counter Sentry.Metrics.count("button.clicks", 1) - Sentry.Metrics.count("button.clicks", 5, unit: "click", attributes: %{button_id: "submit"}) + Sentry.Metrics.count("button.clicks", 5, attributes: %{button_id: "submit"}) # Record a gauge Sentry.Metrics.gauge("memory.usage", 1024, unit: "megabyte") @@ -25,6 +25,23 @@ defmodule Sentry.Metrics do # Record a distribution Sentry.Metrics.distribution("response.time", 42.5, unit: "millisecond") + ## Units + + The `:unit` option takes one of the units Sentry recognizes, so it can format and + convert values: + + * Duration - `"nanosecond"`, `"microsecond"`, `"millisecond"`, `"second"`, + `"minute"`, `"hour"`, `"day"`, `"week"` + * Information - `"bit"`, `"byte"`, and the decimal and binary multiples of + `"byte"`: `"kilobyte"`, `"kibibyte"`, `"megabyte"`, `"mebibyte"`, `"gigabyte"`, + `"gibibyte"`, `"terabyte"`, `"tebibyte"`, `"petabyte"`, `"pebibyte"`, + `"exabyte"`, `"exbibyte"` + * Fraction - `"ratio"`, `"percent"` + * `"none"` - for plain counts and other values without a unit + + Leave out `:unit` or pass `"none"` when you count things such as requests or + clicks. Describe what is counted in the metric name instead. + ## Automatically Collected Metrics The SDK can also report BEAM runtime health on its own, without any calls to the @@ -37,15 +54,19 @@ defmodule Sentry.Metrics do Once enabled, these gauges are reported: - * `elixir.runtime.mem.*` — `total`, `processes`, `processes_used`, `system`, - `atom`, `atom_used`, `binary`, `code` and `ets`, in bytes - * `elixir.runtime.run_queue.*` — `total`, `cpu` and `io`, how many processes are - waiting to run - * `elixir.runtime.process.*`, `elixir.runtime.atom.*` and `elixir.runtime.port.*` — + * `elixir.runtime.memory.used` - memory allocated by the VM, in bytes, reported + once per `elixir.memory.type` attribute value: `processes`, `atom`, `binary`, + `code`, `ets` and `other`. `other` is everything else the VM has allocated. + Summing across the types gives the VM total. + * `elixir.runtime.run_queue.length` - how many processes are waiting to run, + reported once per `elixir.run_queue.type` attribute value: `cpu` for the normal + and dirty CPU scheduler run queues and `io` for the dirty IO run queue. Summing + across the types gives the total run queue length. + * `elixir.runtime.process.*`, `elixir.runtime.atom.*` and `elixir.runtime.port.*` - a `count`, the hard VM `limit`, and the `utilization` ratio between them. The `limit` and `utilization` gauges need telemetry_poller 1.3.0 or later, which is when it started measuring the limits; on older versions only `count` is reported. - * `elixir.runtime.scheduler.utilization` — the busy fraction of scheduler time, as a + * `elixir.runtime.scheduler.utilization` - the busy fraction of scheduler time, as a ratio between `0.0` and `1.0`. Unlike the others this is a delta between two samples, so the first collection only takes a baseline and the first value arrives one collection later. The SDK polls for it itself, at the period configured for @@ -67,9 +88,15 @@ defmodule Sentry.Metrics do ### Runtime Version Attributes - Setting `version_attributes: true` adds `elixir_version` and `otp_release` - attributes to every reported measurement, so metrics can be grouped by runtime - version. It is off by default, because those values change only on upgrade and + Setting `version_attributes: true` adds these attributes to every reported + measurement, so metrics can be grouped by runtime version: + + * `process.runtime.name` - always `"elixir"`. + * `process.runtime.version` - the Elixir version, as returned by `System.version/0`. + * `process.runtime.engine.name` - always `"BEAM"`. + * `process.runtime.engine.version` - the OTP release, such as `"27"`. + + It is off by default, because those values change only on upgrade and attaching them to every point starts a fresh series for each metric on every rolling deploy. @@ -96,13 +123,15 @@ defmodule Sentry.Metrics do ## Options - * `:unit` - The unit of measurement (e.g., "click", "request"). Optional. + * `:unit` - The unit of measurement, one of those listed under + [Units](#module-units). Counters usually have none. Optional. * `:attributes` - A map of key-value pairs to attach to the metric. Optional. ## Examples Sentry.Metrics.count("button.clicks", 1) - Sentry.Metrics.count("http.requests", 5, unit: "request", attributes: %{method: "GET"}) + Sentry.Metrics.count("http.requests", 5, attributes: %{method: "GET"}) + Sentry.Metrics.count("http.response.body.size", 2048, unit: "byte") """ @spec count(String.t(), number(), keyword()) :: :ok @@ -118,7 +147,8 @@ defmodule Sentry.Metrics do ## Options - * `:unit` - The unit of measurement (e.g., "byte", "connection"). Optional. + * `:unit` - The unit of measurement, one of those listed under + [Units](#module-units), such as `"byte"` or `"ratio"`. Optional. * `:attributes` - A map of key-value pairs to attach to the metric. Optional. ## Examples @@ -140,7 +170,8 @@ defmodule Sentry.Metrics do ## Options - * `:unit` - The unit of measurement (e.g., "millisecond", "byte"). Optional. + * `:unit` - The unit of measurement, one of those listed under + [Units](#module-units), such as `"millisecond"` or `"byte"`. Optional. * `:attributes` - A map of key-value pairs to attach to the metric. Optional. ## Examples diff --git a/lib/sentry/metrics/runtime.ex b/lib/sentry/metrics/runtime.ex index 469e7d32a..d78c9f4f8 100644 --- a/lib/sentry/metrics/runtime.ex +++ b/lib/sentry/metrics/runtime.ex @@ -16,7 +16,7 @@ defmodule Sentry.Metrics.Runtime do @events [@memory_event, @run_queue_event, @system_counts_event, @scheduler_event] - @run_queue_keys [:total, :cpu, :io] + @run_queue_types [:cpu, :io] @system_counts [ {"process", :process_count, :process_limit}, @@ -24,17 +24,7 @@ defmodule Sentry.Metrics.Runtime do {"port", :port_count, :port_limit} ] - @memory_keys [ - :total, - :processes, - :processes_used, - :system, - :atom, - :atom_used, - :binary, - :code, - :ets - ] + @memory_types [:processes, :atom, :binary, :code, :ets] @spec attach(keyword()) :: :ok def attach(opts) when is_list(opts) do @@ -59,11 +49,23 @@ defmodule Sentry.Metrics.Runtime do :telemetry.handler_config() ) :: :ok def handle_event(@memory_event, measurements, _metadata, config) do - report_measured(config, measurements, @memory_keys, "elixir.runtime.mem", "byte") + report_by_type( + config, + "elixir.runtime.memory.used", + "byte", + "elixir.memory.type", + memory_by_type(measurements) + ) end def handle_event(@run_queue_event, measurements, _metadata, config) do - report_measured(config, measurements, @run_queue_keys, "elixir.runtime.run_queue", nil) + report_by_type( + config, + "elixir.runtime.run_queue.length", + "none", + "elixir.run_queue.type", + Map.take(measurements, @run_queue_types) + ) end def handle_event(@system_counts_event, measurements, _metadata, config) do @@ -139,23 +141,35 @@ defmodule Sentry.Metrics.Runtime do defp report_count(_config, _name, nil, _limit), do: :ok defp report_count(config, name, count, limit) do - gauge(config, "elixir.runtime.#{name}.count", count, nil) + gauge(config, "elixir.runtime.#{name}.count", count, "none") report_limit(config, name, count, limit) end defp report_limit(_config, _name, _count, nil), do: :ok defp report_limit(config, name, count, limit) do - gauge(config, "elixir.runtime.#{name}.limit", limit, nil) + gauge(config, "elixir.runtime.#{name}.limit", limit, "none") gauge(config, "elixir.runtime.#{name}.utilization", ratio(count, limit), "ratio") end defp ratio(_count, 0), do: 0.0 defp ratio(count, limit), do: count / limit - defp report_measured(config, measurements, keys, prefix, unit) do - Enum.each(Map.take(measurements, keys), fn {key, value} -> - gauge(config, "#{prefix}.#{key}", value, unit) + defp memory_by_type(measurements) do + named = Map.take(measurements, @memory_types) + + case measurements do + %{total: total} when map_size(named) == length(@memory_types) -> + Map.put(named, :other, max(total - Enum.sum(Map.values(named)), 0)) + + _incomplete -> + named + end + end + + defp report_by_type(config, name, unit, type_attribute, values_by_type) do + Enum.each(values_by_type, fn {type, value} -> + gauge(config, name, value, unit, %{type_attribute => Atom.to_string(type)}) end) end @@ -173,12 +187,14 @@ defmodule Sentry.Metrics.Runtime do defp version_attributes(true) do %{ - "elixir_version" => System.version(), - "otp_release" => List.to_string(:erlang.system_info(:otp_release)) + "process.runtime.name" => "elixir", + "process.runtime.version" => System.version(), + "process.runtime.engine.name" => "BEAM", + "process.runtime.engine.version" => List.to_string(:erlang.system_info(:otp_release)) } end - defp gauge(%{attributes: attributes}, name, value, unit) do - Metrics.gauge(name, value, unit: unit, attributes: attributes) + defp gauge(%{attributes: attributes}, name, value, unit, extra_attributes \\ %{}) do + Metrics.gauge(name, value, unit: unit, attributes: Map.merge(attributes, extra_attributes)) end end diff --git a/mix.exs b/mix.exs index 962da6baf..8f64fa598 100644 --- a/mix.exs +++ b/mix.exs @@ -236,10 +236,20 @@ defmodule Sentry.Mixfile do defp aliases do [ test: ["sentry.package_source_code", "test"], - "test.integrations": &run_integration_tests_if_supported/1 + "test.integrations": &run_integration_tests_if_supported/1, + "test.conventions": &run_conventions_check/1 ] end + defp run_conventions_check(args) do + {_, status} = + System.cmd(Path.expand("scripts/check_metrics_conventions.sh"), args, + into: IO.binstream(:stdio, :line) + ) + + if status > 0, do: System.at_exit(fn _ -> exit({:shutdown, status}) end) + end + defp run_integration_tests_if_supported(args) do run_integration_tests("prod_mode", args, env: [{"MIX_ENV", "prod"}]) diff --git a/scripts/check_metrics_conventions.sh b/scripts/check_metrics_conventions.sh new file mode 100755 index 000000000..2fa6494bc --- /dev/null +++ b/scripts/check_metrics_conventions.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +set -euo pipefail + +repository="${CONVENTIONS_REPOSITORY:-getsentry/sentry-conventions}" +ref="${1:-${CONVENTIONS_REF:-feat/metric-model}}" +checkout="${CONVENTIONS_CHECKOUT:-tmp/sentry-conventions}" +definitions="test/fixtures/sentry_conventions/model" +minimum_node_major=22 + +cd "$(dirname "$0")/.." + +for tool in git node; do + if ! command -v "$tool" >/dev/null; then + echo "$tool is required to check metrics against sentry-conventions" >&2 + exit 1 + fi +done + +if [ ! -d "$checkout/.git" ]; then + echo "==> Cloning $repository into $checkout" + git clone --quiet --filter=blob:none "https://github.com/$repository.git" "$checkout" +fi + +echo "==> Checking out $repository@$ref" +git -C "$checkout" fetch --quiet origin "$ref" +git -C "$checkout" checkout --quiet --force --detach FETCH_HEAD +git -C "$checkout" clean --quiet -fd -- model +echo "==> sentry-conventions at $(git -C "$checkout" rev-parse --short HEAD)" + +node_runner=() +if [ "$(node -p 'process.versions.node.split(".")[0]')" -lt "$minimum_node_major" ]; then + pinned_node=$(node -p "require('./$checkout/package.json').volta?.node || '$minimum_node_major'") + + if ! command -v mise >/dev/null; then + echo "sentry-conventions needs Node.js $minimum_node_major or later (it pins $pinned_node), found $(node --version)" >&2 + exit 1 + fi + + echo "==> Using Node.js $pinned_node through mise, $(node --version) is too old for sentry-conventions" + node_runner=(mise exec "node@$pinned_node" --) +fi + +echo "==> Validating $definitions with the sentry-conventions tests" +cp -R "$definitions/." "$checkout/model/" +( + cd "$checkout" + export COREPACK_ENABLE_DOWNLOAD_PROMPT=0 + "${node_runner[@]}" corepack yarn install --frozen-lockfile --silent + "${node_runner[@]}" corepack yarn vitest run test/metrics.test.ts test/attributes.test.ts +) + +echo "==> Checking emitted runtime metrics against $definitions" +SENTRY_CONVENTIONS_PATH="$checkout" MIX_ENV=test mix test test/sentry/metrics/runtime_conventions_test.exs diff --git a/test/fixtures/sentry_conventions/model/attributes/elixir/elixir__memory__type.json b/test/fixtures/sentry_conventions/model/attributes/elixir/elixir__memory__type.json new file mode 100644 index 000000000..04e05560c --- /dev/null +++ b/test/fixtures/sentry_conventions/model/attributes/elixir/elixir__memory__type.json @@ -0,0 +1,17 @@ +{ + "key": "elixir.memory.type", + "brief": "The kind of BEAM memory a measurement covers: processes, atom, binary, code, ets, or other for the rest of the VM total.", + "type": "string", + "apply_scrubbing": { + "key": "never" + }, + "is_in_otel": false, + "visibility": "public", + "examples": ["processes", "binary", "other"], + "changelog": [ + { + "version": "next", + "description": "Added elixir.memory.type attribute" + } + ] +} diff --git a/test/fixtures/sentry_conventions/model/attributes/elixir/elixir__run_queue__type.json b/test/fixtures/sentry_conventions/model/attributes/elixir/elixir__run_queue__type.json new file mode 100644 index 000000000..a09c13889 --- /dev/null +++ b/test/fixtures/sentry_conventions/model/attributes/elixir/elixir__run_queue__type.json @@ -0,0 +1,17 @@ +{ + "key": "elixir.run_queue.type", + "brief": "The kind of BEAM run queue a measurement covers: cpu for the normal and dirty CPU scheduler run queues, io for the dirty IO run queue.", + "type": "string", + "apply_scrubbing": { + "key": "never" + }, + "is_in_otel": false, + "visibility": "public", + "examples": ["cpu", "io"], + "changelog": [ + { + "version": "next", + "description": "Added elixir.run_queue.type attribute" + } + ] +} diff --git a/test/fixtures/sentry_conventions/model/metrics/elixir/elixir__runtime__atom__count.json b/test/fixtures/sentry_conventions/model/metrics/elixir/elixir__runtime__atom__count.json new file mode 100644 index 000000000..95c4b1192 --- /dev/null +++ b/test/fixtures/sentry_conventions/model/metrics/elixir/elixir__runtime__atom__count.json @@ -0,0 +1,8 @@ +{ + "key": "elixir.runtime.atom.count", + "brief": "Number of atoms currently existing in the BEAM.", + "instrument": "gauge", + "unit": "none", + "is_in_otel": false, + "requirement_level": "opt_in" +} diff --git a/test/fixtures/sentry_conventions/model/metrics/elixir/elixir__runtime__atom__limit.json b/test/fixtures/sentry_conventions/model/metrics/elixir/elixir__runtime__atom__limit.json new file mode 100644 index 000000000..fbe6f1f4c --- /dev/null +++ b/test/fixtures/sentry_conventions/model/metrics/elixir/elixir__runtime__atom__limit.json @@ -0,0 +1,8 @@ +{ + "key": "elixir.runtime.atom.limit", + "brief": "Maximum number of atoms that can exist at the same time in the BEAM.", + "instrument": "gauge", + "unit": "none", + "is_in_otel": false, + "requirement_level": "opt_in" +} diff --git a/test/fixtures/sentry_conventions/model/metrics/elixir/elixir__runtime__atom__utilization.json b/test/fixtures/sentry_conventions/model/metrics/elixir/elixir__runtime__atom__utilization.json new file mode 100644 index 000000000..3ca233e4a --- /dev/null +++ b/test/fixtures/sentry_conventions/model/metrics/elixir/elixir__runtime__atom__utilization.json @@ -0,0 +1,8 @@ +{ + "key": "elixir.runtime.atom.utilization", + "brief": "Ratio of existing atoms to the maximum the BEAM allows, between 0.0 and 1.0.", + "instrument": "gauge", + "unit": "ratio", + "is_in_otel": false, + "requirement_level": "opt_in" +} diff --git a/test/fixtures/sentry_conventions/model/metrics/elixir/elixir__runtime__memory__used.json b/test/fixtures/sentry_conventions/model/metrics/elixir/elixir__runtime__memory__used.json new file mode 100644 index 000000000..e0250222b --- /dev/null +++ b/test/fixtures/sentry_conventions/model/metrics/elixir/elixir__runtime__memory__used.json @@ -0,0 +1,9 @@ +{ + "key": "elixir.runtime.memory.used", + "brief": "Memory allocated by the BEAM, broken down by memory type. Summing across types gives the total VM memory.", + "instrument": "gauge", + "unit": "byte", + "is_in_otel": false, + "requirement_level": "opt_in", + "attributes": ["elixir.memory.type"] +} diff --git a/test/fixtures/sentry_conventions/model/metrics/elixir/elixir__runtime__port__count.json b/test/fixtures/sentry_conventions/model/metrics/elixir/elixir__runtime__port__count.json new file mode 100644 index 000000000..f30105967 --- /dev/null +++ b/test/fixtures/sentry_conventions/model/metrics/elixir/elixir__runtime__port__count.json @@ -0,0 +1,8 @@ +{ + "key": "elixir.runtime.port.count", + "brief": "Number of ports currently existing in the BEAM.", + "instrument": "gauge", + "unit": "none", + "is_in_otel": false, + "requirement_level": "opt_in" +} diff --git a/test/fixtures/sentry_conventions/model/metrics/elixir/elixir__runtime__port__limit.json b/test/fixtures/sentry_conventions/model/metrics/elixir/elixir__runtime__port__limit.json new file mode 100644 index 000000000..0d627a24b --- /dev/null +++ b/test/fixtures/sentry_conventions/model/metrics/elixir/elixir__runtime__port__limit.json @@ -0,0 +1,8 @@ +{ + "key": "elixir.runtime.port.limit", + "brief": "Maximum number of ports that can exist at the same time in the BEAM.", + "instrument": "gauge", + "unit": "none", + "is_in_otel": false, + "requirement_level": "opt_in" +} diff --git a/test/fixtures/sentry_conventions/model/metrics/elixir/elixir__runtime__port__utilization.json b/test/fixtures/sentry_conventions/model/metrics/elixir/elixir__runtime__port__utilization.json new file mode 100644 index 000000000..e2e4b442d --- /dev/null +++ b/test/fixtures/sentry_conventions/model/metrics/elixir/elixir__runtime__port__utilization.json @@ -0,0 +1,8 @@ +{ + "key": "elixir.runtime.port.utilization", + "brief": "Ratio of existing ports to the maximum the BEAM allows, between 0.0 and 1.0.", + "instrument": "gauge", + "unit": "ratio", + "is_in_otel": false, + "requirement_level": "opt_in" +} diff --git a/test/fixtures/sentry_conventions/model/metrics/elixir/elixir__runtime__process__count.json b/test/fixtures/sentry_conventions/model/metrics/elixir/elixir__runtime__process__count.json new file mode 100644 index 000000000..9d815c0ff --- /dev/null +++ b/test/fixtures/sentry_conventions/model/metrics/elixir/elixir__runtime__process__count.json @@ -0,0 +1,8 @@ +{ + "key": "elixir.runtime.process.count", + "brief": "Number of processes currently existing in the BEAM.", + "instrument": "gauge", + "unit": "none", + "is_in_otel": false, + "requirement_level": "opt_in" +} diff --git a/test/fixtures/sentry_conventions/model/metrics/elixir/elixir__runtime__process__limit.json b/test/fixtures/sentry_conventions/model/metrics/elixir/elixir__runtime__process__limit.json new file mode 100644 index 000000000..62887178e --- /dev/null +++ b/test/fixtures/sentry_conventions/model/metrics/elixir/elixir__runtime__process__limit.json @@ -0,0 +1,8 @@ +{ + "key": "elixir.runtime.process.limit", + "brief": "Maximum number of processes that can exist at the same time in the BEAM.", + "instrument": "gauge", + "unit": "none", + "is_in_otel": false, + "requirement_level": "opt_in" +} diff --git a/test/fixtures/sentry_conventions/model/metrics/elixir/elixir__runtime__process__utilization.json b/test/fixtures/sentry_conventions/model/metrics/elixir/elixir__runtime__process__utilization.json new file mode 100644 index 000000000..f24a57eae --- /dev/null +++ b/test/fixtures/sentry_conventions/model/metrics/elixir/elixir__runtime__process__utilization.json @@ -0,0 +1,8 @@ +{ + "key": "elixir.runtime.process.utilization", + "brief": "Ratio of existing processes to the maximum the BEAM allows, between 0.0 and 1.0.", + "instrument": "gauge", + "unit": "ratio", + "is_in_otel": false, + "requirement_level": "opt_in" +} diff --git a/test/fixtures/sentry_conventions/model/metrics/elixir/elixir__runtime__run_queue__length.json b/test/fixtures/sentry_conventions/model/metrics/elixir/elixir__runtime__run_queue__length.json new file mode 100644 index 000000000..7322e498f --- /dev/null +++ b/test/fixtures/sentry_conventions/model/metrics/elixir/elixir__runtime__run_queue__length.json @@ -0,0 +1,9 @@ +{ + "key": "elixir.runtime.run_queue.length", + "brief": "Number of processes and ports ready to run but waiting in a run queue, broken down by run queue type.", + "instrument": "gauge", + "unit": "none", + "is_in_otel": false, + "requirement_level": "opt_in", + "attributes": ["elixir.run_queue.type"] +} diff --git a/test/fixtures/sentry_conventions/model/metrics/elixir/elixir__runtime__scheduler__utilization.json b/test/fixtures/sentry_conventions/model/metrics/elixir/elixir__runtime__scheduler__utilization.json new file mode 100644 index 000000000..284d983fa --- /dev/null +++ b/test/fixtures/sentry_conventions/model/metrics/elixir/elixir__runtime__scheduler__utilization.json @@ -0,0 +1,8 @@ +{ + "key": "elixir.runtime.scheduler.utilization", + "brief": "Fraction of wall time the normal BEAM schedulers spent busy since the previous sample, between 0.0 and 1.0.", + "instrument": "gauge", + "unit": "ratio", + "is_in_otel": false, + "requirement_level": "opt_in" +} diff --git a/test/sentry/metrics/runtime_conventions_test.exs b/test/sentry/metrics/runtime_conventions_test.exs new file mode 100644 index 000000000..dc9526e93 --- /dev/null +++ b/test/sentry/metrics/runtime_conventions_test.exs @@ -0,0 +1,171 @@ +defmodule Sentry.Metrics.RuntimeConventionsTest do + use Sentry.Case, async: false + + import Sentry.TestHelpers + + alias Sentry.Metrics.Runtime + alias Sentry.Test, as: SentryTest + + @namespace "elixir" + @definitions Path.expand("../../fixtures/sentry_conventions", __DIR__) + + setup do + SentryTest.setup_sentry() + :ok = Runtime.attach(version_attributes: true) + on_exit(&Runtime.detach/0) + %{metrics: emit_all_runtime_metrics()} + end + + test "every emitted runtime metric has a conventions definition", %{metrics: metrics} do + missing = + for name <- metric_names(metrics), + not File.exists?(metric_definition_path(name)), + do: name + + assert missing == [] + end + + test "every definition in the elixir namespace is emitted", %{metrics: metrics} do + defined = namespace_definitions() |> Enum.map(& &1["key"]) |> Enum.sort() + + assert defined -- metric_names(metrics) == [] + end + + test "emitted type and unit match the definition", %{metrics: metrics} do + mismatches = + for metric <- metrics, + definition = read_metric_definition(metric.name), + definition != nil, + {to_string(metric.type), metric.unit} != + {definition["instrument"], definition["unit"]}, + uniq: true, + do: + {metric.name, + sdk: {to_string(metric.type), metric.unit}, + conventions: {definition["instrument"], definition["unit"]}} + + assert mismatches == [] + end + + test "breakdown attributes are declared by the definition and present on every point", %{ + metrics: metrics + } do + mismatches = + for {name, points} <- Enum.group_by(metrics, & &1.name), + definition = read_metric_definition(name), + definition != nil, + declared = Enum.sort(definition["attributes"] || []), + point <- points, + emitted = namespace_attribute_keys(point), + emitted != declared, + uniq: true, + do: {name, sdk: emitted, conventions: declared} + + assert mismatches == [] + end + + @tag :conventions + test "every emitted attribute is registered in sentry-conventions and not deprecated", %{ + metrics: metrics + } do + conventions = conventions_path!() + keys = metrics |> Enum.flat_map(&Map.keys(&1.attributes)) |> Enum.uniq() |> Enum.sort() + + problems = + for key <- keys, + problem = attribute_problem(conventions, key), + problem != nil, + do: {key, problem} + + assert problems == [] + end + + defp emit_all_runtime_metrics do + :telemetry.execute( + [:vm, :memory], + %{total: 100_000, processes: 40_000, atom: 1_000, binary: 5_000, code: 20_000, ets: 3_000}, + %{} + ) + + :telemetry.execute([:vm, :total_run_queue_lengths], %{total: 7, cpu: 5, io: 2}, %{}) + + :telemetry.execute( + [:vm, :system_counts], + %{ + process_count: 100, + process_limit: 1_000, + atom_count: 50, + atom_limit: 500, + port_count: 4, + port_limit: 200 + }, + %{} + ) + + :ok = Runtime.dispatch_scheduler_utilization() + :ok = Runtime.dispatch_scheduler_utilization() + + flush_telemetry_processor() + + metrics = SentryTest.pop_sentry_metrics() + refute metrics == [] + metrics + end + + defp conventions_path! do + case System.get_env("SENTRY_CONVENTIONS_PATH") do + nil -> flunk("SENTRY_CONVENTIONS_PATH must point at a sentry-conventions checkout") + path -> Path.expand(path) + end + end + + defp metric_names(metrics), do: metrics |> Enum.map(& &1.name) |> Enum.uniq() |> Enum.sort() + + defp namespace_attribute_keys(point) do + point.attributes + |> Map.keys() + |> Enum.filter(&String.starts_with?(&1, @namespace <> ".")) + |> Enum.sort() + end + + defp namespace_definitions do + @definitions + |> Path.join("model/metrics/#{@namespace}/*.json") + |> Path.wildcard() + |> Enum.map(&(&1 |> File.read!() |> Jason.decode!())) + end + + defp read_metric_definition(name) do + path = metric_definition_path(name) + if File.exists?(path), do: path |> File.read!() |> Jason.decode!() + end + + defp metric_definition_path(name) do + Path.join([@definitions, "model/metrics", namespace_of(name), file_name(name)]) + end + + defp attribute_problem(conventions, key) do + path = Enum.find(attribute_paths(conventions, key), &File.exists?/1) + + cond do + path == nil -> :not_registered + get_in(Jason.decode!(File.read!(path)), ["deprecation"]) != nil -> :deprecated + true -> nil + end + end + + defp attribute_paths(conventions, key) do + for root <- [@definitions, conventions] do + Path.join([root, "model/attributes", namespace_of(key), file_name(key)]) + end + end + + defp namespace_of(key) do + case String.split(key, ".", parts: 2) do + [namespace, _rest] -> namespace + [_single] -> "" + end + end + + defp file_name(key), do: String.replace(key, ".", "__") <> ".json" +end diff --git a/test/sentry/metrics/runtime_test.exs b/test/sentry/metrics/runtime_test.exs index fa7f9a004..48dc24d5a 100644 --- a/test/sentry/metrics/runtime_test.exs +++ b/test/sentry/metrics/runtime_test.exs @@ -24,34 +24,60 @@ defmodule Sentry.Metrics.RuntimeTest do end describe "memory metrics" do - test "reports every key of the measurement map as a gauge in bytes" do + test "reports one elixir.runtime.memory.used gauge in bytes per memory type" do metrics = emit_memory() - for {key, value} <- @memory_measurements do - metric = find_metric!(metrics, "elixir.runtime.mem.#{key}") + assert Enum.all?(metrics, &(&1.name == "elixir.runtime.memory.used" and &1.unit == "byte")) - assert metric.value == value - assert metric.unit == "byte" - end + assert memory_by_type(metrics) == %{ + "processes" => 40_000, + "atom" => 1_000, + "binary" => 5_000, + "code" => 20_000, + "ets" => 3_000, + "other" => 31_000 + } end - test "skips keys the measurement map does not carry" do - metrics = emit([:vm, :memory], %{total: 1}) + test "breaks the total down into types that sum back to it" do + metrics = emit_memory() - assert [%{name: "elixir.runtime.mem.total"}] = metrics + assert metrics |> memory_by_type() |> Map.values() |> Enum.sum() == + @memory_measurements.total + end + + test "never reports a negative remainder when the named types exceed the total" do + metrics = + emit([:vm, :memory], %{ + total: 1_000, + processes: 600, + atom: 100, + binary: 200, + code: 100, + ets: 100 + }) + + assert memory_by_type(metrics)["other"] == 0 + end + + test "reports only the types the measurement lets it compute" do + metrics = emit([:vm, :memory], %{total: 1_000, processes: 400, binary: 100}) + + assert memory_by_type(metrics) == %{"processes" => 400, "binary" => 100} end end describe "run queue metrics" do - test "reports the run queue lengths as unitless gauges" do + test "reports one unitless elixir.runtime.run_queue.length gauge per run queue type" do metrics = emit([:vm, :total_run_queue_lengths], %{total: 7, cpu: 5, io: 2}) - for {key, value} <- [total: 7, cpu: 5, io: 2] do - metric = find_metric!(metrics, "elixir.runtime.run_queue.#{key}") + assert Enum.all?( + metrics, + &(&1.name == "elixir.runtime.run_queue.length" and &1.unit == "none") + ) - assert metric.value == value - assert metric.unit == nil - end + assert Map.new(metrics, &{&1.attributes["elixir.run_queue.type"], &1.value}) == + %{"cpu" => 5, "io" => 2} end end @@ -65,12 +91,15 @@ defmodule Sentry.Metrics.RuntimeTest do port_limit: 200 } - test "reports the count, the limit and the ratio between them" do + test "reports the unitless count and limit and the ratio between them" do metrics = emit([:vm, :system_counts], @system_counts) for {name, count, limit} <- [{"process", 100, 1_000}, {"atom", 50, 500}, {"port", 4, 200}] do - assert find_metric!(metrics, "elixir.runtime.#{name}.count").value == count - assert find_metric!(metrics, "elixir.runtime.#{name}.limit").value == limit + assert %{value: ^count, unit: "none"} = + find_metric!(metrics, "elixir.runtime.#{name}.count") + + assert %{value: ^limit, unit: "none"} = + find_metric!(metrics, "elixir.runtime.#{name}.limit") utilization = find_metric!(metrics, "elixir.runtime.#{name}.utilization") assert utilization.value == count / limit @@ -149,6 +178,13 @@ defmodule Sentry.Metrics.RuntimeTest do end describe "metric attributes" do + @runtime_attributes [ + "process.runtime.name", + "process.runtime.version", + "process.runtime.engine.name", + "process.runtime.engine.version" + ] + test "tags every metric with the runtime metrics origin" do for metric <- emit_memory() do assert metric.attributes["sentry.origin"] == "auto.elixir.runtime_metrics" @@ -161,19 +197,24 @@ defmodule Sentry.Metrics.RuntimeTest do end end - test "attaches the Elixir and OTP versions when version_attributes is enabled" do + test "attaches the process.runtime attributes when version_attributes is enabled" do for metric <- emit_memory(version_attributes: true) do - assert metric.attributes["elixir_version"] == System.version() + assert Map.take(metric.attributes, @runtime_attributes) == %{ + "process.runtime.name" => "elixir", + "process.runtime.version" => System.version(), + "process.runtime.engine.name" => "BEAM", + "process.runtime.engine.version" => + List.to_string(:erlang.system_info(:otp_release)) + } - assert metric.attributes["otp_release"] == - List.to_string(:erlang.system_info(:otp_release)) + refute Map.has_key?(metric.attributes, "elixir_version") + refute Map.has_key?(metric.attributes, "otp_release") end end - test "omits the version attributes by default" do + test "omits the process.runtime attributes by default" do for metric <- emit_memory() do - refute Map.has_key?(metric.attributes, "elixir_version") - refute Map.has_key?(metric.attributes, "otp_release") + assert Map.take(metric.attributes, @runtime_attributes) == %{} end end end @@ -187,10 +228,10 @@ defmodule Sentry.Metrics.RuntimeTest do :ok = SentryTest.allow_sentry_reports(self(), poller) collect_once(poller) - metric = assert_sentry_metric(:gauge, name: "elixir.runtime.mem.total") + metric = assert_sentry_metric(:gauge, name: "elixir.runtime.memory.used") assert metric.value > 0 - assert_sentry_metric(:gauge, name: "elixir.runtime.run_queue.total") + assert_sentry_metric(:gauge, name: "elixir.runtime.run_queue.length") assert_sentry_metric(:gauge, name: "elixir.runtime.process.count") assert_sentry_metric(:gauge, name: "elixir.runtime.process.limit") end @@ -237,6 +278,12 @@ defmodule Sentry.Metrics.RuntimeTest do on_exit(fn -> Application.put_env(:telemetry_poller, :default, original) end) end + defp memory_by_type(metrics) do + metrics + |> Enum.filter(&(&1.name == "elixir.runtime.memory.used")) + |> Map.new(&{&1.attributes["elixir.memory.type"], &1.value}) + end + defp find_metric!(metrics, name) do Enum.find(metrics, &(&1.name == name)) || flunk("no #{name} in #{inspect(Enum.map(metrics, & &1.name))}") diff --git a/test/test_helper.exs b/test/test_helper.exs index 7e962a960..1034f0e9e 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -7,6 +7,13 @@ exclude = [otp_25_plus: true] end +exclude = + if System.get_env("SENTRY_CONVENTIONS_PATH") do + exclude + else + [{:conventions, true} | exclude] + end + ExUnit.start(assert_receive_timeout: 1000, exclude: exclude) # Start the default-named RateLimiter globally for the entire test suite. diff --git a/test_integrations/phoenix_app/test/phoenix_app/runtime_metrics_test.exs b/test_integrations/phoenix_app/test/phoenix_app/runtime_metrics_test.exs index a71eca8ca..65512c5d6 100644 --- a/test_integrations/phoenix_app/test/phoenix_app/runtime_metrics_test.exs +++ b/test_integrations/phoenix_app/test/phoenix_app/runtime_metrics_test.exs @@ -6,32 +6,34 @@ defmodule Sentry.Integrations.Phoenix.RuntimeMetricsTest do alias Sentry.Metrics.Runtime alias Sentry.Test, as: SentryTest - @memory_keys [ - :total, - :processes, - :processes_used, - :system, - :atom, - :atom_used, - :binary, - :code, - :ets - ] + @memory_types ["processes", "atom", "binary", "code", "ets", "other"] describe "memory metrics from a real telemetry_poller" do setup do SentryTest.setup_sentry(collect_envelopes: [type: "trace_metric"]) end - test "reports every key erlang:memory/0 measures as a gauge in bytes" do + test "reports one elixir.runtime.memory.used gauge in bytes per memory type" do metrics = collect_runtime_metrics([:memory]) - for key <- @memory_keys do - metric = find_metric!(metrics, "elixir.runtime.mem.#{key}") + memory_by_type = values_by_type(metrics, "elixir.runtime.memory.used", "elixir.memory.type") - assert metric.value > 0 - assert metric.unit == "byte" - end + assert Enum.sort(Map.keys(memory_by_type)) == Enum.sort(@memory_types) + assert Enum.all?(Map.values(memory_by_type), &(&1 > 0)) + + assert metrics + |> Enum.filter(&(&1.name == "elixir.runtime.memory.used")) + |> Enum.all?(&(&1.unit == "byte")) + end + + test "breaks down the erlang:memory/0 total the poller samples so the types sum back to it" do + poller = start_vm_poller([:memory]) + forward_measured_total([:vm, :memory]) + + metrics = collect_metrics_from(poller) + assert_received {:measured_total, ^poller, total} + + assert sum_by_type(metrics, "elixir.runtime.memory.used", "elixir.memory.type") == total end test "delivers the gauges to Sentry as trace_metric envelope items", %{ref: ref} do @@ -39,11 +41,12 @@ defmodule Sentry.Integrations.Phoenix.RuntimeMetricsTest do Sentry.TelemetryProcessor.flush() [batch] = collect_sentry_metric_items(ref, 1, timeout: 2000) - item = Enum.find(batch["items"], &(&1["name"] == "elixir.runtime.mem.total")) + item = Enum.find(batch["items"], &(&1["name"] == "elixir.runtime.memory.used")) assert item["type"] == "gauge" assert item["unit"] == "byte" assert item["value"] > 0 + assert item["attributes"]["elixir.memory.type"]["value"] in @memory_types assert item["attributes"]["sentry.origin"]["value"] == "auto.elixir.runtime_metrics" end end @@ -54,33 +57,38 @@ defmodule Sentry.Integrations.Phoenix.RuntimeMetricsTest do :ok end - test "reports the real run queue lengths as unitless gauges" do + test "reports the cpu and io run queue lengths as unitless gauges" do metrics = collect_runtime_metrics([:total_run_queue_lengths]) - for key <- [:total, :cpu, :io] do - metric = find_metric!(metrics, "elixir.runtime.run_queue.#{key}") + run_queues = + values_by_type(metrics, "elixir.runtime.run_queue.length", "elixir.run_queue.type") - assert is_integer(metric.value) - assert metric.unit == nil - end + assert Enum.sort(Map.keys(run_queues)) == ["cpu", "io"] + assert Enum.all?(Map.values(run_queues), &is_integer/1) + + assert metrics + |> Enum.filter(&(&1.name == "elixir.runtime.run_queue.length")) + |> Enum.all?(&(&1.unit == "none")) end - test "reports the scheduler queue lengths the VM counts as non-negative" do + test "reports the cpu run queue length the VM counts as non-negative" do metrics = collect_runtime_metrics([:total_run_queue_lengths]) - for key <- [:total, :cpu] do - assert find_metric!(metrics, "elixir.runtime.run_queue.#{key}").value >= 0 - end + run_queues = + values_by_type(metrics, "elixir.runtime.run_queue.length", "elixir.run_queue.type") + + assert run_queues["cpu"] >= 0 end - test "forwards the split the poller computes rather than recomputing it" do - metrics = collect_runtime_metrics([:total_run_queue_lengths]) + test "breaks down the poller's total so the run queue types sum back to it" do + poller = start_vm_poller([:total_run_queue_lengths]) + forward_measured_total([:vm, :total_run_queue_lengths]) - total = find_metric!(metrics, "elixir.runtime.run_queue.total").value - cpu = find_metric!(metrics, "elixir.runtime.run_queue.cpu").value - io = find_metric!(metrics, "elixir.runtime.run_queue.io").value + metrics = collect_metrics_from(poller) + assert_received {:measured_total, ^poller, total} - assert total == cpu + io + assert sum_by_type(metrics, "elixir.runtime.run_queue.length", "elixir.run_queue.type") == + total end end @@ -90,12 +98,12 @@ defmodule Sentry.Integrations.Phoenix.RuntimeMetricsTest do :ok end - test "reports the real counts against the limits the VM enforces" do + test "reports the real unitless counts against the limits the VM enforces" do metrics = collect_runtime_metrics([:system_counts]) for name <- ["process", "atom", "port"] do - count = find_metric!(metrics, "elixir.runtime.#{name}.count").value - limit = find_metric!(metrics, "elixir.runtime.#{name}.limit").value + %{value: count, unit: "none"} = find_metric!(metrics, "elixir.runtime.#{name}.count") + %{value: limit, unit: "none"} = find_metric!(metrics, "elixir.runtime.#{name}.limit") utilization = find_metric!(metrics, "elixir.runtime.#{name}.utilization") assert count > 0 @@ -162,7 +170,11 @@ defmodule Sentry.Integrations.Phoenix.RuntimeMetricsTest do end defp collect_runtime_metrics(measurements) do - measurements |> start_vm_poller() |> allow_and_collect() + measurements |> start_vm_poller() |> collect_metrics_from() + end + + defp collect_metrics_from(poller) do + allow_and_collect(poller) Sentry.TelemetryProcessor.flush() metrics = SentryTest.pop_sentry_metrics() @@ -193,6 +205,33 @@ defmodule Sentry.Integrations.Phoenix.RuntimeMetricsTest do :ok end + defp forward_measured_total(event) do + handler_id = "measured-total-#{System.unique_integer([:positive])}" + test_pid = self() + + :ok = + :telemetry.attach( + handler_id, + event, + fn _event, measurements, _metadata, _config -> + send(test_pid, {:measured_total, self(), measurements.total}) + end, + nil + ) + + on_exit(fn -> :telemetry.detach(handler_id) end) + end + + defp sum_by_type(metrics, name, type_attribute) do + metrics |> values_by_type(name, type_attribute) |> Map.values() |> Enum.sum() + end + + defp values_by_type(metrics, name, type_attribute) do + metrics + |> Enum.filter(&(&1.name == name)) + |> Map.new(&{&1.attributes[type_attribute], &1.value}) + end + defp find_metric!(metrics, name) do Enum.find(metrics, &(&1.name == name)) || flunk("no #{name} in #{inspect(Enum.map(metrics, & &1.name))}")