feat(metrics): report process, atom and port counts in runtime metrics

lib/sentry/metrics.ex

@@ -41,6 +41,10 @@ defmodule Sentry.Metrics do
       `atom`, `atom_used`, `binary`, `code` and `ets`, in bytes
     * `elixir.runtime.run_queue.*` — `total`, `cpu` and `io`, how many processes are
       waiting to run
+    * `elixir.runtime.process.*`, `elixir.runtime.atom.*` and `elixir.runtime.port.*` —
+      a `count`, the hard VM `limit`, and the `utilization` ratio between them. The
+      `limit` and `utilization` gauges need telemetry_poller 1.3.0 or later, which is
+      when it started measuring the limits; on older versions only `count` is reported.
 
   ### Collection Frequency
 

lib/sentry/metrics/runtime.ex

@@ -8,11 +8,18 @@ defmodule Sentry.Metrics.Runtime do
 
   @memory_event [:vm, :memory]
   @run_queue_event [:vm, :total_run_queue_lengths]
+  @system_counts_event [:vm, :system_counts]
 
-  @events [@memory_event, @run_queue_event]
+  @events [@memory_event, @run_queue_event, @system_counts_event]
 
   @run_queue_keys [:total, :cpu, :io]
 
+  @system_counts [
+    {"process", :process_count, :process_limit},
+    {"atom", :atom_count, :atom_limit},
+    {"port", :port_count, :port_limit}
+  ]
+
   @memory_keys [
     :total,
     :processes,
@@ -55,6 +62,29 @@ defmodule Sentry.Metrics.Runtime do
     report_measured(config, measurements, @run_queue_keys, "elixir.runtime.run_queue", nil)
   end
 
+  def handle_event(@system_counts_event, measurements, _metadata, config) do
+    Enum.each(@system_counts, fn {name, count_key, limit_key} ->
+      report_count(config, name, measurements[count_key], measurements[limit_key])
+    end)
+  end
+
+  defp report_count(_config, _name, nil, _limit), do: :ok
+
+  defp report_count(config, name, count, limit) do
+    gauge(config, "elixir.runtime.#{name}.count", count, nil)
+    report_limit(config, name, count, limit)
+  end
+
+  defp report_limit(_config, _name, _count, nil), do: :ok
+
+  defp report_limit(config, name, count, limit) do
+    gauge(config, "elixir.runtime.#{name}.limit", limit, nil)
+    gauge(config, "elixir.runtime.#{name}.utilization", ratio(count, limit), "ratio")
+  end
+
+  defp ratio(_count, 0), do: 0.0
+  defp ratio(count, limit), do: count / limit
+
   defp report_measured(config, measurements, keys, prefix, unit) do
     Enum.each(Map.take(measurements, keys), fn {key, value} ->
       gauge(config, "#{prefix}.#{key}", value, unit)

test/sentry/metrics/runtime_test.exs

@@ -55,6 +55,36 @@ defmodule Sentry.Metrics.RuntimeTest do
     end
   end
 
+  describe "system count metrics" do
+    @system_counts %{
+      process_count: 100,
+      process_limit: 1_000,
+      atom_count: 50,
+      atom_limit: 500,
+      port_count: 4,
+      port_limit: 200
+    }
+
+    test "reports the count, the limit and the ratio between them" do
+      metrics = emit([:vm, :system_counts], @system_counts)
+
+      for {name, count, limit} <- [{"process", 100, 1_000}, {"atom", 50, 500}, {"port", 4, 200}] do
+        assert find_metric!(metrics, "elixir.runtime.#{name}.count").value == count
+        assert find_metric!(metrics, "elixir.runtime.#{name}.limit").value == limit
+
+        utilization = find_metric!(metrics, "elixir.runtime.#{name}.utilization")
+        assert utilization.value == count / limit
+        assert utilization.unit == "ratio"
+      end
+    end
+
+    test "omits the limit and the ratio when the poller does not report limits" do
+      metrics = emit([:vm, :system_counts], %{process_count: 100})
+
+      assert [%{name: "elixir.runtime.process.count", value: 100}] = metrics
+    end
+  end
+
   describe "metric attributes" do
     test "tags every metric with the runtime metrics origin" do
       for metric <- emit_memory() do
@@ -88,7 +118,7 @@ defmodule Sentry.Metrics.RuntimeTest do
   describe "wiring against a real telemetry_poller" do
     test "maps the builtin measurements onto Sentry gauges" do
       attach()
-      poller = start_idle_poller([:memory, :total_run_queue_lengths])
+      poller = start_idle_poller([:memory, :total_run_queue_lengths, :system_counts])
 
       :ok = SentryTest.allow_sentry_reports(self(), poller)
       collect_once(poller)
@@ -97,6 +127,8 @@ defmodule Sentry.Metrics.RuntimeTest do
       assert metric.value > 0
 
       assert_sentry_metric(:gauge, name: "elixir.runtime.run_queue.total")
+      assert_sentry_metric(:gauge, name: "elixir.runtime.process.count")
+      assert_sentry_metric(:gauge, name: "elixir.runtime.process.limit")
     end
   end
 

test_integrations/phoenix_app/test/phoenix_app/runtime_metrics_test.exs

@@ -83,6 +83,41 @@ defmodule Sentry.Integrations.Phoenix.RuntimeMetricsTest do
     end
   end
 
+  describe "system count metrics from a real telemetry_poller" do
+    setup do
+      SentryTest.setup_sentry()
+      :ok
+    end
+
+    test "reports the real counts against the limits the VM enforces" do
+      metrics = collect_runtime_metrics([:system_counts])
+
+      for name <- ["process", "atom", "port"] do
+        count = find_metric!(metrics, "elixir.runtime.#{name}.count").value
+        limit = find_metric!(metrics, "elixir.runtime.#{name}.limit").value
+        utilization = find_metric!(metrics, "elixir.runtime.#{name}.utilization")
+
+        assert count > 0
+        assert count < limit
+        assert utilization.value == count / limit
+        assert utilization.unit == "ratio"
+      end
+    end
+
+    test "forwards the hard limits telemetry_poller measures" do
+      metrics = collect_runtime_metrics([:system_counts])
+
+      for {name, key} <- [
+            {"process", :process_limit},
+            {"atom", :atom_limit},
+            {"port", :port_limit}
+          ] do
+        assert find_metric!(metrics, "elixir.runtime.#{name}.limit").value ==
+                 :erlang.system_info(key)
+      end
+    end
+  end
+
   describe "the application wiring" do
     test "attaches the runtime metrics handler at boot" do
       handler_ids = [:vm, :memory] |> :telemetry.list_handlers() |> Enum.map(& &1.id)