feat(metrics): report run queue depth in runtime metrics

lib/sentry/metrics.ex

@@ -39,6 +39,8 @@ defmodule Sentry.Metrics do
 
     * `elixir.runtime.mem.*` — `total`, `processes`, `processes_used`, `system`,
       `atom`, `atom_used`, `binary`, `code` and `ets`, in bytes
+    * `elixir.runtime.run_queue.*` — `total`, `cpu` and `io`, how many processes are
+      waiting to run
 
   ### Collection Frequency
 

lib/sentry/metrics/runtime.ex

@@ -7,8 +7,11 @@ defmodule Sentry.Metrics.Runtime do
   @origin "auto.elixir.runtime_metrics"
 
   @memory_event [:vm, :memory]
+  @run_queue_event [:vm, :total_run_queue_lengths]
 
-  @events [@memory_event]
+  @events [@memory_event, @run_queue_event]
+
+  @run_queue_keys [:total, :cpu, :io]
 
   @memory_keys [
     :total,
@@ -48,6 +51,10 @@ defmodule Sentry.Metrics.Runtime do
     report_measured(config, measurements, @memory_keys, "elixir.runtime.mem", "byte")
   end
 
+  def handle_event(@run_queue_event, measurements, _metadata, config) do
+    report_measured(config, measurements, @run_queue_keys, "elixir.runtime.run_queue", nil)
+  end
+
   defp report_measured(config, measurements, keys, prefix, unit) do
     Enum.each(Map.take(measurements, keys), fn {key, value} ->
       gauge(config, "#{prefix}.#{key}", value, unit)

test/sentry/metrics/runtime_test.exs

@@ -42,6 +42,19 @@ defmodule Sentry.Metrics.RuntimeTest do
     end
   end
 
+  describe "run queue metrics" do
+    test "reports the run queue lengths as unitless gauges" do
+      metrics = emit([:vm, :total_run_queue_lengths], %{total: 7, cpu: 5, io: 2})
+
+      for {key, value} <- [total: 7, cpu: 5, io: 2] do
+        metric = find_metric!(metrics, "elixir.runtime.run_queue.#{key}")
+
+        assert metric.value == value
+        assert metric.unit == nil
+      end
+    end
+  end
+
   describe "metric attributes" do
     test "tags every metric with the runtime metrics origin" do
       for metric <- emit_memory() do
@@ -73,15 +86,17 @@ defmodule Sentry.Metrics.RuntimeTest do
   end
 
   describe "wiring against a real telemetry_poller" do
-    test "maps the builtin memory measurement onto Sentry gauges" do
+    test "maps the builtin measurements onto Sentry gauges" do
       attach()
-      poller = start_idle_poller([:memory])
+      poller = start_idle_poller([:memory, :total_run_queue_lengths])
 
       :ok = SentryTest.allow_sentry_reports(self(), poller)
       collect_once(poller)
 
       metric = assert_sentry_metric(:gauge, name: "elixir.runtime.mem.total")
       assert metric.value > 0
+
+      assert_sentry_metric(:gauge, name: "elixir.runtime.run_queue.total")
     end
   end
 

test_integrations/phoenix_app/test/phoenix_app/runtime_metrics_test.exs

@@ -47,6 +47,42 @@ defmodule Sentry.Integrations.Phoenix.RuntimeMetricsTest do
     end
   end
 
+  describe "run queue metrics from a real telemetry_poller" do
+    setup do
+      SentryTest.setup_sentry()
+      :ok
+    end
+
+    test "reports the real run queue lengths as unitless gauges" do
+      metrics = collect_runtime_metrics([:total_run_queue_lengths])
+
+      for key <- [:total, :cpu, :io] do
+        metric = find_metric!(metrics, "elixir.runtime.run_queue.#{key}")
+
+        assert is_integer(metric.value)
+        assert metric.unit == nil
+      end
+    end
+
+    test "reports the scheduler queue lengths the VM counts as non-negative" do
+      metrics = collect_runtime_metrics([:total_run_queue_lengths])
+
+      for key <- [:total, :cpu] do
+        assert find_metric!(metrics, "elixir.runtime.run_queue.#{key}").value >= 0
+      end
+    end
+
+    test "forwards the split the poller computes rather than recomputing it" do
+      metrics = collect_runtime_metrics([:total_run_queue_lengths])
+
+      total = find_metric!(metrics, "elixir.runtime.run_queue.total").value
+      cpu = find_metric!(metrics, "elixir.runtime.run_queue.cpu").value
+      io = find_metric!(metrics, "elixir.runtime.run_queue.io").value
+
+      assert total == cpu + io
+    end
+  end
+
   describe "the application wiring" do
     test "attaches the runtime metrics handler at boot" do
       handler_ids = [:vm, :memory] |> :telemetry.list_handlers() |> Enum.map(& &1.id)