From 2d583a30f47a1a9ae8355736f83778035516c326 Mon Sep 17 00:00:00 2001 From: kalo <24719519+KaloyanTanev@users.noreply.github.com> Date: Fri, 17 Apr 2026 17:37:55 +0300 Subject: [PATCH 1/8] Update charon overview dashboard --- grafana/dashboards/dash_charon_overview.json | 10322 ++++++++++++---- grafana/dashboards/single_node_dashboard.json | 2854 ----- 2 files changed, 8038 insertions(+), 5138 deletions(-) delete mode 100644 grafana/dashboards/single_node_dashboard.json diff --git a/grafana/dashboards/dash_charon_overview.json b/grafana/dashboards/dash_charon_overview.json index c3f60539..d148e443 100644 --- a/grafana/dashboards/dash_charon_overview.json +++ b/grafana/dashboards/dash_charon_overview.json @@ -24,8 +24,32 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 2, - "id": 389, - "links": [], + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": false, + "title": "Explore Charon Logs", + "tooltip": "Navigate to Charon Logs Explorer", + "type": "link", + "url": "/explore?orgId=1&left=%7B%22datasource%22:%22ALmVcoFVk%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bcluster_name%3D%5C%22$cluster_name%5C%22,%20cluster_hash%3D%5C%22$cluster_hash%5C%22,%20cluster_peer%3D%5C%22$cluster_peer%5C%22%7D%20%7C%20logfmt%20%7C%20line_format%20%60%7B%7B.pretty%7D%7D%60%22,%22queryType%22:%22range%22,%22datasource%22:%7B%22type%22:%22loki%22,%22uid%22:%22ALmVcoFVk%22%7D,%22editorMode%22:%22builder%22%7D%5D,%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], "panels": [ { "collapsed": false, @@ -119,7 +143,7 @@ "steps": [ { "color": "green", - "value": null + "value": 0 }, { "color": "red", @@ -132,7 +156,7 @@ "overrides": [] }, "gridPos": { - "h": 4, + "h": 3, "w": 4, "x": 0, "y": 1 @@ -155,7 +179,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.4.1", + "pluginVersion": "12.4.1", "targets": [ { "datasource": { @@ -165,7 +189,6 @@ "editorMode": "code", "exemplar": false, "expr": "app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", - "hide": false, "instant": true, "interval": "", "legendFormat": "", @@ -173,7 +196,7 @@ "refId": "A" } ], - "title": "Health Status", + "title": "Health Status - Peer", "type": "stat" }, { @@ -181,116 +204,61 @@ "type": "prometheus", "uid": "prometheus" }, - "description": "- **Uptime**: Total time since cluster start time\n- **Slot**: Current slot\n- **Epoch**: Current Epoch\n- **Validators Active**: Number of active validators in the cluster\n- **Charon Peers Connected**: Number of Peers in the cluster\n- **Relays Connected**: Number of Relays connected to the cluster\n- **Beacon Node Peers**: Beacon Node Peers connected, based on app_beacon_node_peers\n- **P2P Reachability**\" Latest P2P reachability status, based on p2p_reachability_status", + "description": "Successful cluster duty percentage by type over the whole time range", "fieldConfig": { "defaults": { "color": { - "fixedColor": "text", - "mode": "fixed" + "mode": "thresholds" }, + "links": [], "mappings": [], - "max": 10, + "max": 1, "min": 0, - "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "P2P Reachability" - }, - "properties": [ - { - "id": "mappings", - "value": [ - { - "options": { - "0": { - "color": "text", - "index": 0, - "text": "Unknown" - }, - "1": { - "color": "green", - "index": 1, - "text": "Public" - }, - "2": { - "color": "yellow", - "index": 2, - "text": "Private" - } - }, - "type": "value" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Uptime" - }, - "properties": [ - { - "id": "unit", - "value": "s" + "color": "red", + "value": 0 }, { - "id": "decimals", - "value": 1 + "color": "#EAB839", + "value": 0.9 }, { - "id": "color", - "value": { - "fixedColor": "text", - "mode": "fixed" - } + "color": "green", + "value": 0.99 } ] - } - ] + }, + "unit": "percentunit" + }, + "overrides": [] }, "gridPos": { "h": 9, - "w": 4, + "w": 14, "x": 4, "y": 1 }, - "id": 113, + "id": 176, "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "percentChangeColorMode": "standard", + "minVizHeight": 200, + "minVizWidth": 200, + "orientation": "auto", "reduceOptions": { "calcs": [ - "last" + "lastNotNull" ], "fields": "", - "values": false - }, - "showPercentChange": false, - "text": { - "titleSize": 15, - "valueSize": 15 + "values": true }, - "textMode": "auto", - "wideLayout": true + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "text": {} }, - "pluginVersion": "11.4.1", + "pluginVersion": "12.4.1", "targets": [ { "datasource": { @@ -299,115 +267,28 @@ }, "editorMode": "code", "exemplar": false, - "expr": "core_scheduler_current_slot{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", - "format": "time_series", - "hide": false, + "expr": "(\n sum(increase(core_tracker_success_duties_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$__range])) by (duty)\n) \n/ \n( \n sum(increase(core_tracker_expect_duties_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$__range])) by (duty) \n) > 0", "instant": true, - "interval": "", - "legendFormat": "Slot", + "interval": "$interval", + "legendFormat": "{{duty}}", "range": false, "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "core_scheduler_current_epoch{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", - "hide": false, - "instant": true, - "legendFormat": "Epoch", - "range": false, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "core_scheduler_validators_active{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", - "hide": false, - "instant": true, - "legendFormat": "Validators Active", - "range": false, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", - "hide": false, - "instant": true, - "legendFormat": "Charon Peers Connected", - "range": false, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(p2p_relay_connections{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) ", - "hide": false, - "instant": true, - "legendFormat": "Relays Connected", - "range": false, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "app_beacon_node_peers{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", - "hide": false, - "instant": true, - "legendFormat": "Beacon Node Peers", - "range": false, - "refId": "H" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "p2p_reachability_status{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", - "hide": false, - "instant": true, - "legendFormat": "P2P Reachability", - "range": false, - "refId": "F" } ], - "title": "Current Status", - "type": "stat" + "title": "Cluster Duty Success(%) ", + "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "description": "Successful cluster duty percentage by type over the whole time range. This metric uses the reporting node's view of the chain, and may not be completely accurate (for example if its peers succeed with the duty without this node taking part), consult a block explorer for a more definitive success rate. ", + "description": "Average beacon node score for the whole time window. The score is a combination of:\n - **errors**: the ratio or errors returned by the beacon node \n - **latency**: how fast or slow the beacon node is responding\n\n\nThe performance of the Beacon node is critical to the performance of the validators and therefore rewards. ", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, - "links": [], "mappings": [], "max": 1, "min": 0, @@ -416,15 +297,15 @@ "steps": [ { "color": "red", - "value": null + "value": 0 }, { - "color": "#EAB839", - "value": 0.9 + "color": "orange", + "value": 0.5 }, { "color": "green", - "value": 0.99 + "value": 0.8 } ] }, @@ -434,11 +315,11 @@ }, "gridPos": { "h": 9, - "w": 16, - "x": 8, + "w": 6, + "x": 18, "y": 1 }, - "id": 176, + "id": 174, "options": { "minVizHeight": 200, "minVizWidth": 200, @@ -448,14 +329,13 @@ "lastNotNull" ], "fields": "", - "values": true + "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true, - "sizing": "auto", - "text": {} + "sizing": "auto" }, - "pluginVersion": "11.4.1", + "pluginVersion": "12.4.1", "targets": [ { "datasource": { @@ -464,15 +344,14 @@ }, "editorMode": "code", "exemplar": false, - "expr": "(\n sum(increase(core_tracker_success_duties_total{\n cluster_name=\"$cluster_name\",\n cluster_hash=\"$cluster_hash\",\n cluster_peer=~\"$cluster_peer\",\n duty=~\"attester|proposer|sync_message\"\n }[$__range])) \n by (duty)\n)\n/\n(\n sum(increase(core_tracker_expect_duties_total{\n cluster_name=\"$cluster_name\",\n cluster_hash=\"$cluster_hash\",\n cluster_peer=~\"$cluster_peer\",\n duty=~\"attester|proposer|sync_message\"\n }[$__range])) \n by (duty)\n) > 0 ", + "expr": "(clamp(app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 1 OR app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 4 OR app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 5 OR app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 6, 1, 1) OR on() vector(0))\n*\n(\n (\n0.5 * (1.0 - 10*(sum(increase(app_eth2_errors_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$__range])) / (sum(increase(app_eth2_latency_seconds_count{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", endpoint !=\"submit_validator_registrations\"}[$__range]))))) OR on() vector(0.5)\n )\n +\n (\n 0.5 * (1.0 - clamp_max(histogram_quantile(0.99, sum(rate(app_eth2_latency_seconds_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", endpoint !=\"submit_validator_registrations\"}[$__range])) by (le)),1))\n )\n)", "instant": true, - "interval": "$interval", - "legendFormat": "{{duty}}", + "legendFormat": "__auto", "range": false, "refId": "A" } ], - "title": "Peer Duty Success(%) ", + "title": "Beacon Node Score", "type": "gauge" }, { @@ -492,7 +371,7 @@ "steps": [ { "color": "text", - "value": null + "value": 0 } ] }, @@ -588,10 +467,10 @@ ] }, "gridPos": { - "h": 5, + "h": 6, "w": 4, "x": 0, - "y": 5 + "y": 4 }, "id": 115, "options": { @@ -615,7 +494,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.4.1", + "pluginVersion": "12.4.1", "targets": [ { "datasource": { @@ -626,7 +505,6 @@ "exemplar": false, "expr": "( 1 * app_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", cluster_network=\"mainnet\"} )\nor \n( 2 * app_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", cluster_network=\"goerli\"} )\nor\n( 3 * app_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", cluster_network=\"sepolia\"} )\nor\n( 4 * app_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", cluster_network=\"gnosis\"} )", "format": "time_series", - "hide": false, "instant": true, "legendFormat": "Network", "range": false, @@ -653,7 +531,6 @@ "editorMode": "code", "exemplar": false, "expr": "cluster_threshold{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", - "hide": false, "instant": true, "legendFormat": "Threshold", "range": false, @@ -667,7 +544,6 @@ "editorMode": "code", "exemplar": false, "expr": "core_scheduler_validators_active{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", - "hide": false, "instant": true, "legendFormat": "Validators Active", "range": false, @@ -681,7 +557,6 @@ "editorMode": "code", "exemplar": false, "expr": "cluster_validators{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", - "hide": false, "instant": true, "legendFormat": "Validators Total", "range": false, @@ -708,12 +583,12 @@ "id": "organize", "options": { "excludeByName": { - "F · lock_hash": false + "F \u00b7 lock_hash": false }, "includeByName": {}, "indexByName": { - "F · lock_hash": 6, - "G · Link": 0, + "F \u00b7 lock_hash": 6, + "G \u00b7 Link": 0, "Network": 1, "Operators/Nodes": 2, "Threshold": 3, @@ -721,8 +596,8 @@ "Validators Total": 5 }, "renameByName": { - "F · lock_hash": "_", - "G · Link": "Cluster Hash", + "F \u00b7 lock_hash": "_", + "G \u00b7 Link": "Cluster Hash", "Network": "", "Threshold": "" } @@ -733,33 +608,36 @@ }, { "datasource": { - "type": "prometheus", - "uid": "prometheus" + "type": "datasource", + "uid": "-- Mixed --" }, - "description": "- **Peer**: The peer's name (inferred from their *charon-enr-private-key*)\n- **You**: ⭐️ is this local charon node\n- **Connected**: Whether you are currently connected to this peer.\n- **Direct**: Whether the connection is *direct* (👍) or *relay* (👎) \n- **Latency**: The time messages take to travel to/from the peer.\n- **Version**: The version of the peer's charon node\n- **Supported**: Whether the peers charon version is supported/compatible or not.\n- **Uptime**\" The duration since peer was restarted.\n- **🚧Attest**: Number of attestation duties missed in the time window.\n- **🚧Propose**: Number of block proposal duties missed in the time window.\n- **🚧Other**: Number of other duties missed in the time window.\n\n", + "description": "Health checks that have failed at least once in the selected time window, based on the app_health_checks_failed_total counter.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { - "align": "center", + "align": "auto", "cellOptions": { "type": "auto" }, - "inspect": false, - "minWidth": 0, - "width": 90 + "footer": { + "reducers": [] + }, + "inspect": false }, - "decimals": 0, "mappings": [], - "noValue": "--", "thresholds": { "mode": "absolute", "steps": [ { - "color": "transparent", - "value": null + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 } ] } @@ -768,776 +646,5418 @@ { "matcher": { "id": "byName", - "options": "Latency (ms)" + "options": "Severity" }, "properties": [ - { - "id": "unit", - "value": "none" - }, { "id": "mappings", "value": [ { "options": { - "from": 0, - "result": { - "color": "green", - "index": 0 + "critical": { + "color": "red", + "index": 2, + "text": "critical" }, - "to": 150 + "info": { + "color": "blue", + "index": 0, + "text": "info" + }, + "warning": { + "color": "orange", + "index": 1, + "text": "warning" + } }, - "type": "range" - }, - { - "options": { - "from": 150, - "result": { - "color": "yellow", - "index": 1 - }, - "to": 300 - }, - "type": "range" - }, - { - "options": { - "from": 300, - "result": { - "color": "orange", - "index": 2 - }, - "to": 500 - }, - "type": "range" - }, - { - "options": { - "from": 500, - "result": { - "color": "red", - "index": 3 - }, - "to": 100000 - }, - "type": "range" + "type": "value" } ] }, { "id": "custom.cellOptions", "value": { - "mode": "lcd", - "type": "gauge", - "valueDisplayMode": "color" + "type": "color-text" } }, { "id": "custom.width", - "value": 200 - }, - { - "id": "max", - "value": 700 + "value": 69 } ] }, { "matcher": { "id": "byName", - "options": "You" + "options": "Check Name" }, "properties": [ - { - "id": "noValue", - "value": "⭐" - }, { "id": "custom.cellOptions", "value": { - "type": "auto" + "type": "color-text" } }, { - "id": "mappings", - "value": [ - { - "options": { - "0": { - "index": 0, - "text": "--" - }, - "1": { - "index": 1, - "text": "--" - } - }, - "type": "value" - } - ] + "id": "color", + "value": { + "fixedColor": "super-light-yellow", + "mode": "fixed" + } }, { "id": "custom.width", - "value": 60 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Connected" - }, - "properties": [ - { - "id": "mappings", - "value": [ - { - "options": { - "0": { - "index": 0, - "text": " ❌" - }, - "1": { - "index": 1, - "text": "✅" - } - }, - "type": "value" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "ClockDiff" - }, - "properties": [ - { - "id": "unit", - "value": "ms" - }, - { - "id": "mappings", - "value": [ - { - "options": { - "0": { - "index": 0, - "text": "🆗" - } - }, - "type": "value" - } - ] + "value": 261 } ] }, { "matcher": { "id": "byName", - "options": "Direct" + "options": "Times Failed" }, "properties": [ { "id": "custom.width", - "value": 55 + "value": 100 }, { - "id": "mappings", - "value": [ - { - "options": { - "0": { - "index": 0, - "text": "--" - } - }, - "type": "value" - }, - { - "options": { - "from": 1, - "result": { - "index": 1, - "text": "👎" - }, - "to": 9 - }, - "type": "range" - }, - { - "options": { - "from": 10, - "result": { - "index": 2, - "text": "👍" - }, - "to": 1000 - }, - "type": "range" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Peer" - }, - "properties": [ - { - "id": "custom.width", - "value": 150 + "id": "decimals", + "value": 0 } ] }, { "matcher": { "id": "byName", - "options": "Uptime" + "options": "Last Reported" }, "properties": [ { "id": "unit", - "value": "s" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Index" - }, - "properties": [ + "value": "dateTimeAsLocal" + }, { "id": "custom.width", - "value": 30 + "value": 160 } ] }, { "matcher": { - "id": "byRegexp", - "options": "🚧.*" + "id": "byName", + "options": "Description" }, "properties": [ { - "id": "unit", - "value": "percentunit" - }, - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "thresholds" - } + "id": "custom.wrapText", + "value": true }, { "id": "custom.cellOptions", "value": { - "type": "color-text" - } - }, - { - "id": "decimals", - "value": 1 - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "#585858", - "value": null - }, - { - "color": "super-light-green", - "value": 0.00001 - }, - { - "color": "yellow", - "value": 0.01 - }, - { - "color": "red", - "value": 0.1 - } - ] + "type": "auto" } - }, - { - "id": "noValue", - "value": "0%" } ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 333, + "options": { + "cellHeight": "lg", + "showHeader": true + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" }, - { - "matcher": { - "id": "byName", - "options": "Supported" - }, - "properties": [ - { - "id": "mappings", - "value": [ - { - "options": { - "0": { - "index": 0, - "text": "❌" - }, - "1": { - "index": 1, - "text": "🆗" - } - }, - "type": "value" - } - ] - } - ] + "editorMode": "code", + "exemplar": false, + "expr": "max by (name, severity, description) (\n app_health_checks_failed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}\n) > 0", + "format": "table", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" }, - { - "matcher": { - "id": "byName", - "options": "Builder API" - }, - "properties": [ - { - "id": "mappings", - "value": [ - { + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\n label_replace(\n label_replace(\n (\n max by (cluster_peer, beacon_id) (\n app_beacon_node_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",beacon_id=~\".+\"}\n )\n ) * on(beacon_id) group_left()\n (\n count by (beacon_id) (\n max by (cluster_peer, beacon_id) (\n app_beacon_node_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",beacon_id=~\".+\"}\n )\n ) > 1\n ),\n \"name\", \"reused_beacon_node\", \"\", \"\"\n ),\n \"severity\", \"warning\", \"\", \"\"\n ),\n \"description\", \"This peer shares its beacon node (beacon_id: $1) with at least one other peer in the cluster\", \"beacon_id\", \"(.*)\"\n)", + "format": "table", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Health Checks History", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "mode": "columns" + } + }, + { + "id": "merge", + "options": {} + }, + { + "id": "groupBy", + "options": { + "fields": { + "Time": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "Value": { + "aggregations": [] + }, + "description": { + "aggregations": [], + "operation": "groupby" + }, + "name": { + "aggregations": [], + "operation": "groupby" + }, + "severity": { + "aggregations": [], + "operation": "groupby" + } + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Value": true + }, + "indexByName": { + "Time (lastNotNull)": 3, + "description": 2, + "name": 0, + "severity": 1 + }, + "renameByName": { + "Time (lastNotNull)": "Last Reported", + "description": "Description", + "name": "Check Name", + "severity": "Severity" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Status returned from the `/readyz` health status endpoint. Unhealthy reasons include beacon node problems, charon p2p problems, or validator client problems.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#ffffff", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "decimals": 0, + "mappings": [ + { + "options": { + "0": { + "color": "text", + "index": 1, + "text": "Unknown" + }, + "1": { + "color": "green", + "index": 0, + "text": "OK" + }, + "2": { + "color": "red", + "index": 3, + "text": "BeaconNode Down" + }, + "3": { + "color": "orange", + "index": 4, + "text": "BeaconNode Syncing" + }, + "4": { + "color": "orange", + "index": 5, + "text": "Insufficient Peers" + }, + "5": { + "color": "light-orange", + "index": 6, + "text": "VC not connected" + }, + "6": { + "color": "orange", + "index": 7, + "text": "VC missing validators" + }, + "7": { + "color": "orange", + "index": 8, + "text": "BeaconNode has zero peers" + }, + "8": { + "index": 9, + "text": "BeaconNode far behind" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "color": "text", + "index": 2, + "text": "Dead" + } + }, + "type": "special" + } + ], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#ccccdb", + "value": 0 + }, + { + "color": "green", + "value": 0.5 + }, + { + "color": "red", + "value": 1.5 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 9, + "x": 8, + "y": 10 + }, + "id": 119, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", + "instant": false, + "interval": "$interval", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Historical Health Status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "center", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "inspect": false, + "minWidth": 0, + "width": 90 + }, + "decimals": 0, + "mappings": [], + "noValue": "--", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": 0 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Peer" + }, + "properties": [ + { + "id": "custom.width", + "value": 120 + }, + { + "id": "custom.align", + "value": "left" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Connected" + }, + "properties": [ + { + "id": "custom.width", + "value": 90 + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "index": 0, + "text": "\u00a0\u274c" + }, + "1": { + "index": 1, + "text": "\u2705" + } + }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Direct" + }, + "properties": [ + { + "id": "custom.width", + "value": 70 + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "index": 0, + "text": "--" + } + }, + "type": "value" + }, + { + "options": { + "from": 1, + "result": { + "index": 1, + "text": "\u00a0\u274c" + }, + "to": 9 + }, + "type": "range" + }, + { + "options": { + "from": 10, + "result": { + "index": 2, + "text": "\u2705" + }, + "to": 1000 + }, + "type": "range" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Latency (ms)" + }, + "properties": [ + { + "id": "unit", + "value": "none" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "lcd", + "type": "gauge", + "valueDisplayMode": "color" + } + }, + { + "id": "custom.width", + "value": 212 + }, + { + "id": "max", + "value": 700 + }, + { + "id": "mappings", + "value": [ + { + "options": { + "from": 0, + "result": { + "color": "green", + "index": 0 + }, + "to": 150 + }, + "type": "range" + }, + { + "options": { + "from": 150, + "result": { + "color": "yellow", + "index": 1 + }, + "to": 300 + }, + "type": "range" + }, + { + "options": { + "from": 300, + "result": { + "color": "orange", + "index": 2 + }, + "to": 500 + }, + "type": "range" + }, + { + "options": { + "from": 500, + "result": { + "color": "red", + "index": 3 + }, + "to": 100000 + }, + "type": "range" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Version" + }, + "properties": [ + { + "id": "custom.width", + "value": 104 + }, + { + "id": "custom.align", + "value": "left" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 7, + "x": 17, + "y": 10 + }, + "id": 1041, + "options": { + "cellHeight": "sm", + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Peer" + } + ] + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer)", + "format": "table", + "instant": true, + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "(max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"direct\"}) by (peer)*10 + max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"relay\"}) by (peer))", + "format": "table", + "instant": true, + "range": false, + "refId": "K" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(p2p_ping_latency_secs_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (le,peer)) * 1000", + "format": "table", + "instant": true, + "range": false, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(app_peerinfo_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer,version)", + "format": "table", + "instant": true, + "range": false, + "refId": "L" + } + ], + "title": "Peer Connectivity", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "peer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time 1": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "Value #L": true, + "cluster_hash 1": true, + "cluster_hash 2": true, + "cluster_hash 3": true, + "cluster_hash 4": true, + "cluster_name 1": true, + "cluster_name 2": true, + "cluster_name 3": true, + "cluster_name 4": true, + "cluster_peer 1": true, + "cluster_peer 2": true, + "cluster_peer 3": true, + "cluster_peer 4": true, + "instance 1": true, + "instance 2": true, + "instance 3": true, + "instance 4": true, + "job 1": true, + "job 2": true, + "job 3": true, + "job 4": true + }, + "indexByName": { + "Value #A": 2, + "Value #B": 4, + "Value #K": 3, + "peer": 0, + "version": 1 + }, + "renameByName": { + "Value #A": "Connected", + "Value #B": "Latency (ms)", + "Value #K": "Direct", + "peer": "Peer", + "version": "Version" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 17 + }, + "id": 1032, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "showPercentChange": false, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(label_replace(app_execution_layer_version{cluster_name=~\"$cluster_name\",cluster_hash=~\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}, \"version\", \"$1\", \"version\", \"(.*)\")) by (version)", + "instant": true, + "legendFormat": "{{version}}", + "refId": "A" + } + ], + "title": "Execution Layer", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": 0 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 17 + }, + "id": 1031, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "showPercentChange": false, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(app_beacon_node_version{cluster_name=~\"$cluster_name\",cluster_hash=~\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (version)", + "instant": true, + "legendFormat": "{{version}}", + "refId": "A" + } + ], + "title": "Beacon Node", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 17 + }, + "id": 1030, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "showPercentChange": false, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(app_peerinfo_version{cluster_name=~\"$cluster_name\",cluster_hash=~\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (version)", + "instant": true, + "legendFormat": "{{version}}", + "refId": "A" + } + ], + "title": "Charon", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": 0 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 17 + }, + "id": 1033, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "showPercentChange": false, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(label_replace(core_validatorapi_vc_user_agent{cluster_name=~\"$cluster_name\",cluster_hash=~\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}, \"version\", \"$1\", \"user_agent\", \"(.*)\")) by (version)", + "instant": true, + "legendFormat": "{{version}}", + "refId": "A" + } + ], + "title": "Validator Client", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 193, + "panels": [], + "title": "Overview Advanced", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "- **Uptime**: Total time since cluster start time\n- **Slot**: Current slot\n- **Epoch**: Current Epoch\n- **Validators Active**: Number of active validators in the cluster\n- **Charon Peers Connected**: Number of Peers in the cluster\n- **Relays Connected**: Number of Relays connected to the cluster\n- **Beacon Node Peers**: Beacon Node Peers connected, based on app_beacon_node_peers\n- **P2P Reachability**\" Latest P2P reachability status, based on p2p_reachability_status", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "text", + "mode": "fixed" + }, + "mappings": [], + "max": 10, + "min": 0, + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "P2P Reachability" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "color": "text", + "index": 0, + "text": "Unknown" + }, + "1": { + "color": "green", + "index": 1, + "text": "Public" + }, + "2": { + "color": "yellow", + "index": 2, + "text": "Private" + } + }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Uptime" + }, + "properties": [ + { + "id": "unit", + "value": "s" + }, + { + "id": "decimals", + "value": 1 + }, + { + "id": "color", + "value": { + "fixedColor": "text", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 0, + "y": 22 + }, + "id": 113, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "titleSize": 15, + "valueSize": 15 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "time() - max(app_start_time_secs{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", + "instant": true, + "legendFormat": "Uptime", + "range": false, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "core_scheduler_current_slot{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", + "format": "time_series", + "instant": true, + "interval": "", + "legendFormat": "Slot", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "core_scheduler_current_epoch{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", + "instant": true, + "legendFormat": "Epoch", + "range": false, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "core_scheduler_validators_active{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", + "instant": true, + "legendFormat": "Validators Active", + "range": false, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", + "instant": true, + "legendFormat": "Charon Peers Connected", + "range": false, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(p2p_relay_connections{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) ", + "instant": true, + "legendFormat": "Relays Connected", + "range": false, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "app_beacon_node_peers{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", + "instant": true, + "legendFormat": "Beacon Node Peers", + "range": false, + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "p2p_reachability_status{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", + "instant": true, + "legendFormat": "P2P Reachability", + "range": false, + "refId": "F" + } + ], + "title": "Current Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Status and balance of each validator by public key with links to beaconcha.in.\n\nIf present, the \"\u2757\ufe0fPeer Exits\" column indicate the number of peers that submitted partially signed exits for each validator.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "light-blue", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Balance" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "mappings", + "value": [ + { "options": { "0": { - "index": 0, - "text": "❌" - }, - "1": { - "index": 1, - "text": "✅" + "color": "transparent", + "index": 3 } }, "type": "value" + }, + { + "options": { + "from": 32, + "result": { + "color": "green", + "index": 0 + }, + "to": 100 + }, + "type": "range" + }, + { + "options": { + "from": 30, + "result": { + "color": "orange", + "index": 1 + }, + "to": 32 + }, + "type": "range" + }, + { + "options": { + "from": 1, + "result": { + "color": "red", + "index": 2 + }, + "to": 30 + }, + "type": "range" + } + ] + }, + { + "id": "unit", + "value": "ETH" + }, + { + "id": "custom.width", + "value": 122 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": " Public Key" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "beaconcha.in", + "url": "http://${__data.fields[\"cluster_network\"]}.beaconcha.in/validator/${__data.fields[\"pubkey_full\"]}" } ] }, { - "id": "custom.width", - "value": 99 + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "custom.width", + "value": 119 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "custom.width", + "value": 130 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*Exits" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "__hide__" + }, + "properties": [ + { + "id": "custom.hideFrom.viz", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #B" + }, + "properties": [ + { + "id": "custom.footer.reducers", + "value": [ + "sum" + ] + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 4, + "y": 22 + }, + "id": 134, + "options": { + "cellHeight": "sm", + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Balance" + } + ] + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(core_scheduler_validator_balance_gwei{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} / 1000000000) by (pubkey_full,pubkey, cluster_network)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "{{pubkey}}", + "range": false, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(core_scheduler_validator_status{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 1) by (pubkey, status)", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(core_parsigdb_exit_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (pubkey) ", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "C" + } + ], + "title": "Validators", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "pubkey", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Time 3": true, + "Value #A": true, + "Value #B": false, + "cluster_network": false, + "job": true, + "pubkey_full": false + }, + "indexByName": { + "Time 1": 2, + "Time 2": 7, + "Time 3": 9, + "Value #A": 8, + "Value #B": 6, + "Value #C": 4, + "cluster_network": 3, + "pubkey": 0, + "pubkey_full": 1, + "status": 5 + }, + "renameByName": { + "Value #A": "", + "Value #B": "Balance", + "Value #C": "\u2757\ufe0fPeer Exits", + "cluster_network": "__hide__", + "pubkey": " Public Key", + "pubkey_full": "__hide__", + "status": "Status" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Warning (w) and error (e) logs grouped by topic. \n\nOccasional warns and errors are normal, but a significant increase indicate problems that require operators to investigate by looking at the actual logs entries.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "links": [ + { + "targetBlank": true, + "title": "Show in log explorer", + "url": "/explore?orgId=1&left=%7B%22datasource%22:%22loki%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22editorMode%22:%22builder%22,%22expr%22:%22%7Bcompose_service%3D%5C%22$job%5C%22%7D%20%7C%3D%20%60${__field.labels.msg}%60%20%7C%20logfmt%20%7C%20line_format%20%60%7B%7B.pretty%7D%7D%60%22,%22queryType%22:%22range%22%7D%5D,%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D" + } + ], + "mappings": [], + "min": 0, + "noValue": "No errors or warnings", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 100 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 117, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(app_log_error_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (topic) > 0 ", + "interval": "$interval", + "legendFormat": "e:{{topic}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(app_log_warn_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (topic) > 0 ", + "interval": "$interval", + "legendFormat": "w:{{topic}}", + "range": true, + "refId": "A" + } + ], + "title": "Charon Warning(w) and Error(e) logs (per $interval)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "What percentage of Blocks are seen after the attestation kickoff time of 4 seconds. Validators that attest with data from this CL at that mark, before it is finished processing a valid block, are likely to see reduced rewards for an inaccurate head vote attestation as a result. Please seek to optimise your beacon node performance if this number is high. ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "min": 0, + "noValue": "?", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": 0 + }, + { + "color": "green", + "value": 1 + }, + { + "color": "yellow", + "value": 2 + }, + { + "color": "dark-orange", + "value": 3 + }, + { + "color": "red", + "value": 4 + }, + { + "color": "dark-red", + "value": 5 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 5, + "x": 0, + "y": 29 + }, + "id": 306, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sort_desc(100 * (\n 1 - \n sum by (cluster_peer, nickname) (rate(app_beacon_node_sse_head_delay_bucket{cluster_name=\"$cluster_name\", cluster_hash=\"$cluster_hash\", le=~\"4|4.0\"}[$__range])) \n / ignoring(le) \n sum by (cluster_peer, nickname) (rate(app_beacon_node_sse_head_delay_bucket{cluster_name=\"$cluster_name\", cluster_hash=\"$cluster_hash\", le=\"+Inf\"}[$__range]))\n))", + "instant": false, + "legendFormat": "{{nickname}} ({{cluster_peer}})", + "range": true, + "refId": "D" + } + ], + "title": "% of Blocks Processed after 4s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Any peer which is running on its secondary beacon nodes rather than its primary will be marked true here. ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 2, + "pointSize": 8, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "decimals": 0, + "mappings": [ + { + "options": { + "0": { + "index": 1, + "text": "False" + }, + "1": { + "index": 0, + "text": "True" + } + }, + "type": "value" + } + ], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "orange", + "value": 0.5 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "DV Labs (nervous-pillow)" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": true, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 7, + "x": 5, + "y": 29 + }, + "id": 294, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "editorMode": "code", + "expr": "app_eth2_using_fallback{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\"}", + "legendFormat": "{{nickname}} ({{cluster_peer}})", + "range": true, + "refId": "A" + } + ], + "title": "Using Fallback Beacon Node?", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "{__name__=\"app_feature_flags\", charon_version=\"v1.7.0-dev\", cluster_hash=\"e0fa07c\", cluster_name=\"test\", cluster_network=\"testnet\", cluster_peer=\"jittery-metal\", feature_flags=\"fetch_only_commidx_0\", instance=\"node0:3620\", job=\"charon-0\"}" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [] + }, + { + "matcher": { + "id": "byName", + "options": "cluster_peer" + }, + "properties": [ + { + "id": "custom.width", + "value": 127 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "nickname" + }, + "properties": [ + { + "id": "custom.width", + "value": 148 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 29 + }, + "id": 298, + "options": { + "cellHeight": "sm", + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "editorMode": "code", + "expr": "app_feature_flags {cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Feature Flags", + "transformations": [ + { + "id": "timeSeriesTable", + "options": {} + }, + { + "id": "groupBy", + "options": { + "fields": { + "cluster_peer": { + "aggregations": [], + "operation": "groupby" + }, + "feature_flags": { + "aggregations": [ + "allValues" + ], + "operation": "aggregate" + }, + "nickname": { + "aggregations": [], + "operation": "groupby" + } + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisPlacement": "auto", + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineWidth": 0, + "spanNulls": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "D" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 1021, + "options": { + "alignValue": "left", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "mergeValues": true, + "rowHeight": 0.9, + "showValue": "never", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(app_beacon_node_version{cluster_name=~\"$cluster_name\",cluster_hash=~\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (version)", + "instant": false, + "legendFormat": "BN: {{version}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(label_replace(app_execution_layer_version{cluster_name=~\"$cluster_name\",cluster_hash=~\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}, \"version\", \"$1\", \"version\", \"(.*)\")) by (version)", + "instant": false, + "legendFormat": "EL: {{version}}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(label_replace(core_validatorapi_vc_user_agent{cluster_name=~\"$cluster_name\",cluster_hash=~\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}, \"version\", \"$1\", \"user_agent\", \"(.*)\")) by (version)", + "instant": false, + "legendFormat": "VC: {{version}}", + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(app_peerinfo_version{cluster_name=~\"$cluster_name\",cluster_hash=~\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (version)", + "instant": false, + "legendFormat": "Charon: {{version}}", + "refId": "D" + } + ], + "title": "Stack Version History (\ud83d\udfe1=BN, \ud83d\udd34=EL, \ud83d\udd35=VC, \ud83d\udfe2=DV)", + "type": "state-timeline" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 46 + }, + "id": 178, + "panels": [], + "title": "Duties", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Completed duties by type over time.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "palette-classic" + }, + "custom": { + "axisPlacement": "auto", + "fillOpacity": 100, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 0 + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 47 + }, + "id": 92, + "options": { + "colWidth": 0.5, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "rowHeight": 0.43, + "showValue": "auto", + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(core_tracker_success_duties_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (duty) > 0", + "instant": false, + "interval": "$interval", + "legendFormat": "{{duty}}", + "range": true, + "refId": "A" + } + ], + "title": "\u2705 Completed duties by type ", + "type": "status-history" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Successful duty participation per peer (filtered by selected duty) over time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisPlacement": "auto", + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 81, + "options": { + "colWidth": 0.9, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "rowHeight": 0.9, + "showValue": "never", + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(core_tracker_participation_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=~\"$peer\",duty=~\"$duty\"}[$interval])) by (peer) > 0 ", + "interval": "$interval", + "legendFormat": "{{peer}}", + "range": true, + "refId": "A" + } + ], + "title": "\u2705 Successful Peer Participation: Duty=$duty", + "type": "status-history" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Failed duties by type over time. Use data link to show logs.\nProposer duties are non-failing after being successfully included on-chain.\nOther broadcasted to the beacon node duties (i.e.: attester), are non-failing after being successfully broadcasted.\nAny other duties not broadcasted to the beacon node (i.e.: randao), are non-failing after consensus is reached.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "palette-classic" + }, + "custom": { + "axisPlacement": "auto", + "fillOpacity": 100, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 0 + }, + "links": [ + { + "targetBlank": true, + "title": "Show in log explorer", + "url": "/explore?orgId=1&left=%7B%22datasource%22:%22loki%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22editorMode%22:%22builder%22,%22expr%22:%22%7Bcompose_service%3D%5C%22$job%5C%22%7D%20%7C%3D%20%60Duty%20failed%60%20%7C%3D%20%60${__field.labels.duty}%60%20%7C%20logfmt%20%7C%20line_format%20%60%7B%7B.pretty%7D%7D%60%22,%22queryType%22:%22range%22%7D%5D,%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D" + } + ], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 53 + }, + "id": 91, + "options": { + "colWidth": 0.5, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "rowHeight": 0.43, + "showValue": "auto", + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(core_tracker_failed_duties_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (duty) > 0", + "instant": false, + "interval": "$interval", + "legendFormat": "{{duty}}", + "range": true, + "refId": "A" + } + ], + "title": "\u274c Failed duties by type (per $interval)", + "type": "status-history" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Missed duty participation per peer (filtered by selected duty) over time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisPlacement": "auto", + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 53 + }, + "id": 185, + "options": { + "colWidth": 0.9, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "rowHeight": 0.9, + "showValue": "never", + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(core_tracker_participation_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=~\"$peer\",duty=~\"$duty\"}[$interval])) by (peer) > 0 ", + "interval": "$interval", + "legendFormat": "{{peer}}", + "range": true, + "refId": "A" + } + ], + "title": "\ud83d\udea7 Missed Peer Participation: Duty=$duty", + "type": "status-history" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Failed duties percentage by type over time", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 59 + }, + "id": 180, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "(\n sum(increase(core_tracker_failed_duties_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (duty)\n) \n/ \n(\n sum(increase(core_tracker_failed_duties_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (duty) \n + \n sum(increase(core_bcast_broadcast_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (duty) \n) > 0 ", + "instant": false, + "interval": "$interval", + "legendFormat": "{{duty}}", + "range": true, + "refId": "A" + } + ], + "title": "\u274c Failed duty percentage by type (per $interval)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Missed duty participation per duty (filtered by peer) over time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisPlacement": "auto", + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 59 + }, + "id": 186, + "options": { + "colWidth": 0.9, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "rowHeight": 0.9, + "showValue": "never", + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "repeat": "peer", + "repeatDirection": "v", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(core_tracker_participation_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=\"$peer\"}[$interval])) by (duty) > 0 ", + "interval": "$interval", + "legendFormat": "{{duty}}", + "range": true, + "refId": "A" + } + ], + "title": "\ud83d\udea7 Missed Peer Participation: Peer=$peer", + "type": "status-history" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of failed duties by reason per time window. Note this is only available from v0.16.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "mappings": [], + "noValue": "No failures \ud83c\udf89", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 65 + }, + "id": 222, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(core_tracker_failed_duty_reasons_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",duty=~\"$duty\"}[$interval])) by (reason) > 0 ", + "interval": "$interval", + "legendFormat": "{{.duty}}", + "range": true, + "refId": "A" + } + ], + "title": "\ud83d\udea8 Duty failure reason: $duty (per $interval)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Duties that was successfully broadcast to the Beacon Node, but that was never included on-chain. This could be due to broadcasting too late (too slow), but it could also just be to aspects of consensus layer out-of-our-control. See https://eth2book.info/bellatrix/part2/incentives/rewards/#remarks.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 71 + }, + "id": 203, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "(sum(increase(core_tracker_inclusion_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=~\"$duty\"}[$interval])) by (duty)) > 0 ", + "instant": false, + "interval": "$interval", + "legendFormat": "{{duty}}", + "range": true, + "refId": "A" + } + ], + "title": "\ud83d\udca5 Broadcasted duties never included on-chain (per $interval)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "When a signature from a peer or own VC was registered at the parsigdb, compared to minimum expected time.\n\nMinimum expected time is:\n- attester 4s after slot start;\n- aggregator 8s after slot start;\n- sync_contribution 8s after slot start;\n- all other duties 0s.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "min": -1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "+Inf" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "12" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "10" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "8" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "6" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4.5" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3.5" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4.0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5.0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "6.0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "8.0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "10.0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "12.0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 77 + }, + "id": 309, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile(0.90, sum(rate(core_parsigdb_store_bucket{cluster_name=\"$cluster_name\", cluster_hash=\"$cluster_hash\", cluster_peer=\"$cluster_peer\", duty=~\"$duty\"}[$interval])) by (le, peer_idx))", + "format": "heatmap", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Signature received from peer since expected time: Duty=$duty", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "When a signature of attestation from a peer or own VC was registered at the parsigdb, compared to minimum expected time.\n\nMinimum expected time for attestation signature is 4s after slot start.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "fieldMinMax": false, + "mappings": [], + "min": -1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "#EAB839", + "value": 2 + }, + { + "color": "red", + "value": 4 + }, + { + "color": "dark-red", + "value": 8 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "+Inf" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "12" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "10" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "8" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "6" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4.5" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3.5" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4.0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5.0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "6.0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "8.0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "10.0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "12.0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 83 + }, + "id": 317, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile(0.90, sum(rate(core_parsigdb_store_bucket{cluster_name=\"$cluster_name\", cluster_hash=\"$cluster_hash\", cluster_peer=\"$cluster_peer\",duty=\"attester\"}[$interval])) by (le, peer_idx)) ", + "format": "heatmap", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Signature received from peer since expected time: Duty=attester", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "When a signature of attestation from a peer or own VC was registered at the parsigdb, compared to minimum expected time.\n\nMinimum expected time for attestation signature is 4s after slot start.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "fieldMinMax": false, + "mappings": [], + "min": -1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "#EAB839", + "value": 3 + }, + { + "color": "red", + "value": 4 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "+Inf" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "12" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "10" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "8" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "6" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4.5" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3.5" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4.0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5.0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "6.0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "8.0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "10.0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "12.0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 89 + }, + "id": 318, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile(0.90, sum(rate(core_parsigdb_store_bucket{cluster_name=\"$cluster_name\", cluster_hash=\"$cluster_hash\", cluster_peer=\"$cluster_peer\",duty=\"proposer\"}[$interval])) by (le, peer_idx)) ", + "format": "heatmap", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Signature received from peer since expected time: Duty=proposer", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The delay since the expected duty submission time.\nFor duties broadcasted to the beacon node (i.e.: proposer), successful beacon node response is taken as completion time.\nFor duties not broadcasted to the beacon node (i.e.: randao), completed consensus is taken as completion time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 12, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 2 + }, + { + "color": "orange", + "value": 4 + }, + { + "color": "red", + "value": 8 } ] }, - { - "matcher": { - "id": "byName", - "options": "Version" + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 95 + }, + "id": 218, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(core_bcast_broadcast_delay_seconds_sum{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",duty=~\"$duty\"}[$interval]) / rate(core_bcast_broadcast_delay_seconds_count{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",duty=~\"$duty\"}[$interval])", + "interval": "$interval", + "legendFormat": "{{duty}}", + "range": true, + "refId": "A" + } + ], + "title": "Submission delay: $duty", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 102 + }, + "id": 320, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The leader of final consensus round represented as its cluster peer index. This is the node who proposed the value the cluster came to consensus on to sign. ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 8, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } }, - "properties": [ - { - "id": "custom.width", - "value": 104 + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 254 + }, + "id": 293, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "core_consensus_decided_leader_index{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=\"proposer\"}", + "format": "time_series", + "instant": false, + "legendFormat": "Leader Index", + "range": true, + "refId": "A" + } + ], + "title": "Node that chose the proposal data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The leader of final consensus round represented as its cluster peer index. This is the node who proposed the value the cluster came to consensus on to sign. ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 8, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] } - ] + }, + "overrides": [] }, - { - "matcher": { - "id": "byName", - "options": "Nickname" + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 254 + }, + "id": 296, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "properties": [ - { - "id": "custom.width", - "value": 147 + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "core_consensus_decided_leader_index{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=\"attester\"}", + "format": "time_series", + "instant": false, + "legendFormat": "Leader Index", + "range": true, + "refId": "A" + } + ], + "title": "Node that chose the attester data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "How long in total did it take to come to consensus on this/these duties? Duties generally have a 4 second window to be completed (1.25s on gnosis chain), so consensus should not take more than half of that time for good performance. ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "orange", + "value": 1.5 + }, + { + "color": "red", + "value": 2.5 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 352 + }, + "id": 261, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(core_consensus_duration_seconds_sum{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",duty=~\"$duty\"}[$interval]) / rate(core_consensus_duration_seconds_count{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",duty=~\"$duty\"}[$interval])", + "interval": "$interval", + "legendFormat": "{{duty}}", + "range": true, + "refId": "A" + } + ], + "title": "Consensus duration: $duty", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The number of consensus rounds it takes to achieve consensus on a batch of duties. When every node is online, this should be 1 round, if it is taking more rounds than the fault tolerance of the cluster to come to consensus, there may be an issue with node operators in the cluster, or the latency of the cluster. ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 3, + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "orange", + "value": 2 + }, + { + "color": "red", + "value": 3 + } + ] } - ] + }, + "overrides": [] }, - { - "matcher": { - "id": "byName", - "options": "Fallback" + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 352 + }, + "id": 268, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "properties": [ - { - "id": "mappings", - "value": [ + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(core_consensus_decided_rounds{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",duty=~\"$duty\"}) by (timer,duty)", + "interval": "$interval", + "legendFormat": "{{duty}}", + "range": true, + "refId": "A" + } + ], + "title": "Consensus Rounds: $duty", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of consensus timeouts per time window.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "mappings": [], + "noValue": "No timeouts \ud83c\udf89", + "thresholds": { + "mode": "absolute", + "steps": [ { - "options": { - "0": { - "color": "dark-green", - "index": 0, - "text": "False" - }, - "1": { - "color": "dark-red", - "index": 1, - "text": "True" - } - }, - "type": "value" + "color": "red", + "value": 0 } ] } - ] - } - ] - }, + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 359 + }, + "id": 189, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(core_consensus_timeout_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",duty=~\"$duty\"}[$interval])) by (timer) > 0", + "interval": "$interval", + "legendFormat": "{{.duty}}", + "range": true, + "refId": "B" + } + ], + "title": "\ud83d\udea8 Consensus Timeouts (per $interval)", + "type": "timeseries" + } + ], + "title": "Consensus and leaders", + "type": "row" + }, + { + "collapsed": true, "gridPos": { - "h": 9, + "h": 1, "w": 24, "x": 0, - "y": 10 + "y": 103 }, - "id": 85, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "frameIndex": 0, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "🚧Other" - } - ] - }, - "pluginVersion": "11.4.1", - "targets": [ + "id": 123, + "panels": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer)", - "format": "table", - "instant": true, - "range": false, - "refId": "A" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 637 + }, + "id": 127, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(go_memstats_alloc_bytes{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", + "interval": "$interval", + "legendFormat": "Allocated Memory", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(go_memstats_heap_inuse_bytes{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", + "hide": true, + "legendFormat": "Inuse Heap", + "range": true, + "refId": "B" + } + ], + "title": "Memory", + "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(0.90, sum(rate(p2p_ping_latency_secs_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$__rate_interval])) by (le,peer)) * 1000", - "format": "table", - "hide": false, - "instant": true, - "range": false, - "refId": "B" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 637 + }, + "id": 129, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(irate(process_cpu_seconds_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval]))", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "CPU", + "metric": "go_gc_duration_seconds", + "range": true, + "refId": "A", + "step": 4 + } + ], + "title": "CPU", + "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer)", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "", - "range": false, - "refId": "F" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "File Descriptors" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 644 + }, + "id": 125, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(go_goroutines{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", + "interval": "$interval", + "legendFormat": "Go Routines", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(process_open_fds{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", + "interval": "$interval", + "legendFormat": "File Descriptors", + "range": true, + "refId": "B" + } + ], + "title": "Go Routines and File Descriptors", + "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "editorMode": "code", - "exemplar": false, - "expr": "(max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"direct\"}) by (peer)*10 + \nmax(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"relay\"}) by (peer))", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "", - "range": false, - "refId": "K" + "description": "Total time spent on garbage collection per interval", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 644 + }, + "id": 130, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(increase(go_gc_duration_seconds_sum{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) ", + "interval": "$interval", + "intervalFactor": 2, + "legendFormat": "__auto", + "metric": "go_gc_duration_seconds", + "range": true, + "refId": "A", + "step": 4 + } + ], + "title": "Garbage Collection Duration (per $interval)", + "type": "timeseries" + } + ], + "title": "Process", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 104 + }, + "id": 141, + "panels": [], + "title": "Upstream Beacon Node", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Beacon node score over time", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#ffffff", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "orange", + "value": 0.5 + }, + { + "color": "green", + "value": 0.8 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 105 + }, + "id": 145, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", - "exemplar": false, - "expr": "sum(app_peerinfo_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer,version) ", - "format": "table", - "hide": false, - "instant": true, + "expr": "clamp_min((clamp(app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 1 OR app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 4 OR app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 5 OR app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 6, 1, 1) OR on() vector(0))\n*\n(\n (\n0.5 * (1.0 - 10*(sum(increase(app_eth2_errors_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) / (sum(increase(app_eth2_latency_seconds_count{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval]))))) OR on() vector(0.5)\n )\n +\n (\n 0.5 * (1.0 - clamp_max(histogram_quantile(0.99, sum(rate(app_eth2_latency_seconds_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (le)),1))\n )\n), 0)", + "interval": "$interval", "legendFormat": "__auto", - "range": false, - "refId": "L" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" + "range": true, + "refId": "A" + } + ], + "title": "Beacon Node Score", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of consensus layer P2P peers the beacon node is connected to", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#fffefe", + "mode": "fixed" }, - "editorMode": "code", - "exemplar": false, - "expr": "time() - max(app_peerinfo_start_time_secs{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} > 0) by (peer)", - "format": "table", - "hide": false, - "instant": true, - "range": false, - "refId": "Q" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } }, - "editorMode": "code", - "exemplar": false, - "expr": "max(app_peerinfo_index{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer)", - "format": "table", - "hide": false, - "instant": true, - "range": false, - "refId": "R" + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "#EAB839", + "value": 20 + }, + { + "color": "green", + "value": 40 + } + ] + } }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "(\n sum(increase(core_tracker_participation_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=\"attester\"}[$__range])) by (peer) \n/\n sum(increase(core_tracker_participation_expected_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=\"attester\"}[$__range])) by (peer) \n) OR on() vector(0)", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "C" + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 105 + }, + "id": 147, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", - "exemplar": false, - "expr": "(\n sum(increase(core_tracker_participation_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=~\"proposer\"}[$__range])) by (peer) \n/\n sum(increase(core_tracker_participation_expected_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=~\"proposer\"}[$__range])) by (peer) \n) OR on() vector(0)", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" + "expr": "app_beacon_node_peers{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", + "interval": "$interval", + "legendFormat": "Peer count", + "range": true, + "refId": "A" + } + ], + "title": "Beacon node peer count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Upstream beacon node API request rate per second by endpoint", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "editorMode": "code", - "exemplar": false, - "expr": "(\n sum(increase(core_tracker_participation_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=\"sync_message\"}[$__range])) by (peer) \n/\n sum(increase(core_tracker_participation_expected_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=\"sync_message\"}[$__range])) by (peer) \n) OR on() vector(0)", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, - "editorMode": "code", - "exemplar": false, - "expr": "max(app_peerinfo_version_support{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer) ", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "", - "range": false, - "refId": "G" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "max(app_peerinfo_builder_api_enabled{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer)", - "format": "table", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "H", - "useBackend": false + "unit": "reqps" }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(app_peerinfo_nickname{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer,peer_nickname)", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "I" + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 112 + }, + "id": 142, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.4.1", + "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", - "expr": "max(app_beacon_node_version{cluster_name=~\"$cluster_name\",cluster_hash=~\"$cluster_hash\",cluster_network=~\"$cluster_network\"}) by (cluster_peer, version)", - "hide": false, - "instant": false, - "legendFormat": "__auto", + "exemplar": true, + "expr": "max(rate(app_eth2_latency_seconds_count{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (endpoint) > 0 ", + "interval": "$interval", + "legendFormat": "{{endpoint}}", "range": true, - "refId": "M" - } - ], - "title": "Peer Connectivity and Missed Duties (🚧)", - "transformations": [ - { - "id": "seriesToColumns", - "options": { - "byField": "peer" - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time 1": true, - "Time 10": true, - "Time 11": true, - "Time 12": true, - "Time 13": true, - "Time 14": true, - "Time 15": true, - "Time 16": true, - "Time 17": true, - "Time 2": true, - "Time 3": true, - "Time 4": true, - "Time 5": true, - "Time 6": true, - "Time 7": true, - "Time 8": true, - "Time 9": true, - "Value #F": false, - "Value #L": true, - "Value #M": true - }, - "includeByName": {}, - "indexByName": { - "Time 1": 15, - "Time 10": 24, - "Time 11": 25, - "Time 12": 26, - "Time 13": 27, - "Time 2": 16, - "Time 3": 17, - "Time 4": 18, - "Time 5": 19, - "Time 6": 20, - "Time 7": 21, - "Time 8": 22, - "Time 9": 23, - "Value #A": 4, - "Value #B": 6, - "Value #C": 10, - "Value #D": 11, - "Value #E": 12, - "Value #F": 2, - "Value #G": 8, - "Value #H": 13, - "Value #I": 14, - "Value #K": 5, - "Value #L": 28, - "Value #Q": 9, - "Value #R": 1, - "peer": 0, - "peer_nickname": 3, - "version": 7 - }, - "renameByName": { - "Time 1": "", - "Value #A": "Connected", - "Value #B": "Latency (ms)", - "Value #C": "🚧Attest", - "Value #D": "🚧Propose", - "Value #E": "🚧Sync Message", - "Value #F": "You", - "Value #G": "Supported", - "Value #H": "Builder API", - "Value #I": "Exit", - "Value #J": "Fallback", - "Value #K": "Direct", - "Value #L": "", - "Value #N": "SyncMsg", - "Value #O": "SyncContrib", - "Value #P": "PrepareContrib", - "Value #Q": "Uptime", - "Value #R": "Index", - "git_hash": "GitCommit", - "nickname": "Nickname", - "peer": "Peer", - "peer_nickname": "Nickname", - "version": "Version" - } - } - }, - { - "id": "sortBy", - "options": { - "fields": {}, - "sort": [ - { - "field": "Peer" - } - ] - } + "refId": "A" } ], - "type": "table" + "title": "Beacon Node API requests rate", + "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "description": "Status returned from the `/readyz` health status endpoint. Unhealthy reasons include beacon node problems, charon p2p problems, or validator client problems.", + "description": "Upstream beacon node API request latency by endpoint by timewindow", "fieldConfig": { "defaults": { "color": { - "fixedColor": "#ffffff", - "mode": "fixed" + "mode": "palette-classic" }, "custom": { "axisBorderShow": false, @@ -1556,131 +6076,72 @@ "viz": false }, "insertNulls": false, - "lineInterpolation": "stepAfter", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 3, + "lineInterpolation": "linear", + "lineWidth": 2, "pointSize": 5, "scaleDistribution": { - "type": "linear" + "log": 2, + "type": "log" }, - "showPoints": "never", + "showPoints": "auto", + "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { - "mode": "area" + "mode": "dashed+area" } }, - "decimals": 0, - "mappings": [ - { - "options": { - "0": { - "color": "text", - "index": 1, - "text": "Unknown" - }, - "1": { - "color": "green", - "index": 0, - "text": "OK" - }, - "2": { - "color": "red", - "index": 3, - "text": "BeaconNode Down" - }, - "3": { - "color": "orange", - "index": 4, - "text": "BeaconNode Syncing" - }, - "4": { - "color": "orange", - "index": 5, - "text": "Insufficient Peers" - }, - "5": { - "color": "light-orange", - "index": 6, - "text": "VC not connected" - }, - "6": { - "color": "orange", - "index": 7, - "text": "VC missing validators" - }, - "7": { - "color": "orange", - "index": 8, - "text": "BeaconNode has zero peers" - }, - "8": { - "index": 9, - "text": "BeaconNode far behind" - } - }, - "type": "value" - }, - { - "options": { - "match": "null", - "result": { - "color": "text", - "index": 2, - "text": "Dead" - } - }, - "type": "special" - } - ], - "min": 0, + "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "#ccccdb", - "value": null + "color": "green", + "value": 0 }, { - "color": "green", - "value": 0.5 + "color": "yellow", + "value": 1 + }, + { + "color": "orange", + "value": 2 }, { "color": "red", - "value": 1.5 + "value": 3 } ] }, - "unit": "none" + "unit": "s" }, "overrides": [] }, "gridPos": { - "h": 6, - "w": 7, - "x": 0, - "y": 19 + "h": 7, + "w": 12, + "x": 12, + "y": 112 }, - "id": 119, + "id": 138, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": false + "showLegend": true }, "tooltip": { + "hideZeros": false, "maxHeight": 600, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, - "pluginVersion": "11.4.1", + "pluginVersion": "12.4.1", "targets": [ { "datasource": { @@ -1688,17 +6149,15 @@ "uid": "prometheus" }, "editorMode": "code", - "exemplar": false, - "expr": "max(app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", - "hide": false, - "instant": false, + "exemplar": true, + "expr": "rate(app_eth2_latency_seconds_sum{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval]) / rate(app_eth2_latency_seconds_count{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])", "interval": "$interval", - "legendFormat": "__auto", + "legendFormat": "{{endpoint}}", "range": true, "refId": "A" } ], - "title": "Historical Health Status", + "title": "Beacon Node API request latency", "type": "timeseries" }, { @@ -1706,31 +6165,53 @@ "type": "prometheus", "uid": "prometheus" }, - "description": "The running Charon's version and git commit hash", + "description": "Number of errors per endpoint per time window", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { - "fillOpacity": 70, + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, - "lineWidth": 0, - "spanNulls": false + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", - "value": null + "value": 0 } ] }, @@ -1739,29 +6220,27 @@ "overrides": [] }, "gridPos": { - "h": 6, - "w": 7, - "x": 7, - "y": 19 + "h": 7, + "w": 12, + "x": 0, + "y": 119 }, - "id": 83, + "id": 144, "options": { - "alignValue": "left", "legend": { + "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": false + "showLegend": true }, - "mergeValues": true, - "rowHeight": 0.9, - "showValue": "never", "tooltip": { + "hideZeros": false, "maxHeight": 600, - "mode": "none", + "mode": "multi", "sort": "none" } }, - "pluginVersion": "11.4.1", + "pluginVersion": "12.4.1", "targets": [ { "datasource": { @@ -1769,116 +6248,111 @@ "uid": "prometheus" }, "editorMode": "code", - "exemplar": false, - "expr": "max(app_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (version)", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{version}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(app_git_commit{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (git_hash)", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "{{git_hash}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "rate(app_start_time_secs{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) > 0", - "hide": false, - "instant": false, + "exemplar": true, + "expr": "max(increase(app_eth2_errors_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (endpoint) > 0 ", "interval": "$interval", - "legendFormat": "Restarts", + "legendFormat": "{{endpoint}}", "range": true, - "refId": "C" + "refId": "A" } ], - "title": "Versions and Git Hashes", - "type": "state-timeline" + "title": "\ud83d\udea8 Beacon Node API error count (per $interval)", + "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "description": "Type of connections to each peer in the cluster during the time window:\n - 🟢 Green: Direct connection (this is the preferred)\n - 🟡 Yellow: Indirect relay connection (this is ok if only temporary)\n - 🔴 Red: No connection to peer (this is a problem)\n\nSee https://docs.obol.tech/docs/charon/networking#external-p2p-network", + "description": "Upstream beacon node API proxied request latency by endpoint by timewindow", "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "mode": "palette-classic" }, "custom": { - "fillOpacity": 70, + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "insertNulls": false, - "lineWidth": 0, - "spanNulls": false + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "red", - "value": null + "color": "green", + "value": 0 }, { - "color": "#EAB839", + "color": "yellow", "value": 1 }, { - "color": "green", - "value": 80 + "color": "orange", + "value": 2 + }, + { + "color": "red", + "value": 3 } ] - } + }, + "unit": "s" }, "overrides": [] }, "gridPos": { - "h": 6, - "w": 10, - "x": 14, - "y": 19 + "h": 7, + "w": 12, + "x": 12, + "y": 119 }, - "id": 87, + "id": 300, "options": { - "alignValue": "left", "legend": { + "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": false + "showLegend": true }, - "mergeValues": true, - "rowHeight": 0.9, - "showValue": "never", "tooltip": { + "hideZeros": false, "maxHeight": 600, - "mode": "single", - "sort": "none" + "mode": "multi", + "sort": "desc" } }, - "pluginVersion": "11.4.1", + "pluginVersion": "12.4.1", "targets": [ { "datasource": { @@ -1886,111 +6360,278 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "(max(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer))\n+\n(1 * max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"relay\"}) by (peer))\n+ \n(100 * max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"direct\"}) by (peer))", - "format": "time_series", + "exemplar": true, + "expr": "rate(core_validatorapi_proxy_request_latency_seconds_sum{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval]) / rate(core_validatorapi_proxy_request_latency_seconds_count{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])", "interval": "$interval", - "legendFormat": "__auto", + "legendFormat": "{{path}}", "range": true, "refId": "A" } ], - "title": "Peer Connections (🟢=direct, 🟡=relay, 🔴=not connected)", - "type": "state-timeline" + "title": "Beacon Node API proxied request latency", + "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "description": "This panels shows any health checks that are failing. See for help troubleshooting. \n\nNote the different severities:\n- **info**: Only informational, might not be an actual problem.\n- **warning**: Performance is degraded, operator intervention only required if this persists for a long time.\n- **critical**: Significant problem, validators probably not operation at all. Immediate operator intervention required.\n \n\n", + "description": "An overview of when in the slot a block is finished processing by the connected CL. Blocks processed after 4 seconds (i.e. in the 4.5 bar and higher) miss the attestation kick-off, so if your cluster is attesting to this slot, and this node is the leader, it will propose an attestation to an incorrect head of the chain, and result in reduced rewards.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, + "fieldMinMax": false, "mappings": [], - "noValue": "All checks are passing 🎉", + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 + "value": 0 } ] - } + }, + "unit": "locale" }, "overrides": [ { "matcher": { "id": "byName", - "options": "Severity" + "options": "+Inf" }, "properties": [ { - "id": "mappings", - "value": [ - { - "options": { - "critical": { - "color": "red", - "index": 2, - "text": "critical" - }, - "info": { - "color": "blue", - "index": 0, - "text": "info" - }, - "warning": { - "color": "orange", - "index": 1, - "text": "warning" - } - }, - "type": "value" - } - ] - }, + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "12" + }, + "properties": [ { - "id": "custom.cellOptions", + "id": "color", "value": { - "type": "color-text" + "fixedColor": "dark-red", + "mode": "fixed" } - }, + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "10" + }, + "properties": [ { - "id": "custom.width", - "value": 75 + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "8" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "6" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4.5" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3.5" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "4.5" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [] + }, + { + "matcher": { + "id": "byName", + "options": "4.0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5.0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "6.0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "8.0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } } ] }, { "matcher": { "id": "byName", - "options": "Check Name" + "options": "10.0" }, "properties": [ { - "id": "custom.cellOptions", + "id": "color", "value": { - "type": "color-text" + "fixedColor": "dark-red", + "mode": "fixed" } - }, + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "12.0" + }, + "properties": [ { "id": "color", "value": { - "fixedColor": "super-light-yellow", + "fixedColor": "dark-red", "mode": "fixed" } } @@ -1999,266 +6640,272 @@ ] }, "gridPos": { - "h": 8, - "w": 8, + "h": 7, + "w": 12, "x": 0, - "y": 25 + "y": 126 }, - "id": 233, + "id": 299, "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" ], - "show": false + "fields": "", + "values": false }, - "showHeader": true + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" }, - "pluginVersion": "11.4.1", + "pluginVersion": "12.4.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, "editorMode": "code", "exemplar": false, - "expr": "sum(app_health_checks{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (name,severity) > 0", - "format": "table", - "instant": true, + "expr": "sum(increase(app_beacon_node_sse_block_bucket{cluster_name=\"$cluster_name\", cluster_hash=\"$cluster_hash\", cluster_peer=\"$cluster_peer\" }[$__range])) by (le)", + "format": "heatmap", + "instant": false, "legendFormat": "__auto", - "range": false, + "range": true, "refId": "A" } ], - "title": "Failing Health Checks", - "transformations": [ - { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Value": true - }, - "indexByName": {}, - "renameByName": { - "Value": "", - "name": "Check Name", - "severity": "Severity" - } - } - } - ], - "type": "table" + "title": "When in a slot a block finishes processing", + "type": "bargauge" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "description": "Status and balance of each validator by public key with links to beaconcha.in.\n\nIf present, the \"❗️Peer Exits\" column indicate the number of peers that submitted partially signed exits for each validator.", + "description": "Time in seconds between block gossip and head events, indicating block processing time, averaged across all connected beacon nodes.", "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "mode": "palette-classic" }, "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "inspect": false + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "light-blue", - "value": null + "color": "green", + "value": 0 }, { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Balance" - }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "type": "color-text" - } + "color": "yellow", + "value": 0.5 }, { - "id": "mappings", - "value": [ - { - "options": { - "0": { - "color": "transparent", - "index": 3 - } - }, - "type": "value" - }, - { - "options": { - "from": 32, - "result": { - "color": "green", - "index": 0 - }, - "to": 100 - }, - "type": "range" - }, - { - "options": { - "from": 30, - "result": { - "color": "orange", - "index": 1 - }, - "to": 32 - }, - "type": "range" - }, - { - "options": { - "from": 1, - "result": { - "color": "red", - "index": 2 - }, - "to": 30 - }, - "type": "range" - } - ] + "color": "orange", + "value": 1 }, { - "id": "unit", - "value": "ETH" + "color": "dark-orange", + "value": 2 }, { - "id": "custom.width", - "value": 150 + "color": "red", + "value": 4 } ] }, - { - "matcher": { - "id": "byName", - "options": " Public Key" - }, - "properties": [ - { - "id": "links", - "value": [ - { - "targetBlank": true, - "title": "beaconcha.in", - "url": "http://${__data.fields[\"cluster_network\"]}.beaconcha.in/validator/${__data.fields[\"pubkey_full\"]}" - } - ] - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-text" - } - }, - { - "id": "custom.width" - } - ] + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 126 + }, + "id": 310, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "avg by () (histogram_quantile(\n 0.50,\n increase(app_beacon_node_sse_block_processing_time_bucket{\n cluster_hash=\"$cluster_hash\",\n cluster_name=\"$cluster_name\",\n cluster_peer=\"$cluster_peer\"\n }[$__range])\n))", + "instant": false, + "legendFormat": "P50", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "avg by () (histogram_quantile(\n 0.90,\n increase(app_beacon_node_sse_block_processing_time_bucket{\n cluster_hash=\"$cluster_hash\",\n cluster_name=\"$cluster_name\",\n cluster_peer=\"$cluster_peer\"\n }[$__range])\n))", + "instant": false, + "legendFormat": "P90", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "avg by () (histogram_quantile(\n 1,\n increase(app_beacon_node_sse_block_processing_time_bucket{\n cluster_hash=\"$cluster_hash\",\n cluster_name=\"$cluster_name\",\n cluster_peer=\"$cluster_peer\"\n }[$__range])\n))", + "instant": false, + "legendFormat": "Max", + "range": true, + "refId": "C" + }, + { + "editorMode": "code", + "expr": "avg by () (rate(app_beacon_node_sse_block_processing_time_sum{cluster_hash=\"$cluster_hash\",cluster_name=\"$cluster_name\",cluster_peer=\"$cluster_peer\"}[$__range])/rate(app_beacon_node_sse_block_processing_time_count{cluster_hash=\"$cluster_hash\",cluster_name=\"$cluster_name\",cluster_peer=\"$cluster_peer\"}[$__range]))", + "legendFormat": "Average", + "range": true, + "refId": "A" + } + ], + "title": "Block Processing Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Charon caches certain calls to the beacon node. The bigger the amount of cache hits seen in this panel, the more requests Charon saved to the BN and improved its health.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "matcher": { - "id": "byName", - "options": "Status" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "properties": [ - { - "id": "custom.width", - "value": 150 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*Exits" + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" }, - "properties": [ - { - "id": "custom.cellOptions", - "value": { - "type": "color-text" - } - }, - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] + "thresholdsStyle": { + "mode": "off" + } }, - { - "matcher": { - "id": "byName", - "options": "__hide__" - }, - "properties": [ + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ { - "id": "custom.hidden", - "value": true + "color": "green", + "value": 0 } ] - } - ] + }, + "unit": "none" + }, + "overrides": [] }, "gridPos": { - "h": 8, - "w": 9, - "x": 8, - "y": 25 + "h": 7, + "w": 12, + "x": 0, + "y": 133 }, - "id": 134, + "id": 315, "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": [ - "Value #B" - ], - "reducer": [ - "sum" - ], - "show": true + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "showHeader": true, - "sortBy": [ - { - "desc": true, - "displayName": "Status" - } - ] + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 }, - "pluginVersion": "11.4.1", + "pluginVersion": "12.4.1", "targets": [ { "datasource": { @@ -2266,1162 +6913,2278 @@ "uid": "prometheus" }, "editorMode": "code", - "exemplar": false, - "expr": "max(core_scheduler_validator_balance_gwei{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} / 1000000000) by (pubkey_full,pubkey, cluster_network)", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "{{pubkey}}", - "range": false, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(core_scheduler_validator_status{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"} == 1) by (pubkey, status)", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, + "exemplar": true, + "expr": "max(increase(app_cache_hits_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (endpoint) > 0 ", + "interval": "$interval", + "legendFormat": "{{endpoint}}", + "range": true, "refId": "A" + } + ], + "title": "Cache hits total", + "type": "barchart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Charon caches certain calls to the beacon node. The lower the number of misses the better. It is expected to have 1-2 misses every epoch.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 133 + }, + "id": 314, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "12.4.1", + "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", - "exemplar": false, - "expr": "max(core_parsigdb_exit_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (pubkey) ", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "C" + "exemplar": true, + "expr": "max(increase(app_cache_misses_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (endpoint) > 0 ", + "interval": "$interval", + "legendFormat": "{{endpoint}}", + "range": true, + "refId": "A" } ], - "title": "Validators", - "transformations": [ - { - "id": "joinByField", - "options": { - "byField": "pubkey", - "mode": "outer" - } + "title": "Cache misses total", + "type": "barchart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Charon caches certain calls to the beacon node. On reorg event from the beacon node, Charon invalidates its cache.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 140 + }, + "id": 316, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "12.4.1", + "targets": [ { - "id": "organize", - "options": { - "excludeByName": { - "Time": true, - "Time 1": true, - "Time 2": true, - "Time 3": true, - "Value #A": true, - "Value #B": false, - "cluster_network": false, - "job": true, - "pubkey_full": false - }, - "indexByName": { - "Time 1": 2, - "Time 2": 7, - "Time 3": 9, - "Value #A": 8, - "Value #B": 6, - "Value #C": 4, - "cluster_network": 3, - "pubkey": 0, - "pubkey_full": 1, - "status": 5 - }, - "renameByName": { - "Value #A": "", - "Value #B": "Balance", - "Value #C": "❗️Peer Exits", - "cluster_network": "__hide__", - "pubkey": " Public Key", - "pubkey_full": "__hide__", - "status": "Status" - } - } + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(increase(app_cache_invalidated_reorg_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (endpoint) > 0 ", + "interval": "$interval", + "legendFormat": "{{endpoint}}", + "range": true, + "refId": "A" } ], - "type": "table" + "title": "Cache invalidation due reorg", + "type": "barchart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 147 + }, + "id": 319, + "panels": [], + "title": "MEV", + "type": "row" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "description": "Number of charon nodes running particular consensus client versions", + "description": "Whether this node has a growing number of relay registration failures, indicating there may be a risk to an upcoming MEV block production opportunity. No data may mean MEV registration is working correctly, or that it is not enabled at all.", "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "fixedColor": "#ffffff", + "mode": "palette-classic" }, "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "inspect": false, - "width": 300 + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } }, - "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", - "value": null + "value": 0 + }, + { + "color": "green", + "value": 0.1 + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "orange", + "value": 200 }, { "color": "red", - "value": 80 + "value": 500 } ] - } + }, + "unit": "none" }, "overrides": [ { "matcher": { "id": "byName", - "options": "Peer" + "options": "Total" }, "properties": [ { - "id": "custom.width", - "value": 200 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "nickname" - }, - "properties": [ + "id": "custom.axisPlacement", + "value": "right" + }, { - "id": "custom.width", - "value": 164 + "id": "unit", + "value": "none" + }, + { + "id": "color", + "value": { + "fixedColor": "#808080c4", + "mode": "fixed" + } } ] }, { + "__systemRef": "hideSeriesFrom", "matcher": { - "id": "byName", - "options": "Beacon Node Version" - }, - "properties": [ - { - "id": "custom.width", - "value": 224 - } - ] - } - ] - }, - "gridPos": { - "h": 4, - "w": 7, - "x": 17, - "y": 25 - }, - "id": 293, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true, - "sortBy": [] - }, - "pluginVersion": "11.4.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "f65e1e2f-12d7-4547-a93d-8b2ab35b38d3" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(app_beacon_node_version{cluster_name=~\"$cluster_name\",cluster_hash=~\"$cluster_hash\",cluster_network=~\"$cluster_network\", cluster_peer=~\"$cluster_peer\"}) by (cluster_name, cluster_hash, cluster_peer, version, nickname)", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Beacon Node Version", - "transformations": [ - { - "id": "labelsToFields", - "options": { - "keepLabels": [ - "cluster_peer", - "version", - "nickname" - ], - "mode": "columns" - } - }, - { - "id": "merge", - "options": {} - }, - { - "id": "groupBy", - "options": { - "fields": { - "Beacon Node Version": { - "aggregations": [ - "lastNotNull" - ], - "operation": "aggregate" - }, - "Charon Version": { - "aggregations": [ - "lastNotNull" - ], - "operation": "aggregate" - }, - "Time": { - "aggregations": [ - "lastNotNull" - ], - "operation": "aggregate" - }, - "Value": { - "aggregations": [] - }, - "charon_version": { - "aggregations": [ - "lastNotNull" - ], - "operation": "groupby" - }, - "cluster_peer": { - "aggregations": [] - }, - "nickname": { - "aggregations": [], - "operation": "groupby" - }, - "version": { - "aggregations": [ - "lastNotNull" + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Etherfi-Obol-curated-EU-03: DV Labs (nervous-pillow)" ], - "operation": "groupby" + "prefix": "All except:", + "readOnly": true } - } - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Value": true, - "charon_version": true, - "charon_version (lastNotNull)": true - }, - "includeByName": {}, - "indexByName": { - "Lighthouse/v5.1.0-10a38a8/x86_64-linux": 1, - "cluster_peer": 0 }, - "renameByName": { - "Lighthouse/v5.1.0-10a38a8/x86_64-linux": "", - "Time (lastNotNull)": "Last Time Reported", - "charon_version": "Charon Version", - "charon_version (lastNotNull)": "Charon Version", - "cluster_peer": "Peer", - "version": "Beacon Node Version", - "version (lastNotNull)": "Beacon Node Version" - } - } - }, - { - "id": "sortBy", - "options": { - "fields": {}, - "sort": [ + "properties": [ { - "desc": false, - "field": "Peer" + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": true, + "viz": true + } } ] } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 148 + }, + "id": 242, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(increase(app_eth2_errors_total{endpoint=~\"submit_validator_registrations\", cluster_name=\"$cluster_name\", cluster_hash=\"$cluster_hash\", cluster_peer=\"$cluster_peer\"}[$__range]))", + "legendFormat": "{{cluster_name}}: {{nickname}} ({{cluster_peer}})", + "range": true, + "refId": "A" } ], - "type": "table" + "title": "MEV Relay Submit Registrations Failure Growth", + "type": "timeseries" }, { "datasource": { - "default": true, "type": "prometheus", "uid": "prometheus" }, + "description": "The fetch proposal from peer was locally built or built by MEV (blinded).", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" + "axisPlacement": "auto", + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "inspect": false + "insertNulls": false, + "lineWidth": 0, + "spanNulls": false }, - "mappings": [], + "mappings": [ + { + "options": { + "0": { + "color": "#808080", + "index": 0, + "text": "N/A" + }, + "1": { + "color": "green", + "index": 1, + "text": "Blinded" + }, + "2": { + "color": "red", + "index": 2, + "text": "Local" + } + }, + "type": "value" + } + ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 + "value": 0 } ] } }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "VC Version" - }, - "properties": [ - { - "id": "mappings", - "value": [ - { - "options": { - "node": { - "index": 0, - "text": "Lodestar" - } - }, - "type": "value" - } - ] - } - ] - } - ] + "overrides": [] }, "gridPos": { - "h": 4, - "w": 7, - "x": 17, - "y": 29 + "h": 7, + "w": 12, + "x": 12, + "y": 148 }, - "id": 294, + "id": 307, "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false + "alignValue": "left", + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false }, - "showHeader": true + "mergeValues": false, + "rowHeight": 0.9, + "showValue": "auto", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } }, - "pluginVersion": "11.4.1", + "pluginVersion": "12.4.1", "targets": [ { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, "editorMode": "code", - "expr": "count(core_validatorapi_vc_user_agent{cluster_name=~\"$cluster_name\",cluster_hash=~\"$cluster_hash\",cluster_network=~\"$cluster_network\", cluster_peer=~\"$cluster_peer\"}) by (cluster_peer, user_agent, nickname)", + "exemplar": false, + "expr": "sum(core_fetcher_proposal_blinded{cluster_name=\"$cluster_name\", cluster_hash=\"$cluster_hash\", cluster_peer=\"$cluster_peer\"})", + "format": "time_series", "instant": false, - "legendFormat": "__auto", + "interval": "", + "legendFormat": " ", "range": true, "refId": "A" } ], - "title": "Validator Client Version", - "transformations": [ - { - "id": "labelsToFields", - "options": {} - }, - { - "id": "merge", - "options": {} - }, + "title": "Blinded or Local block received", + "type": "state-timeline" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 155 + }, + "id": 132, + "panels": [ { - "id": "groupBy", - "options": { - "fields": { - "Time": { - "aggregations": [ - "lastNotNull" - ], - "operation": "aggregate" + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "cluster_peer": { - "aggregations": [] + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, - "nickname": { - "aggregations": [], - "operation": "groupby" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] }, - "user_agent": { - "aggregations": [], - "operation": "groupby" + "unit": "reqps" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "propose_block_v3" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": true, + "viz": true + } + } + ] } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 1282 + }, + "id": 136, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" } - } + }, + "pluginVersion": "11.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(rate(core_validatorapi_request_latency_seconds_count{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (endpoint) > 0", + "interval": "$interval", + "legendFormat": "{{endpoint}}", + "range": true, + "refId": "A" + } + ], + "title": "Validator API requests rate", + "type": "timeseries" }, { - "id": "organize", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "#EAB839", + "value": 1 + }, + { + "color": "orange", + "value": 2 + }, + { + "color": "red", + "value": 3 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 1282 + }, + "id": 143, "options": { - "excludeByName": {}, - "includeByName": {}, - "indexByName": {}, - "renameByName": { - "Time (lastNotNull)": "Last Reported", - "cluster_peer": "Peer", - "user_agent": "VC Version" + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" } - } + }, + "pluginVersion": "11.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(core_validatorapi_request_latency_seconds_sum{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$__interval]) / rate(core_validatorapi_request_latency_seconds_count{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$__interval])", + "interval": "$interval", + "legendFormat": "{{endpoint}}", + "range": true, + "refId": "A" + } + ], + "title": "Validator API request latency", + "type": "timeseries" }, { - "id": "sortBy", - "options": { - "fields": {}, - "sort": [ + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of errors per endpoint per time window", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [ { - "field": "Peer" + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "propose_block_v3" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": true, + "viz": true + } + } + ] } ] - } + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 1289 + }, + "id": 139, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(increase(core_validatorapi_request_error_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (endpoint) > 0 ", + "interval": "$interval", + "legendFormat": "{{endpoint}}", + "range": true, + "refId": "A" + } + ], + "title": "\ud83d\udea8 Validator API error count (per $interval)", + "type": "timeseries" } ], - "type": "table" + "title": "Downstream Validator", + "type": "row" }, { - "collapsed": false, + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 33 - }, - "id": 178, - "panels": [], - "title": "Duties", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" + "y": 156 }, - "description": "Completed duties by type over time.", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "red", - "mode": "palette-classic" + "id": 149, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" }, - "custom": { - "fillOpacity": 100, - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + "description": "Type of connections to each peer in the cluster during the time window:\n - \ud83d\udfe2 Green: Direct connection (this is the preferred)\n - \ud83d\udfe1 Yellow: Indirect relay connection (this is ok if only temporary)\n - \ud83d\udd34 Red: No connection to peer (this is a problem)\n\nSee https://docs.obol.tech/docs/charon/networking#external-p2p-network", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisPlacement": "auto", + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineWidth": 0, + "spanNulls": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "#EAB839", + "value": 1 + }, + { + "color": "green", + "value": 80 + } + ] + } }, - "lineWidth": 0 + "overrides": [] }, - "links": [], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 655 + }, + "id": 195, + "options": { + "alignValue": "left", + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "mergeValues": true, + "rowHeight": 0.9, + "showValue": "never", + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" }, - { - "color": "red", - "value": 80 - } - ] - } + "editorMode": "code", + "expr": "(max(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer))\n+\n(1 * max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"relay\"}) by (peer))\n+ \n(100 * max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"direct\"}) by (peer))", + "format": "time_series", + "interval": "$interval", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Peer Connections (\ud83d\udfe2=direct, \ud83d\udfe1=relay, \ud83d\udd34=not connected)", + "type": "state-timeline" }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 34 - }, - "id": 92, - "options": { - "colWidth": 0.5, - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of new libp2p connections per peer per time window", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 655 + }, + "id": 163, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(increase(p2p_peer_connection_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=~\"$peer\"}[$interval])) by (peer) > 0 ", + "interval": "$interval", + "legendFormat": "{{peer}}", + "range": true, + "refId": "A" + } + ], + "title": "New P2P Connections (per $interval)", + "type": "timeseries" }, - "rowHeight": 0.43, - "showValue": "auto", - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.4.1", - "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(increase(core_bcast_broadcast_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (duty) > 0", - "instant": false, - "interval": "$interval", - "legendFormat": "{{duty}}", - "range": true, - "refId": "A" - } - ], - "title": "✅ Completed duties by type ", - "type": "status-history" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Failed duties percentage by type over time", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "red", - "mode": "palette-classic" + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 662 + }, + "id": 160, + "options": { + "calculate": false, + "cellGap": 5, + "cellValues": { + "unit": "s" }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "max": 0.5, + "min": 0, + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlGn", + "steps": 82 }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "exemplars": { + "color": "rgba(255,0,255,0.7)" }, - "thresholdsStyle": { - "mode": "off" + "filterValues": { + "le": 1e-09 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false } }, - "links": [], - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 34 - }, - "id": 180, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.90, sum(rate(p2p_ping_latency_secs_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=~\"$peer\"}[$interval])) by (le,peer))", + "interval": "", + "legendFormat": "{{peer}}", + "range": true, + "refId": "A" + } + ], + "title": "Peer ping latency (90%) (heatmap)", + "type": "heatmap" }, - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.4.1", - "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "editorMode": "code", - "exemplar": false, - "expr": "(\n sum(increase(core_tracker_failed_duties_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (duty)\n) \n/ \n(\n sum(increase(core_tracker_failed_duties_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (duty) \n + \n sum(increase(core_bcast_broadcast_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (duty) \n) > 0 ", - "instant": false, - "interval": "$interval", - "legendFormat": "{{duty}}", - "range": true, - "refId": "A" - } - ], - "title": "❌ Failed duty percentage by type (per $interval)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Failed duties by type over time. Use data link to show logs.\nProposer duties are non-failing after being successfully included on-chain.\nOther broadcasted to the beacon node duties (i.e.: attester), are non-failing after being successfully broadcasted.\nAny other duties not broadcasted to the beacon node (i.e.: randao), are non-failing after consensus is reached.", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "red", - "mode": "palette-classic" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 0.15 + }, + { + "color": "orange", + "value": 0.3 + }, + { + "color": "red", + "value": 0.5 + } + ] + }, + "unit": "s" + }, + "overrides": [] }, - "custom": { - "fillOpacity": 100, - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 662 + }, + "id": 25, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "lineWidth": 0 + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "single", + "sort": "none" + } }, - "links": [ + "pluginVersion": "12.4.1", + "targets": [ { - "targetBlank": true, - "title": "Show in log explorer", - "url": "/explore?orgId=1&left=%7B%22datasource%22:%22loki%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22editorMode%22:%22builder%22,%22expr%22:%22%7Bcompose_service%3D%5C%22$job%5C%22%7D%20%7C%3D%20%60Duty%20failed%60%20%7C%3D%20%60${__field.labels.duty}%60%20%7C%20logfmt%20%7C%20line_format%20%60%7B%7B.pretty%7D%7D%60%22,%22queryType%22:%22range%22%7D%5D,%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D" + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "histogram_quantile(0.90, sum(rate(p2p_ping_latency_secs_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=~\"$peer\"}[$interval])) by (le,peer)) ", + "interval": "$interval", + "legendFormat": "{{peer}}", + "range": true, + "refId": "A" } ], - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 40 - }, - "id": 91, - "options": { - "colWidth": 0.5, - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false + "title": "Peer ping latency (90%)", + "type": "timeseries" }, - "rowHeight": 0.43, - "showValue": "auto", - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.4.1", - "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(increase(core_tracker_failed_duties_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (duty) > 0", - "instant": false, - "interval": "$interval", - "legendFormat": "{{duty}}", - "range": true, - "refId": "A" - } - ], - "title": "❌ Failed duties by type (per $interval)", - "type": "status-history" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Which operators are having a growing number of relay registrations failures, indicating there may be a risk to an upcoming MEV block production opportunity. Be conscious that absence of an operator from this list may mean their MEV registration is working, or it may mean it is not enabled at all and thus not failing. ", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "#ffffff", - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "opacity", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 1, - "scaleDistribution": { - "type": "linear" + "description": "Time windows that contain ping errors per peer", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisPlacement": "auto", + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineWidth": 0, + "spanNulls": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + } + ] + } }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 669 + }, + "id": 166, + "options": { + "alignValue": "left", + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false }, - "thresholdsStyle": { - "mode": "area" + "mergeValues": true, + "rowHeight": 0.9, + "showValue": "never", + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "single", + "sort": "none" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" }, - { - "color": "green", - "value": 0.1 + "editorMode": "code", + "expr": "sum(increase(p2p_ping_error_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=~\"$peer\"}[$interval])) by (peer) > 0", + "interval": "$interval", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Ping Errors", + "type": "state-timeline" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Avg Rate of P2P network bytes sent plus received per peer (bytes/second) ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "color": "yellow", - "value": 10 + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, - { - "color": "orange", - "value": 200 + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": 0 + } + ] }, + "unit": "Bps" + }, + "overrides": [ { - "color": "red", - "value": 500 + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "decbytes" + } + ] } ] }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Total" + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 669 + }, + "id": 153, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "none" + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" }, - { - "id": "color", - "value": { - "fixedColor": "#808080c4", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 40 - }, - "id": 242, - "options": { - "legend": { - "calcs": [ - "lastNotNull" + "editorMode": "code", + "expr": "sum(rate(p2p_peer_network_receive_bytes_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (peer, transport) + sum(rate(p2p_peer_network_sent_bytes_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (peer, transport)", + "interval": "$interval", + "legendFormat": "{{peer}} ({{transport}})", + "range": true, + "refId": "A" + } ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true + "title": "P2P Network Avg Rate Rx+Tx", + "type": "timeseries" }, - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.4.1", - "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "editorMode": "code", - "expr": "max(increase(app_eth2_errors_total{endpoint=~\"submit_validator_registrations\", cluster_name=~\"$cluster_name\"}[$__range])) by (cluster_peer, cluster_name)", - "hide": false, - "legendFormat": "{{cluster_name}}: {{cluster_peer}}", - "range": true, - "refId": "A" - } - ], - "title": "⚠️ MEV Relay Submit Registrations Failure Growth by Operator", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "The number of consensus rounds it takes to achieve consensus on a batch of duties. When every node is online, this should be 1 round, if it is taking more rounds than the fault tolerance of the cluster to come to consensus, there may be an issue with node operators in the cluster, or the latency of the cluster. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 3, - "axisSoftMin": 0, - "barAlignment": -1, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 675 + }, + "id": 158, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "dashed+area" + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "single", + "sort": "none" } }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "orange", - "value": 2 + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" }, - { - "color": "red", - "value": 3 - } - ] - } + "editorMode": "code", + "exemplar": true, + "expr": "max(rate(p2p_ping_latency_secs_count{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",peer=~\"$peer\"}[$interval])) by (peer) ", + "interval": "$interval", + "legendFormat": "{{peer}}", + "range": true, + "refId": "A" + } + ], + "title": "Ping Rate", + "type": "timeseries" }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "aggregator", - "attester", - "proposer" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Total P2P network bytes sent plus received per peer per time window (bytes per interval)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, - "viz": true + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": 0 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "decbytes" + } + ] } ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 46 - }, - "id": 268, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 675 + }, + "id": 154, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(p2p_peer_network_receive_bytes_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (peer, transport) + sum(increase(p2p_peer_network_sent_bytes_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (peer,transport)", + "interval": "$interval", + "legendFormat": "{{peer}} ({{transport}})", + "range": true, + "refId": "A" + } + ], + "title": "P2P Network Bytes Rx+Tx (per $interval)", + "type": "timeseries" }, - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.4.1", - "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "editorMode": "code", - "expr": "max(core_consensus_decided_rounds{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",duty=~\"$duty\"}) by (timer, duty)", - "hide": false, - "interval": "$interval", - "legendFormat": "{{duty}}", - "range": true, - "refId": "A" - } - ], - "title": "Consensus Rounds: $duty", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "How long in total did it take to come to consensus on this/these duties? Duties generally have a 4 second window to be completed (1.25s on gnosis chain), so consensus should not take more than half of that time for good performance. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "yellow", + "value": -0.5 + }, + { + "color": "green", + "value": -0.2 + }, + { + "color": "#EAB839", + "value": 0.2 + }, + { + "color": "red", + "value": 0.5 + } + ] + }, + "unit": "s" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 681 + }, + "id": 151, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "thresholdsStyle": { - "mode": "dashed+area" + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(app_peerinfo_clock_offset_seconds{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=~\"$cluster_peer\"}) by (peer) ", + "interval": "$interval", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Clock Offset", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "An overview of when in the slot a block is finished processing by the connected CL. Blocks processed after 4 seconds (i.e. in the 4.5 bar and higher) miss the attestation kick-off, so if your cluster is attesting to this slot, and this node is the leader, it will propose an attestation to an incorrect head of the chain, and result in reduced rewards.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "locale" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "+Inf" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "12" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "10" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "8" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "6" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, { - "color": "green", - "value": null + "matcher": { + "id": "byName", + "options": "4.5" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] }, { - "color": "yellow", - "value": 0.3 + "matcher": { + "id": "byName", + "options": "4" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] }, { - "color": "orange", - "value": 0.6 + "matcher": { + "id": "byName", + "options": "3.5" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] }, { - "color": "red", - "value": 1 + "matcher": { + "id": "byName", + "options": "4.0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5.0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "6.0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "8.0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "10.0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "12.0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] } ] }, - "unit": "s" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "aggregator" - ], - "prefix": "All except:", - "readOnly": true - } + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 681 + }, + "id": 311, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "libp2p_swarm_handshake_latency_seconds_sum{cluster_name=\"$cluster_name\",cluster_peer=\"$cluster_peer\"}/\nlibp2p_swarm_handshake_latency_seconds_count{cluster_name=\"$cluster_name\",cluster_peer=\"$cluster_peer\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{transport}}", + "range": true, + "refId": "A" + } + ], + "title": "Avg Handshake duration by Transport", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisPlacement": "auto", + "fillOpacity": 70, + "hideFrom": { "legend": false, "tooltip": false, - "viz": true - } + "viz": false + }, + "insertNulls": false, + "lineWidth": 0, + "spanNulls": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": 0 + } + ] } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 46 - }, - "id": 261, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 687 + }, + "id": 156, + "options": { + "alignValue": "left", + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "mergeValues": true, + "rowHeight": 0.9, + "showValue": "never", + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(p2p_relay_connections{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer) > 0 ", + "interval": "$interval", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Connected Relays", + "type": "state-timeline" }, - "tooltip": { - "maxHeight": 600, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.4.1", - "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "editorMode": "code", - "expr": "rate(core_consensus_duration_seconds_sum{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",duty=~\"$duty\"}[$interval]) / rate(core_consensus_duration_seconds_count{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",duty=~\"$duty\"}[$interval])", - "hide": false, - "interval": "$interval", - "legendFormat": "{{duty}}", - "range": true, - "refId": "A" + "description": "The number of active libp2p streams by peer or relay. Note this can be further grouped by direction and protocol.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 64 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 694 + }, + "id": 226, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(p2p_peer_streams{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer, transport)", + "interval": "$interval", + "legendFormat": "{{peer}} ({{transport}})", + "range": true, + "refId": "A" + } + ], + "title": "Active libp2p streams", + "type": "timeseries" } ], - "title": "Consensus duration: $duty", - "type": "timeseries" + "title": "P2P Networking", + "type": "row" } ], "preload": false, "refresh": "1m", - "schemaVersion": 40, + "schemaVersion": 42, "tags": [], "templating": { "list": [ @@ -3429,11 +9192,9 @@ "allValue": "all", "current": { "text": [ - "mainnet", "hoodi" ], "value": [ - "mainnet", "hoodi" ] }, @@ -3441,7 +9202,7 @@ "type": "prometheus", "uid": "prometheus" }, - "definition": "label_values(app_peerinfo_nickname,cluster_network)", + "definition": "label_values(app_peer_name,cluster_network)", "description": "", "includeAll": true, "label": "Cluster Network", @@ -3450,175 +9211,154 @@ "options": [], "query": { "qryType": 1, - "query": "label_values(app_peerinfo_nickname,cluster_network)", + "query": "label_values(app_peer_name,cluster_network)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", + "regexApplyTo": "value", "type": "query" }, { "current": { - "text": "01node", - "value": "01node" + "text": "test", + "value": "test" }, "datasource": { "type": "prometheus", "uid": "prometheus" }, - "definition": "label_values(app_peerinfo_nickname{cluster_network=~\"$cluster_network\"},cluster_name)", + "definition": "label_values(app_peer_name{cluster_network=~\"$cluster_network\"},cluster_name)", "description": "", "includeAll": false, "label": "Cluster Name", "name": "cluster_name", "options": [], "query": { - "qryType": 1, - "query": "label_values(app_peerinfo_nickname{cluster_network=~\"$cluster_network\"},cluster_name)", + "query": "label_values(app_peer_name{cluster_network=~\"$cluster_network\"},cluster_name)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 2, "regex": "", + "regexApplyTo": "value", "sort": 1, "type": "query" }, { "current": { - "text": "c2ce374", - "value": "c2ce374" + "text": "24716d9", + "value": "24716d9" }, "datasource": { "type": "prometheus", "uid": "prometheus" }, - "definition": "label_values(app_peerinfo_nickname{cluster_name=\"$cluster_name\"},cluster_hash)", + "definition": "label_values(app_peer_name{cluster_name=\"$cluster_name\"}, cluster_hash)", "description": "", "includeAll": false, "label": "Cluster Hash", "name": "cluster_hash", "options": [], "query": { - "qryType": 1, - "query": "label_values(app_peerinfo_nickname{cluster_name=\"$cluster_name\"},cluster_hash)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" + "query": "label_values(app_peer_name{cluster_name=\"$cluster_name\"}, cluster_hash)", + "refId": "StandardVariableQuery" }, "refresh": 2, "regex": "", + "regexApplyTo": "value", "sort": 1, "type": "query" }, { "current": { - "text": "busy-hospital", - "value": "busy-hospital" + "text": "colorful-food", + "value": "colorful-food" }, "datasource": { "type": "prometheus", "uid": "prometheus" }, - "definition": "label_values(app_peerinfo_nickname{cluster_name=\"$cluster_name\", cluster_hash=\"$cluster_hash\"},cluster_peer)", + "definition": "label_values(app_peer_name{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\"}, cluster_peer)", "description": "", "includeAll": false, "label": "Cluster Peer", "name": "cluster_peer", "options": [], "query": { - "qryType": 1, - "query": "label_values(app_peerinfo_nickname{cluster_name=\"$cluster_name\", cluster_hash=\"$cluster_hash\"},cluster_peer)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" + "query": "label_values(app_peer_name{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\"}, cluster_peer)", + "refId": "StandardVariableQuery" }, "refresh": 2, "regex": "", + "regexApplyTo": "value", "sort": 1, "type": "query" }, { - "allValue": ".*", "current": { - "text": "$__all", + "text": "charon", + "value": "charon" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(app_peer_name{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"},job)", + "description": "Prometheus job (can be ignored)", + "hide": 2, + "includeAll": false, + "label": "--", + "name": "job", + "options": [], + "query": { + "query": "label_values(app_peer_name{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"},job)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "regexApplyTo": "value", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "All", "value": "$__all" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(app_peerinfo_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\", cluster_peer=\"$cluster_peer\"}, peer)", + "description": "", + "hide": 2, "includeAll": true, - "label": "Duty", - "name": "duty", - "options": [ - { - "selected": false, - "text": "attester", - "value": "attester" - }, - { - "selected": false, - "text": "prepare_aggregator", - "value": "prepare_aggregator" - }, - { - "selected": false, - "text": "aggregator", - "value": "aggregator" - }, - { - "selected": false, - "text": "sync_message", - "value": "sync_message" - }, - { - "selected": false, - "text": "prepare_sync_contribution", - "value": "prepare_sync_contribution" - }, - { - "selected": false, - "text": "sync_contribution", - "value": "sync_contribution" - }, - { - "selected": false, - "text": "proposer", - "value": "proposer" - }, - { - "selected": false, - "text": "randao", - "value": "randao" - }, - { - "selected": false, - "text": "exit", - "value": "exit" - }, - { - "selected": false, - "text": "builder_proposer", - "value": "builder_proposer" - }, - { - "selected": false, - "text": "builder_registration", - "value": "builder_registration" - }, - { - "selected": false, - "text": "info_sync", - "value": "info_sync" - } - ], - "query": "attester,prepare_aggregator,aggregator,sync_message,prepare_sync_contribution,sync_contribution,proposer,randao,exit,builder_proposer,builder_registration,info_sync", - "type": "custom" + "label": "P2P Peer", + "name": "peer", + "options": [], + "query": { + "query": "label_values(app_peerinfo_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\", cluster_peer=\"$cluster_peer\"}, peer)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "regexApplyTo": "value", + "sort": 1, + "type": "query" }, { "auto": true, "auto_count": 30, "auto_min": "1m", "current": { - "text": "1m", - "value": "1m" + "text": "$__auto", + "value": "$__auto" }, "label": "Interval", "name": "interval", "options": [ { - "selected": true, + "selected": false, "text": "1m", "value": "1m" }, @@ -3671,17 +9411,31 @@ "query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", "refresh": 2, "type": "interval" + }, + { + "allValue": ".*", + "current": { + "text": "$__all", + "value": "$__all" + }, + "includeAll": true, + "label": "Duty", + "name": "duty", + "options": [], + "query": "attester,prepare_aggregator,aggregator,sync_message,prepare_sync_contribution,sync_contribution,proposer,randao,exit,builder_proposer,builder_registration,info_sync", + "type": "custom", + "valuesFormat": "csv" } ] }, "time": { - "from": "now-1h", - "to": "now" + "from": "2026-04-16T13:50:00.000Z", + "to": "2026-04-16T14:00:00.000Z" }, "timepicker": {}, - "timezone": "", - "title": "Single Charon Client Overview", - "uid": "charonoverview", - "version": 66, + "timezone": "utc", + "title": "Charon Overview", + "uid": "charon_overview", + "version": 46, "weekStart": "" } \ No newline at end of file diff --git a/grafana/dashboards/single_node_dashboard.json b/grafana/dashboards/single_node_dashboard.json deleted file mode 100644 index ddc1906c..00000000 --- a/grafana/dashboards/single_node_dashboard.json +++ /dev/null @@ -1,2854 +0,0 @@ -{ - "annotations": { - "list": [ - { - "$$hashKey": "object:139", - "builtIn": 1, - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "loki", - "uid": "loki" - }, - "description": "", - "gridPos": { - "h": 3, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 261, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "### ⚠️⚠️⚠️ This panel has been deprecated ⚠️⚠️⚠️\n\nPlease use the Charon overview dashboard for the latest metrics and visualisations about your node and cluster.", - "mode": "markdown" - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "loki" - }, - "expr": "", - "queryType": "range", - "refId": "A" - } - ], - "transparent": true, - "type": "text" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 3 - }, - "id": 74, - "panels": [], - "title": "Execution Client", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 4 - }, - "id": 76, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "none", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "1", - "refId": "A" - } - ], - "title": "GETH", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "red", - "index": 1, - "text": "Inactive" - }, - "1": { - "color": "green", - "index": 0, - "text": "Active" - } - }, - "type": "value" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 0, - "y": 5 - }, - "id": 84, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "up{job=\"geth\"}", - "refId": "A" - } - ], - "title": "Status", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Peers connected with our execution client. Also number of peers we dialled and we are serving.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 4, - "y": 5 - }, - "id": 78, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "p2p_peers", - "legendFormat": "peers", - "range": true, - "refId": "A" - } - ], - "title": "Peers", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Memory used by execution client.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 5 - }, - "id": 80, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "rate(system_memory_allocs[1m])", - "legendFormat": "alloc", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "system_memory_used", - "hide": false, - "legendFormat": "used", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "system_memory_held", - "hide": false, - "legendFormat": "held", - "range": true, - "refId": "C" - } - ], - "title": "Memory", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 5 - }, - "id": 86, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "9.0.7", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "system_cpu_sysload", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "system", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "system_cpu_syswait", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "iowait", - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "system_cpu_procload", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "geth", - "refId": "C" - } - ], - "title": "CPU", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "locale" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 0, - "y": 8 - }, - "id": 82, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "chain_head_header", - "refId": "A" - } - ], - "title": "Latest Chain Header", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 11 - }, - "id": 53, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "none", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "1", - "refId": "A" - } - ], - "title": "LIGHTHOUSE", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 12 - }, - "id": 55, - "panels": [], - "title": "Beacon Client", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "decimals": 0, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 0, - "y": 13 - }, - "id": 64, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "8.5.3", - "targets": [ - { - "expr": "sync_range_chains", - "interval": "", - "legendFormat": "{{range_type}}", - "refId": "A" - } - ], - "title": "Syncing chain counts", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Lighthouse's reported current slot (unsafe not finalised)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "color": "red", - "index": 0, - "text": "Unknown" - } - }, - "type": "special" - } - ], - "noValue": "Unknown", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "locale" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 3, - "x": 6, - "y": 13 - }, - "id": 58, - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "expr": "max(slotclock_present_slot)", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "title": "Current Slot", - "transparent": true, - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "The sync status for lighthouse's peers", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "mappings": [ - { - "options": { - "Ahead": { - "color": "red", - "index": 2 - }, - "Behind": { - "color": "orange", - "index": 1 - }, - "Synced": { - "color": "green", - "index": 0 - } - }, - "type": "value" - } - ] - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 9, - "y": 13 - }, - "id": 60, - "options": { - "displayLabels": [ - "value" - ], - "legend": { - "displayMode": "table", - "placement": "right", - "showLegend": true, - "values": [ - "value" - ] - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "8.1.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "exemplar": false, - "expr": "sync_peers_per_status", - "instant": true, - "interval": "", - "legendFormat": "{{sync_status}}", - "refId": "A" - } - ], - "title": "Peer Sync Status", - "transparent": true, - "type": "piechart" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Peers that have dialed us. Indicates if our node is reachable", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "mappings": [] - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 13, - "y": 13 - }, - "id": 62, - "options": { - "displayLabels": [ - "value" - ], - "legend": { - "displayMode": "table", - "placement": "right", - "showLegend": true, - "values": [ - "value" - ] - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "8.1.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "exemplar": false, - "expr": "network_inbound_peers", - "instant": true, - "interval": "", - "legendFormat": "Inbound", - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "exemplar": false, - "expr": "network_outbound_peers", - "format": "time_series", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "Outbound", - "refId": "B" - } - ], - "title": "Peer Type", - "transparent": true, - "type": "piechart" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Identifies if there is an upstream execution client connected", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 0, - "mappings": [ - { - "options": { - "0": { - "color": "red", - "index": 1, - "text": "Not Connected" - }, - "1": { - "color": "green", - "index": 0, - "text": "Connected" - } - }, - "type": "value" - }, - { - "options": { - "match": "null", - "result": { - "color": "orange", - "index": 2, - "text": "Unknown" - } - }, - "type": "special" - }, - { - "options": { - "match": "empty", - "result": { - "color": "red", - "index": 3, - "text": "Not Connected" - } - }, - "type": "special" - } - ], - "min": 0, - "noValue": "Not Connected", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 17, - "y": 13 - }, - "id": 65, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "sync_eth1_connected", - "interval": "", - "legendFormat": "{{range_type}}", - "refId": "A" - } - ], - "title": "Eth1 Execution Client Connected", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Identifies if the consensus client is synced", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 0, - "mappings": [ - { - "options": { - "0": { - "color": "red", - "index": 1, - "text": "Not Synced" - }, - "1": { - "color": "green", - "index": 0, - "text": "Synced" - } - }, - "type": "value" - } - ], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 21, - "y": 13 - }, - "id": 66, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "sync_eth2_synced", - "interval": "", - "legendFormat": "{{range_type}}", - "refId": "A" - } - ], - "title": "Eth2 Consensus Client Synced", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 18 - }, - "id": 48, - "panels": [], - "title": "Charon", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 19 - }, - "id": 56, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "none", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "expr": "1", - "refId": "A" - } - ], - "title": "CHARON", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Status returned from the `/readyz` health status endpoint. Unhealthy reasons include beacon node problems, charon p2p problems, or validator client problems.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "text", - "index": 1, - "text": "Unknown" - }, - "1": { - "color": "green", - "index": 0, - "text": "OK" - }, - "2": { - "color": "red", - "index": 3, - "text": "BeaconNode Down" - }, - "3": { - "color": "orange", - "index": 4, - "text": "BeaconNode Syncing" - }, - "4": { - "color": "orange", - "index": 5, - "text": "Insufficient Peers" - }, - "5": { - "color": "orange", - "index": 6, - "text": "VC not connected" - }, - "6": { - "color": "orange", - "index": 7, - "text": "VC missing validators" - }, - "7": { - "color": "orange", - "index": 8, - "text": "BeaconNode has zero peers" - }, - "8": { - "color": "orange", - "index": 9, - "text": "BeaconNode far behind" - } - }, - "type": "value" - }, - { - "options": { - "match": "null", - "result": { - "color": "text", - "index": 2, - "text": "Dead" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 0, - "y": 20 - }, - "id": 95, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "app_monitoring_readyz{job=\"charon\"}", - "hide": false, - "instant": true, - "interval": "", - "legendFormat": "", - "range": false, - "refId": "A" - } - ], - "title": "Health Status", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Successful cluster duty percentage by type over the whole time range", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "#EAB839", - "value": 0.9 - }, - { - "color": "green", - "value": 0.99 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 14, - "x": 4, - "y": 20 - }, - "id": 100, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "(\n sum(increase(core_tracker_success_duties_total{job=\"charon\"}[$__range])) by (duty)\n) \n/ \n( \n sum(increase(core_tracker_expect_duties_total{job=\"charon\"}[$__range])) by (duty) \n) > 0", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Cluster Duty Success(%) ", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Average beacon node score for the whole time window. The score is a combination of:\n - **errors**: the ratio or errors returned by the beacon node \n - **latency**: how fast or slow the beacon node is responding\n\n\nThe performance of the Beacon node is critical to the performance of the validators and therefore rewards. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "orange", - "value": 0.5 - }, - { - "color": "green", - "value": 0.8 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 20 - }, - "id": 101, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "(clamp(app_monitoring_readyz{job=\"charon\"} == 1 OR app_monitoring_readyz{job=\"charon\"} == 4 OR app_monitoring_readyz{job=\"charon\"} == 5 OR app_monitoring_readyz{job=\"charon\"} == 6, 1, 1) OR on() vector(0))\n*\n(\n (\n0.5 * (1.0 - 10*(sum(increase(app_eth2_errors_total{job=\"charon\"}[$__range])) / (sum(increase(app_eth2_latency_seconds_count{job=\"charon\"}[$__range]))))) OR on() vector(0.5)\n )\n +\n (\n 0.5 * (1.0 - clamp_max(histogram_quantile(0.99, sum(rate(app_eth2_latency_seconds_bucket{job=\"charon\"}[$__range])) by (le)),1))\n )\n)", - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "A" - } - ], - "title": "Beacon Node Score", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "Information about the charon cluster.\n\n- **Network**: The blockchain network \n- **Operators/Nodes**: The number of operators or nodes or peers in the charon DV cluster.\n- **Threshold**: The minimum number of participating nodes (also quorum).\n- **Validators**: The number of validators added to the cluster, each validator is staking 32 ETH. ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "text", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Network" - }, - "properties": [ - { - "id": "mappings", - "value": [ - { - "options": { - "1": { - "color": "blue", - "index": 0, - "text": "Mainnet" - }, - "2": { - "color": "purple", - "index": 1, - "text": "Goerli" - }, - "3": { - "color": "purple", - "index": 2, - "text": "Sepolia" - }, - "4": { - "color": "purple", - "index": 3, - "text": "Gnosis" - } - }, - "type": "value" - } - ] - } - ] - } - ] - }, - "gridPos": { - "h": 4, - "w": 4, - "x": 0, - "y": 23 - }, - "id": 96, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "text": { - "titleSize": 15, - "valueSize": 15 - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "( 1 * app_version{job=\"charon\", cluster_network=\"mainnet\"} )\nor \n( 2 * app_version{job=\"charon\", cluster_network=\"goerli\"} )\nor\n( 3 * app_version{job=\"charon\", cluster_network=\"sepolia\"} )\nor\n( 4 * app_version{job=\"charon\", cluster_network=\"gnosis\"} )", - "format": "time_series", - "hide": false, - "instant": true, - "legendFormat": "Network", - "range": false, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "cluster_operators{job=\"charon\"}", - "instant": true, - "legendFormat": "Operators/Nodes", - "range": false, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "cluster_threshold{job=\"charon\"}", - "hide": false, - "instant": true, - "legendFormat": "Threshold", - "range": false, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "core_scheduler_validators_active{job=\"charon\"}", - "hide": false, - "instant": true, - "legendFormat": "Validators Active", - "range": false, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "cluster_validators{job=\"charon\"}", - "hide": false, - "instant": true, - "legendFormat": "Validators Total", - "range": false, - "refId": "C" - } - ], - "title": "Cluster Info", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "- **Peer**: The peer's name (inferred from their *charon-enr-private-key*)\n- **You**: ⭐️ is this local charon node\n- **Connected**: Whether you are currently connected to this peer.\n- **Direct**: Whether the connection is *direct* (👍) or *relay* (👎) \n- **Latency**: The time messages take to travel to/from the peer.\n- **Version**: The version of the peer's charon node\n- **Supported**: Whether the peers charon version is supported/compatible or not.\n- **Uptime**\" The duration since peer was restarted.\n- **🚧Attest**: Number of attestation duties missed in the time window.\n- **🚧Propose**: Number of block proposal duties missed in the time window.\n- **🚧Other**: Number of other duties missed in the time window.\n\n", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "center", - "cellOptions": { - "type": "auto" - }, - "inspect": false, - "minWidth": 0, - "width": 90 - }, - "decimals": 0, - "mappings": [], - "noValue": "--", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "transparent", - "value": null - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Latency (ms)" - }, - "properties": [ - { - "id": "unit", - "value": "none" - }, - { - "id": "mappings", - "value": [ - { - "options": { - "from": 0, - "result": { - "color": "green", - "index": 0 - }, - "to": 150 - }, - "type": "range" - }, - { - "options": { - "from": 150, - "result": { - "color": "yellow", - "index": 1 - }, - "to": 300 - }, - "type": "range" - }, - { - "options": { - "from": 300, - "result": { - "color": "orange", - "index": 2 - }, - "to": 500 - }, - "type": "range" - }, - { - "options": { - "from": 500, - "result": { - "color": "red", - "index": 3 - }, - "to": 100000 - }, - "type": "range" - } - ] - }, - { - "id": "custom.cellOptions", - "value": { - "mode": "lcd", - "type": "gauge", - "valueDisplayMode": "color" - } - }, - { - "id": "custom.width", - "value": 200 - }, - { - "id": "max", - "value": 700 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "You" - }, - "properties": [ - { - "id": "noValue", - "value": "⭐" - }, - { - "id": "custom.cellOptions", - "value": { - "type": "auto" - } - }, - { - "id": "mappings", - "value": [ - { - "options": { - "0": { - "index": 0, - "text": "--" - }, - "1": { - "index": 1, - "text": "--" - } - }, - "type": "value" - } - ] - }, - { - "id": "custom.width", - "value": 60 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Connected" - }, - "properties": [ - { - "id": "mappings", - "value": [ - { - "options": { - "0": { - "index": 0, - "text": "❌" - }, - "1": { - "index": 1, - "text": "✅" - } - }, - "type": "value" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "ClockDiff" - }, - "properties": [ - { - "id": "unit", - "value": "ms" - }, - { - "id": "mappings", - "value": [ - { - "options": { - "0": { - "index": 0, - "text": "🆗" - } - }, - "type": "value" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Direct" - }, - "properties": [ - { - "id": "custom.width", - "value": 55 - }, - { - "id": "mappings", - "value": [ - { - "options": { - "0": { - "index": 0, - "text": "--" - } - }, - "type": "value" - }, - { - "options": { - "from": 1, - "result": { - "index": 1, - "text": "👎" - }, - "to": 9 - }, - "type": "range" - }, - { - "options": { - "from": 10, - "result": { - "index": 2, - "text": "👍" - }, - "to": 1000 - }, - "type": "range" - } - ] - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Peer" - }, - "properties": [ - { - "id": "custom.width", - "value": 150 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Uptime" - }, - "properties": [ - { - "id": "unit", - "value": "s" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Index" - }, - "properties": [ - { - "id": "custom.width", - "value": 30 - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": "🚧.*" - }, - "properties": [ - { - "id": "unit", - "value": "percentunit" - }, - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "thresholds" - } - }, - { - "id": "custom.cellOptions", - "value": { - "type": "color-text" - } - }, - { - "id": "decimals", - "value": 1 - }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "#585858", - "value": null - }, - { - "color": "super-light-green", - "value": 0.00001 - }, - { - "color": "yellow", - "value": 0.01 - }, - { - "color": "red", - "value": 0.1 - } - ] - } - }, - { - "id": "noValue", - "value": "0%" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Supported" - }, - "properties": [ - { - "id": "mappings", - "value": [ - { - "options": { - "0": { - "index": 0, - "text": "❌" - }, - "1": { - "index": 1, - "text": "🆗" - } - }, - "type": "value" - } - ] - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 27 - }, - "id": 99, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "frameIndex": 0, - "showHeader": true, - "sortBy": [ - { - "desc": false, - "displayName": "You" - } - ] - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(p2p_ping_success{job=\"charon\"}) by (peer)", - "format": "table", - "instant": true, - "range": false, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(0.90, sum(rate(p2p_ping_latency_secs_bucket{job=\"charon\"}[$__rate_interval])) by (le,peer)) * 1000", - "format": "table", - "hide": false, - "instant": true, - "range": false, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(p2p_ping_success{job=\"charon\"}) by (peer)", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "", - "range": false, - "refId": "F" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "(max(p2p_peer_connection_types{job=\"charon\",type=\"direct\"}) by (peer)*10 + \nmax(p2p_peer_connection_types{job=\"charon\",type=\"relay\"}) by (peer))", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "", - "range": false, - "refId": "K" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(app_peerinfo_version{job=\"charon\"}) by (peer,version) ", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "L" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "time() - max(app_peerinfo_start_time_secs{job=\"charon\"} > 0) by (peer)", - "format": "table", - "hide": false, - "instant": true, - "range": false, - "refId": "Q" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(app_peerinfo_index{job=\"charon\"}) by (peer)", - "format": "table", - "hide": false, - "instant": true, - "range": false, - "refId": "R" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "(\n sum(increase(core_tracker_participation_missed_total{job=\"charon\", duty=\"attester\"}[$__range])) by (peer) \n/\n sum(increase(core_tracker_participation_expected_total{job=\"charon\", duty=\"attester\"}[$__range])) by (peer) \n) OR on() vector(0)", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "(\n sum(increase(core_tracker_participation_missed_total{job=\"charon\", duty=~\".*proposer\"}[$__range])) by (peer) \n/\n sum(increase(core_tracker_participation_expected_total{job=\"charon\", duty=~\".*proposer\"}[$__range])) by (peer) \n) OR on() vector(0)", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "(\n sum(increase(core_tracker_participation_missed_total{job=\"charon\", duty!~\"(proposer|attester)\"}[$__range])) by (peer) \n/\n sum(increase(core_tracker_participation_expected_total{job=\"charon\", duty!~\"(proposer|attester)\"}[$__range])) by (peer) \n) OR on() vector(0)", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(app_peerinfo_version_support{job=\"charon\"}) by (peer) ", - "format": "table", - "hide": false, - "instant": true, - "legendFormat": "", - "range": false, - "refId": "G" - } - ], - "title": "Peer Connectivity and Missed Duties (🚧)", - "transformations": [ - { - "id": "seriesToColumns", - "options": { - "byField": "peer" - } - }, - { - "id": "organize", - "options": { - "excludeByName": { - "Time 1": true, - "Time 10": true, - "Time 11": true, - "Time 12": true, - "Time 13": true, - "Time 14": true, - "Time 15": true, - "Time 16": true, - "Time 17": true, - "Time 2": true, - "Time 3": true, - "Time 4": true, - "Time 5": true, - "Time 6": true, - "Time 7": true, - "Time 8": true, - "Time 9": true, - "Value #F": false, - "Value #L": true, - "Value #M": true - }, - "indexByName": { - "Time 1": 9, - "Time 10": 22, - "Time 11": 23, - "Time 2": 11, - "Time 3": 12, - "Time 4": 13, - "Time 5": 14, - "Time 6": 15, - "Time 7": 16, - "Time 8": 17, - "Time 9": 21, - "Value #A": 3, - "Value #B": 5, - "Value #C": 18, - "Value #D": 19, - "Value #E": 20, - "Value #F": 2, - "Value #G": 7, - "Value #K": 4, - "Value #L": 10, - "Value #Q": 8, - "Value #R": 1, - "peer": 0, - "version": 6 - }, - "renameByName": { - "Value #A": "Connected", - "Value #B": "Latency (ms)", - "Value #C": "🚧Attest", - "Value #D": "🚧Propose", - "Value #E": "🚧Other", - "Value #F": "You", - "Value #G": "Supported", - "Value #H": "Aggregate", - "Value #I": "Exit", - "Value #J": "ClockDiff", - "Value #K": "Direct", - "Value #L": "", - "Value #N": "SyncMsg", - "Value #O": "SyncContrib", - "Value #P": "PrepareContrib", - "Value #Q": "Uptime", - "Value #R": "Index", - "git_hash": "GitCommit", - "peer": "Peer", - "version": "Version" - } - } - } - ], - "type": "table" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 34 - }, - "id": 50, - "panels": [], - "title": "Validator", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "description": "A lodestar validator client serving as part of this distributed validator cluster.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 35 - }, - "id": 36, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "none", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "1", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "LODESTAR", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "match": "null", - "result": { - "text": "N/A" - } - }, - "type": "special" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#F2495C", - "value": null - }, - { - "color": "rgba(237, 129, 40, 0.89)", - "value": 0 - }, - { - "color": "#299c46", - "value": 1 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 36 - }, - "id": 25, - "maxDataPoints": 100, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "vc_indices_count", - "interval": "", - "legendFormat": "", - "range": true, - "refId": "A" - } - ], - "title": "Number of validators", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "yellow", - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-blue", - "value": null - }, - { - "color": "semi-dark-green", - "value": 0 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "ERROR" - }, - "properties": [ - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - } - ] - } - ] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 36 - }, - "id": 29, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "vc_published_attestations_total", - "interval": "", - "legendFormat": "{{outcome}}", - "range": true, - "refId": "A" - } - ], - "title": "Attestations Published", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "dark-yellow", - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "blue", - "value": 0 - }, - { - "color": "blue", - "value": 1 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 36 - }, - "id": 30, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "vc_block_published_total", - "interval": "", - "legendFormat": "", - "range": true, - "refId": "A" - } - ], - "title": "Blocks Published", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "yellow", - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-blue", - "value": null - }, - { - "color": "purple", - "value": 0 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "ERROR" - }, - "properties": [ - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 1 - } - ] - } - } - ] - } - ] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 36 - }, - "id": 94, - "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "vc_published_aggregates_total", - "interval": "", - "legendFormat": "{{outcome}}", - "range": true, - "refId": "A" - } - ], - "title": "Aggregate Attestations Published", - "type": "stat" - } - ], - "refresh": "10s", - "revision": 1, - "schemaVersion": 39, - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ] - }, - "timezone": "", - "title": "Single Charon Node Dashboard", - "uid": "singlenode", - "version": 2, - "weekStart": "" -} From df4f819af86f6f59191c106dd445b2832f6e395d Mon Sep 17 00:00:00 2001 From: kalo <24719519+KaloyanTanev@users.noreply.github.com> Date: Fri, 17 Apr 2026 17:38:16 +0300 Subject: [PATCH 2/8] Rename --- .../{dash_charon_overview.json => charon_overview_dashboard.json} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename grafana/dashboards/{dash_charon_overview.json => charon_overview_dashboard.json} (100%) diff --git a/grafana/dashboards/dash_charon_overview.json b/grafana/dashboards/charon_overview_dashboard.json similarity index 100% rename from grafana/dashboards/dash_charon_overview.json rename to grafana/dashboards/charon_overview_dashboard.json From 65879a56642e73ef1d20f5bc72f2a08a3e0f1203 Mon Sep 17 00:00:00 2001 From: kalo <24719519+KaloyanTanev@users.noreply.github.com> Date: Tue, 21 Apr 2026 12:24:53 +0300 Subject: [PATCH 3/8] Fix default service owner --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index c97a4aac..17ec9287 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -177,7 +177,7 @@ services: networks: [dvnode] environment: PROM_REMOTE_WRITE_TOKEN: ${PROM_REMOTE_WRITE_TOKEN} - SERVICE_OWNER: ${SERVICE_OWNER:-"obol-cdvn"} + SERVICE_OWNER: ${SERVICE_OWNER:-obol-cdvn} CLUSTER_NAME: ${CLUSTER_NAME:-} CLUSTER_PEER: ${CLUSTER_PEER:-} volumes: From 6ef720e1b712b9ea898ee2d582d031f3dec000b2 Mon Sep 17 00:00:00 2001 From: kalo <24719519+KaloyanTanev@users.noreply.github.com> Date: Tue, 21 Apr 2026 12:25:11 +0300 Subject: [PATCH 4/8] Add cl-lighthouse and vc-lodestar to scraping --- prometheus/prometheus.yml.example | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/prometheus/prometheus.yml.example b/prometheus/prometheus.yml.example index 0b8ee241..86df3a99 100644 --- a/prometheus/prometheus.yml.example +++ b/prometheus/prometheus.yml.example @@ -21,7 +21,7 @@ scrape_configs: - targets: ["nethermind:8008"] - job_name: "lighthouse" static_configs: - - targets: ["lighthouse:5054"] + - targets: ["lighthouse:5054", "cl-lighthouse:5054"] - job_name: "charon" static_configs: - targets: ["charon:3620"] @@ -34,7 +34,7 @@ scrape_configs: - job_name: "lodestar" static_configs: - - targets: ["lodestar:5064"] + - targets: ["lodestar:5064","vc-lodestar:5064"] # Debug - job_name: "node-exporter" static_configs: From b761cebbeaffd133de58a969b652796e187f297d Mon Sep 17 00:00:00 2001 From: kalo <24719519+KaloyanTanev@users.noreply.github.com> Date: Mon, 27 Apr 2026 12:43:08 +0300 Subject: [PATCH 5/8] Add Prysm and Nimbus CLs --- compose-cl.yml | 90 +++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 82 insertions(+), 8 deletions(-) diff --git a/compose-cl.yml b/compose-cl.yml index e1287a78..a608f087 100644 --- a/compose-cl.yml +++ b/compose-cl.yml @@ -11,7 +11,7 @@ services: # |___/ cl-grandine: - profiles: [cl-grandine] + profiles: [ cl-grandine ] image: sifrai/grandine:${GRANDINE_VERSION:-2.0.3} restart: unless-stopped labels: @@ -36,7 +36,7 @@ services: volumes: - ./data/cl-grandine:/root/.grandine - ./jwt:/jwt:ro - networks: [dvnode] + networks: [ dvnode ] # _ _ _ _ _ # | (_) __ _| |__ | |_| |__ ___ _ _ ___ ___ @@ -46,7 +46,7 @@ services: # |___/ cl-lighthouse: - profiles: [cl-lighthouse] + profiles: [ cl-lighthouse ] image: sigp/lighthouse:${LIGHTHOUSE_VERSION:-v8.1.3} restart: unless-stopped labels: @@ -74,7 +74,7 @@ services: volumes: - ./data/lighthouse:/opt/app/beacon # Keep data in lighthouse and not cl-lighthouse for backwards compatibility - ./jwt:/opt/jwt - networks: [dvnode] + networks: [ dvnode ] # _ _ # | |_ ___| | ___ _ @@ -83,7 +83,7 @@ services: # \__\___|_|\_\\__,_| cl-teku: - profiles: [cl-teku] + profiles: [ cl-teku ] image: consensys/teku:${CL_TEKU_VERSION:-26.3.0} restart: unless-stopped labels: @@ -110,7 +110,7 @@ services: volumes: - ./data/cl-teku:/opt/teku/data - ./jwt:/jwt:ro - networks: [dvnode] + networks: [ dvnode ] # _ _ _ # | | ___ __| | ___ ___| |_ __ _ _ __ @@ -119,7 +119,7 @@ services: # |_|\___/ \__,_|\___||___/\__\__,_|_| cl-lodestar: - profiles: [cl-lodestar] + profiles: [ cl-lodestar ] image: chainsafe/lodestar:${CL_LODESTAR_VERSION:-v1.40.0} restart: unless-stopped labels: @@ -146,4 +146,78 @@ services: volumes: - ./data/cl-lodestar:/opt/lodestar/data - ./jwt:/jwt:ro - networks: [dvnode] + networks: [ dvnode ] + + # _ __ _ __ _ _ ___ _ __ ___ + # | '_ \| '__| | | / __| '_ ` _ \ + # | |_) | | | |_| \__ \ | | | | | + # | .__/|_| \__, |___/_| |_| |_| + # |_| |___/ + + cl-prysm: + profiles: [ cl-prysm ] + image: gcr.io/prysmaticlabs/prysm/beacon-chain:${CL_PRYSM_VERSION:-v7.1.3} + restart: unless-stopped + labels: + - "alloy-monitored=${CL_PRYSM_ALLOY_MONITORED:-true}" + command: + - --accept-terms-of-use + - --${NETWORK} + - --datadir=/data + - --checkpoint-sync-url=${LIGHTHOUSE_CHECKPOINT_SYNC_URL} + - --genesis-beacon-api-url=${LIGHTHOUSE_CHECKPOINT_SYNC_URL} + - --execution-endpoint=http://${EL}:8551 + - --jwt-secret=/jwt/jwt.hex + - --http-mev-relay=http://${MEV}:18550 + - --http-host=0.0.0.0 + - --http-port=5052 + - --disable-monitoring=false + - --monitoring-host=0.0.0.0 + - --monitoring-port=5054 + ports: + - ${CL_PORT_P2P:-9000}:13000/tcp + - ${CL_PORT_P2P:-9000}:12000/udp + - ${CL_IP_HTTP:-127.0.0.1}:${CL_PORT_HTTP:-5052}:5052 + volumes: + - ./data/cl-prysm:/data + - ./jwt:/jwt:ro + networks: [ dvnode ] + + # _ _ + # _ __ (_)_ __ ___ | |__ _ _ ___ + # | '_ \| | '_ ` _ \| '_ \| | | / __| + # | | | | | | | | | | |_) | |_| \__ \ + # |_| |_|_|_| |_| |_|_.__/ \__,_|___/ + + cl-nimbus: + profiles: [ cl-nimbus ] + image: statusim/nimbus-eth2:${CL_NIMBUS_VERSION:-multiarch-v26.3.1} + restart: unless-stopped + labels: + - "alloy-monitored=${CL_NIMBUS_ALLOY_MONITORED:-true}" + # Nimbus has no runtime checkpoint-sync flag — to checkpoint-sync from an + # empty data dir, run the `trustedNodeSync` subcommand once before `up`: + # docker compose run --rm --no-deps cl-nimbus trustedNodeSync \ + # --network=$NETWORK --data-dir=/data \ + # --trusted-node-url=$LIGHTHOUSE_CHECKPOINT_SYNC_URL --backfill=false + command: + - --network=${NETWORK} + - --data-dir=/data + - --el=http://${EL}:8551 + - --jwt-secret=/jwt/jwt.hex + - --rest + - --rest-address=0.0.0.0 + - --rest-port=5052 + - --metrics + - --metrics-address=0.0.0.0 + - --metrics-port=5054 + - --payload-builder=true + - --payload-builder-url=http://${MEV}:18550 + ports: + - ${CL_PORT_P2P:-9000}:9000/tcp + - ${CL_PORT_P2P:-9000}:9000/udp + - ${CL_IP_HTTP:-127.0.0.1}:${CL_PORT_HTTP:-5052}:5052 + volumes: + - ./data/cl-nimbus:/data + - ./jwt:/jwt:ro + networks: [ dvnode ] From 7c4dffb79b581f867e1dd5b35fd142a725fdaf47 Mon Sep 17 00:00:00 2001 From: kalo <24719519+KaloyanTanev@users.noreply.github.com> Date: Mon, 27 Apr 2026 12:45:51 +0300 Subject: [PATCH 6/8] Update prometheus to scrape all BNs --- prometheus/prometheus.yml.example | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/prometheus/prometheus.yml.example b/prometheus/prometheus.yml.example index 86df3a99..227db665 100644 --- a/prometheus/prometheus.yml.example +++ b/prometheus/prometheus.yml.example @@ -1,5 +1,6 @@ global: scrape_interval: 30s # Set the scrape interval to every 30 seconds. + scrape_timeout: 25s evaluation_interval: 30s # Evaluate rules every 30 seconds. external_labels: service_owner: "$SERVICE_OWNER" # replace this with your Operator name you want to be identified by, it helps us route alerts and metrics to your notification channels easily @@ -19,9 +20,16 @@ scrape_configs: - job_name: "nethermind" static_configs: - targets: ["nethermind:8008"] - - job_name: "lighthouse" + - job_name: "consensus-client" static_configs: - - targets: ["lighthouse:5054", "cl-lighthouse:5054"] + - targets: + - "lighthouse:5054" + - "cl-lighthouse:5054" + - "cl-prysm:5054" + - "cl-teku:5054" + - "cl-nimbus:5054" + - "cl-lodestar:5054" + - "cl-grandine:5054" - job_name: "charon" static_configs: - targets: ["charon:3620"] @@ -32,7 +40,7 @@ scrape_configs: static_configs: - targets: ["mev-mevboost:18551"] - - job_name: "lodestar" + - job_name: "validator-client" static_configs: - targets: ["lodestar:5064","vc-lodestar:5064"] # Debug From 75bd243df7f6efd24b10cedd11f4ce74b0c279b3 Mon Sep 17 00:00:00 2001 From: kalo <24719519+KaloyanTanev@users.noreply.github.com> Date: Mon, 27 Apr 2026 12:46:03 +0300 Subject: [PATCH 7/8] Update dashboards --- .../dashboards/charon_overview_dashboard.json | 72 +- .../dashboards/node_overview_dashboard.json | 4593 +++++++++++++++++ 2 files changed, 4635 insertions(+), 30 deletions(-) create mode 100644 grafana/dashboards/node_overview_dashboard.json diff --git a/grafana/dashboards/charon_overview_dashboard.json b/grafana/dashboards/charon_overview_dashboard.json index d148e443..6d0f2d0a 100644 --- a/grafana/dashboards/charon_overview_dashboard.json +++ b/grafana/dashboards/charon_overview_dashboard.json @@ -583,12 +583,12 @@ "id": "organize", "options": { "excludeByName": { - "F \u00b7 lock_hash": false + "F · lock_hash": false }, "includeByName": {}, "indexByName": { - "F \u00b7 lock_hash": 6, - "G \u00b7 Link": 0, + "F · lock_hash": 6, + "G · Link": 0, "Network": 1, "Operators/Nodes": 2, "Threshold": 3, @@ -596,8 +596,8 @@ "Validators Total": 5 }, "renameByName": { - "F \u00b7 lock_hash": "_", - "G \u00b7 Link": "Cluster Hash", + "F · lock_hash": "_", + "G · Link": "Cluster Hash", "Network": "", "Threshold": "" } @@ -707,7 +707,7 @@ }, { "id": "custom.width", - "value": 261 + "value": 157 } ] }, @@ -758,6 +758,18 @@ "value": { "type": "auto" } + }, + { + "id": "custom.inspect", + "value": true + }, + { + "id": "custom.minWidth", + "value": 400 + }, + { + "id": "custom.width", + "value": 489 } ] } @@ -1112,11 +1124,11 @@ "options": { "0": { "index": 0, - "text": "\u00a0\u274c" + "text": " ❌" }, "1": { "index": 1, - "text": "\u2705" + "text": "✅" } }, "type": "value" @@ -1152,7 +1164,7 @@ "from": 1, "result": { "index": 1, - "text": "\u00a0\u274c" + "text": " ❌" }, "to": 9 }, @@ -1163,7 +1175,7 @@ "from": 10, "result": { "index": 2, - "text": "\u2705" + "text": "✅" }, "to": 1000 }, @@ -1893,7 +1905,7 @@ "type": "prometheus", "uid": "prometheus" }, - "description": "Status and balance of each validator by public key with links to beaconcha.in.\n\nIf present, the \"\u2757\ufe0fPeer Exits\" column indicate the number of peers that submitted partially signed exits for each validator.", + "description": "Status and balance of each validator by public key with links to beaconcha.in.\n\nIf present, the \"❗️Peer Exits\" column indicate the number of peers that submitted partially signed exits for each validator.", "fieldConfig": { "defaults": { "color": { @@ -2184,7 +2196,7 @@ "renameByName": { "Value #A": "", "Value #B": "Balance", - "Value #C": "\u2757\ufe0fPeer Exits", + "Value #C": "❗️Peer Exits", "cluster_network": "__hide__", "pubkey": " Public Key", "pubkey_full": "__hide__", @@ -2875,7 +2887,7 @@ "refId": "D" } ], - "title": "Stack Version History (\ud83d\udfe1=BN, \ud83d\udd34=EL, \ud83d\udd35=VC, \ud83d\udfe2=DV)", + "title": "Stack Version History (🟡=BN, 🔴=EL, 🔵=VC, 🟢=DV)", "type": "state-timeline" }, { @@ -2971,7 +2983,7 @@ "refId": "A" } ], - "title": "\u2705 Completed duties by type ", + "title": "✅ Completed duties by type ", "type": "status-history" }, { @@ -3051,7 +3063,7 @@ "refId": "A" } ], - "title": "\u2705 Successful Peer Participation: Duty=$duty", + "title": "✅ Successful Peer Participation: Duty=$duty", "type": "status-history" }, { @@ -3136,7 +3148,7 @@ "refId": "A" } ], - "title": "\u274c Failed duties by type (per $interval)", + "title": "❌ Failed duties by type (per $interval)", "type": "status-history" }, { @@ -3216,7 +3228,7 @@ "refId": "A" } ], - "title": "\ud83d\udea7 Missed Peer Participation: Duty=$duty", + "title": "🚧 Missed Peer Participation: Duty=$duty", "type": "status-history" }, { @@ -3319,7 +3331,7 @@ "refId": "A" } ], - "title": "\u274c Failed duty percentage by type (per $interval)", + "title": "❌ Failed duty percentage by type (per $interval)", "type": "timeseries" }, { @@ -3401,7 +3413,7 @@ "refId": "A" } ], - "title": "\ud83d\udea7 Missed Peer Participation: Peer=$peer", + "title": "🚧 Missed Peer Participation: Peer=$peer", "type": "status-history" }, { @@ -3451,7 +3463,7 @@ } }, "mappings": [], - "noValue": "No failures \ud83c\udf89", + "noValue": "No failures 🎉", "thresholds": { "mode": "absolute", "steps": [ @@ -3500,7 +3512,7 @@ "refId": "A" } ], - "title": "\ud83d\udea8 Duty failure reason: $duty (per $interval)", + "title": "🚨 Duty failure reason: $duty (per $interval)", "type": "timeseries" }, { @@ -3603,7 +3615,7 @@ "refId": "A" } ], - "title": "\ud83d\udca5 Broadcasted duties never included on-chain (per $interval)", + "title": "💥 Broadcasted duties never included on-chain (per $interval)", "type": "timeseries" }, { @@ -5198,7 +5210,7 @@ } }, "mappings": [], - "noValue": "No timeouts \ud83c\udf89", + "noValue": "No timeouts 🎉", "thresholds": { "mode": "absolute", "steps": [ @@ -5247,7 +5259,7 @@ "refId": "B" } ], - "title": "\ud83d\udea8 Consensus Timeouts (per $interval)", + "title": "🚨 Consensus Timeouts (per $interval)", "type": "timeseries" } ], @@ -6256,7 +6268,7 @@ "refId": "A" } ], - "title": "\ud83d\udea8 Beacon Node API error count (per $interval)", + "title": "🚨 Beacon Node API error count (per $interval)", "type": "timeseries" }, { @@ -7745,7 +7757,7 @@ "refId": "A" } ], - "title": "\ud83d\udea8 Validator API error count (per $interval)", + "title": "🚨 Validator API error count (per $interval)", "type": "timeseries" } ], @@ -7767,7 +7779,7 @@ "type": "prometheus", "uid": "prometheus" }, - "description": "Type of connections to each peer in the cluster during the time window:\n - \ud83d\udfe2 Green: Direct connection (this is the preferred)\n - \ud83d\udfe1 Yellow: Indirect relay connection (this is ok if only temporary)\n - \ud83d\udd34 Red: No connection to peer (this is a problem)\n\nSee https://docs.obol.tech/docs/charon/networking#external-p2p-network", + "description": "Type of connections to each peer in the cluster during the time window:\n - 🟢 Green: Direct connection (this is the preferred)\n - 🟡 Yellow: Indirect relay connection (this is ok if only temporary)\n - 🔴 Red: No connection to peer (this is a problem)\n\nSee https://docs.obol.tech/docs/charon/networking#external-p2p-network", "fieldConfig": { "defaults": { "color": { @@ -7846,7 +7858,7 @@ "refId": "A" } ], - "title": "Peer Connections (\ud83d\udfe2=direct, \ud83d\udfe1=relay, \ud83d\udd34=not connected)", + "title": "Peer Connections (🟢=direct, 🟡=relay, 🔴=not connected)", "type": "state-timeline" }, { @@ -8001,7 +8013,7 @@ "color": "rgba(255,0,255,0.7)" }, "filterValues": { - "le": 1e-09 + "le": 1e-9 }, "legend": { "show": true @@ -9436,6 +9448,6 @@ "timezone": "utc", "title": "Charon Overview", "uid": "charon_overview", - "version": 46, + "version": 2, "weekStart": "" } \ No newline at end of file diff --git a/grafana/dashboards/node_overview_dashboard.json b/grafana/dashboards/node_overview_dashboard.json new file mode 100644 index 00000000..fa890cd4 --- /dev/null +++ b/grafana/dashboards/node_overview_dashboard.json @@ -0,0 +1,4593 @@ +{ + "title": "Node overview", + "uid": "node_overview", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "liveNow": false, + "schemaVersion": 42, + "style": "dark", + "tags": [ + "beacon", + "charon", + "node-overview" + ], + "templating": { + "list": [ + { + "allValue": "all", + "current": { + "text": [ + "hoodi" + ], + "value": [ + "hoodi" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(app_peer_name,cluster_network)", + "description": "", + "includeAll": true, + "label": "Cluster Network", + "multi": true, + "name": "cluster_network", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(app_peer_name,cluster_network)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "regexApplyTo": "value", + "type": "query" + }, + { + "current": { + "text": "test", + "value": "test" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(app_peer_name{cluster_network=~\"$cluster_network\"},cluster_name)", + "description": "", + "includeAll": false, + "label": "Cluster Name", + "name": "cluster_name", + "options": [], + "query": { + "query": "label_values(app_peer_name{cluster_network=~\"$cluster_network\"},cluster_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "regexApplyTo": "value", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "24716d9", + "value": "24716d9" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(app_peer_name{cluster_name=\"$cluster_name\"}, cluster_hash)", + "description": "", + "includeAll": false, + "label": "Cluster Hash", + "name": "cluster_hash", + "options": [], + "query": { + "query": "label_values(app_peer_name{cluster_name=\"$cluster_name\"}, cluster_hash)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "regexApplyTo": "value", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "colorful-food", + "value": "colorful-food" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(app_peer_name{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\"}, cluster_peer)", + "description": "", + "includeAll": false, + "label": "Cluster Peer", + "name": "cluster_peer", + "options": [], + "query": { + "query": "label_values(app_peer_name{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\"}, cluster_peer)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "regexApplyTo": "value", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "charon", + "value": "charon" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(app_peer_name{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"},job)", + "description": "Prometheus job (can be ignored)", + "hide": 2, + "includeAll": false, + "label": "--", + "name": "job", + "options": [], + "query": { + "query": "label_values(app_peer_name{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"},job)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "regexApplyTo": "value", + "sort": 1, + "type": "query" + }, + { + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(app_peerinfo_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\", cluster_peer=\"$cluster_peer\"}, peer)", + "description": "", + "hide": 2, + "includeAll": true, + "label": "P2P Peer", + "name": "peer", + "options": [], + "query": { + "query": "label_values(app_peerinfo_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\", cluster_peer=\"$cluster_peer\"}, peer)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "regexApplyTo": "value", + "sort": 1, + "type": "query" + }, + { + "auto": true, + "auto_count": 30, + "auto_min": "1m", + "current": { + "text": "$__auto", + "value": "$__auto" + }, + "label": "Interval", + "name": "interval", + "options": [ + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "refresh": 2, + "type": "interval" + }, + { + "allValue": ".*", + "current": { + "text": "$__all", + "value": "$__all" + }, + "includeAll": true, + "label": "Duty", + "name": "duty", + "options": [], + "query": "attester,prepare_aggregator,aggregator,sync_message,prepare_sync_contribution,sync_contribution,proposer,randao,exit,builder_proposer,builder_registration,info_sync", + "type": "custom", + "valuesFormat": "csv" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "refresh": "30s", + "version": 1, + "weekStart": "", + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "panels": [ + { + "id": 100, + "type": "stat", + "title": "Beacon node", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 3, + "w": 12, + "x": 0, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "textMode": "name", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "wideLayout": true + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "label_replace(last_over_time(lighthouse_info{job=\"consensus-client\"}[1m]), \"version\", \"$1\", \"version\", \"Lighthouse/(.+)\")", + "instant": true, + "range": false, + "legendFormat": "Lighthouse {{version}}" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "last_over_time(prysm_version{job=\"consensus-client\"}[1m])", + "instant": true, + "range": false, + "legendFormat": "Prysm {{version}}" + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "last_over_time(beacon_teku_version_total{job=\"consensus-client\"}[1m])", + "instant": true, + "range": false, + "legendFormat": "Teku {{version}}" + }, + { + "refId": "D", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "last_over_time(version{job=\"consensus-client\"}[1m])", + "instant": true, + "range": false, + "legendFormat": "Nimbus {{version}}" + }, + { + "refId": "E", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "last_over_time(lodestar_version{job=\"consensus-client\"}[1m])", + "instant": true, + "range": false, + "legendFormat": "Lodestar {{version}}" + }, + { + "refId": "F", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "last_over_time(GRANDINE_VERSION{job=\"consensus-client\"}[1m])", + "instant": true, + "range": false, + "legendFormat": "Grandine {{version}}" + } + ], + "transparent": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 1030, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "showPercentChange": false, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "max(app_peerinfo_version{cluster_name=~\"$cluster_name\",cluster_hash=~\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (version)", + "instant": true, + "legendFormat": "{{version}}", + "refId": "A" + } + ], + "title": "Charon", + "type": "stat", + "transparent": false + }, + { + "id": 1000, + "type": "row", + "title": "Beacon node", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "panels": [] + }, + { + "id": 10, + "type": "stat", + "title": "Head Slot", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 4 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "special", + "options": { + "match": "null", + "result": { + "text": "N/A" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "locale", + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "wideLayout": true + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "beacon_head_slot", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "description": "Latest slot of the beacon chain head. Should advance by 1 every ~12 seconds (one slot). If it lags the wall-clock slot, this node is falling behind and cannot attest or propose." + }, + { + "id": 11, + "type": "stat", + "title": "Finalized Epoch", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 4 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "special", + "options": { + "match": "null", + "result": { + "text": "N/A" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "locale", + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "wideLayout": true + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "beacon_finalized_epoch", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "description": "Most recent epoch that has been economically finalized by a 2/3+ validator supermajority. Under normal conditions this sits 2 epochs behind the current epoch. A widening gap (> 4 epochs) signals finality is delayed — typically caused by participation issues or a minority client bug." + }, + { + "id": 12, + "type": "stat", + "title": "Current Justified Epoch", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 4 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "special", + "options": { + "match": "null", + "result": { + "text": "N/A" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "locale", + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "wideLayout": true + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "beacon_current_justified_epoch", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "description": "The most recent epoch that has received 2/3+ attesting support. Justification is the step before finalization; this should normally equal `current_epoch - 1`." + }, + { + "id": 13, + "type": "stat", + "title": "Previous Justified Epoch", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 4 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "special", + "options": { + "match": "null", + "result": { + "text": "N/A" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "locale", + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "wideLayout": true + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "beacon_previous_justified_epoch", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "description": "The second-most-recent justified epoch. Shown alongside current justified so you can see the justification cursor advance epoch-by-epoch." + }, + { + "id": 20, + "type": "timeseries", + "title": "Head Slot over time", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 8 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "axisSoftMin": 0, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "locale", + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "label_replace(\n beacon_head_slot,\n \"cl\", \"$2\", \"instance\", \"(cl-)?([a-z]+):[0-9]+\"\n)", + "legendFormat": "{{cl}}", + "range": true, + "refId": "A" + } + ], + "description": "Latest slot of the beacon chain head. Should advance by 1 every ~12 seconds (one slot). If it lags the wall-clock slot, this node is falling behind and cannot attest or propose." + }, + { + "id": 21, + "type": "timeseries", + "title": "Finalized Epoch over time", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 8 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "axisSoftMin": 0, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "locale", + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "label_replace(\n beacon_finalized_epoch,\n \"cl\", \"$2\", \"instance\", \"(cl-)?([a-z]+):[0-9]+\"\n)", + "legendFormat": "{{cl}}", + "range": true, + "refId": "A" + } + ], + "description": "Most recent epoch that has been economically finalized by a 2/3+ validator supermajority. Under normal conditions this sits 2 epochs behind the current epoch. A widening gap (> 4 epochs) signals finality is delayed — typically caused by participation issues or a minority client bug." + }, + { + "id": 22, + "type": "timeseries", + "title": "Current Justified Epoch over time", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 8 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "axisSoftMin": 0, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "locale", + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "label_replace(\n beacon_current_justified_epoch,\n \"cl\", \"$2\", \"instance\", \"(cl-)?([a-z]+):[0-9]+\"\n)", + "legendFormat": "{{cl}}", + "range": true, + "refId": "A" + } + ], + "description": "The most recent epoch that has received 2/3+ attesting support. Justification is the step before finalization; this should normally equal `current_epoch - 1`." + }, + { + "id": 23, + "type": "timeseries", + "title": "Previous Justified Epoch over time", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 8 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "axisSoftMin": 0, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "locale", + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "label_replace(\n beacon_previous_justified_epoch,\n \"cl\", \"$2\", \"instance\", \"(cl-)?([a-z]+):[0-9]+\"\n)", + "legendFormat": "{{cl}}", + "range": true, + "refId": "A" + } + ], + "description": "The second-most-recent justified epoch. Shown alongside current justified so you can see the justification cursor advance epoch-by-epoch." + }, + { + "id": 30, + "type": "stat", + "title": "Active Validators", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 15 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "special", + "options": { + "match": "null", + "result": { + "text": "N/A" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "none", + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "wideLayout": true + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "beacon_current_active_validators", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "description": "Current total active validators in the active set this epoch — i.e. validators eligible to attest and propose. Universal across all clients." + }, + { + "id": 32, + "type": "stat", + "title": "Total Validators", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 15 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "special", + "options": { + "match": "null", + "result": { + "text": "N/A" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "none", + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "wideLayout": true + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "beacon_current_validators\n or on() last_over_time(beacon_head_state_total_validators_total[5m])\n or on() last_over_time(beacon_current_active_validators[5m])", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "description": "Count of all validators in status `pending | active | exited | withdrawable` in the current epoch — i.e. the total validator registry size.\n\n**Coverage gap**: Prysm does not expose a total-validator count. Lighthouse, Teku, Nimbus and Grandine approximate with related metrics — values may differ slightly from the strict spec semantics." + }, + { + "id": 40, + "type": "timeseries", + "title": "Active Validators over time", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 19 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "axisSoftMin": 0, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none", + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "label_replace(\n beacon_current_active_validators,\n \"cl\", \"$2\", \"instance\", \"(cl-)?([a-z]+):[0-9]+\"\n)", + "legendFormat": "{{cl}}", + "range": true, + "refId": "A" + } + ], + "description": "Current total active validators in the active set this epoch — i.e. validators eligible to attest and propose. Universal across all clients." + }, + { + "id": 42, + "type": "timeseries", + "title": "Total Validators over time", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 19 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "axisSoftMin": 0, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none", + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "label_replace(\n \n max by (instance) (\n beacon_current_validators\n or on() last_over_time(beacon_head_state_total_validators_total[5m])\n or on() last_over_time(beacon_current_active_validators[5m])\n ),\n \"cl\", \"$2\", \"instance\", \"(cl-)?([a-z]+):[0-9]+\"\n)", + "legendFormat": "{{cl}}", + "range": true, + "refId": "A" + } + ], + "description": "Count of all validators in status `pending | active | exited | withdrawable` in the current epoch — i.e. the total validator registry size.\n\n**Coverage gap**: Prysm does not expose a total-validator count. Lighthouse, Teku, Nimbus and Grandine approximate with related metrics — values may differ slightly from the strict spec semantics." + }, + { + "id": 50, + "type": "stat", + "title": "libp2p Peers", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 26 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "special", + "options": { + "match": "null", + "result": { + "text": "N/A" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "#EAB839", + "value": 20 + }, + { + "color": "green", + "value": 40 + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "wideLayout": true + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(libp2p_peers) or on() sum(last_over_time(connected_libp2p_peers[5m]))", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "description": "Current count of libp2p peers connected to this beacon node. A healthy node typically maintains 30–100 peers. Sustained drops usually mean networking issues, a misconfigured port, or peer-scoring problems." + }, + { + "id": 51, + "type": "stat", + "title": "Gossipsub Mesh Peers", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 26 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "type": "special", + "options": { + "match": "null", + "result": { + "text": "N/A" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "#EAB839", + "value": 4 + }, + { + "color": "green", + "value": 6 + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "wideLayout": true + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(gossipsub_mesh_peer_counts)\n or on() (sum(last_over_time(p2p_topic_peer_count[5m])))\n or on() (sum(last_over_time(network_subnet_peer_count[5m])))\n or on() (sum(last_over_time(libp2p_gossipsub_peers_per_topic_mesh[5m])))\n or on() (sum(last_over_time(gossipsub_mesh_peer_count[5m])))\n or on() (sum(last_over_time(gossipsub_mesh_peer_inclusion_events_subscribed_total[5m])))", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "description": "Number of partial-message-capable peers in the gossipsub mesh, summed across all topics. Mesh quality determines how fast this node sees new blocks and attestations — low counts lead to block-delay spikes.\n\n**Caveat**: Teku does not expose a direct gossipsub mesh peer gauge. On Teku this panel falls back to `network_subnet_peer_count`, which counts subnet peers rather than per-topic mesh peers — similar magnitude but different semantics." + }, + { + "id": 60, + "type": "timeseries", + "title": "libp2p Peers over time", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 30 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "#fffefe" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "auto", + "spanNulls": true, + "axisPlacement": "auto", + "axisSoftMin": 0, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "gradientMode": "none", + "thresholdsStyle": { + "mode": "dashed+area" + }, + "pointSize": 5 + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "#EAB839", + "value": 20 + }, + { + "color": "green", + "value": 40 + } + ] + }, + "unit": "short", + "decimals": 0, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "label_replace(\n sum by (instance) (libp2p_peers)\n or on(instance) sum by (instance) (last_over_time(connected_libp2p_peers[5m])),\n \"cl\", \"$2\", \"instance\", \"(cl-)?([a-z]+):[0-9]+\"\n)", + "legendFormat": "{{cl}}", + "range": true, + "refId": "A" + } + ], + "description": "Current count of libp2p peers connected to this beacon node. A healthy node typically maintains 30–100 peers. Sustained drops usually mean networking issues, a misconfigured port, or peer-scoring problems." + }, + { + "id": 61, + "type": "timeseries", + "title": "Gossipsub Mesh Peers over time", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 30 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "#fffefe" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "auto", + "spanNulls": true, + "axisPlacement": "auto", + "axisSoftMin": 0, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "gradientMode": "none", + "thresholdsStyle": { + "mode": "dashed+area" + }, + "pointSize": 5 + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "#EAB839", + "value": 4 + }, + { + "color": "green", + "value": 6 + } + ] + }, + "unit": "short", + "decimals": 0, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "label_replace(\n sum by (instance) (gossipsub_mesh_peer_counts)\n or on(instance) sum by (instance) (last_over_time(p2p_topic_peer_count[5m]))\n or on(instance) sum by (instance) (last_over_time(network_subnet_peer_count[5m]))\n or on(instance) sum by (instance) (last_over_time(libp2p_gossipsub_peers_per_topic_mesh[5m]))\n or on(instance) sum by (instance) (last_over_time(gossipsub_mesh_peer_count[5m]))\n or on(instance) sum by (instance) (last_over_time(gossipsub_mesh_peer_inclusion_events_subscribed_total[5m])),\n \"cl\", \"$2\", \"instance\", \"(cl-)?([a-z]+):[0-9]+\"\n)", + "legendFormat": "{{cl}}", + "range": true, + "refId": "A" + } + ], + "description": "Number of partial-message-capable peers in the gossipsub mesh, summed across all topics. Mesh quality determines how fast this node sees new blocks and attestations — low counts lead to block-delay spikes." + }, + { + "id": 70, + "type": "timeseries", + "title": "Reorgs Total (cumulative)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 48 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "axisSoftMin": 0, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "label_replace(\n \n max by (instance) (\n beacon_reorgs_total\n or on() last_over_time(beacon_fork_choice_changed_head_total[5m])\n ),\n \"cl\", \"$2\", \"instance\", \"(cl-)?([a-z]+):[0-9]+\"\n)", + "legendFormat": "{{cl}}", + "range": true, + "refId": "A" + } + ], + "description": "Cumulative count of chain reorganizations observed by this node (fork-choice head changes). Small numbers (< 5 per day) are normal; sudden spikes indicate network-wide consensus instability or this node being on a minority fork." + }, + { + "id": 71, + "type": "timeseries", + "title": "Deposits Total (cumulative)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 48 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "axisSoftMin": 0, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "label_replace(\n \n max by (instance) (\n beacon_processed_deposits_total\n or on() last_over_time(beacon_eth1_deposit_total[5m])\n ),\n \"cl\", \"$2\", \"instance\", \"(cl-)?([a-z]+):[0-9]+\"\n)", + "legendFormat": "{{cl}}", + "range": true, + "refId": "A" + } + ], + "description": "Number of validator deposits processed from the deposit contract on the execution layer.\n\n**Note**: This is a **since-start** counter — it reflects deposits observed by the running beacon-node process, not a genesis-cumulative total. Restarting the BN resets the value to 0 until new deposits occur. Verified on Lighthouse (`beacon_processed_deposits_total`) and Teku (`beacon_eth1_deposit_total`) — both have this semantic." + }, + { + "id": 72, + "type": "timeseries", + "title": "Block Delay (p90)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 55 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "#fffefe" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 0, + "showPoints": "auto", + "spanNulls": true, + "axisPlacement": "auto", + "axisSoftMin": 0, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "gradientMode": "none", + "thresholdsStyle": { + "mode": "dashed+area" + }, + "pointSize": 5 + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "#EAB839", + "value": 3.5 + }, + { + "color": "red", + "value": 4.0 + } + ] + }, + "unit": "s", + "decimals": 3, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "label_replace(\n histogram_quantile(0.90, sum(rate(block_arrival_latency_milliseconds_bucket[5m])) by (le, instance)) / 1000\n or on() histogram_quantile(0.90, sum(rate(beacon_block_delay_bucket[5m])) by (le, instance))\n or on() histogram_quantile(0.90, sum(rate(lodestar_gossip_block_elapsed_time_till_received_bucket[5m])) by (le, instance))\n or on() histogram_quantile(0.90, sum(rate(beacon_block_gossip_slot_start_delay_time_bucket[5m])) by (le, instance)),\n \"cl\", \"$2\", \"instance\", \"(cl-)?([a-z]+):[0-9]+\"\n)", + "legendFormat": "{{cl}}", + "range": true, + "refId": "A" + } + ], + "description": "90th-percentile block arrival delay relative to the expected slot start. Values above 4 s mean blocks are consistently late to this node — could be a sync issue, poor peer quality, or slow propagation.\n\n**Coverage gap**: Lighthouse and Teku do not expose a histogram-based block-delay metric. The panel will be empty when either is running. Prysm reports delay in milliseconds (normalised to seconds by the panel). Nimbus, Lodestar and Grandine expose native second-based histograms." + }, + { + "id": 73, + "type": "timeseries", + "title": "State Data Cache Misses (rate)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 55 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "axisSoftMin": 0, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops", + "decimals": 2 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "label_replace(\n \n sum by (instance) (\n rate(beacon_state_data_cache_misses_total[5m])\n or on() rate(store_beacon_historic_state_cache_miss_total[5m])\n or on() rate(hot_state_cache_miss[5m])\n or on() rate(storage_block_cache_data_miss[5m])\n or on() rate(beacon_state_data_cache_misses[5m])\n ),\n \"cl\", \"$2\", \"instance\", \"(cl-)?([a-z]+):[0-9]+\"\n)", + "legendFormat": "{{cl}}", + "range": true, + "refId": "A" + } + ], + "description": "Rate of state-data cache misses per second. High rates indicate cache pressure — the beacon node is falling back to disk for state reads, which slows down attestation processing and head-block updates.\n\n**Coverage gap**: Grandine does not expose a state-cache miss metric; panel will be empty on Grandine.\n\n**Caveat**: Lodestar's fallback (`lodestar_state_cache_size`) is cache *size*, not miss *count* — its numeric meaning differs from the other clients." + }, + { + "id": 74, + "type": "timeseries", + "title": "Engine getBlobs Request Dur (p90)", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 55 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": true, + "axisPlacement": "auto", + "axisSoftMin": 0, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s", + "decimals": 3 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "label_replace(\n histogram_quantile(0.90, sum(rate(beacon_engine_getBlobsV3_request_duration_seconds_bucket[5m])) by (le, instance))\n or on() histogram_quantile(0.90, sum(rate(beacon_engine_getBlobsV2_request_duration_seconds_bucket[5m])) by (le, instance))\n or on() histogram_quantile(0.90, sum(rate(execution_layer_request_times_bucket[5m])) by (le, instance))\n or on() histogram_quantile(0.90, sum(rate(engine_api_request_duration_seconds_bucket[5m])) by (le, instance)),\n \"cl\", \"$2\", \"instance\", \"(cl-)?([a-z]+):[0-9]+\"\n)", + "legendFormat": "{{cl}}", + "range": true, + "refId": "A" + } + ], + "description": "90th-percentile latency of `engine_getBlobsV3` requests to the execution layer. Elevated latency can indicate slow EL processing, network congestion, or EL saturation.\n\n**Coverage gap**: Prysm does not expose this metric; panel will be empty on Prysm.\n\n**Caveat**: Lighthouse and Nimbus fall back to generic engine-API latency metrics rather than blob-specific ones, so their values include non-blob engine calls too." + }, + { + "id": 62, + "type": "stat", + "title": "Peers by Client Type", + "description": "Current count of connected libp2p peers grouped by the remote peer's client-type.\n\n**Caveat**: Lighthouse, Prysm and Lodestar expose per-peer-type metrics. Teku, Nimbus and Grandine do not — panel will be empty on those clients.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 37 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "short", + "decimals": 0, + "mappings": [ + { + "type": "special", + "options": { + "match": "null", + "result": { + "text": "0" + } + } + } + ] + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#808080" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^[Ll]ighthouse$" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "orange" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^[Pp]rysm$" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^[Tt]eku$" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "purple" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^[Nn]imbus$" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#58a6ff" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^[Ll]odestar$" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^[Gg]randine$" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "yellow" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^[Cc]aplin$" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#ae67ff" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^[Uu]nknown$" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "text" + } + } + ] + } + ] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "textMode": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "wideLayout": false + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "label_replace(libp2p_peers_per_client, \"client_type\", \"$1\", \"Client\", \"(.+)\")\nor\nlabel_replace(connected_libp2p_peers, \"client_type\", \"$1\", \"agent\", \"(.+)\")\nor\nlabel_replace(lodestar_peers_by_client_count, \"client_type\", \"$1\", \"client\", \"(.+)\")", + "instant": true, + "range": false, + "legendFormat": "{{client_type}}", + "refId": "A" + } + ] + }, + { + "id": 63, + "type": "timeseries", + "title": "Peers by Client Type over time", + "description": "History of connected libp2p peers grouped by the remote peer's client-type. Useful for tracking whether one client type is disappearing from your peer list over time.\n\n**Caveat**: Lighthouse, Prysm and Lodestar expose per-peer-type metrics. Teku, Nimbus and Grandine do not — panel will be empty on those clients.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 41 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "smooth", + "lineWidth": 2, + "fillOpacity": 20, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "normal", + "group": "A" + }, + "axisPlacement": "auto", + "axisSoftMin": 0, + "scaleDistribution": { + "type": "linear" + } + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#808080" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^[Ll]ighthouse$" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "orange" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^[Pp]rysm$" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^[Tt]eku$" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "purple" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^[Nn]imbus$" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#58a6ff" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^[Ll]odestar$" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^[Gg]randine$" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "yellow" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^[Cc]aplin$" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#ae67ff" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^[Uu]nknown$" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "text" + } + } + ] + } + ] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "label_replace(libp2p_peers_per_client, \"client_type\", \"$1\", \"Client\", \"(.+)\")\nor\nlabel_replace(connected_libp2p_peers, \"client_type\", \"$1\", \"agent\", \"(.+)\")\nor\nlabel_replace(lodestar_peers_by_client_count, \"client_type\", \"$1\", \"client\", \"(.+)\")", + "legendFormat": "{{client_type}}", + "range": true, + "refId": "A" + } + ] + }, + { + "id": 2000, + "type": "row", + "title": "Charon", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 62 + }, + "panels": [] + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Status returned from the `/readyz` health status endpoint. Unhealthy reasons include beacon node problems, charon p2p problems, or validator client problems.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "text", + "index": 1, + "text": "Unknown" + }, + "1": { + "color": "green", + "index": 0, + "text": "OK" + }, + "2": { + "color": "red", + "index": 3, + "text": "BeaconNode Down" + }, + "3": { + "color": "orange", + "index": 4, + "text": "BeaconNode Syncing" + }, + "4": { + "color": "orange", + "index": 5, + "text": "Insufficient Peers" + }, + "5": { + "color": "orange", + "index": 6, + "text": "VC not connected" + }, + "6": { + "color": "orange", + "index": 7, + "text": "VC missing validators" + }, + "7": { + "color": "orange", + "index": 8, + "text": "BeaconNode has zero peers" + }, + "8": { + "color": "orange", + "index": 9, + "text": "BeaconNode far behind" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "color": "text", + "index": 2, + "text": "Dead" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 63 + }, + "id": 170, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", + "instant": true, + "interval": "", + "legendFormat": "", + "range": false, + "refId": "A" + } + ], + "title": "Health Status - Peer", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Successful cluster duty percentage by type over the whole time range", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "links": [], + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "#EAB839", + "value": 0.9 + }, + { + "color": "green", + "value": 0.99 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 20, + "x": 4, + "y": 63 + }, + "id": 176, + "options": { + "minVizHeight": 200, + "minVizWidth": 200, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "text": {} + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "(\n sum(increase(core_tracker_success_duties_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$__range])) by (duty)\n) \n/ \n( \n sum(increase(core_tracker_expect_duties_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$__range])) by (duty) \n) > 0", + "instant": true, + "interval": "$interval", + "legendFormat": "{{duty}}", + "range": false, + "refId": "A" + } + ], + "title": "Cluster Duty Success(%) ", + "type": "gauge" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "description": "Information about the charon cluster.\n\n- **Network**: The blockchain network \n- **Operators/Nodes**: The number of operators or nodes or peers in the charon DV cluster.\n- **Threshold**: The minimum number of participating nodes (also quorum).\n- **Validators**: The number of validators added to the cluster, each validator is staking 32 ETH. ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": 0 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Network" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "1": { + "color": "blue", + "index": 0, + "text": "Mainnet" + }, + "2": { + "color": "purple", + "index": 1, + "text": "Goerli" + }, + "3": { + "color": "purple", + "index": 2, + "text": "Sepolia" + }, + "4": { + "color": "purple", + "index": 3, + "text": "Gnosis" + } + }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cluster Hash" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "Launchpad", + "url": "https://launchpad.obol.org/cluster/details/?lockHash=${__data.fields[\"_\"]}" + } + ] + }, + { + "id": "unit", + "value": "string" + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "_" + }, + "properties": [ + { + "id": "unit", + "value": "string" + }, + { + "id": "color", + "value": { + "fixedColor": "transparent", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 66 + }, + "id": 115, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "text": { + "titleSize": 14, + "valueSize": 14 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "( 1 * app_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", cluster_network=\"mainnet\"} )\nor \n( 2 * app_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", cluster_network=\"goerli\"} )\nor\n( 3 * app_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", cluster_network=\"sepolia\"} )\nor\n( 4 * app_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", cluster_network=\"gnosis\"} )", + "format": "time_series", + "instant": true, + "legendFormat": "Network", + "range": false, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "cluster_operators{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", + "instant": true, + "legendFormat": "Operators/Nodes", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "cluster_threshold{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", + "instant": true, + "legendFormat": "Threshold", + "range": false, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "core_scheduler_validators_active{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", + "instant": true, + "legendFormat": "Validators Active", + "range": false, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "cluster_validators{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}", + "instant": true, + "legendFormat": "Validators Total", + "range": false, + "refId": "C" + } + ], + "title": "Cluster Info", + "transformations": [ + { + "id": "reduce", + "options": { + "includeTimeField": false, + "mode": "reduceFields", + "reducers": [ + "lastNotNull" + ] + } + }, + { + "id": "concatenate", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": { + "F · lock_hash": false + }, + "includeByName": {}, + "indexByName": { + "F · lock_hash": 6, + "G · Link": 0, + "Network": 1, + "Operators/Nodes": 2, + "Threshold": 3, + "Validators Active": 4, + "Validators Total": 5 + }, + "renameByName": { + "F · lock_hash": "_", + "G · Link": "Cluster Hash", + "Network": "", + "Threshold": "" + } + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "description": "Health checks that have failed at least once in the selected time window, based on the app_health_checks_failed_total counter.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Severity" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "critical": { + "color": "red", + "index": 2, + "text": "critical" + }, + "info": { + "color": "blue", + "index": 0, + "text": "info" + }, + "warning": { + "color": "orange", + "index": 1, + "text": "warning" + } + }, + "type": "value" + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "custom.width", + "value": 69 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Check Name" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "color", + "value": { + "fixedColor": "super-light-yellow", + "mode": "fixed" + } + }, + { + "id": "custom.width", + "value": 157 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Times Failed" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + }, + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Last Reported" + }, + "properties": [ + { + "id": "unit", + "value": "dateTimeAsLocal" + }, + { + "id": "custom.width", + "value": 160 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Description" + }, + "properties": [ + { + "id": "custom.wrapText", + "value": true + }, + { + "id": "custom.cellOptions", + "value": { + "type": "auto" + } + }, + { + "id": "custom.inspect", + "value": true + }, + { + "id": "custom.minWidth", + "value": 400 + }, + { + "id": "custom.width", + "value": 489 + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 72 + }, + "id": 333, + "options": { + "cellHeight": "lg", + "showHeader": true + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max by (name, severity, description) (\n app_health_checks_failed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}\n) > 0", + "format": "table", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\n label_replace(\n label_replace(\n (\n max by (cluster_peer, beacon_id) (\n app_beacon_node_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",beacon_id=~\".+\"}\n )\n ) * on(beacon_id) group_left()\n (\n count by (beacon_id) (\n max by (cluster_peer, beacon_id) (\n app_beacon_node_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",beacon_id=~\".+\"}\n )\n ) > 1\n ),\n \"name\", \"reused_beacon_node\", \"\", \"\"\n ),\n \"severity\", \"warning\", \"\", \"\"\n ),\n \"description\", \"This peer shares its beacon node (beacon_id: $1) with at least one other peer in the cluster\", \"beacon_id\", \"(.*)\"\n)", + "format": "table", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Health Checks History", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "mode": "columns" + } + }, + { + "id": "merge", + "options": {} + }, + { + "id": "groupBy", + "options": { + "fields": { + "Time": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "Value": { + "aggregations": [] + }, + "description": { + "aggregations": [], + "operation": "groupby" + }, + "name": { + "aggregations": [], + "operation": "groupby" + }, + "severity": { + "aggregations": [], + "operation": "groupby" + } + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Value": true + }, + "indexByName": { + "Time (lastNotNull)": 3, + "description": 2, + "name": 0, + "severity": 1 + }, + "renameByName": { + "Time (lastNotNull)": "Last Reported", + "description": "Description", + "name": "Check Name", + "severity": "Severity" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Status returned from the `/readyz` health status endpoint. Unhealthy reasons include beacon node problems, charon p2p problems, or validator client problems.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "#ffffff", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "stepAfter", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "decimals": 0, + "mappings": [ + { + "options": { + "0": { + "color": "text", + "index": 1, + "text": "Unknown" + }, + "1": { + "color": "green", + "index": 0, + "text": "OK" + }, + "2": { + "color": "red", + "index": 3, + "text": "BeaconNode Down" + }, + "3": { + "color": "orange", + "index": 4, + "text": "BeaconNode Syncing" + }, + "4": { + "color": "orange", + "index": 5, + "text": "Insufficient Peers" + }, + "5": { + "color": "light-orange", + "index": 6, + "text": "VC not connected" + }, + "6": { + "color": "orange", + "index": 7, + "text": "VC missing validators" + }, + "7": { + "color": "orange", + "index": 8, + "text": "BeaconNode has zero peers" + }, + "8": { + "index": 9, + "text": "BeaconNode far behind" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "color": "text", + "index": 2, + "text": "Dead" + } + }, + "type": "special" + } + ], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#ccccdb", + "value": 0 + }, + { + "color": "green", + "value": 0.5 + }, + { + "color": "red", + "value": 1.5 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 9, + "x": 8, + "y": 72 + }, + "id": 119, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(app_monitoring_readyz{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"})", + "instant": false, + "interval": "$interval", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Historical Health Status", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "center", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "inspect": false, + "minWidth": 0, + "width": 90 + }, + "decimals": 0, + "mappings": [], + "noValue": "--", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": 0 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Peer" + }, + "properties": [ + { + "id": "custom.width", + "value": 120 + }, + { + "id": "custom.align", + "value": "left" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Connected" + }, + "properties": [ + { + "id": "custom.width", + "value": 90 + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "index": 0, + "text": " ❌" + }, + "1": { + "index": 1, + "text": "✅" + } + }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Direct" + }, + "properties": [ + { + "id": "custom.width", + "value": 70 + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "index": 0, + "text": "--" + } + }, + "type": "value" + }, + { + "options": { + "from": 1, + "result": { + "index": 1, + "text": " ❌" + }, + "to": 9 + }, + "type": "range" + }, + { + "options": { + "from": 10, + "result": { + "index": 2, + "text": "✅" + }, + "to": 1000 + }, + "type": "range" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Latency (ms)" + }, + "properties": [ + { + "id": "unit", + "value": "none" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "lcd", + "type": "gauge", + "valueDisplayMode": "color" + } + }, + { + "id": "custom.width", + "value": 212 + }, + { + "id": "max", + "value": 700 + }, + { + "id": "mappings", + "value": [ + { + "options": { + "from": 0, + "result": { + "color": "green", + "index": 0 + }, + "to": 150 + }, + "type": "range" + }, + { + "options": { + "from": 150, + "result": { + "color": "yellow", + "index": 1 + }, + "to": 300 + }, + "type": "range" + }, + { + "options": { + "from": 300, + "result": { + "color": "orange", + "index": 2 + }, + "to": 500 + }, + "type": "range" + }, + { + "options": { + "from": 500, + "result": { + "color": "red", + "index": 3 + }, + "to": 100000 + }, + "type": "range" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Version" + }, + "properties": [ + { + "id": "custom.width", + "value": 104 + }, + { + "id": "custom.align", + "value": "left" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 7, + "x": 17, + "y": 72 + }, + "id": 1041, + "options": { + "cellHeight": "sm", + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Peer" + } + ] + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer)", + "format": "table", + "instant": true, + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "(max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"direct\"}) by (peer)*10 + max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"relay\"}) by (peer))", + "format": "table", + "instant": true, + "range": false, + "refId": "K" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum(rate(p2p_ping_latency_secs_bucket{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (le,peer)) * 1000", + "format": "table", + "instant": true, + "range": false, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(app_peerinfo_version{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer,version)", + "format": "table", + "instant": true, + "range": false, + "refId": "L" + } + ], + "title": "Peer Connectivity", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "peer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time 1": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "Value #L": true, + "cluster_hash 1": true, + "cluster_hash 2": true, + "cluster_hash 3": true, + "cluster_hash 4": true, + "cluster_name 1": true, + "cluster_name 2": true, + "cluster_name 3": true, + "cluster_name 4": true, + "cluster_peer 1": true, + "cluster_peer 2": true, + "cluster_peer 3": true, + "cluster_peer 4": true, + "instance 1": true, + "instance 2": true, + "instance 3": true, + "instance 4": true, + "job 1": true, + "job 2": true, + "job 3": true, + "job 4": true + }, + "indexByName": { + "Value #A": 2, + "Value #B": 4, + "Value #K": 3, + "peer": 0, + "version": 1 + }, + "renameByName": { + "Value #A": "Connected", + "Value #B": "Latency (ms)", + "Value #K": "Direct", + "peer": "Peer", + "version": "Version" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Completed duties by type over time.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "palette-classic" + }, + "custom": { + "axisPlacement": "auto", + "fillOpacity": 100, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 0 + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 79 + }, + "id": 92, + "options": { + "colWidth": 0.5, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "rowHeight": 0.43, + "showValue": "auto", + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(core_tracker_success_duties_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (duty) > 0", + "instant": false, + "interval": "$interval", + "legendFormat": "{{duty}}", + "range": true, + "refId": "A" + } + ], + "title": "✅ Completed duties by type ", + "type": "status-history" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Failed duties by type over time. Use data link to show logs.\nProposer duties are non-failing after being successfully included on-chain.\nOther broadcasted to the beacon node duties (i.e.: attester), are non-failing after being successfully broadcasted.\nAny other duties not broadcasted to the beacon node (i.e.: randao), are non-failing after consensus is reached.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "palette-classic" + }, + "custom": { + "axisPlacement": "auto", + "fillOpacity": 100, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 0 + }, + "links": [ + { + "targetBlank": true, + "title": "Show in log explorer", + "url": "/explore?orgId=1&left=%7B%22datasource%22:%22loki%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22editorMode%22:%22builder%22,%22expr%22:%22%7Bcompose_service%3D%5C%22$job%5C%22%7D%20%7C%3D%20%60Duty%20failed%60%20%7C%3D%20%60${__field.labels.duty}%60%20%7C%20logfmt%20%7C%20line_format%20%60%7B%7B.pretty%7D%7D%60%22,%22queryType%22:%22range%22%7D%5D,%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D" + } + ], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 79 + }, + "id": 91, + "options": { + "colWidth": 0.5, + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "rowHeight": 0.43, + "showValue": "auto", + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(core_tracker_failed_duties_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (duty) > 0", + "instant": false, + "interval": "$interval", + "legendFormat": "{{duty}}", + "range": true, + "refId": "A" + } + ], + "title": "❌ Failed duties by type (per $interval)", + "type": "status-history" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Failed duties percentage by type over time", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 85 + }, + "id": 180, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "(\n sum(increase(core_tracker_failed_duties_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (duty)\n) \n/ \n(\n sum(increase(core_tracker_failed_duties_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (duty) \n + \n sum(increase(core_bcast_broadcast_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}[$interval])) by (duty) \n) > 0 ", + "instant": false, + "interval": "$interval", + "legendFormat": "{{duty}}", + "range": true, + "refId": "A" + } + ], + "title": "❌ Failed duty percentage by type (per $interval)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of failed duties by reason per time window. Note this is only available from v0.16.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "mappings": [], + "noValue": "No failures 🎉", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 85 + }, + "id": 222, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(core_tracker_failed_duty_reasons_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",duty=~\"$duty\"}[$interval])) by (reason) > 0 ", + "interval": "$interval", + "legendFormat": "{{.duty}}", + "range": true, + "refId": "A" + } + ], + "title": "🚨 Duty failure reason: $duty (per $interval)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Duties that was successfully broadcast to the Beacon Node, but that was never included on-chain. This could be due to broadcasting too late (too slow), but it could also just be to aspects of consensus layer out-of-our-control. See https://eth2book.info/bellatrix/part2/incentives/rewards/#remarks.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 91 + }, + "id": 203, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "(sum(increase(core_tracker_inclusion_missed_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\", duty=~\"$duty\"}[$interval])) by (duty)) > 0 ", + "instant": false, + "interval": "$interval", + "legendFormat": "{{duty}}", + "range": true, + "refId": "A" + } + ], + "title": "💥 Broadcasted duties never included on-chain (per $interval)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Number of consensus timeouts per time window.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed+area" + } + }, + "mappings": [], + "noValue": "No timeouts 🎉", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 91 + }, + "id": 189, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(increase(core_consensus_timeout_total{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",duty=~\"$duty\"}[$interval])) by (timer) > 0", + "interval": "$interval", + "legendFormat": "{{.duty}}", + "range": true, + "refId": "B" + } + ], + "title": "🚨 Consensus Timeouts (per $interval)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Type of connections to each peer in the cluster during the time window:\n - 🟢 Green: Direct connection (this is the preferred)\n - 🟡 Yellow: Indirect relay connection (this is ok if only temporary)\n - 🔴 Red: No connection to peer (this is a problem)\n\nSee https://docs.obol.tech/docs/charon/networking#external-p2p-network", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisPlacement": "auto", + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineWidth": 0, + "spanNulls": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "#EAB839", + "value": 1 + }, + { + "color": "green", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 97 + }, + "id": 195, + "options": { + "alignValue": "left", + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "mergeValues": true, + "rowHeight": 0.9, + "showValue": "never", + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "(max(p2p_ping_success{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer))\n+\n(1 * max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"relay\"}) by (peer))\n+ \n(100 * max(p2p_peer_connection_types{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\",type=\"direct\"}) by (peer))", + "format": "time_series", + "interval": "$interval", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Peer Connections (🟢=direct, 🟡=relay, 🔴=not connected)", + "type": "state-timeline" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisPlacement": "auto", + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineWidth": 0, + "spanNulls": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": 0 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 104 + }, + "id": 156, + "options": { + "alignValue": "left", + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "mergeValues": true, + "rowHeight": 0.9, + "showValue": "never", + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(p2p_relay_connections{cluster_name=\"$cluster_name\",cluster_hash=\"$cluster_hash\",cluster_peer=\"$cluster_peer\"}) by (peer) > 0 ", + "interval": "$interval", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Connected Relays", + "type": "state-timeline" + } + ] +} From 137ff27647166e06991400afa829670cac2352d8 Mon Sep 17 00:00:00 2001 From: kalo <24719519+KaloyanTanev@users.noreply.github.com> Date: Mon, 27 Apr 2026 12:52:48 +0300 Subject: [PATCH 8/8] Update .env samples --- .env.sample.hoodi | 10 ++++++++++ .env.sample.mainnet | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/.env.sample.hoodi b/.env.sample.hoodi index b9a0151c..56400c3b 100644 --- a/.env.sample.hoodi +++ b/.env.sample.hoodi @@ -80,6 +80,14 @@ LIGHTHOUSE_CHECKPOINT_SYNC_URL=https://checkpoint-sync.hoodi.ethpandaops.io/ # See available tags https://hub.docker.com/r/chainsafe/lodestar/tags #CL_LODESTAR_VERSION= +# Prysm beacon node docker container image version. +# See available tags https://console.cloud.google.com/artifacts/docker/prysmaticlabs/us/gcr.io/prysm%2Fbeacon-chain +#CL_PRYSM_VERSION= + +# Nimbus beacon node docker container image version. +# See available tags https://hub.docker.com/r/statusim/nimbus-eth2/tags +#CL_NIMBUS_VERSION= + ######### Validator Client Config ######### # Override prometheus metrics port for validator client. @@ -210,6 +218,8 @@ MEV_RELAYS=https://0x98f0ef62f00780cf8eb06701a7d22725b9437d4768bb19b363e882ae871 #CL_GRANDINE_ALLOY_MONITORED=false #CL_TEKU_ALLOY_MONITORED=false #CL_LODESTAR_ALLOY_MONITORED=false +#CL_PRYSM_ALLOY_MONITORED=false +#CL_NIMBUS_ALLOY_MONITORED=false #CHARON_ALLOY_MONITORED=false #VC_LODESTAR_ALLOY_MONITORED=false #VC_NIMBUS_ALLOY_MONITORED=false diff --git a/.env.sample.mainnet b/.env.sample.mainnet index 5b6ebcd2..2758a65b 100644 --- a/.env.sample.mainnet +++ b/.env.sample.mainnet @@ -80,6 +80,14 @@ LIGHTHOUSE_CHECKPOINT_SYNC_URL=https://mainnet.checkpoint.sigp.io/ # See available tags https://hub.docker.com/r/chainsafe/lodestar/tags #CL_LODESTAR_VERSION= +# Prysm beacon node docker container image version. +# See available tags https://console.cloud.google.com/artifacts/docker/prysmaticlabs/us/gcr.io/prysm%2Fbeacon-chain +#CL_PRYSM_VERSION= + +# Nimbus beacon node docker container image version. +# See available tags https://hub.docker.com/r/statusim/nimbus-eth2/tags +#CL_NIMBUS_VERSION= + ######### Validator Client Config ######### # Override prometheus metrics port for validator client. @@ -210,6 +218,8 @@ MEV_RELAYS=https://0xa15b52576bcbf1072f4a011c0f99f9fb6c66f3e1ff321f11f461d15e31b #CL_GRANDINE_ALLOY_MONITORED=false #CL_TEKU_ALLOY_MONITORED=false #CL_LODESTAR_ALLOY_MONITORED=false +#CL_PRYSM_ALLOY_MONITORED=false +#CL_NIMBUS_ALLOY_MONITORED=false #CHARON_ALLOY_MONITORED=false #VC_LODESTAR_ALLOY_MONITORED=false #VC_NIMBUS_ALLOY_MONITORED=false