From fb0f65b8d0f3c2ae2bc2976af271edc8d2e70e86 Mon Sep 17 00:00:00 2001 From: Eric Warehime Date: Tue, 26 Nov 2024 07:55:41 -0800 Subject: [PATCH] Perf test changes --- .../files/grafana/config/config.ini | 3 +- .../grafana/config/dashboard-backup.json | 1037 +++++++++++++++++ .../files/grafana/config/dashboard.json | 260 ++++- core/monitoring/grafana.go | 6 + core/monitoring/grafana_snapshot.go | 16 +- core/monitoring/prometheus.go | 4 +- 6 files changed, 1274 insertions(+), 52 deletions(-) create mode 100644 core/monitoring/files/grafana/config/dashboard-backup.json diff --git a/core/monitoring/files/grafana/config/config.ini b/core/monitoring/files/grafana/config/config.ini index 4ceaa82..b72abe7 100644 --- a/core/monitoring/files/grafana/config/config.ini +++ b/core/monitoring/files/grafana/config/config.ini @@ -34,4 +34,5 @@ org_name = Main Org. enabled = true external_enabled = true external_snapshot_url = https://snapshots.raintank.io -public_mode = true \ No newline at end of file +public_mode = true +include_datasource_query_result = true \ No newline at end of file diff --git a/core/monitoring/files/grafana/config/dashboard-backup.json b/core/monitoring/files/grafana/config/dashboard-backup.json new file mode 100644 index 0000000..14aa0a2 --- /dev/null +++ b/core/monitoring/files/grafana/config/dashboard-backup.json @@ -0,0 +1,1037 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 8, + "panels": [], + "title": "Consensus stats", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "petri_prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 2, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "petri_prometheus" + }, + "editorMode": "code", + "expr": "avg(rate(cometbft_consensus_block_interval_seconds_sum[$__rate_interval]) / rate(cometbft_consensus_block_interval_seconds_count[$__rate_interval]))", + "instant": false, + "legendFormat": "Average seconds per block", + "range": true, + "refId": "A" + } + ], + "title": "Block production rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "petri_prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "petri_prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "avg(cometbft_consensus_rounds)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Consensus rounds", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Consensus rounds", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "petri_prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "petri_prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(cometbft_consensus_validator_missed_blocks[$__rate_interval]) * 60)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Average block miss rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "petri_prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "txs", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "petri_prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "avg(cometbft_mempool_size)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Average mempool size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "petri_prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "txs", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "petri_prometheus" + }, + "editorMode": "code", + "expr": "rate(cometbft_consensus_total_txs[$__rate_interval]) * 60", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Transaction rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "petri_prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "txs", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "petri_prometheus" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "cometbft_mempool_size", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{job}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Mempool size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "petri_prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "petri_prometheus" + }, + "editorMode": "code", + "expr": "sum by (instance) (rate(cometbft_p2p_message_receive_bytes_total[$__rate_interval]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Node received bytes per second", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "petri_prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "petri_prometheus" + }, + "editorMode": "code", + "expr": "sum by (instance) (rate(cometbft_p2p_message_send_bytes_total[$__rate_interval]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Node transmitted bytes per second", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "petri_prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 33 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "petri_prometheus" + }, + "editorMode": "code", + "expr": "avg by (message_type) (sum by (instance, message_type) (rate(cometbft_p2p_message_receive_bytes_total[$__rate_interval])))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Node received bytes per second by message type", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "petri_prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 33 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "petri_prometheus" + }, + "editorMode": "code", + "expr": "avg by (message_type) (sum by (job, message_type) (rate(cometbft_p2p_message_send_bytes_total[$__rate_interval])))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Node transmitted bytes per second by message type", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "BlockSDK performance tests", + "uid": "b8ff6e6f-5b4b-4d5e-bc50-91bbbf10f436", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/core/monitoring/files/grafana/config/dashboard.json b/core/monitoring/files/grafana/config/dashboard.json index 14aa0a2..d9dfce0 100644 --- a/core/monitoring/files/grafana/config/dashboard.json +++ b/core/monitoring/files/grafana/config/dashboard.json @@ -18,9 +18,8 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 2, + "id": 1, "links": [], - "liveNow": false, "panels": [ { "collapsed": false, @@ -52,6 +51,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -78,8 +78,6 @@ } }, "mappings": [], - "max": 2, - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -92,9 +90,7 @@ "value": 80 } ] - }, - "unit": "s", - "unitScale": true + } }, "overrides": [] }, @@ -104,7 +100,7 @@ "x": 0, "y": 1 }, - "id": 10, + "id": 18, "options": { "legend": { "calcs": [], @@ -117,21 +113,82 @@ "sort": "none" } }, + "pluginVersion": "11.4.0-207486", "targets": [ + { + "editorMode": "code", + "expr": "avg(rate(cometbft_consensus_step_duration_seconds_sum{step=\"Prevote\"}[$__rate_interval]) / rate(cometbft_consensus_step_duration_seconds_count{step=\"Prevote\"}[$__rate_interval]))", + "legendFormat": "Prevote", + "range": true, + "refId": "Prevote" + }, { "datasource": { "type": "prometheus", "uid": "petri_prometheus" }, "editorMode": "code", - "expr": "avg(rate(cometbft_consensus_block_interval_seconds_sum[$__rate_interval]) / rate(cometbft_consensus_block_interval_seconds_count[$__rate_interval]))", + "expr": "avg(rate(cometbft_consensus_step_duration_seconds_sum{step=\"Commit\"}[$__rate_interval]) / rate(cometbft_consensus_step_duration_seconds_count{step=\"Commit\"}[$__rate_interval]))", + "hide": false, "instant": false, - "legendFormat": "Average seconds per block", + "legendFormat": "Commit", "range": true, - "refId": "A" + "refId": "Commit" + }, + { + "datasource": { + "type": "prometheus", + "uid": "petri_prometheus" + }, + "editorMode": "code", + "expr": "avg(rate(cometbft_consensus_step_duration_seconds_sum{step=\"NewHeight\"}[$__rate_interval]) / rate(cometbft_consensus_step_duration_seconds_count{step=\"NewHeight\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "NewHeight", + "range": true, + "refId": "NewHeight" + }, + { + "datasource": { + "type": "prometheus", + "uid": "petri_prometheus" + }, + "editorMode": "code", + "expr": "avg(rate(cometbft_consensus_step_duration_seconds_sum{step=\"NewRound\"}[$__rate_interval]) / rate(cometbft_consensus_step_duration_seconds_count{step=\"NewRound\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "NewRound", + "range": true, + "refId": "NewRound" + }, + { + "datasource": { + "type": "prometheus", + "uid": "petri_prometheus" + }, + "editorMode": "code", + "expr": "avg(rate(cometbft_consensus_step_duration_seconds_sum{step=\"Precommit\"}[$__rate_interval]) / rate(cometbft_consensus_step_duration_seconds_count{step=\"Precommit\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "Precommit", + "range": true, + "refId": "Precommit" + }, + { + "datasource": { + "type": "prometheus", + "uid": "petri_prometheus" + }, + "editorMode": "code", + "expr": "avg(rate(cometbft_consensus_step_duration_seconds_sum{step=\"Propose\"}[$__rate_interval]) / rate(cometbft_consensus_step_duration_seconds_count{step=\"Propose\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "Propose", + "range": true, + "refId": "Propose" } ], - "title": "Block production rate", + "title": "Avg Step Duration", "type": "timeseries" }, { @@ -151,6 +208,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -189,8 +247,7 @@ "value": 80 } ] - }, - "unitScale": true + } }, "overrides": [] }, @@ -213,6 +270,7 @@ "sort": "none" } }, + "pluginVersion": "11.4.0-207486", "targets": [ { "datasource": { @@ -251,6 +309,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -277,6 +336,7 @@ } }, "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -290,7 +350,7 @@ } ] }, - "unitScale": true + "unit": "s" }, "overrides": [] }, @@ -300,7 +360,7 @@ "x": 0, "y": 9 }, - "id": 7, + "id": 10, "options": { "legend": { "calcs": [], @@ -313,6 +373,7 @@ "sort": "none" } }, + "pluginVersion": "11.4.0-207486", "targets": [ { "datasource": { @@ -320,14 +381,14 @@ "uid": "petri_prometheus" }, "editorMode": "code", - "expr": "sum(rate(cometbft_consensus_validator_missed_blocks[$__rate_interval]) * 60)", + "expr": "avg(rate(cometbft_consensus_block_interval_seconds_sum[$__rate_interval]) / rate(cometbft_consensus_block_interval_seconds_count[$__rate_interval]))", "instant": false, - "legendFormat": "__auto", + "legendFormat": "Average seconds per block", "range": true, "refId": "A" } ], - "title": "Average block miss rate", + "title": "Block production rate", "type": "timeseries" }, { @@ -347,6 +408,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -386,8 +448,7 @@ } ] }, - "unit": "txs", - "unitScale": true + "unit": "txs" }, "overrides": [] }, @@ -410,6 +471,7 @@ "sort": "none" } }, + "pluginVersion": "11.4.0-207486", "targets": [ { "datasource": { @@ -448,6 +510,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -486,9 +549,7 @@ "value": 80 } ] - }, - "unit": "txs", - "unitScale": true + } }, "overrides": [] }, @@ -498,7 +559,7 @@ "x": 0, "y": 17 }, - "id": 11, + "id": 7, "options": { "legend": { "calcs": [], @@ -511,6 +572,7 @@ "sort": "none" } }, + "pluginVersion": "11.4.0-207486", "targets": [ { "datasource": { @@ -518,14 +580,14 @@ "uid": "petri_prometheus" }, "editorMode": "code", - "expr": "rate(cometbft_consensus_total_txs[$__rate_interval]) * 60", + "expr": "sum(rate(cometbft_consensus_validator_missed_blocks[$__rate_interval]) * 60)", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], - "title": "Transaction rate", + "title": "Average block miss rate", "type": "timeseries" }, { @@ -545,6 +607,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -584,8 +647,7 @@ } ] }, - "unit": "txs", - "unitScale": true + "unit": "txs" }, "overrides": [] }, @@ -608,6 +670,7 @@ "sort": "none" } }, + "pluginVersion": "11.4.0-207486", "targets": [ { "datasource": { @@ -646,6 +709,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -685,8 +749,7 @@ } ] }, - "unit": "Bps", - "unitScale": true + "unit": "txs" }, "overrides": [] }, @@ -696,7 +759,7 @@ "x": 0, "y": 25 }, - "id": 14, + "id": 11, "options": { "legend": { "calcs": [], @@ -709,6 +772,7 @@ "sort": "none" } }, + "pluginVersion": "11.4.0-207486", "targets": [ { "datasource": { @@ -716,14 +780,14 @@ "uid": "petri_prometheus" }, "editorMode": "code", - "expr": "sum by (instance) (rate(cometbft_p2p_message_receive_bytes_total[$__rate_interval]))", + "expr": "rate(cometbft_consensus_total_txs[$__rate_interval]) * 60", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], - "title": "Node received bytes per second", + "title": "Transaction rate", "type": "timeseries" }, { @@ -743,6 +807,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -782,8 +847,7 @@ } ] }, - "unit": "Bps", - "unitScale": true + "unit": "Bps" }, "overrides": [] }, @@ -806,6 +870,7 @@ "sort": "none" } }, + "pluginVersion": "11.4.0-207486", "targets": [ { "datasource": { @@ -840,6 +905,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -879,8 +945,7 @@ } ] }, - "unit": "Bps", - "unitScale": true + "unit": "Bps" }, "overrides": [] }, @@ -890,7 +955,7 @@ "x": 0, "y": 33 }, - "id": 16, + "id": 14, "options": { "legend": { "calcs": [], @@ -903,6 +968,7 @@ "sort": "none" } }, + "pluginVersion": "11.4.0-207486", "targets": [ { "datasource": { @@ -910,14 +976,14 @@ "uid": "petri_prometheus" }, "editorMode": "code", - "expr": "avg by (message_type) (sum by (instance, message_type) (rate(cometbft_p2p_message_receive_bytes_total[$__rate_interval])))", + "expr": "sum by (instance) (rate(cometbft_p2p_message_receive_bytes_total[$__rate_interval]))", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], - "title": "Node received bytes per second by message type", + "title": "Node received bytes per second", "type": "timeseries" }, { @@ -937,6 +1003,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -976,8 +1043,7 @@ } ] }, - "unit": "Bps", - "unitScale": true + "unit": "Bps" }, "overrides": [] }, @@ -1000,6 +1066,7 @@ "sort": "none" } }, + "pluginVersion": "11.4.0-207486", "targets": [ { "datasource": { @@ -1016,22 +1083,121 @@ ], "title": "Node transmitted bytes per second by message type", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "petri_prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 41 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.4.0-207486", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "petri_prometheus" + }, + "editorMode": "code", + "expr": "avg by (message_type) (sum by (instance, message_type) (rate(cometbft_p2p_message_receive_bytes_total[$__rate_interval])))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Node received bytes per second by message type", + "type": "timeseries" } ], + "preload": false, "refresh": "", - "schemaVersion": 39, + "schemaVersion": 40, "tags": [], "templating": { "list": [] }, "time": { - "from": "now-15m", + "from": "now-10m", "to": "now" }, "timepicker": {}, "timezone": "", - "title": "BlockSDK performance tests", + "title": "Comet Performance", "uid": "b8ff6e6f-5b4b-4d5e-bc50-91bbbf10f436", - "version": 1, + "version": 5, "weekStart": "" } \ No newline at end of file diff --git a/core/monitoring/grafana.go b/core/monitoring/grafana.go index 5ebeefc..5f350c2 100644 --- a/core/monitoring/grafana.go +++ b/core/monitoring/grafana.go @@ -28,6 +28,12 @@ var grafanaDatasourceTemplate string //go:embed files/grafana/config/dashboards.yml var grafanaDashboardProvisioningConfig string +//go:embed files/grafana/config/dashboard.json +var DefaultDashboardJSON string + +// DefaultDashboardUID is the UID of the dashboard in DefaultDashboardJSON +var DefaultDashboardUID = "b8ff6e6f-5b4b-4d5e-bc50-91bbbf10f436" + // SetupGrafanaTask sets up and configures (but does not start) a Grafana task. // Additionally, it creates a Prometheus datasource and a dashboard (given the DashboardJSON in GrafanaOptions). func SetupGrafanaTask(ctx context.Context, logger *zap.Logger, p provider.Provider, opts GrafanaOptions) (*provider.Task, error) { diff --git a/core/monitoring/grafana_snapshot.go b/core/monitoring/grafana_snapshot.go index 5715f1a..de2f80a 100644 --- a/core/monitoring/grafana_snapshot.go +++ b/core/monitoring/grafana_snapshot.go @@ -78,7 +78,19 @@ func SnapshotGrafanaDashboard(ctx context.Context, uid, grafanaURL string) (stri return "", err } - shareDashboardButton, err := page.Element("[data-testid='data-testid share-button']") + err = page.SetViewport(&proto.EmulationSetDeviceMetricsOverride{ + Width: 1920 * 5, + Height: 1080 * 5, + DeviceScaleFactor: 1, + }) + if err != nil { + return "", err + } + + // The page takes a long time to render, and if you create the snapshot before the data loads it will be empty. + time.Sleep(time.Second * 10) + + shareDashboardButton, err := page.Element("[data-testid='data-testid Share dashboard']") if err != nil { return "", err } @@ -88,7 +100,7 @@ func SnapshotGrafanaDashboard(ctx context.Context, uid, grafanaURL string) (stri return "", err } - snapshotButton, err := page.Element("[aria-label='Tab Snapshot']") + snapshotButton, err := page.Element("[data-testid='data-testid Tab Snapshot']") if err != nil { return "", err } diff --git a/core/monitoring/prometheus.go b/core/monitoring/prometheus.go index 68570f3..08d73ce 100644 --- a/core/monitoring/prometheus.go +++ b/core/monitoring/prometheus.go @@ -35,10 +35,10 @@ func SetupPrometheusTask(ctx context.Context, logger *zap.Logger, p provider.Pro Ports: []string{ "3000", }, - DataDir: "/prometheus_config", + DataDir: "/prometheus", Entrypoint: []string{ "/bin/prometheus", - "--config.file=/prometheus_config/prometheus.yml", + "--config.file=/prometheus/prometheus.yml", "--storage.tsdb.path=/prometheus", "--web.console.libraries=/usr/share/prometheus/console_libraries", "--web.console.templates=/usr/share/prometheus/consoles",