{ "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": 1141, "links": [], "panels": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 14, "w": 12, "x": 0, "y": 0 }, "id": 4, "options": { "legend": { "calcs": ["lastNotNull"], "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.1.1", "targets": [ { "disableTextWrap": false, "editorMode": "code", "expr": "histogram_quantile(0.95, sum by(le, namespace,job,store_name,store_party,synchronizer_id) (rate(splice_store_signal_when_ingested_latency_duration_seconds{namespace=~\"$namespace\", node_name=~\"$node_name\", store_name=~\"$store_name\", store_party=~\"$store_party\"}[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": false, "legendFormat": "95% - {{namespace}} {{job}} {{store_name}} {{store_party}} {{synchronizer_id}}", "range": true, "refId": "A", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "disableTextWrap": false, "editorMode": "code", "expr": "histogram_avg(rate(splice_store_signal_when_ingested_latency_duration_seconds{namespace=~\"$namespace\", node_name=~\"$node_name\", store_name=~\"$store_name\", store_party=~\"$store_party\"}[$__rate_interval]))", "fullMetaSearch": false, "hide": false, "includeNullMetadata": false, "legendFormat": "avg - {{namespace}} {{job}} {{store_name}} {{store_party}} {{synchronizer_id}}", "range": true, "refId": "B", "useBackend": false } ], "title": "Ingestion timing", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 14, "w": 12, "x": 12, "y": 0 }, "id": 3, "options": { "legend": { "calcs": [], "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "hideZeros": true, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.1.1", "targets": [ { "editorMode": "code", "expr": "histogram_count(rate(splice_store_signal_when_ingested_latency_duration_seconds{namespace=~\"$namespace\",node_name=~\"$node_name\",store_name=~\"$store_name\",store_party=~\"$store_party\"}[$__rate_interval]))", "legendFormat": "{{namespace}} {{job}} {{store_name}} {{store_party}} {{synchronizer_id}}", "range": true, "refId": "A" } ], "title": "Ingestion rate", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 9, "w": 24, "x": 0, "y": 14 }, "id": 15, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.1.1", "targets": [ { "editorMode": "code", "expr": "histogram_quantile(0.99, sum by (le, namespace,job,store_name,store_party,synchronizer_id) (rate(splice_store_ingestion_batch_size{namespace=~\"$namespace\", node_name=~\"$node_name\", store_name=~\"$store_name\", store_party=~\"$store_party\"}[$__rate_interval])))", "legendFormat": "99% - {{namespace}} {{job}} {{store_name}} {{store_party}} {{synchronizer_id}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_avg(sum by (le, namespace,job,store_name,store_party,synchronizer_id) (rate(splice_store_ingestion_batch_size{namespace=~\"$namespace\", node_name=~\"$node_name\", store_name=~\"$store_name\", store_party=~\"$store_party\"}[$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "avg - {{namespace}} {{job}} {{store_name}} {{store_party}} {{synchronizer_id}}", "range": true, "refId": "B" } ], "title": "Batch size", "type": "timeseries" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 }, "id": 12, "panels": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "How many events (created / archived) were ingested per $rate_interval", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 77 }, "id": 14, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.1.1", "targets": [ { "editorMode": "code", "expr": "rate(splice_store_event_count{namespace=~\"$namespace\", node_name=~\"$node_name\", store_name=~\"$store_name\", store_party=~\"$store_party\", event_type=\"archived\"}[$__rate_interval])", "hide": false, "legendFormat": "__auto", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "rate(splice_store_event_count{namespace=~\"$namespace\", node_name=~\"$node_name\", store_name=~\"$store_name\", store_party=~\"$store_party\", event_type=\"created\"}[$__rate_interval])", "hide": false, "instant": false, "legendFormat": "__auto", "range": true, "refId": "B" } ], "title": "Ingestion rate events", "type": "timeseries" } ], "title": "Events", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 24 }, "id": 7, "panels": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "The time difference between the last ingested record time and the current wallclock time.\n\nNote that the last ingested record time metric only updates when the store ingests a new transaction so if there is no activity, the last ingested record time will not advance. For a party performing reward collection, e.g., the validator operator party you expect at least one transaction every round so the lag should not go above 20min.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 13, "w": 12, "x": 0, "y": 25 }, "id": 2, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.1.1", "targets": [ { "editorMode": "code", "expr": " time() * 1000 - splice_store_last_ingested_record_time_ms{namespace=~\"$namespace\",node_name=~\"$node_name\",store_name=~\"$store_name\",store_party=~\"$store_party\"}", "hide": false, "legendFormat": "{{namespace}} {{job}} {{store_name}} {{store_party}} {{synchronizer_id}}", "range": true, "refId": "A" } ], "title": "Last Ingested Record Time Lag", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "The last ingested record time for each store.\n\nNote that the last ingested record time metric only updates when the store ingests a new transaction so if no transaction is ingested the last ingested time will not advance. For a party performing reward collection, e.g., the validator operator party you expect at least one transaction every round so the last ingested time should be at most 20min ago.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "dateTimeAsIso" }, "overrides": [] }, "gridPos": { "h": 13, "w": 12, "x": 12, "y": 25 }, "id": 1, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "asc" } }, "pluginVersion": "12.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "splice_store_last_ingested_record_time_ms{namespace=~\"$namespace\",node_name=~\"$node_name\",store_name=~\"$store_name\",store_party=~\"$store_party\"}", "legendFormat": "{{namespace}} {{job}} {{store_name}} {{store_party}} {{synchronizer_id}}", "range": true, "refId": "A" } ], "title": "Last Ingested Record Time", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "The time difference between the last seen record time and the current wallclock time.\n\nThe last seen record time updates regardless of whether an update was ingested or filtered out.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 13, "w": 12, "x": 0, "y": 38 }, "id": 5, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.1.1", "targets": [ { "editorMode": "code", "expr": " time() * 1000 - splice_store_last_seen_record_time_ms{namespace=~\"$namespace\",node_name=~\"$node_name\",store_name=~\"$store_name\",store_party=~\"$store_party\"}", "hide": false, "legendFormat": "{{namespace}} {{job}} {{store_name}} {{store_party}} {{synchronizer_id}}", "range": true, "refId": "A" } ], "title": "Last Seen Record Time Lag", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "The most recent record time seen by this store for each synchronizer in milliseconds.\n\nThis updates for every entry seen, regardless of whether it was ingested or filtered out.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] }, "unit": "dateTimeAsIso" }, "overrides": [] }, "gridPos": { "h": 13, "w": 12, "x": 12, "y": 38 }, "id": 6, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "asc" } }, "pluginVersion": "12.1.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "splice_store_last_seen_record_time_ms{namespace=~\"$namespace\",node_name=~\"$node_name\",store_name=~\"$store_name\",store_party=~\"$store_party\"}", "legendFormat": "{{namespace}} {{job}} {{store_name}} {{store_party}} {{synchronizer_id}}", "range": true, "refId": "A" } ], "title": "Last Seen Record Time", "type": "timeseries" } ], "title": "Record time metrics", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 25 }, "id": 9, "panels": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "How many txlog entries were ingested per $rate_interval", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": 0 }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 102 }, "id": 10, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.1.1", "targets": [ { "editorMode": "code", "expr": "rate(splice_store_ingested_tx_log_entries_total{namespace=~\"$namespace\", node_name=~\"$node_name\", store_name=~\"$store_name\", store_party=~\"$store_party\"}[$__rate_interval])", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Ingestion rate txlog entries", "type": "timeseries" } ], "title": "TxLog", "type": "row" } ], "preload": false, "refresh": "1m", "schemaVersion": 41, "tags": [], "templating": { "list": [ { "allValue": ".*", "current": { "text": ["sv-1"], "value": ["sv-1"] }, "definition": "label_values(splice_store_last_ingested_record_time_ms,namespace)", "includeAll": true, "multi": true, "name": "namespace", "options": [], "query": { "qryType": 1, "query": "label_values(splice_store_last_ingested_record_time_ms,namespace)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", "type": "query" }, { "allValue": ".*", "current": { "text": "All", "value": ["$__all"] }, "definition": "label_values(splice_store_last_ingested_record_time_ms,node_name)", "includeAll": true, "multi": true, "name": "node_name", "options": [], "query": { "qryType": 1, "query": "label_values(splice_store_last_ingested_record_time_ms,node_name)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", "type": "query" }, { "allValue": ".*", "current": { "text": "All", "value": ["$__all"] }, "definition": "label_values(splice_store_last_ingested_record_time_ms,store_name)", "description": "The name of the store", "includeAll": true, "multi": true, "name": "store_name", "options": [], "query": { "qryType": 1, "query": "label_values(splice_store_last_ingested_record_time_ms,store_name)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", "type": "query" }, { "allValue": ".*", "current": { "text": "All", "value": ["$__all"] }, "definition": "label_values(splice_store_last_ingested_record_time_ms,store_party)", "description": "The party that the store is ingesting data for", "includeAll": true, "multi": true, "name": "store_party", "options": [], "query": { "qryType": 1, "query": "label_values(splice_store_last_ingested_record_time_ms,store_party)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", "type": "query" } ] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": {}, "timezone": "UTC", "title": "Splice Stores Ingestion", "uid": "dec17fpqzdwcge", "version": 10 }