{ "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "description": "Track the trigger automations being run by Splice Apps", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": 8872, "links": [], "panels": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "runs / second" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "/.*failure/" }, "properties": [ { "id": "custom.fillOpacity", "value": 50 } ] } ] }, "gridPos": { "h": 10, "w": 24, "x": 0, "y": 0 }, "id": 18, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "right", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "disableTextWrap": false, "editorMode": "code", "expr": "rate(splice_trigger_completed_total{namespace=~\"$namespace\", node_type=~\"$app\", trigger_name=~\"$trigger_name\", outcome=~\"success|failure\"}[$__rate_interval])", "fullMetaSearch": false, "includeNullMetadata": false, "instant": false, "legendFormat": "{{node_name}} {{trigger_name}} {{outcome}} ", "range": true, "refId": "A", "useBackend": false } ], "title": "Runs", "type": "timeseries" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 10 }, "id": 3, "panels": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "The rate of successful iterations completed", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 4, "x": 0, "y": 11 }, "id": 5, "options": { "colorMode": "none", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "exemplar": false, "expr": "sum(histogram_sum(rate(splice_trigger_latency_duration_seconds{namespace=~\"$namespace\", node_type=~\"$app\", trigger_name=~\"$trigger_name\", trigger_type=\"$trigger_type\"}[$__rate_interval])))", "hide": false, "instant": true, "range": false, "refId": "A" } ], "title": "Traffic", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "The rate at which the trigger(s) are failing with retryable errors", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 4, "x": 4, "y": 11 }, "id": 7, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(rate(splice_retries_failures{namespace=~\"$namespace\", node_type=~\"$app\", trigger_name=~\"$trigger_name\", trigger_type=\"$trigger_type\", error_kind=\"transient\"}[$__rate_interval]))", "instant": false, "range": true, "refId": "A" } ], "title": "Retries", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "The rate at which trigger(s) are failing with fatal/non-retryable errors", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 4, "x": 8, "y": 11 }, "id": 8, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "sum(rate(splice_retries_failures{namespace=~\"$namespace\", node_type=~\"$app\", trigger_name=~\"$trigger_name\", trigger_type=\"$trigger_type\", error_kind=\"fatal\"}[$__rate_interval]))", "instant": false, "range": true, "refId": "A" } ], "title": "Errors", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "The time it took to perform an iteration", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 11 }, "id": 6, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "expr": "histogram_quantile(0.95, rate(splice_trigger_latency_duration_seconds{namespace=~\"$namespace\", node_type=~\"$app\", trigger_name=~\"$trigger_name\", trigger_type=\"$trigger_type\"}[$__rate_interval]))", "instant": false, "legendFormat": "{{namespace}} {{node_type}} {{service}} {{trigger_name}}", "range": true, "refId": "A" } ], "title": "Latency", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "continuous-RdYlGr" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 }, { "color": "#EAB839", "value": 90 }, { "color": "#6ED0E0", "value": 100 }, { "color": "#EF843C", "value": 110 }, { "color": "#E24D42", "value": 120 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 18 }, "id": 10, "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "frameIndex": 0, "showHeader": true, "sortBy": [] }, "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "exemplar": false, "expr": "sum by(trigger_name) (histogram_count(rate(splice_trigger_latency_duration_seconds{namespace=~\"$namespace\", node_type=~\"$app\", trigger_type=\"$trigger_type\"}[$__rate_interval])))", "format": "time_series", "instant": false, "range": true, "refId": "A" } ], "title": "Top 5 Active Triggers", "transformations": [ { "id": "reduce", "options": { "includeTimeField": false, "labelsToFields": true, "mode": "seriesToRows", "reducers": [ "sum" ] } }, { "id": "sortBy", "options": { "fields": {}, "sort": [ { "desc": true, "field": "Total" } ] } }, { "id": "organize", "options": { "excludeByName": { "Field": true }, "indexByName": {}, "renameByName": { "Total": "Traffic", "trigger_name": "Trigger" } } }, { "id": "limit", "options": { "limitField": 5 } } ], "type": "table" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "description": "How busy triggers are within a certain time window", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 16, "x": 8, "y": 18 }, "id": 9, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "right", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "editorMode": "code", "exemplar": false, "expr": "sum by(trigger_name, instance, node_type, trigger_type) (histogram_sum(rate(splice_trigger_latency_duration_seconds{namespace=~\"$namespace\", node_type=~\"$app\", trigger_name=~\"$trigger_name\", trigger_type=\"$trigger_type\"}[$__rate_interval])))", "instant": false, "legendFormat": "{{node_type}} {{instance}} {{trigger_name}}", "range": true, "refId": "A" } ], "title": "Saturation", "type": "timeseries" } ], "repeat": "trigger_type", "title": "Triggers $trigger_type", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 12 }, "id": 22, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0", "targets": [ { "disableTextWrap": false, "editorMode": "code", "expr": "sum(rate(splice_trigger_attempted_total{isDsoDelegateTrigger=~\"$isDsoDelegateTrigger\", trigger_name=~\"$trigger_name\", namespace=~\"$namespace\", node_type=~\"$app\"}[$__rate_interval]))", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "total attempts", "range": true, "refId": "A", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "disableTextWrap": false, "editorMode": "code", "expr": "sum(rate(splice_trigger_attempted_total{statusCode!~\"OK\", isDsoDelegateTrigger=~\"$isDsoDelegateTrigger\", namespace=~\"$namespace\", node_type=~\"$app\", trigger_name=~\"$trigger_name\", contentionFailure=\"true\"}[$__rate_interval]))", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "legendFormat": "total contentions", "range": true, "refId": "B", "useBackend": false } ], "title": "Trigger attempts", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 20 }, "id": 20, "options": { "legend": { "calcs": [ "last" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "disableTextWrap": false, "editorMode": "code", "expr": "sum by(errorCodeId, namespace, node_type, trigger_name) (rate(splice_trigger_attempted_total{namespace=~\"$namespace\", node_type=~\"$app\", trigger_name=~\"$trigger_name\", isDsoDelegateTrigger=~\"$isDsoDelegateTrigger\", contentionFailure=\"true\"}[$__rate_interval]))", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, "legendFormat": "__auto", "range": true, "refId": "A", "useBackend": false } ], "title": "Failed trigger attempts with contention errors", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 28 }, "id": 21, "options": { "legend": { "calcs": [ "last" ], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, "disableTextWrap": false, "editorMode": "code", "expr": "sum by (errorCodeId, namespace, node_type, trigger_name) (rate(splice_trigger_attempted_total{namespace=~\"$namespace\",node_type=~\"$app\",trigger_name=~\"$trigger_name\",isDsoDelegateTrigger=~\"$isDsoDelegateTrigger\", statusCode!~\"OK\", contentionFailure=\"false\"}[$__rate_interval]))", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, "legendFormat": "__auto", "range": true, "refId": "B", "useBackend": false } ], "title": "Failed trigger attempts with non-contention errors", "type": "timeseries" } ], "preload": false, "refresh": "", "schemaVersion": 40, "tags": [], "templating": { "list": [ { "current": { "text": "All", "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "prometheus" }, "definition": "label_values(splice_trigger_completed_total,namespace)", "includeAll": true, "label": "Namespace", "name": "namespace", "options": [], "query": { "qryType": 1, "query": "label_values(splice_trigger_completed_total,namespace)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", "type": "query" }, { "current": { "text": "All", "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "prometheus" }, "definition": "label_values(splice_trigger_completed_total{namespace=~\"$namespace\"},node_type)", "includeAll": true, "label": "App", "name": "app", "options": [], "query": { "qryType": 1, "query": "label_values(splice_trigger_completed_total{namespace=~\"$namespace\"},node_type)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", "type": "query" }, { "current": { "text": "All", "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "prometheus" }, "definition": "label_values(splice_trigger_completed_total{namespace=~\"$namespace\", node_type=~\"$app\"},trigger_name)", "includeAll": true, "label": "Trigger Name", "name": "trigger_name", "options": [], "query": { "qryType": 1, "query": "label_values(splice_trigger_completed_total{namespace=~\"$namespace\", node_type=~\"$app\"},trigger_name)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", "type": "query" }, { "current": { "text": [ "taskbased", "polling" ], "value": [ "taskbased", "polling" ] }, "includeAll": false, "multi": true, "name": "trigger_type", "options": [ { "selected": true, "text": "taskbased", "value": "taskbased" }, { "selected": true, "text": "polling", "value": "polling" } ], "query": "taskbased, polling", "type": "custom" }, { "current": { "text": "All", "value": "$__all" }, "definition": "label_values(isDsoDelegateTrigger)", "includeAll": true, "label": "isDsoDelegateTrigger", "name": "isDsoDelegateTrigger", "options": [], "query": { "qryType": 1, "query": "label_values(isDsoDelegateTrigger)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", "type": "query" } ] }, "time": { "from": "now-3h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Automations", "uid": "a3e1385f-6f03-46d9-908c-34aca0f507a6", "version": 5, "weekStart": "" }