{
    "title": "SRE Operations",
    "description": "Infrastructure and operational health — queue depth, webhook delivery, session health, stuck detection, and tool loop monitoring. The Tier 3 SRE deep-dive in the OpenClaw observability hierarchy.",
    "tags": [
        "openclaw",
        "grafana-lens",
        "sre-operations"
    ],
    "timezone": "browser",
    "editable": true,
    "schemaVersion": 39,
    "time": {
        "from": "now-6h",
        "to": "now"
    },
    "refresh": "30s",
    "links": [
        {
            "title": "LLM Command Center",
            "tags": [
                "openclaw",
                "grafana-lens",
                "command-center"
            ],
            "type": "dashboards",
            "asDropdown": false,
            "icon": "external link"
        },
        {
            "title": "Session Explorer",
            "tags": [
                "openclaw",
                "grafana-lens",
                "session-explorer"
            ],
            "type": "dashboards",
            "asDropdown": false,
            "icon": "external link"
        },
        {
            "title": "Tool Performance",
            "tags": [
                "openclaw",
                "grafana-lens",
                "tool-performance"
            ],
            "type": "dashboards",
            "asDropdown": false,
            "icon": "external link"
        },
        {
            "title": "GenAI Observability",
            "tags": [
                "openclaw",
                "grafana-lens",
                "genai-observability"
            ],
            "type": "dashboards",
            "asDropdown": false,
            "icon": "external link"
        },
        {
            "title": "OpenClaw Dashboards",
            "tags": [
                "openclaw",
                "grafana-lens"
            ],
            "type": "dashboards",
            "asDropdown": true,
            "icon": "external link"
        }
    ],
    "annotations": {
        "list": [
            {
                "builtIn": 1,
                "datasource": {
                    "type": "grafana",
                    "uid": "-- Grafana --"
                },
                "enable": true,
                "hide": true,
                "iconColor": "rgba(0, 211, 255, 1)",
                "name": "Annotations & Alerts",
                "type": "dashboard"
            }
        ]
    },
    "templating": {
        "list": [
            {
                "name": "prometheus",
                "label": "Prometheus",
                "type": "datasource",
                "query": "prometheus",
                "current": {},
                "hide": 0
            },
            {
                "name": "loki",
                "label": "Loki",
                "type": "datasource",
                "query": "loki",
                "current": {},
                "hide": 0
            }
        ]
    },
    "panels": [
        {
            "id": 1,
            "title": "Health Overview",
            "type": "row",
            "gridPos": {
                "h": 1,
                "w": 24,
                "x": 0,
                "y": 0
            },
            "collapsed": false,
            "panels": []
        },
        {
            "id": 2,
            "title": "Queue Depth",
            "description": "Current message queue depth.",
            "type": "stat",
            "gridPos": {
                "h": 4,
                "w": 4,
                "x": 0,
                "y": 1
            },
            "datasource": {
                "type": "prometheus",
                "uid": "$prometheus"
            },
            "targets": [
                {
                    "expr": "sum(openclaw_lens_queue_depth) or vector(0)",
                    "legendFormat": "depth",
                    "refId": "A"
                }
            ],
            "fieldConfig": {
                "defaults": {
                    "unit": "short",
                    "decimals": 0,
                    "thresholds": {
                        "mode": "absolute",
                        "steps": [
                            {
                                "color": "green",
                                "value": null
                            },
                            {
                                "color": "yellow",
                                "value": 5
                            },
                            {
                                "color": "red",
                                "value": 20
                            }
                        ]
                    },
                    "noValue": "0"
                }
            },
            "options": {
                "graphMode": "area",
                "colorMode": "background"
            }
        },
        {
            "id": 3,
            "title": "Stuck Sessions",
            "description": "Number of sessions detected as stuck.",
            "type": "stat",
            "gridPos": {
                "h": 4,
                "w": 4,
                "x": 4,
                "y": 1
            },
            "datasource": {
                "type": "prometheus",
                "uid": "$prometheus"
            },
            "targets": [
                {
                    "expr": "sum(openclaw_lens_sessions_stuck) or vector(0)",
                    "legendFormat": "stuck",
                    "refId": "A"
                }
            ],
            "fieldConfig": {
                "defaults": {
                    "unit": "short",
                    "decimals": 0,
                    "thresholds": {
                        "mode": "absolute",
                        "steps": [
                            {
                                "color": "green",
                                "value": null
                            },
                            {
                                "color": "red",
                                "value": 1
                            }
                        ]
                    },
                    "noValue": "0"
                }
            },
            "options": {
                "graphMode": "none",
                "colorMode": "background"
            }
        },
        {
            "id": 4,
            "title": "Max Stuck Age",
            "description": "Maximum age (ms) of any stuck session.",
            "type": "stat",
            "gridPos": {
                "h": 4,
                "w": 4,
                "x": 8,
                "y": 1
            },
            "datasource": {
                "type": "prometheus",
                "uid": "$prometheus"
            },
            "targets": [
                {
                    "expr": "sum(openclaw_lens_stuck_session_max_age_ms) or vector(0)",
                    "legendFormat": "max age",
                    "refId": "A"
                }
            ],
            "fieldConfig": {
                "defaults": {
                    "unit": "ms",
                    "decimals": 0,
                    "thresholds": {
                        "mode": "absolute",
                        "steps": [
                            {
                                "color": "green",
                                "value": null
                            },
                            {
                                "color": "yellow",
                                "value": 30000
                            },
                            {
                                "color": "red",
                                "value": 120000
                            }
                        ]
                    },
                    "noValue": "0ms"
                }
            },
            "options": {
                "graphMode": "none",
                "colorMode": "background"
            }
        },
        {
            "id": 5,
            "title": "Tool Loops",
            "description": "Currently active tool loop detections (warning + critical).",
            "type": "stat",
            "gridPos": {
                "h": 4,
                "w": 4,
                "x": 12,
                "y": 1
            },
            "datasource": {
                "type": "prometheus",
                "uid": "$prometheus"
            },
            "targets": [
                {
                    "expr": "sum(openclaw_lens_tool_loops_active) or vector(0)",
                    "legendFormat": "loops",
                    "refId": "A"
                }
            ],
            "fieldConfig": {
                "defaults": {
                    "unit": "short",
                    "decimals": 0,
                    "thresholds": {
                        "mode": "absolute",
                        "steps": [
                            {
                                "color": "green",
                                "value": null
                            },
                            {
                                "color": "red",
                                "value": 1
                            }
                        ]
                    },
                    "noValue": "0"
                }
            },
            "options": {
                "graphMode": "none",
                "colorMode": "background"
            }
        },
        {
            "id": 6,
            "title": "Webhook Errors",
            "description": "Webhook error rate over the last 5 minutes.",
            "type": "stat",
            "gridPos": {
                "h": 4,
                "w": 4,
                "x": 16,
                "y": 1
            },
            "datasource": {
                "type": "prometheus",
                "uid": "$prometheus"
            },
            "targets": [
                {
                    "expr": "sum(rate(openclaw_lens_webhook_error_total[5m])) * 60 or vector(0)",
                    "legendFormat": "errors/min",
                    "refId": "A"
                }
            ],
            "fieldConfig": {
                "defaults": {
                    "unit": "short",
                    "decimals": 1,
                    "thresholds": {
                        "mode": "absolute",
                        "steps": [
                            {
                                "color": "green",
                                "value": null
                            },
                            {
                                "color": "red",
                                "value": 0.1
                            }
                        ]
                    },
                    "noValue": "0"
                }
            },
            "options": {
                "graphMode": "area",
                "colorMode": "background"
            }
        },
        {
            "id": 7,
            "title": "Active Sessions",
            "description": "Currently active agent sessions from heartbeat snapshot.",
            "type": "stat",
            "gridPos": {
                "h": 4,
                "w": 4,
                "x": 20,
                "y": 1
            },
            "datasource": {
                "type": "prometheus",
                "uid": "$prometheus"
            },
            "targets": [
                {
                    "expr": "sum(openclaw_lens_sessions_active_snapshot) or vector(0)",
                    "legendFormat": "active",
                    "refId": "A"
                }
            ],
            "fieldConfig": {
                "defaults": {
                    "unit": "short",
                    "decimals": 0,
                    "noValue": "0"
                }
            },
            "options": {
                "graphMode": "area"
            }
        },
        {
            "id": 21,
            "title": "Sessions Today",
            "description": "Total sessions completed today.",
            "type": "stat",
            "gridPos": {
                "h": 4,
                "w": 8,
                "x": 0,
                "y": 5
            },
            "datasource": {
                "type": "prometheus",
                "uid": "$prometheus"
            },
            "targets": [
                {
                    "expr": "sum(increase(openclaw_lens_sessions_completed_total[1d])) or vector(0)",
                    "legendFormat": "sessions",
                    "refId": "A"
                }
            ],
            "fieldConfig": {
                "defaults": {
                    "unit": "short",
                    "decimals": 0,
                    "noValue": "0"
                }
            },
            "options": {
                "graphMode": "area"
            }
        },
        {
            "id": 22,
            "title": "Context Pressure",
            "description": "Context window utilization. >60% yellow, >80% red — compaction imminent.",
            "type": "gauge",
            "gridPos": {
                "h": 4,
                "w": 8,
                "x": 8,
                "y": 5
            },
            "datasource": {
                "type": "prometheus",
                "uid": "$prometheus"
            },
            "targets": [
                {
                    "expr": "openclaw_lens_context_tokens{type=\"used\"} / openclaw_lens_context_tokens{type=\"limit\"} * 100",
                    "legendFormat": "pressure %",
                    "refId": "A"
                }
            ],
            "fieldConfig": {
                "defaults": {
                    "unit": "percent",
                    "min": 0,
                    "max": 100,
                    "decimals": 0,
                    "thresholds": {
                        "mode": "absolute",
                        "steps": [
                            {
                                "color": "green",
                                "value": null
                            },
                            {
                                "color": "yellow",
                                "value": 60
                            },
                            {
                                "color": "red",
                                "value": 80
                            }
                        ]
                    },
                    "noValue": "0%"
                }
            },
            "options": {
                "reduceOptions": {
                    "calcs": [
                        "lastNotNull"
                    ]
                },
                "showThresholdLabels": false,
                "showThresholdMarkers": true
            }
        },
        {
            "id": 23,
            "title": "Messages/hr",
            "description": "Current message processing rate projected to hourly.",
            "type": "stat",
            "gridPos": {
                "h": 4,
                "w": 8,
                "x": 16,
                "y": 5
            },
            "datasource": {
                "type": "prometheus",
                "uid": "$prometheus"
            },
            "targets": [
                {
                    "expr": "sum(rate(openclaw_lens_messages_processed_total[5m])) * 3600 or vector(0)",
                    "legendFormat": "msg/hr",
                    "refId": "A"
                }
            ],
            "fieldConfig": {
                "defaults": {
                    "unit": "short",
                    "decimals": 0,
                    "noValue": "0"
                }
            },
            "options": {
                "graphMode": "area"
            }
        },
        {
            "id": 8,
            "title": "Queue Health",
            "type": "row",
            "gridPos": {
                "h": 1,
                "w": 24,
                "x": 0,
                "y": 9
            },
            "collapsed": false,
            "panels": []
        },
        {
            "id": 9,
            "title": "Queue Depth",
            "description": "Message queue depth over time.",
            "type": "timeseries",
            "gridPos": {
                "h": 8,
                "w": 8,
                "x": 0,
                "y": 10
            },
            "datasource": {
                "type": "prometheus",
                "uid": "$prometheus"
            },
            "targets": [
                {
                    "expr": "openclaw_lens_queue_depth",
                    "legendFormat": "depth",
                    "refId": "A"
                }
            ],
            "fieldConfig": {
                "defaults": {
                    "unit": "short",
                    "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 20,
                        "spanNulls": true
                    }
                }
            },
            "options": {
                "legend": {
                    "displayMode": "list",
                    "placement": "bottom"
                }
            }
        },
        {
            "id": 10,
            "title": "Enqueue vs Dequeue Rate",
            "description": "Message queue enqueue and dequeue rates. Divergence indicates queue buildup.",
            "type": "timeseries",
            "gridPos": {
                "h": 8,
                "w": 8,
                "x": 8,
                "y": 10
            },
            "datasource": {
                "type": "prometheus",
                "uid": "$prometheus"
            },
            "targets": [
                {
                    "expr": "sum(rate(openclaw_lens_queue_lane_enqueue_total[5m]))",
                    "legendFormat": "enqueue/s",
                    "refId": "A"
                },
                {
                    "expr": "sum(rate(openclaw_lens_queue_lane_dequeue_total[5m]))",
                    "legendFormat": "dequeue/s",
                    "refId": "B"
                }
            ],
            "fieldConfig": {
                "defaults": {
                    "unit": "cps",
                    "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 10,
                        "spanNulls": true
                    }
                },
                "overrides": [
                    {
                        "matcher": {
                            "id": "byName",
                            "options": "enqueue/s"
                        },
                        "properties": [
                            {
                                "id": "color",
                                "value": {
                                    "fixedColor": "blue",
                                    "mode": "fixed"
                                }
                            }
                        ]
                    },
                    {
                        "matcher": {
                            "id": "byName",
                            "options": "dequeue/s"
                        },
                        "properties": [
                            {
                                "id": "color",
                                "value": {
                                    "fixedColor": "green",
                                    "mode": "fixed"
                                }
                            }
                        ]
                    }
                ]
            },
            "options": {
                "legend": {
                    "displayMode": "list",
                    "placement": "bottom"
                }
            }
        },
        {
            "id": 11,
            "title": "Queue Wait Time (p50/p95)",
            "description": "How long messages wait in the queue before processing.",
            "type": "timeseries",
            "gridPos": {
                "h": 8,
                "w": 8,
                "x": 16,
                "y": 10
            },
            "datasource": {
                "type": "prometheus",
                "uid": "$prometheus"
            },
            "targets": [
                {
                    "expr": "histogram_quantile(0.50, sum(rate(openclaw_lens_queue_wait_ms_bucket[5m])) by (le))",
                    "legendFormat": "p50",
                    "refId": "A"
                },
                {
                    "expr": "histogram_quantile(0.95, sum(rate(openclaw_lens_queue_wait_ms_bucket[5m])) by (le))",
                    "legendFormat": "p95",
                    "refId": "B"
                }
            ],
            "fieldConfig": {
                "defaults": {
                    "unit": "ms",
                    "custom": {
                        "drawStyle": "line",
                        "spanNulls": true
                    }
                }
            },
            "options": {
                "legend": {
                    "displayMode": "list",
                    "placement": "bottom"
                }
            }
        },
        {
            "id": 12,
            "title": "Webhooks & Delivery",
            "type": "row",
            "gridPos": {
                "h": 1,
                "w": 24,
                "x": 0,
                "y": 18
            },
            "collapsed": false,
            "panels": []
        },
        {
            "id": 13,
            "title": "Webhook Events",
            "description": "Webhook received and error rates over time.",
            "type": "timeseries",
            "gridPos": {
                "h": 8,
                "w": 12,
                "x": 0,
                "y": 19
            },
            "datasource": {
                "type": "prometheus",
                "uid": "$prometheus"
            },
            "targets": [
                {
                    "expr": "sum(rate(openclaw_lens_webhook_received_total[5m]))",
                    "legendFormat": "received/s",
                    "refId": "A"
                },
                {
                    "expr": "sum(rate(openclaw_lens_webhook_error_total[5m]))",
                    "legendFormat": "errors/s",
                    "refId": "B"
                }
            ],
            "fieldConfig": {
                "defaults": {
                    "unit": "cps",
                    "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 10,
                        "spanNulls": true
                    }
                },
                "overrides": [
                    {
                        "matcher": {
                            "id": "byName",
                            "options": "errors/s"
                        },
                        "properties": [
                            {
                                "id": "color",
                                "value": {
                                    "fixedColor": "red",
                                    "mode": "fixed"
                                }
                            }
                        ]
                    }
                ]
            },
            "options": {
                "legend": {
                    "displayMode": "list",
                    "placement": "bottom"
                }
            }
        },
        {
            "id": 14,
            "title": "Webhook Duration (p50/p95)",
            "description": "Webhook processing time percentiles.",
            "type": "timeseries",
            "gridPos": {
                "h": 8,
                "w": 12,
                "x": 12,
                "y": 19
            },
            "datasource": {
                "type": "prometheus",
                "uid": "$prometheus"
            },
            "targets": [
                {
                    "expr": "histogram_quantile(0.50, sum(rate(openclaw_lens_webhook_duration_ms_bucket[5m])) by (le))",
                    "legendFormat": "p50",
                    "refId": "A"
                },
                {
                    "expr": "histogram_quantile(0.95, sum(rate(openclaw_lens_webhook_duration_ms_bucket[5m])) by (le))",
                    "legendFormat": "p95",
                    "refId": "B"
                }
            ],
            "fieldConfig": {
                "defaults": {
                    "unit": "ms",
                    "custom": {
                        "drawStyle": "line",
                        "spanNulls": true
                    }
                }
            },
            "options": {
                "legend": {
                    "displayMode": "list",
                    "placement": "bottom"
                }
            }
        },
        {
            "id": 15,
            "title": "Session Health",
            "type": "row",
            "gridPos": {
                "h": 1,
                "w": 24,
                "x": 0,
                "y": 27
            },
            "collapsed": false,
            "panels": []
        },
        {
            "id": 16,
            "title": "Session State Changes",
            "description": "Session state transition events. Tracks idle→processing→idle lifecycle.",
            "type": "logs",
            "gridPos": {
                "h": 8,
                "w": 12,
                "x": 0,
                "y": 28
            },
            "datasource": {
                "type": "loki",
                "uid": "$loki"
            },
            "targets": [
                {
                    "expr": "{service_name=\"openclaw\"} | event_name=~\"session\\\\.(start|state)\"",
                    "refId": "A"
                }
            ],
            "options": {
                "showTime": true,
                "showLabels": false,
                "showCommonLabels": false,
                "wrapLogMessage": true,
                "prettifyLogMessage": false,
                "enableLogDetails": true,
                "sortOrder": "Descending",
                "dedupStrategy": "none"
            }
        },
        {
            "id": 17,
            "title": "Stuck Sessions & Tool Loop Alerts",
            "description": "Events for stuck sessions and tool loop detections — should be empty on healthy systems.",
            "type": "logs",
            "gridPos": {
                "h": 8,
                "w": 12,
                "x": 12,
                "y": 28
            },
            "datasource": {
                "type": "loki",
                "uid": "$loki"
            },
            "targets": [
                {
                    "expr": "{service_name=\"openclaw\"} | event_name=~\"session\\\\.stuck|tool\\\\.loop|tool\\\\.repeated_failure\"",
                    "refId": "A"
                }
            ],
            "options": {
                "showTime": true,
                "showLabels": false,
                "showCommonLabels": false,
                "wrapLogMessage": true,
                "prettifyLogMessage": false,
                "enableLogDetails": true,
                "sortOrder": "Descending",
                "dedupStrategy": "none"
            }
        },
        {
            "id": 18,
            "title": "GenAI SRE Signals",
            "type": "row",
            "gridPos": {
                "h": 1,
                "w": 24,
                "x": 0,
                "y": 36
            },
            "collapsed": false,
            "panels": []
        },
        {
            "id": 19,
            "title": "LLM Error Rate (gen_ai standard)",
            "description": "Error rate from gen_ai standard metrics — percentage of LLM calls returning errors. Cross-validate with openclaw-specific error metrics.",
            "type": "timeseries",
            "gridPos": {
                "h": 8,
                "w": 12,
                "x": 0,
                "y": 37
            },
            "datasource": {
                "type": "prometheus",
                "uid": "$prometheus"
            },
            "targets": [
                {
                    "expr": "sum(rate(gen_ai_client_operation_duration_seconds_count{error_type!=\"\"}[5m])) / sum(rate(gen_ai_client_operation_duration_seconds_count[5m])) * 100 or vector(0)",
                    "legendFormat": "gen_ai error %",
                    "refId": "A"
                },
                {
                    "expr": "sum(rate(openclaw_lens_messages_processed_total{outcome=\"error\"}[5m])) / sum(rate(openclaw_lens_messages_processed_total[5m])) * 100 or vector(0)",
                    "legendFormat": "message error %",
                    "refId": "B"
                }
            ],
            "fieldConfig": {
                "defaults": {
                    "unit": "percent",
                    "decimals": 1,
                    "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 10,
                        "spanNulls": true
                    },
                    "thresholds": {
                        "mode": "absolute",
                        "steps": [
                            {
                                "color": "green",
                                "value": null
                            },
                            {
                                "color": "yellow",
                                "value": 1
                            },
                            {
                                "color": "red",
                                "value": 5
                            }
                        ]
                    }
                },
                "overrides": [
                    {
                        "matcher": {
                            "id": "byName",
                            "options": "gen_ai error %"
                        },
                        "properties": [
                            {
                                "id": "color",
                                "value": {
                                    "fixedColor": "red",
                                    "mode": "fixed"
                                }
                            }
                        ]
                    },
                    {
                        "matcher": {
                            "id": "byName",
                            "options": "message error %"
                        },
                        "properties": [
                            {
                                "id": "color",
                                "value": {
                                    "fixedColor": "orange",
                                    "mode": "fixed"
                                }
                            }
                        ]
                    }
                ]
            },
            "options": {
                "legend": {
                    "displayMode": "table",
                    "placement": "bottom",
                    "calcs": [
                        "lastNotNull",
                        "mean",
                        "max"
                    ]
                }
            }
        },
        {
            "id": 20,
            "title": "Context Window Pressure",
            "description": "Context token usage vs limit. When usage approaches the limit, sessions will trigger compaction. Watch for sustained high pressure.",
            "type": "timeseries",
            "gridPos": {
                "h": 8,
                "w": 12,
                "x": 12,
                "y": 37
            },
            "datasource": {
                "type": "prometheus",
                "uid": "$prometheus"
            },
            "targets": [
                {
                    "expr": "openclaw_lens_context_tokens{type=\"used\"}",
                    "legendFormat": "used",
                    "refId": "A"
                },
                {
                    "expr": "openclaw_lens_context_tokens{type=\"limit\"}",
                    "legendFormat": "limit",
                    "refId": "B"
                }
            ],
            "fieldConfig": {
                "defaults": {
                    "unit": "short",
                    "custom": {
                        "drawStyle": "line",
                        "fillOpacity": 10,
                        "spanNulls": true
                    }
                },
                "overrides": [
                    {
                        "matcher": {
                            "id": "byName",
                            "options": "used"
                        },
                        "properties": [
                            {
                                "id": "color",
                                "value": {
                                    "fixedColor": "blue",
                                    "mode": "fixed"
                                }
                            },
                            {
                                "id": "custom.fillOpacity",
                                "value": 20
                            }
                        ]
                    },
                    {
                        "matcher": {
                            "id": "byName",
                            "options": "limit"
                        },
                        "properties": [
                            {
                                "id": "color",
                                "value": {
                                    "fixedColor": "red",
                                    "mode": "fixed"
                                }
                            },
                            {
                                "id": "custom.lineStyle",
                                "value": {
                                    "fill": "dash",
                                    "dash": [
                                        10,
                                        10
                                    ]
                                }
                            },
                            {
                                "id": "custom.fillOpacity",
                                "value": 0
                            },
                            {
                                "id": "custom.lineWidth",
                                "value": 2
                            }
                        ]
                    }
                ]
            },
            "options": {
                "legend": {
                    "displayMode": "table",
                    "placement": "bottom",
                    "calcs": [
                        "lastNotNull",
                        "mean",
                        "max"
                    ]
                }
            }
        }
    ]
}