apiVersion: 1
groups:
  - orgId: 1
    name: automation
    folder: canton-network
    interval: 5m
    rules:
      - uid: fe73c0e7-dcb3-4975-a7d1-04ed8da087be
        title: Automation Failures
        condition: threshold
        data:
          - refId: total
            relativeTimeRange:
              from: 600
              to: 0
            datasourceUid: prometheus
            model:
              editorMode: code
              expr: sum by(namespace, node_type, trigger_name, migration) (delta(splice_trigger_completed_total{trigger_name=~".+", outcome=~".+"}[10m]))
              instant: true
              intervalMs: 1000
              legendFormat: __auto
              maxDataPoints: 43200
              range: false
              refId: total
          - refId: failures
            relativeTimeRange:
              from: 600
              to: 0
            datasourceUid: prometheus
            model:
              datasource:
                type: prometheus
                uid: prometheus
              editorMode: code
              expr: sum by(namespace, node_type, trigger_name, migration) (delta(splice_trigger_completed_total{trigger_name=~".+", outcome=~"failure"}[10m])) or on() vector(0)
              hide: false
              instant: true
              intervalMs: 1000
              legendFormat: __auto
              maxDataPoints: 43200
              range: false
              refId: failures
          - refId: failure_pct
            datasourceUid: __expr__
            model:
              conditions:
                - evaluator:
                    params:
                      - 0
                      - 0
                    type: gt
                  operator:
                    type: and
                  query:
                    params: []
                  reducer:
                    params: []
                    type: avg
                  type: query
              datasource:
                name: Expression
                type: __expr__
                uid: __expr__
              expression: ${failures} / ${total} * 100
              hide: false
              intervalMs: 1000
              maxDataPoints: 43200
              refId: failure_pct
              type: math
          - refId: threshold
            datasourceUid: __expr__
            model:
              conditions:
                - evaluator:
                    params:
                      - 0
                      - 0
                    type: gt
                  operator:
                    type: and
                  query:
                    params: []
                  reducer:
                    params: []
                    type: avg
                  type: query
              datasource:
                name: Expression
                type: __expr__
                uid: __expr__
              expression: failure_pct
              hide: false
              intervalMs: 1000
              maxDataPoints: 43200
              refId: threshold
              type: threshold
        dashboardUid: a3e1385f-6f03-46d9-908c-34aca0f507a6
        panelId: 14
        noDataState: $NODATA
        execErrState: Alerting
        for: 5m
        annotations:
          __dashboardUid__: a3e1385f-6f03-46d9-908c-34aca0f507a6
          __panelId__: "14"
          description: The {{ index $labels "trigger_name" }} for the {{ index $labels "node_type" }} app in the {{ index $labels "namespace" }} namespace on migration id {{ index $labels "migration" }} experienced {{ index $values "failure_pct" }}% failures in the last 10 minutes.
          severity: |-
            {{- if (gt $values.failure_pct.Value 50.0) -}}
            critical
            {{- else -}}
            warning
            {{- end -}}
          summary: '{{ index $values "failure_pct" }}% fatal errors occurred in {{ index $labels "namespace" }} - {{ index $labels "trigger_name" }} automation trigger'
        labels:
          gcloud_filter: 'resource.labels.namespace_name=%22{{ index $labels "namespace" }}%22%0A%22{{ index $labels "trigger_name" }}%22'
        isPaused: false
      - uid: adwt1yr5xuscge
        title: ACS snapshot taking too long
        condition: too_long
        data:
          - refId: latency
            relativeTimeRange:
              from: 600
              to: 0
            datasourceUid: prometheus
            model:
              editorMode: code
              expr: histogram_quantile(0.99, rate(splice_trigger_latency_duration_seconds{trigger_name="AcsSnapshotTrigger", namespace="sv-1"}[10m]))
              instant: true
              intervalMs: 1000
              legendFormat: __auto
              maxDataPoints: 43200
              range: false
              refId: latency
          - refId: too_long
            relativeTimeRange:
              from: 600
              to: 0
            datasourceUid: __expr__
            model:
              conditions:
                - evaluator:
                    params:
                      - 600
                      - 0
                    type: gt
                  operator:
                    type: and
                  query:
                    params: [ ]
                  reducer:
                    params: [ ]
                    type: avg
                  type: query
              datasource:
                name: Expression
                type: __expr__
                uid: __expr__
              expression: latency
              intervalMs: 1000
              maxDataPoints: 43200
              refId: too_long
              type: threshold
        noDataState: OK
        execErrState: Alerting
        for: 5m
        annotations:
          description: ""
          runbook_url: ""
          severity: warning
          summary: ACS snapshot took longer than 10m in {{ index $labels "namespace" }}'s Scan
        labels:
          "": ""
          gcloud_filter: resource.labels.namespace_name=%22{{ index  "namespace" }}%22%0A%22{{ index  "trigger_name" }}%22
        isPaused: false
      - uid: ady2ks9ehbw1sb
        title: Busy task-based automation
        condition: threshold
        data:
          - refId: runs
            relativeTimeRange:
              from: 300
              to: 0
            datasourceUid: prometheus
            model:
              editorMode: code
              expr: sum by(namespace, node_type, node_name, job, trigger_name, migration, party) (rate(splice_trigger_completed_total{trigger_name!~"ScanVerdictStoreIngestion|ScanHistoryBackfillingTrigger|AcsSnapshotTrigger|ScanBackfillAggregatesTrigger|TxLogBackfillingTrigger|ReportValidatorLicenseMetricsExportTrigger"}[5m]))
              instant: true
              intervalMs: 1000
              legendFormat: __auto
              maxDataPoints: 43200
              range: false
              refId: runs
          - refId: threshold
            relativeTimeRange:
              from: 300
              to: 0
            datasourceUid: __expr__
            model:
              conditions:
                  - evaluator:
                      params:
                          - 1
                          - 0
                      type: gt
                    operator:
                      type: and
                    query:
                      params: []
                    reducer:
                      params: []
                      type: avg
                    type: query
              datasource:
                  name: Expression
                  type: __expr__
                  uid: __expr__
              expression: runs
              intervalMs: 1000
              maxDataPoints: 43200
              refId: threshold
              type: threshold
        dashboardUid: a3e1385f-6f03-46d9-908c-34aca0f507a6
        panelId: 14
        noDataState: OK
        execErrState: OK
        for: 5m
        annotations:
          __dashboardUid__: a3e1385f-6f03-46d9-908c-34aca0f507a6
          __panelId__: "14"
          description: The {{ index $labels "trigger_name" }} for the {{ index $labels "node_type" }} app in the {{ index $labels "namespace" }} namespace on migration id {{ index $labels "migration" }} experienced {{ index $values "runs" }} runs per second in the last 5 minutes.
          runbook_url: ""
          severity: |-
              {{- if and $values.runs (gt $values.runs.Value 2.0) -}}
              critical
              {{- else -}}
              warning
              {{- end -}}
          summary: '{{ index $values "runs" }} trigger runs per second occurred in {{ index $labels "namespace" }} - {{ index $labels "trigger_name" }} automation trigger'
        labels:
          "": ""
          gcloud_filter: resource.labels.namespace_name=%22{{ index  "namespace" }}%22%0A%22{{ index  "trigger_name" }}%22
        isPaused: false
      - uid: edz6eq1kc543ke
        title: Busy polling-based automation
        condition: threshold
        data:
          - refId: runs
            relativeTimeRange:
              from: 300
              to: 0
            datasourceUid: prometheus
            model:
              editorMode: code
              expr: sum by(namespace, node_type, node_name, job, trigger_name, migration, party) (rate(splice_trigger_iterations_total{trigger_name!~"ScanVerdictStoreIngestion|ScanHistoryBackfillingTrigger|AcsSnapshotTrigger|ScanBackfillAggregatesTrigger|TxLogBackfillingTrigger"}[5m]))
              instant: true
              intervalMs: 1000
              legendFormat: __auto
              maxDataPoints: 43200
              range: false
              refId: runs
          - refId: threshold
            relativeTimeRange:
              from: 300
              to: 0
            datasourceUid: __expr__
            model:
              conditions:
                  - evaluator:
                      params:
                          - 1
                          - 0
                      type: gt
                    operator:
                      type: and
                    query:
                      params: []
                    reducer:
                      params: []
                      type: avg
                    type: query
              datasource:
                  name: Expression
                  type: __expr__
                  uid: __expr__
              expression: runs
              intervalMs: 1000
              maxDataPoints: 43200
              refId: threshold
              type: threshold
        dashboardUid: a3e1385f-6f03-46d9-908c-34aca0f507a6
        panelId: 14
        noDataState: OK
        execErrState: OK
        for: 5m
        annotations:
          __dashboardUid__: a3e1385f-6f03-46d9-908c-34aca0f507a6
          __panelId__: "14"
          description: The {{ index $labels "trigger_name" }} for the {{ index $labels "node_type" }} app in the {{ index $labels "namespace" }} namespace on migration id {{ index $labels "migration" }} experienced {{ index $values "runs" }} runs per second in the last 5 minutes.
          runbook_url: ""
          severity: |-
              {{- if and $values.runs (gt $values.runs.Value 2.0) -}}
              critical
              {{- else -}}
              warning
              {{- end -}}
          summary: '{{ index $values "runs" }} trigger runs per second occurred in {{ index $labels "namespace" }} - {{ index $labels "trigger_name" }} automation trigger'
        labels:
          "": ""
          gcloud_filter: resource.labels.namespace_name=%22{{ index  "namespace" }}%22%0A%22{{ index  "trigger_name" }}%22
        isPaused: false
      - uid: fe12i7xur3eo0d
        title: Backfilling not progressing
        condition: C
        data:
          - refId: Max (rate vs completed)
            relativeTimeRange:
              from: 600
              to: 0
            datasourceUid: prometheus
            model:
              editorMode: code
              expr: max by(namespace)(rate(splice_history_backfilling_transaction_count[5m]) > 0 or splice_history_backfilling_completed)
              instant: true
              intervalMs: 1000
              legendFormat: __auto
              maxDataPoints: 43200
              range: false
              refId: Max (rate vs completed)
          - refId: C
            relativeTimeRange:
              from: 600
              to: 0
            datasourceUid: __expr__
            model:
              conditions:
                - evaluator:
                    params:
                      - 1e-11
                    type: lt
                  operator:
                    type: and
                  query:
                    params:
                      - C
                  reducer:
                    params: [ ]
                    type: last
                  type: query
              datasource:
                type: __expr__
                uid: __expr__
              expression: Max (rate vs completed)
              intervalMs: 1000
              maxDataPoints: 43200
              refId: C
              type: threshold
        noDataState: $NODATA
        execErrState: Alerting
        for: 5m
        annotations:
          description: ""
          runbook_url: ""
          summary: History backfilling is not making any progress in {{ index $labels "namespace" }}
        labels:
          "": ""
        isPaused: false
      - uid: bel66uf182ha8e
        title: TxLog backfilling not progressing
        condition: C
        data:
          - refId: Max (rate vs completed)
            relativeTimeRange:
              from: 600
              to: 0
            datasourceUid: prometheus
            model:
              editorMode: code
              expr: |-
                max by(namespace)(
                  rate(splice_history_txlog_backfilling_transaction_count[5m]) > 0 or
                  splice_history_txlog_backfilling_completed or
                  # TxLog backfilling can't start until history backfilling is completed
                  absent(splice_history_backfilling_completed) or
                  # Note: "== bool" is a boolean operator, "==" is a filter
                  (splice_history_backfilling_completed == bool 0)
                )
              instant: true
              intervalMs: 1000
              legendFormat: __auto
              maxDataPoints: 43200
              range: false
              refId: Max (rate vs completed)
          - refId: C
            relativeTimeRange:
              from: 600
              to: 0
            datasourceUid: __expr__
            model:
              conditions:
                - evaluator:
                    params:
                      - 1e-11
                    type: lt
                  operator:
                    type: and
                  query:
                    params:
                      - C
                  reducer:
                    params: [ ]
                    type: last
                  type: query
              datasource:
                type: __expr__
                uid: __expr__
              expression: Max (rate vs completed)
              intervalMs: 1000
              maxDataPoints: 43200
              refId: C
              type: threshold
        noDataState: $NODATA
        execErrState: Alerting
        for: 5m
        annotations:
          description: ""
          runbook_url: ""
          summary: TxLog backfilling is not making any progress in {{ index $labels "namespace" }}
        labels:
          "": ""
        isPaused: false
      - uid: fentlrcbcrsaoa
        title: Delegateless trigger contention
        condition: threshold
        data:
          - refId: total_attempts
            relativeTimeRange:
              from: 600
              to: 0
            datasourceUid: prometheus
            model:
              editorMode: code
              expr: sum(rate(splice_trigger_attempted_total{isDsoDelegateTrigger="true", node_type="sv"}[30m])) by (namespace)
              instant: true
              intervalMs: 1000
              legendFormat: __auto
              maxDataPoints: 43200
              range: false
              refId: total_attempts
          - refId: total_contention
            relativeTimeRange:
              from: 600
              to: 0
            datasourceUid: prometheus
            model:
              datasource:
                type: prometheus
                uid: prometheus
              editorMode: code
              expr: sum(rate(splice_trigger_attempted_total{statusCode!~"OK", isDsoDelegateTrigger="true", node_type="sv", contentionFailure="true"}[30m])) by (namespace)
              instant: true
              intervalMs: 1000
              legendFormat: __auto
              maxDataPoints: 43200
              range: false
              refId: total_contention
          - refId: threshold
            datasourceUid: __expr__
            model:
              conditions:
                - evaluator:
                    params:
                      - 0
                      - 0
                    type: gt
                  operator:
                    type: and
                  query:
                    params: [ ]
                  reducer:
                    params: [ ]
                    type: avg
                  type: query
              datasource:
                name: Expression
                type: __expr__
                uid: __expr__
              expression: (${total_contention} > $CONTENTION_THRESHOLD_PERCENTAGE_PER_NAMESPACE * ${total_attempts}) && (${total_contention} > 0)
              intervalMs: 1000
              maxDataPoints: 43200
              refId: threshold
              type: math
        # Alert also reports noData when there is no contention.
        noDataState: OK
        execErrState: Alerting
        for: 5m
        annotations:
          description: '{{ index $labels "namespace" }} had more than 5% contention in the last 30 minutes.'
        isPaused: false
      - uid: df3t10f23cm68c
        title: 'Ingestion saturation'
        condition: C
        data:
          - refId: A
            relativeTimeRange:
              from: 600
              to: 0
            datasourceUid: prometheus
            model:
              editorMode: code
              expr: histogram_avg(sum by (le, namespace,job,store_name,store_party,synchronizer_id) (rate(splice_store_ingestion_batch_size{namespace=~"sv"}[20m])))
              instant: true
              intervalMs: 1000
              legendFormat: __auto
              maxDataPoints: 43200
              range: false
              refId: A
          - refId: C
            datasourceUid: __expr__
            model:
              conditions:
                - evaluator:
                    params:
                      - $INGESTION_ENTRIES_PER_BATCH_THRESHOLD
                    type: gt
                  operator:
                    type: and
                  query:
                    params:
                      - C
                  reducer:
                    params: [ ]
                    type: last
                  type: query
              datasource:
                type: __expr__
                uid: __expr__
              expression: A
              intervalMs: 1000
              maxDataPoints: 43200
              refId: C
              type: threshold
        noDataState: OK
        execErrState: OK
        for: 30m
        annotations:
          description: |
            The ingestion batch size has exceeded $INGESTION_ENTRIES_PER_BATCH_THRESHOLD entries/batch for a prolonged time in {{ index $labels "namespace" }}'s {{ index $labels "node_type" }}.
            This might mean that ingestion is close to not being able to keep up with ledger activity.
            This can also happen when an SV was just onboarded and is catching up with the ACS.
          summary: The ingestion batch size has exceeded the expected size for a prolonged time.
        isPaused: false