lukegb/depot
1
0
Fork 0
Code Issues 1 Pull requests Projects Packages Activity Actions

totoro: add alert for BFD session failure

Browse source
This commit is contained in:
Luke Granger-Brown 2021-09-02 18:35:18 +00:00
parent 3abe727604
commit 683e6ffc21
1 changed files with 11 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
ops/nixos/totoro/default.nix
View file

@ -300,7 +300,7 @@ in {
severity: page severity: page
annotations: annotations:
summary: "Average packet loss from {{ $labels.system }} high" summary: "Average packet loss from {{ $labels.system }} high"
description: "The average packet loss from {{ $labels.system }} is {{ $value | humanize }}%, which is too high." description: "The average packet loss from {{ $labels.system }} is {{ $value | humanizePercentage }}%, which is too high."
- alert: SmokepingPacketLossVeryHigh - alert: SmokepingPacketLossVeryHigh
expr: ((rate(smokeping_requests_total{host=~"(([a-z0-9]+.)+[a-z]+|([0-9]+.){3}[0-9]+)"}[5m]) - rate(smokeping_response_duration_seconds_count[5m])) / rate(smokeping_requests_total[5m])) >= 0.10 expr: ((rate(smokeping_requests_total{host=~"(([a-z0-9]+.)+[a-z]+|([0-9]+.){3}[0-9]+)"}[5m]) - rate(smokeping_response_duration_seconds_count[5m])) / rate(smokeping_requests_total[5m])) >= 0.10
for: 10m for: 10m
@ -308,7 +308,7 @@ in {
severity: page severity: page
annotations: annotations:
summary: "Packet loss to {{ $labels.host }} from {{ $labels.system }} high" summary: "Packet loss to {{ $labels.host }} from {{ $labels.system }} high"
description: "The packet loss from {{ $labels.system }} to {{ $labels.host }} (IP: {{ $labels.ip }}) is very high ({{ $value | humanize }}%)." description: "The packet loss from {{ $labels.system }} to {{ $labels.host }} (IP: {{ $labels.ip }}) is very high ({{ $value | humanizePercentage }}%)."
# Ping latency # Ping latency
- alert: Smokeping95LatencyHigh - alert: Smokeping95LatencyHigh
@ -319,6 +319,15 @@ in {
annotations: annotations:
summary: "Ping latency from {{ $labels.system }} to {{ $labels.host }} high" summary: "Ping latency from {{ $labels.system }} to {{ $labels.host }} high"
description: "The 95th-percentile ping latency from {{ $labels.system }} to {{ $labels.host }} is {{ $value }}." description: "The 95th-percentile ping latency from {{ $labels.system }} to {{ $labels.host }} is {{ $value }}."
# Internet connectivity
- alert: MaldenRoadInternetConnectivityFailure
expr: sum(bird_bfd_session_state{state="Up"} * on(instance,name,neighbor_address,system) group_left(device) bird_bfd_session_device) by (instance,neighbor_address,device,state,system) < 1
labels:
severity: page
annotations:
summary: "Device {{ $labels.device }} on {{ $labels.system }} reports BFD down to neighbour {{ $labels.neighbor_address }}"
description: "Ruh roh, Raggy"
'' ''
]; ];