lots of stuff
This commit is contained in:
@@ -0,0 +1,242 @@
|
||||
alert_groups:
|
||||
- name: backup
|
||||
interval: 5m
|
||||
rules:
|
||||
- uid: backup-daily-stale
|
||||
no_data: Alerting
|
||||
title: Daily backup stale
|
||||
expr: '(time() - max(backup_last_success_timestamp_seconds{backup_job="daily"})) / 3600'
|
||||
op: gt
|
||||
threshold: 26
|
||||
for: 15m
|
||||
severity: critical
|
||||
summary: No successful daily backup for more than 26 hours.
|
||||
|
||||
- uid: backup-weekly-stale
|
||||
no_data: Alerting
|
||||
title: Weekly backup stale
|
||||
expr: '(time() - max(backup_last_success_timestamp_seconds{backup_job="weekly"})) / 3600'
|
||||
op: gt
|
||||
threshold: 192
|
||||
for: 30m
|
||||
severity: critical
|
||||
summary: No successful weekly backup for more than 8 days.
|
||||
|
||||
- uid: backup-restoretest-stale
|
||||
no_data: Alerting
|
||||
title: Restore test stale
|
||||
expr: '(time() - max(backup_last_success_timestamp_seconds{backup_job="restoretest"})) / 3600'
|
||||
op: gt
|
||||
threshold: 192
|
||||
for: 30m
|
||||
severity: warning
|
||||
summary: The vaultwarden restore test has not passed in more than 8 days, backups are unverified.
|
||||
|
||||
- uid: backup-offsite-stale
|
||||
no_data: Alerting
|
||||
title: Offsite sync stale
|
||||
expr: '(time() - max(backup_last_success_timestamp_seconds{backup_job="offsite"})) / 3600'
|
||||
op: gt
|
||||
threshold: 26
|
||||
for: 15m
|
||||
severity: critical
|
||||
summary: Backups have not reached the Hetzner storage box in more than 26 hours.
|
||||
|
||||
- uid: backup-job-failed
|
||||
no_data: Alerting
|
||||
title: Backup job failed
|
||||
expr: 'min by (backup_job) (backup_last_status)'
|
||||
op: lt
|
||||
threshold: 1
|
||||
for: 5m
|
||||
severity: critical
|
||||
summary: The last run of this backup job exited with a failure.
|
||||
|
||||
- uid: backup-archive-shrunk
|
||||
title: Backup archive shrunk
|
||||
expr: 'backup_archive_newest_bytes{kind="daily"} / avg_over_time(backup_archive_newest_bytes{kind="daily"}[7d])'
|
||||
op: lt
|
||||
threshold: 0.5
|
||||
for: 1h
|
||||
severity: warning
|
||||
summary: The newest daily archive is less than half the size of the weekly average, data may be missing.
|
||||
|
||||
- uid: backup-raid-unmounted
|
||||
no_data: Alerting
|
||||
title: Backup RAID unmounted
|
||||
expr: 'min(backup_raid_mounted)'
|
||||
op: lt
|
||||
threshold: 1
|
||||
for: 5m
|
||||
severity: critical
|
||||
summary: The backup RAID is not mounted, backups cannot be written.
|
||||
|
||||
- uid: backup-raid-space
|
||||
title: Backup RAID low on space
|
||||
expr: 'min(node_filesystem_avail_bytes{mountpoint="/home/beeshare/raid"} / node_filesystem_size_bytes{mountpoint="/home/beeshare/raid"}) * 100'
|
||||
op: lt
|
||||
threshold: 15
|
||||
for: 30m
|
||||
severity: warning
|
||||
summary: Less than 15 percent free space left on the backup RAID.
|
||||
|
||||
- uid: offsite-empty
|
||||
no_data: Alerting
|
||||
title: Offsite storage empty
|
||||
expr: 'max(hetzner_remote_files{path="beepi"})'
|
||||
op: lt
|
||||
threshold: 1
|
||||
for: 30m
|
||||
severity: critical
|
||||
summary: The Hetzner storage box holds no completed backup files, offsite recovery is not possible.
|
||||
|
||||
- uid: offsite-unreachable
|
||||
no_data: Alerting
|
||||
title: Offsite storage unreachable
|
||||
expr: 'max(hetzner_reachable)'
|
||||
op: lt
|
||||
threshold: 1
|
||||
for: 2h
|
||||
severity: warning
|
||||
summary: The Hetzner storage box could not be listed, check external reachability on the box.
|
||||
|
||||
- uid: offsite-content-stale
|
||||
title: Offsite content stale
|
||||
expr: 'min(hetzner_remote_newest_age_seconds{path="beepi"}) / 3600'
|
||||
op: gt
|
||||
threshold: 48
|
||||
for: 1h
|
||||
severity: critical
|
||||
summary: The newest backup on the Hetzner storage box is more than 48 hours old.
|
||||
|
||||
- uid: backup-log-errors
|
||||
title: Backup log errors
|
||||
no_data: OK
|
||||
source: loki
|
||||
expr: 'sum(count_over_time({job="backup"} |~ "(?i)(FAILED|verify failed|refusing to sync)" [15m]))'
|
||||
op: gt
|
||||
threshold: 0
|
||||
for: 0s
|
||||
severity: critical
|
||||
summary: The backup log reported a failure in the last 15 minutes.
|
||||
|
||||
- name: services
|
||||
interval: 1m
|
||||
rules:
|
||||
- uid: service-probe-down
|
||||
title: Service probe failing
|
||||
expr: 'min by (service) (probe_success{vantage="local"}) and on (service) (max by (service) (service_enabled) == 1)'
|
||||
op: lt
|
||||
threshold: 1
|
||||
for: 5m
|
||||
severity: critical
|
||||
summary: A service that should be running is not answering its health probe.
|
||||
|
||||
- uid: service-unit-down
|
||||
title: Service unit not running
|
||||
expr: 'min by (service) (service_units_active / service_units_total) and on (service) (max by (service) (service_enabled) == 1)'
|
||||
op: lt
|
||||
threshold: 1
|
||||
for: 5m
|
||||
severity: critical
|
||||
summary: A unit or container backing an enabled service is not running.
|
||||
|
||||
- uid: service-vhost-down
|
||||
title: Public vhost failing
|
||||
expr: 'min by (service) (probe_success{vantage="nginx"}) and on (service) (max by (service) (service_enabled) == 1)'
|
||||
op: lt
|
||||
threshold: 1
|
||||
for: 5m
|
||||
severity: critical
|
||||
summary: An enabled service is not reachable through nginx and TLS.
|
||||
|
||||
- uid: external-check-down
|
||||
title: External check failing
|
||||
expr: 'min by (monitor) (uptimerobot_monitor_up) and on (monitor) (max by (monitor) (uptimerobot_monitor_enabled) == 1)'
|
||||
op: lt
|
||||
threshold: 1
|
||||
for: 5m
|
||||
severity: critical
|
||||
summary: UptimeRobot reports this endpoint as unreachable from outside the network.
|
||||
|
||||
- uid: service-cert-expiry
|
||||
title: TLS certificate expiring
|
||||
expr: 'min by (instance) (probe_ssl_earliest_cert_expiry - time()) / 86400'
|
||||
op: lt
|
||||
threshold: 14
|
||||
for: 1h
|
||||
severity: warning
|
||||
summary: A TLS certificate expires in less than 14 days.
|
||||
|
||||
- uid: scrape-target-down
|
||||
title: Scrape target down
|
||||
expr: 'min by (job, instance) (up{host!~"{{ alert_excluded_hosts }}"})'
|
||||
op: lt
|
||||
threshold: 1
|
||||
for: 10m
|
||||
severity: warning
|
||||
summary: Prometheus cannot scrape this target.
|
||||
|
||||
- name: infra
|
||||
interval: 1m
|
||||
rules:
|
||||
- uid: ingress-down
|
||||
title: Ingress host down
|
||||
expr: 'min(up{job="node",host="ingress"})'
|
||||
op: lt
|
||||
threshold: 1
|
||||
for: 5m
|
||||
severity: critical
|
||||
summary: The ingress VPS is not answering scrapes, inbound traffic is likely broken.
|
||||
|
||||
- uid: tunnel-peer-down
|
||||
title: WireGuard tunnel down
|
||||
expr: 'min by (peer) (wireguard_peer_up{peer=~"beepi|ingress"})'
|
||||
op: lt
|
||||
threshold: 1
|
||||
for: 5m
|
||||
severity: critical
|
||||
summary: The site to site WireGuard peer has not handshaked in the last five minutes.
|
||||
|
||||
- uid: tunnel-probe-down
|
||||
title: Tunnel path failing
|
||||
expr: 'min by (probe) (probe_success{vantage="ingress"})'
|
||||
op: lt
|
||||
threshold: 1
|
||||
for: 5m
|
||||
severity: critical
|
||||
summary: The ingress host cannot reach this port on beepi through the tunnel.
|
||||
|
||||
- uid: host-disk-space
|
||||
title: Host low on disk
|
||||
expr: >-
|
||||
min by (host, device)
|
||||
(node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|ramfs",host!~"{{ alert_excluded_hosts }}"}
|
||||
/ node_filesystem_size_bytes{fstype!~"tmpfs|overlay|ramfs",host!~"{{ alert_excluded_hosts }}"}) * 100
|
||||
op: lt
|
||||
threshold: 10
|
||||
for: 30m
|
||||
severity: warning
|
||||
summary: Less than 10 percent free space left on this filesystem.
|
||||
|
||||
- uid: host-memory
|
||||
title: Host low on memory
|
||||
expr: >-
|
||||
min by (host) (node_memory_MemAvailable_bytes{host!~"{{ alert_excluded_hosts }}"}
|
||||
/ node_memory_MemTotal_bytes{host!~"{{ alert_excluded_hosts }}"}) * 100
|
||||
op: lt
|
||||
threshold: 10
|
||||
for: 15m
|
||||
severity: warning
|
||||
summary: Less than 10 percent available memory on this host.
|
||||
|
||||
- uid: host-systemd-failed
|
||||
title: Failed systemd units
|
||||
expr: >-
|
||||
sum by (host) (node_systemd_unit_state{state="failed",
|
||||
host!~"{{ alert_excluded_hosts }}", name!~"{{ alert_ignored_units }}"})
|
||||
op: gt
|
||||
threshold: 0
|
||||
for: 15m
|
||||
severity: warning
|
||||
summary: A systemd unit outside the known-broken list is in a failed state on this host.
|
||||
Reference in New Issue
Block a user