lots of stuff

.vscode/tasks.json

@@ -10,6 +10,36 @@
         "command": "ansible-vault encrypt_string ${input:pw} --name ${input:name} --vault-password-file .vault-password",
         "problemMatcher": []
     },
+    {
+        "label": "Show vault secrets in current file",
+        "type": "shell",
+        "options": {
+            "cwd": "${workspaceFolder}"
+        },
+        "command": "python3 vaultShow.py ${relativeFile}",
+        "presentation": {
+            "reveal": "always",
+            "panel": "dedicated",
+            "echo": false,
+            "clear": true
+        },
+        "problemMatcher": []
+    },
+    {
+        "label": "Show one vault secret in current file",
+        "type": "shell",
+        "options": {
+            "cwd": "${workspaceFolder}"
+        },
+        "command": "python3 vaultShow.py ${relativeFile} ${input:varName}",
+        "presentation": {
+            "reveal": "always",
+            "panel": "dedicated",
+            "echo": false,
+            "clear": true
+        },
+        "problemMatcher": []
+    },
     {
         "label": "Deploy: nginx",
         "type": "shell",
@@ -234,6 +264,11 @@
         "id": "name",
         "type": "promptString",
         "description": "Name for the string"
+    },
+    {
+        "id": "varName",
+        "type": "promptString",
+        "description": "Variable name to decrypt"
     }
   ]
 }

group_vars/all.yml

@@ -0,0 +1,198 @@
+wireguard_ingress_ip: 10.10.1.1
+wireguard_beepi_ip: 10.10.1.2
+
+node_exporter_textfile_dir: /var/lib/node_exporter/textfile
+
+geoip_max_age_seconds: 86400
+
+alert_excluded_hosts: beeserver
+
+alert_ignored_units: exim4.service|openipmi.service|smartmontools.service|wg-quick@wg0.service
+
+discord_webhook: !vault |
+          $ANSIBLE_VAULT;1.1;AES256
+          66663162633962616438306339616364343634633030663665393934323831303162326632613637
+          3531373835656236326436623036636665356433303762350a613366376231353362353961393566
+          36643136616537666632343463333334636136383232346233353963303334616261613265396632
+          3835316435393935350a373265366239393435373734396538323965326366343832373561366539
+          64613966326233303336653738386161376331373962623764633261353462636231666439643735
+          35333564323831626430373030313861396132363336326639303962326661386533383661333264
+          34663435646434316431343936343131326462646632393833623132666335386134373037323433
+          33373738303162386430616465383766363266333765313533373161376262306532346265303062
+          30393164313431613734323832626637346537656334623232373531633139653064666132373863
+          3762666465303665303666396537323062396139313764353263
+
+uptimerobot_api_key: !vault |
+          $ANSIBLE_VAULT;1.1;AES256
+          62643666393833356630356130373262336130336135393062626130343530363430616532303338
+          6265393131663863323832616334353835386332373037640a333164303034656664643330626532
+          66643238393436653665346538343339346435643934333830636136306565363537336364363038
+          3131653631623961340a373263323362323533353638313361353566383039613662306434366633
+          62386430663466646430373735636132356164393265666634326535333265616233346530663063
+          3935356539346134303038333632373163316438633136396466
+
+grafana_prometheus_uid: ffkc0x7oj7g1sc
+grafana_loki_uid: bfkfq6kdpznr4e
+grafana_infinity_uid: beepitopology
+grafana_uptimerobot_uid: beepiuptimerobot
+grafana_folder: Beepi
+grafana_folder_uid: efva4v6nh301sd
+grafana_general_folder: General Dashboards
+grafana_general_folder_uid: bfkfzgwdpkqgwc
+
+wireguard_peer_names:
+  r0XaxIE3AVud9Z42UPZF3mwoh2R0B0M1EUDY9cvuyDA=: ingress
+  oI69uWkbkwtpuIYxDNGmzg4nRjY1BIZFVFjZwguIvwE=: beepi
+  Jq1FhThPekHlc69t5UcugV9e7Z/Es9lpPR9Ttc0GICk=: phone
+
+monitored_services:
+  - name: gitea
+    title: Gitea
+    group: apps
+    enabled: true
+    owner: gitea
+    units:
+      - gitea-db.service
+      - gitea-server.service
+    vhost: git.secretbee.buzz
+    probe: http://127.0.0.1:3000/api/healthz
+    tcp:
+      - name: gitea-ssh
+        target: 127.0.0.1:2222
+
+  - name: immich
+    title: Immich
+    group: apps
+    enabled: true
+    owner: immich
+    units:
+      - immich-db.service
+      - immich-redis.service
+      - immich-server.service
+    vhost: gallery.secretbee.buzz
+    probe: http://127.0.0.1:2283/api/server/ping
+
+  - name: vaultwarden
+    title: Vaultwarden
+    group: apps
+    enabled: true
+    owner: vaultwarden
+    units:
+      - vaultwarden.service
+    vhost: vault.secretbee.buzz
+    probe: http://127.0.0.1:8222/alive
+
+  - name: ott
+    title: OpenTogetherTube
+    group: apps
+    enabled: false
+    owner: ott
+    units:
+      - ott-db.service
+      - ott-redis.service
+      - ott-server.service
+    vhost: ott.secretbee.buzz
+    probe: http://127.0.0.1:8080/
+
+  - name: matrix
+    title: Matrix
+    group: apps
+    enabled: true
+    containers:
+      - matrix_homeserver_1
+    vhost: matrix.secretbee.buzz
+    probe: http://127.0.0.1:8008/_matrix/client/versions
+    tcp:
+      - name: matrix-federation
+        target: 127.0.0.1:8448
+
+  - name: mail
+    title: Mail
+    group: apps
+    enabled: true
+    containers:
+      - mailserver
+    tcp:
+      - name: smtp
+        target: 127.0.0.1:25
+      - name: submission
+        target: 127.0.0.1:587
+      - name: smtps
+        target: 127.0.0.1:465
+      - name: imaps
+        target: 127.0.0.1:993
+
+  - name: nginx
+    title: Nginx
+    group: platform
+    enabled: true
+    system_units:
+      - nginx.service
+    vhost: secretbee.buzz
+
+  - name: prometheus
+    title: Prometheus
+    group: platform
+    enabled: true
+    containers:
+      - prometheus
+    probe: http://127.0.0.1:9090/-/healthy
+
+  - name: grafana
+    title: Grafana
+    group: platform
+    enabled: true
+    containers:
+      - grafana
+    vhost: grafana.secretbee.buzz
+    probe: http://127.0.0.1:4000/api/health
+
+  - name: loki
+    title: Loki
+    group: platform
+    enabled: true
+    containers:
+      - loki
+    probe: http://127.0.0.1:3100/ready
+
+  - name: alloy
+    title: Alloy
+    group: platform
+    enabled: true
+    containers:
+      - alloy
+    probe: http://127.0.0.1:12345/-/ready
+
+ingress_probe_targets:
+  - name: nginx-http
+    target: 10.10.1.2:80
+  - name: nginx-https
+    target: 10.10.1.2:443
+  - name: matrix-federation
+    target: 10.10.1.2:8448
+  - name: smtp
+    target: 10.10.1.2:25
+  - name: backup-ssh
+    target: 10.10.1.2:4444
+
+backup_jobs:
+  - name: daily
+    title: Daily backup
+    max_age_hours: 26
+  - name: weekly
+    title: Weekly backup
+    max_age_hours: 192
+  - name: restoretest
+    title: Restore test
+    max_age_hours: 192
+  - name: offsite
+    title: Offsite sync
+    max_age_hours: 26
+
+hetzner_host: u651287.your-storagebox.de
+hetzner_user: u651287
+hetzner_port: 23
+hetzner_box_bytes: 1099511627776
+
+backup_archive_dir: /home/beeshare/raid/serverBackup
+backup_raid_mount: /home/beeshare/raid

makefile

@@ -8,4 +8,7 @@ push-%: build-%
 	podman push $(REGISTRY)/bee/$*:latest
 
 deploy-%:
-	ansible-playbook -i $(INVENTORY) playbooks/$*/$*.yml --vault-password-file .vault-password $(ARGS)
+	ansible-playbook -i $(INVENTORY) playbooks/$*/$*.yml --vault-password-file .vault-password $(ARGS)
+
+secrets-%:
+	@python3 vaultShow.py playbooks/$*/$*.yml $(ARGS)

playbooks/backup/backup.yml

@@ -41,12 +41,72 @@
         dest: /opt/backup/restoreTest.sh
         mode: '0700'
 
+    - name: Copy metrics library
+      ansible.builtin.template:
+        src: metrics.sh.j2
+        dest: /opt/backup/metrics.sh
+        mode: '0700'
+
+    - name: Copy inventory collector
+      ansible.builtin.template:
+        src: backupInventory.sh.j2
+        dest: /opt/backup/backupInventory.sh
+        mode: '0700'
+
     - name: Copy passphrase
       ansible.builtin.template:
         src: passphrase.txt.j2
         dest: /opt/backup/passphrase.txt
         mode: '0600'
 
+    - name: Create textfile directory
+      ansible.builtin.file:
+        path: "{{ node_exporter_textfile_dir }}"
+        state: directory
+        mode: '0755'
+
+    - name: Install inventory timer
+      ansible.builtin.copy:
+        dest: "/etc/systemd/system/beepi-backup-inventory.{{ item.kind }}"
+        content: "{{ item.content }}"
+        mode: '0644'
+      loop:
+        - kind: service
+          content: |
+            [Unit]
+            Description=Collect backup inventory metrics
+
+            [Service]
+            Type=oneshot
+            ExecStart=/opt/backup/backupInventory.sh
+        - kind: timer
+          content: |
+            [Unit]
+            Description=Collect backup inventory metrics
+
+            [Timer]
+            OnBootSec=2min
+            OnUnitActiveSec=15min
+            AccuracySec=1min
+
+            [Install]
+            WantedBy=timers.target
+      notify: Reload systemd
+
+    - name: Flush handlers
+      ansible.builtin.meta: flush_handlers
+
+    - name: Enable inventory timer
+      ansible.builtin.systemd_service:
+        name: beepi-backup-inventory.timer
+        state: started
+        enabled: true
+
+    - name: Run inventory collector once
+      changed_when: true
+      ansible.builtin.command:
+        cmd: /opt/backup/backupInventory.sh
+
     - name: Set path
       ansible.builtin.cron:
         name: PATH
@@ -78,3 +138,8 @@
         hour: '6'
         weekday: '0'
         job: /opt/backup/restoreTest.sh >> /var/log/beepi-backup.log 2>&1
+
+  handlers:
+    - name: Reload systemd
+      ansible.builtin.systemd_service:
+        daemon_reload: true

playbooks/backup/backupDaily.sh

@@ -2,6 +2,8 @@
 
 set -euo pipefail
 
+source /opt/backup/metrics.sh
+
 now=$(date +"%Y%m%d%H%M")
 
 declare -a arr=("mail" "matrix" "monitoring")
@@ -12,6 +14,8 @@ declare -a immich=("systemd-immich-pgdata" "systemd-immich-upload")
 gitea_uid=$(id -u gitea)
 immich_uid=$(id -u immich)
 
+metric_init daily
+
 restore() {
     for i in "${arr[@]}"; do
         cd /opt/beeserver/${i}
@@ -32,10 +36,21 @@ restore() {
     rm -f /home/immich/systemd-immich-*.tar
 }
 
-trap restore EXIT
+finish() {
+    local raw=$?
+    local status
+
+    status=$(metric_status ${raw})
+    restore
+    metric_finish ${status}
+    exit ${status}
+}
+
+trap finish EXIT
 
 for i in "${arr[@]}"
 do
+    log "staging ${i}"
     cd /opt/beeserver/${i}
     podman-compose down -t 60
     if [[ "$i" == "monitoring" ]]
@@ -57,10 +72,12 @@ do
     podman-compose up -d
 done
 
+log "staging gitea"
 systemctl --user --machine gitea@.host stop gitea-server.service gitea-db.service
 rsync -aH /home/gitea/data /opt/backup/beeserver/gitea
 systemctl --user --machine gitea@.host start gitea-db.service gitea-server.service
 
+log "staging immich"
 systemctl --user --machine immich@.host stop immich-server.service immich-db.service immich-redis.service
 for v in "${immich[@]}"
 do
@@ -70,17 +87,22 @@ systemctl --user --machine immich@.host start immich-db.service immich-redis.ser
 mkdir -p /opt/backup/beeserver/immich/
 mv /home/immich/systemd-immich-*.tar /opt/backup/beeserver/immich/
 
+log "staging vaultwarden"
 systemctl --user --machine vaultwarden@.host stop vaultwarden.service
 rsync -aH /home/vaultwarden/data /opt/backup/beeserver/vaultwarden
 systemctl --user --machine vaultwarden@.host start vaultwarden.service
 
 archive=/home/beeshare/raid/serverBackup/beeserver_daily_${now}.tar.gz.gpg
 
+log "encrypting ${archive}"
 tar -czf - /opt/backup/beeserver | gpg --batch --passphrase-file /opt/backup/passphrase.txt -c -o ${archive}
 
+log "verifying ${archive}"
 if ! gpg --batch --passphrase-file /opt/backup/passphrase.txt -d ${archive} | tar -tzf - >/dev/null
 then
-    echo "[${now}] verify failed, discarding ${archive}"
+    log "verify failed, discarding ${archive}"
     rm -f ${archive}
     exit 1
 fi
+
+metric_archive ${archive}

playbooks/backup/backupInventory.sh.j2

@@ -0,0 +1,65 @@
+#!/bin/bash
+
+set -uo pipefail
+
+textfile_dir={{ node_exporter_textfile_dir }}
+archive_dir={{ backup_archive_dir }}
+raid_mount={{ backup_raid_mount }}
+
+file=${textfile_dir}/backup_inventory.prom
+tmp=${file}.$$
+now=$(date +%s)
+
+mkdir -p ${textfile_dir}
+
+mounted=0
+if mountpoint -q ${raid_mount}
+then
+    mounted=1
+fi
+
+emit_kind() {
+    local kind=$1
+    local newest
+    local count=0
+    local age=0
+    local bytes=0
+    local total=0
+
+    if [ ${mounted} -eq 1 ]
+    then
+        count=$(find ${archive_dir} -maxdepth 1 -name "beeserver_${kind}_*.tar.gz.gpg" -type f 2>/dev/null | wc -l)
+        newest=$(find ${archive_dir} -maxdepth 1 -name "beeserver_${kind}_*.tar.gz.gpg" -type f -printf '%T@ %p\n' 2>/dev/null | sort -rn | head -1 | cut -d' ' -f2-)
+        total=$(find ${archive_dir} -maxdepth 1 -name "beeserver_${kind}_*.tar.gz.gpg" -type f -printf '%s\n' 2>/dev/null | awk '{ s += $1 } END { print s + 0 }')
+
+        if [ -n "${newest:-}" ]
+        then
+            bytes=$(stat -c %s "${newest}")
+            age=$((now - $(stat -c %Y "${newest}")))
+        fi
+    fi
+
+    echo "backup_archive_count{kind=\"${kind}\"} ${count}"
+    echo "backup_archive_newest_age_seconds{kind=\"${kind}\"} ${age}"
+    echo "backup_archive_newest_bytes{kind=\"${kind}\"} ${bytes}"
+    echo "backup_archive_total_bytes{kind=\"${kind}\"} ${total}"
+}
+
+{
+    echo "# HELP backup_raid_mounted Whether the backup raid is mounted."
+    echo "# TYPE backup_raid_mounted gauge"
+    echo "backup_raid_mounted ${mounted}"
+    echo "# HELP backup_archive_count Number of retained archives."
+    echo "# TYPE backup_archive_count gauge"
+    echo "# HELP backup_archive_newest_age_seconds Age of the newest archive."
+    echo "# TYPE backup_archive_newest_age_seconds gauge"
+    echo "# HELP backup_archive_newest_bytes Size of the newest archive."
+    echo "# TYPE backup_archive_newest_bytes gauge"
+    echo "# HELP backup_archive_total_bytes Size of all retained archives."
+    echo "# TYPE backup_archive_total_bytes gauge"
+    emit_kind daily
+    emit_kind weekly
+} > "${tmp}"
+
+chmod 0644 "${tmp}"
+mv "${tmp}" "${file}"

playbooks/backup/backupWeekly.sh

@@ -2,20 +2,30 @@
 
 set -euo pipefail
 
+source /opt/backup/metrics.sh
+
 now=$(date +"%Y%m%d%H%M")
 
 archive=/home/beeshare/raid/serverBackup/beeserver_weekly_${now}.tar.gz.gpg
 
+metric_init weekly
+trap metric_guard EXIT
+
+log "encrypting ${archive}"
 tar -czf - /opt/backup/beeserver | gpg --batch --passphrase-file /opt/backup/passphrase.txt -c -o ${archive}
 
+log "verifying ${archive}"
 if ! gpg --batch --passphrase-file /opt/backup/passphrase.txt -d ${archive} | tar -tzf - >/dev/null
 then
-    echo "[${now}] verify failed, discarding ${archive} and keeping staging"
+    log "verify failed, discarding ${archive} and keeping staging"
     rm -f ${archive}
     exit 1
 fi
 
+metric_archive ${archive}
+
 rm -rf /opt/backup/beeserver
 
+log "pruning expired archives"
 find /home/beeshare/raid/serverBackup -maxdepth 1 -name 'beeserver_weekly_*.tar.gz.gpg' -mtime +30 -delete
-find /home/beeshare/raid/serverBackup -maxdepth 1 -name 'beeserver_daily_*.tar.gz.gpg' -mtime +7 -delete
+find /home/beeshare/raid/serverBackup -maxdepth 1 -name 'beeserver_daily_*.tar.gz.gpg' -mtime +7 -delete

playbooks/backup/metrics.sh.j2

@@ -0,0 +1,104 @@
+#!/bin/bash
+
+textfile_dir={{ node_exporter_textfile_dir }}
+
+metric_job=""
+metric_started=0
+metric_bytes=0
+metric_signalled=""
+
+metric_on_signal() {
+    metric_signalled=$1
+    exit $((128 + $1))
+}
+
+metric_install_traps() {
+    trap 'metric_on_signal 1' HUP
+    trap 'metric_on_signal 2' INT
+    trap 'metric_on_signal 15' TERM
+}
+
+metric_status() {
+    if [ -n "${metric_signalled}" ]
+    then
+        echo $((128 + metric_signalled))
+    else
+        echo "$1"
+    fi
+}
+
+log() {
+    echo "[$(date --rfc-3339=seconds)] ${metric_job}: $*"
+}
+
+metric_previous_success() {
+    local file=${textfile_dir}/backup_${metric_job}.prom
+
+    if [ -r "${file}" ]
+    then
+        grep "^backup_last_success_timestamp_seconds{backup_job=\"${metric_job}\"} " "${file}" | awk '{ print $2 }'
+    fi
+}
+
+metric_init() {
+    metric_job=$1
+    metric_started=$(date +%s)
+    mkdir -p ${textfile_dir}
+    metric_install_traps
+    log "start"
+}
+
+metric_archive() {
+    metric_bytes=$(stat -c %s "$1" 2>/dev/null || echo 0)
+}
+
+metric_finish() {
+    local status=$1
+    local now
+    local success
+    local previous
+    local file=${textfile_dir}/backup_${metric_job}.prom
+    local tmp=${file}.$$
+
+    now=$(date +%s)
+    previous=$(metric_previous_success)
+
+    if [ "${status}" -eq 0 ]
+    then
+        success=${now}
+        log "done in $((now - metric_started))s"
+    else
+        success=${previous:-0}
+        log "FAILED with status ${status} after $((now - metric_started))s"
+    fi
+
+    cat > "${tmp}" <<EOF
+# HELP backup_last_run_timestamp_seconds Unix time of the last run of this backup job.
+# TYPE backup_last_run_timestamp_seconds gauge
+backup_last_run_timestamp_seconds{backup_job="${metric_job}"} ${now}
+# HELP backup_last_success_timestamp_seconds Unix time of the last successful run of this backup job.
+# TYPE backup_last_success_timestamp_seconds gauge
+backup_last_success_timestamp_seconds{backup_job="${metric_job}"} ${success}
+# HELP backup_last_duration_seconds Duration of the last run of this backup job.
+# TYPE backup_last_duration_seconds gauge
+backup_last_duration_seconds{backup_job="${metric_job}"} $((now - metric_started))
+# HELP backup_last_status Exit status of the last run of this backup job, 1 is success.
+# TYPE backup_last_status gauge
+backup_last_status{backup_job="${metric_job}"} $([ "${status}" -eq 0 ] && echo 1 || echo 0)
+# HELP backup_last_archive_bytes Size of the archive produced by the last run of this backup job.
+# TYPE backup_last_archive_bytes gauge
+backup_last_archive_bytes{backup_job="${metric_job}"} ${metric_bytes}
+EOF
+
+    chmod 0644 "${tmp}"
+    mv "${tmp}" "${file}"
+}
+
+metric_guard() {
+    local raw=$?
+    local status
+
+    status=$(metric_status ${raw})
+    metric_finish ${status}
+    exit ${status}
+}

playbooks/backup/restoreTest.sh

@@ -2,6 +2,8 @@
 
 set -euo pipefail
 
+source /opt/backup/metrics.sh
+
 now=$(date +"%Y%m%d%H%M")
 backup_dir=/home/beeshare/raid/serverBackup
 image=docker.io/vaultwarden/server:1.37.1
@@ -9,15 +11,23 @@ container=vaultwarden-restoretest
 port=18222
 work=$(mktemp -d)
 
+metric_init restoretest
+
 cleanup() {
+    local raw=$?
+    local status
+
+    status=$(metric_status ${raw})
     podman rm -f ${container} >/dev/null 2>&1 || true
     rm -rf "${work}"
+    metric_finish ${status}
+    exit ${status}
 }
 
 trap cleanup EXIT
 
 latest=$(ls -1t ${backup_dir}/beeserver_*.tar.gz.gpg | head -1)
-echo "[${now}] restore test using ${latest}"
+log "restore test using ${latest}"
 
 gpg --batch --passphrase-file /opt/backup/passphrase.txt -d "${latest}" \
     | tar -xzf - -C "${work}" opt/backup/beeserver/vaultwarden
@@ -26,15 +36,15 @@ data="${work}/opt/backup/beeserver/vaultwarden/data"
 
 [[ -s "${data}/db.sqlite3" ]]
 [[ -s "${data}/rsa_key.pem" ]]
-echo "  db and rsa_key present"
+log "db and rsa_key present"
 
 integrity=$(sqlite3 "${data}/db.sqlite3" "PRAGMA integrity_check;")
 [[ "${integrity}" == "ok" ]]
-echo "  integrity_check ok"
+log "integrity_check ok"
 
 users=$(sqlite3 "${data}/db.sqlite3" "select count(*) from users;")
 [[ "${users}" -ge 1 ]]
-echo "  users ${users}"
+log "users ${users}"
 
 email=$(sqlite3 "${data}/db.sqlite3" "select email from users limit 1;")
 
@@ -51,12 +61,14 @@ done
 
 config=$(curl -sf http://127.0.0.1:${port}/api/config)
 [[ "${config}" == *'"version"'* ]]
-echo "  api/config ok"
+log "api/config ok"
 
 prelogin=$(curl -sf -X POST http://127.0.0.1:${port}/identity/accounts/prelogin \
     -H 'Content-Type: application/json' \
     -d "{\"email\":\"${email}\"}")
 [[ "${prelogin}" == *[Kk]df* ]]
-echo "  prelogin ok for ${email}"
+log "prelogin ok for ${email}"
 
-echo "[${now}] restore test PASSED"
+metric_archive "${latest}"
+
+log "restore test PASSED"

playbooks/dashboards/dashboards.yml

@@ -0,0 +1,84 @@
+- name: Dashboards
+  hosts: beepi
+  become: true
+  vars:
+    dashboard_dir: /opt/beeserver/monitoring/grafana/dashboards
+    dashboard_general_dir: /opt/beeserver/monitoring/grafana/dashboards-general
+    provisioning_dir: /opt/beeserver/monitoring/grafana/provisioning/dashboards
+  tasks:
+    - name: Determine whether uptimerobot is configured
+      ansible.builtin.set_fact:
+        uptimerobot_enabled: "{{ uptimerobot_api_key is defined and 'REPLACE_ME' not in uptimerobot_api_key }}"
+
+    - name: Check that grafana is running
+      ansible.builtin.command:
+        cmd: podman container inspect -f '{{ '{{' }}.State.Running{{ '}}' }}' grafana
+      register: grafana_running
+      changed_when: false
+      failed_when: false
+
+    - name: Require a running grafana
+      ansible.builtin.assert:
+        that:
+          - grafana_running.stdout | trim == 'true'
+        fail_msg: >-
+          grafana is not running, so provisioned changes cannot be picked up.
+          Run make deploy-monitoring first.
+
+    - name: Create dashboard directories
+      ansible.builtin.file:
+        path: "{{ item }}"
+        state: directory
+        mode: '0755'
+      loop:
+        - "{{ dashboard_dir }}"
+        - "{{ dashboard_general_dir }}"
+        - "{{ provisioning_dir }}"
+
+    - name: Copy dashboard provider
+      ansible.builtin.template:
+        src: ../monitoring/grafana/provisioning/dashboards/dashboards.yml.j2
+        dest: "{{ provisioning_dir }}/dashboards.yml"
+        mode: '0644'
+      notify: Restart grafana
+
+    - name: Copy generated dashboards
+      ansible.builtin.template:
+        src: "../monitoring/grafana/dashboards/{{ item }}.json.j2"
+        dest: "{{ dashboard_dir }}/{{ item }}.json"
+        mode: '0644'
+        validate: "python3 -c \"import json; json.load(open('%s'))\""
+      loop:
+        - infrastructure-map
+        - backups
+        - services
+        - ingress
+
+    - name: Copy uptimerobot dashboard
+      ansible.builtin.template:
+        src: ../monitoring/grafana/dashboards/uptimerobot.json.j2
+        dest: "{{ dashboard_dir }}/uptimerobot.json"
+        mode: '0644'
+        validate: "python3 -c \"import json; json.load(open('%s'))\""
+      when: uptimerobot_enabled
+
+    - name: Copy hand made dashboards
+      ansible.builtin.copy:
+        src: ../monitoring/grafana/dashboards-general/
+        dest: "{{ dashboard_general_dir }}/"
+        mode: '0644'
+
+    - name: Debug
+      ansible.builtin.debug:
+        msg:
+          - "Dashboards are polled by grafana every 30s, no restart needed."
+          - "Grafana: https://grafana.secretbee.buzz/"
+          - "Errors: sudo podman logs --tail 40 grafana | grep -i provision"
+      tags:
+        - always
+
+  handlers:
+    - name: Restart grafana
+      changed_when: true
+      ansible.builtin.command:
+        cmd: podman restart grafana

playbooks/hetzner/hetzner.yml

@@ -6,7 +6,6 @@
     user_home: /home/hetzner
 
     hetzner_raid_mount: /home/beeshare/raid
-    hetzner_port: 23
 
     hetzner_paths:
       - src: /home/beeshare/raid/serverBackup
@@ -14,10 +13,6 @@
       - src: /home/beeshare/raid/foundry/backup
         dest: ./foundry
 
-    # Fill in after creating the storage box sub-account, then re-run
-    hetzner_host: "u651287.your-storagebox.de"
-    hetzner_user: "u651287"
-
   tasks:
     - name: Install rsync
       ansible.builtin.apt:
@@ -88,6 +83,58 @@
         job: /opt/backup/hetznerSync.sh >> /var/log/beepi-backup.log 2>&1
       when: hetzner_host | length > 0
 
+    - name: Copy metrics collector
+      ansible.builtin.template:
+        src: hetznerMetrics.sh.j2
+        dest: /opt/backup/hetznerMetrics.sh
+        mode: '0700'
+      when: hetzner_host | length > 0
+
+    - name: Install metrics timer
+      ansible.builtin.copy:
+        dest: "/etc/systemd/system/beepi-hetzner-metrics.{{ item.kind }}"
+        content: "{{ item.content }}"
+        mode: '0644'
+      loop:
+        - kind: service
+          content: |
+            [Unit]
+            Description=Collect hetzner storage box metrics
+
+            [Service]
+            Type=oneshot
+            ExecStart=/opt/backup/hetznerMetrics.sh
+        - kind: timer
+          content: |
+            [Unit]
+            Description=Collect hetzner storage box metrics
+
+            [Timer]
+            OnBootSec=5min
+            OnUnitActiveSec=1h
+            AccuracySec=5min
+
+            [Install]
+            WantedBy=timers.target
+      when: hetzner_host | length > 0
+      notify: Reload systemd
+
+    - name: Flush handlers
+      ansible.builtin.meta: flush_handlers
+
+    - name: Enable metrics timer
+      ansible.builtin.systemd_service:
+        name: beepi-hetzner-metrics.timer
+        state: started
+        enabled: true
+      when: hetzner_host | length > 0
+
+    - name: Run metrics collector once
+      changed_when: true
+      ansible.builtin.command:
+        cmd: /opt/backup/hetznerMetrics.sh
+      when: hetzner_host | length > 0
+
     - name: Debug
       ansible.builtin.debug:
         msg:
@@ -95,3 +142,8 @@
           - "Read logs: sudo tail -50 /var/log/beepi-backup.log"
       tags:
         - always
+
+  handlers:
+    - name: Reload systemd
+      ansible.builtin.systemd_service:
+        daemon_reload: true

playbooks/hetzner/hetznerMetrics.sh.j2

@@ -0,0 +1,77 @@
+#!/bin/bash
+
+set -uo pipefail
+
+textfile_dir={{ node_exporter_textfile_dir }}
+file=${textfile_dir}/hetzner.prom
+tmp=${file}.$$
+now=$(date +%s)
+
+ssh_opts="-p {{ hetzner_port }} -i {{ user_home }}/.ssh/id_ed25519 -o UserKnownHostsFile={{ user_home }}/.ssh/known_hosts -o StrictHostKeyChecking=yes -o BatchMode=yes -o ConnectTimeout=15"
+
+mkdir -p ${textfile_dir}
+
+reachable=0
+body=""
+
+emit_path() {
+    local label=$1
+    local remote=$2
+    local listing
+    local files
+    local bytes
+    local pending
+    local newest
+    local age=-1
+
+    if ! listing=$(sudo -u {{ user }} rsync --list-only -e "ssh ${ssh_opts}" {{ hetzner_user }}@{{ hetzner_host }}:${remote}/ 2>/dev/null)
+    then
+        body+="hetzner_remote_listing_ok{path=\"${label}\"} 0"$'\n'
+        return
+    fi
+
+    reachable=1
+
+    files=$(echo "${listing}" | awk '$1 ~ /^-/ && substr($5,1,1) != "." { c++ } END { print c + 0 }')
+    bytes=$(echo "${listing}" | awk '$1 ~ /^-/ && substr($5,1,1) != "." { gsub(",", "", $2); s += $2 } END { printf "%.0f", s + 0 }')
+    pending=$(echo "${listing}" | awk '$1 ~ /^-/ && substr($5,1,1) == "." { gsub(",", "", $2); s += $2 } END { printf "%.0f", s + 0 }')
+    newest=$(echo "${listing}" | awk '$1 ~ /^-/ && substr($5,1,1) != "." { print $3" "$4 }' | sort -r | head -1)
+
+    if [ -n "${newest}" ]
+    then
+        age=$((now - $(date -d "$(echo ${newest} | tr '/' '-')" +%s)))
+    fi
+
+    body+="hetzner_remote_listing_ok{path=\"${label}\"} 1"$'\n'
+    body+="hetzner_remote_files{path=\"${label}\"} ${files}"$'\n'
+    body+="hetzner_remote_bytes{path=\"${label}\"} ${bytes}"$'\n'
+    body+="hetzner_remote_pending_bytes{path=\"${label}\"} ${pending}"$'\n'
+    body+="hetzner_remote_newest_age_seconds{path=\"${label}\"} ${age}"$'\n'
+}
+
+{% for path in hetzner_paths %}
+emit_path "{{ path.dest | replace('./', '') }}" "{{ path.dest }}"
+{% endfor %}
+
+{
+    echo "# HELP hetzner_reachable Whether the storage box answered at least one listing."
+    echo "# TYPE hetzner_reachable gauge"
+    echo "hetzner_reachable ${reachable}"
+    echo "# HELP hetzner_last_check_timestamp_seconds Unix time of the last storage box check."
+    echo "# TYPE hetzner_last_check_timestamp_seconds gauge"
+    echo "hetzner_last_check_timestamp_seconds ${now}"
+    echo "# HELP hetzner_remote_listing_ok Whether this remote path could be listed."
+    echo "# TYPE hetzner_remote_listing_ok gauge"
+    echo "# HELP hetzner_remote_files Number of completed files stored at this remote path."
+    echo "# TYPE hetzner_remote_files gauge"
+    echo "# HELP hetzner_remote_bytes Bytes of completed files stored at this remote path."
+    echo "# TYPE hetzner_remote_bytes gauge"
+    echo "# HELP hetzner_remote_pending_bytes Bytes of in flight partial transfers at this remote path."
+    echo "# TYPE hetzner_remote_pending_bytes gauge"
+    echo "# HELP hetzner_remote_newest_age_seconds Age of the newest completed file at this remote path."
+    echo "# TYPE hetzner_remote_newest_age_seconds gauge"
+    printf '%s' "${body}"
+} > "${tmp}"
+
+chmod 0644 "${tmp}"
+mv "${tmp}" "${file}"

playbooks/hetzner/hetznerSync.sh.j2

@@ -2,11 +2,22 @@
 
 set -euo pipefail
 
-now=$(date +"%Y%m%d%H%M")
+exec 9>/var/lock/hetznerSync.lock
+
+if ! flock -n 9
+then
+    echo "[$(date --rfc-3339=seconds)] offsite: another sync is still running, skipping this run"
+    exit 0
+fi
+
+source /opt/backup/metrics.sh
+
+metric_init offsite
+trap metric_guard EXIT
 
 if ! mountpoint -q {{ hetzner_raid_mount }}
 then
-    echo "[${now}] {{ hetzner_raid_mount }} not mounted, refusing to sync"
+    log "{{ hetzner_raid_mount }} not mounted, refusing to sync"
     exit 1
 fi
 
@@ -16,19 +27,17 @@ sync_path() {
 
     if [ -z "$(ls -A ${src} 2>/dev/null)" ]
     then
-        echo "[${now}] ${src} empty, refusing to sync"
+        log "${src} empty, refusing to sync"
         exit 1
     fi
 
-    echo "[${now}] syncing ${src} to ${dest}"
+    log "syncing ${src} to ${dest}"
     sudo -u {{ user }} rsync -a --delete --partial \
         -e "ssh -p {{ hetzner_port }} -i {{ user_home }}/.ssh/id_ed25519 -o UserKnownHostsFile={{ user_home }}/.ssh/known_hosts -o StrictHostKeyChecking=yes" \
         ${src}/ \
         {{ hetzner_user }}@{{ hetzner_host }}:${dest}/
 }
 
-echo "[${now}] offsite sync start"
 {% for path in hetzner_paths %}
 sync_path {{ path.src }} {{ path.dest }}
 {% endfor %}
-echo "[${now}] offsite sync done"

playbooks/monitoring-ingress/config.alloy.j2

@@ -0,0 +1,102 @@
+loki.relabel "journal" {
+  forward_to = []
+
+  rule {
+    source_labels = ["__journal__systemd_unit"]
+    target_label  = "unit"
+  }
+
+  rule {
+    source_labels = ["__journal_priority_keyword"]
+    target_label  = "level"
+  }
+}
+
+loki.source.journal "system" {
+  forward_to    = [loki.process.journal.receiver]
+  relabel_rules = loki.relabel.journal.rules
+  labels        = {
+    job = "systemd-journal",
+    host = "ingress",
+  }
+}
+
+loki.process "journal" {
+  forward_to = [loki.write.default.receiver]
+
+  stage.match {
+    selector = `{job="systemd-journal"} |~ "\\[UFW AUDIT\\]"`
+    action   = "drop"
+  }
+
+  stage.match {
+    selector = `{job="systemd-journal"} |~ "\\[UFW BLOCK\\]"`
+
+    stage.regex {
+      expression = `\[UFW\s(?P<action>\w+)\].*?SRC=(?P<ip>[0-9.]+).*?DST=(?P<dst>[0-9.]+).*?PROTO=(?P<proto>\w+)(?:\sSPT=(?P<spt>\d+))?(?:\sDPT=(?P<dpt>\d+))?`
+    }
+
+    stage.geoip {
+      db      = "/opt/database/GeoLite2-City.mmdb"
+      source  = "ip"
+      db_type = "city"
+    }
+
+    stage.labels {
+      values = {
+        action                   = "",
+        proto                    = "",
+        dpt                      = "",
+        geoip_country_name       = "",
+        geoip_country_code       = "",
+        geoip_location_latitude  = "",
+        geoip_location_longitude = "",
+      }
+    }
+
+    stage.static_labels {
+      values = {
+        job = "ufw",
+      }
+    }
+  }
+}
+
+local.file_match "fail2ban" {
+  path_targets = [{"__path__" = "/var/log/fail2ban.log", job = "fail2ban", host = "ingress"}]
+}
+
+loki.source.file "fail2ban_source" {
+  targets    = local.file_match.fail2ban.targets
+  forward_to = [loki.process.fail2ban.receiver]
+}
+
+loki.process "fail2ban" {
+  forward_to = [loki.write.default.receiver]
+
+  stage.regex {
+    expression = `^.*:\s(?<level>\w*)\s*\[(?<service>\w*)\]\s(?<type>\w*)\s(?<ip>(?:(?:2(?:[0-4][0-9]|5[0-5])|[0-1]?[0-9]?[0-9])\.){3}(?:(?:2([0-4][0-9]|5[0-5])|[0-1]?[0-9]?[0-9]))).*$`
+    labels_from_groups = true
+  }
+
+  stage.geoip {
+    db      = "/opt/database/GeoLite2-City.mmdb"
+    source  = "ip"
+    db_type = "city"
+  }
+
+  stage.labels {
+    values = {
+      geoip_country_name       = "",
+      geoip_country_code       = "",
+      geoip_location_latitude  = "",
+      geoip_location_longitude = "",
+    }
+  }
+}
+
+loki.write "default" {
+  endpoint {
+    url = "http://{{ wireguard_beepi_ip }}:3100/loki/api/v1/push"
+  }
+}

playbooks/monitoring-ingress/monitoring-ingress.yml

@@ -0,0 +1,297 @@
+- name: Monitoring-ingress
+  hosts: animeistrash
+  become: true
+  vars:
+    geoip_username: !vault |
+              $ANSIBLE_VAULT;1.1;AES256
+              30656365643463396265633130356339386433633331613130633134336633643637653761623962
+              3633336466326564616432373031303530376166353436610a323033353164313436613363613836
+              30303931323234306162343361643162313238313234383837323530343936343134326365666166
+              3438643238626266320a333732363461613834613833303436363534393031323333613534333534
+              3438
+    geoip_license: !vault |
+              $ANSIBLE_VAULT;1.1;AES256
+              32336437653435396231663666643631373638616533653538333934356339666661313439343566
+              6462303439666233316162653430363239393031633362630a323533393233633164613031646633
+              62366362383162306563383235373262326262323337303333313163653838613436316337633063
+              3630356262666338320a333137333063303534386437343762616665383034336365663732343566
+              36376463313964333834373435653564346333396538306337333762613737643363636364353139
+              3032383564333638613566633363666430356230326231633636
+  tasks:
+    - name: Install exporters
+      ansible.builtin.apt:
+        name:
+          - prometheus-node-exporter
+          - prometheus-blackbox-exporter
+          - wireguard-tools
+        state: present
+        update_cache: true
+
+    - name: Create textfile directory
+      ansible.builtin.file:
+        path: "{{ node_exporter_textfile_dir }}"
+        state: directory
+        mode: '0755'
+
+    - name: Configure node exporter
+      ansible.builtin.copy:
+        dest: /etc/default/prometheus-node-exporter
+        content: |
+          ARGS="--web.listen-address={{ wireguard_ingress_ip }}:9100 --collector.textfile.directory={{ node_exporter_textfile_dir }} --collector.systemd"
+        mode: '0644'
+      notify: Restart node exporter
+
+    - name: Configure blackbox exporter
+      ansible.builtin.copy:
+        dest: /etc/default/prometheus-blackbox-exporter
+        content: |
+          ARGS="--config.file=/etc/prometheus/blackbox.yml --web.listen-address={{ wireguard_ingress_ip }}:9115"
+        mode: '0644'
+      notify: Restart blackbox exporter
+
+    - name: Copy blackbox config
+      ansible.builtin.copy:
+        src: ../monitoring/blackbox.yml
+        dest: /etc/prometheus/blackbox.yml
+        mode: '0644'
+      notify: Restart blackbox exporter
+
+    - name: Look up the prometheus group
+      ansible.builtin.getent:
+        database: group
+        key: prometheus
+
+    - name: Allow the prometheus group to send icmp
+      ansible.builtin.copy:
+        dest: /etc/sysctl.d/99-blackbox-icmp.conf
+        content: |
+          net.ipv4.ping_group_range = {{ prometheus_gid }} {{ prometheus_gid }}
+        mode: '0644'
+      vars:
+        prometheus_gid: "{{ ansible_facts.getent_group.prometheus[1] }}"
+      notify: Apply sysctl
+
+    - name: Create blackbox override directory
+      ansible.builtin.file:
+        path: /etc/systemd/system/prometheus-blackbox-exporter.service.d
+        state: directory
+        mode: '0755'
+
+    - name: Allow blackbox to send icmp
+      ansible.builtin.copy:
+        dest: /etc/systemd/system/prometheus-blackbox-exporter.service.d/capabilities.conf
+        content: |
+          [Service]
+          AmbientCapabilities=CAP_NET_RAW
+          CapabilityBoundingSet=CAP_NET_RAW
+        mode: '0644'
+      notify:
+        - Reload systemd
+        - Restart blackbox exporter
+
+    - name: Copy wireguard collector
+      ansible.builtin.template:
+        src: ../monitoring/textfile/wg_metrics.sh.j2
+        dest: /usr/local/bin/wg_metrics.sh
+        mode: '0755'
+
+    - name: Install collector timer
+      ansible.builtin.copy:
+        dest: "/etc/systemd/system/ingress-wg-metrics.{{ item.kind }}"
+        content: "{{ item.content }}"
+        mode: '0644'
+      loop:
+        - kind: service
+          content: |
+            [Unit]
+            Description=Collect wireguard peer metrics
+
+            [Service]
+            Type=oneshot
+            ExecStart=/usr/local/bin/wg_metrics.sh
+        - kind: timer
+          content: |
+            [Unit]
+            Description=Collect wireguard peer metrics
+
+            [Timer]
+            OnBootSec=1min
+            OnUnitActiveSec=30s
+            AccuracySec=5s
+
+            [Install]
+            WantedBy=timers.target
+      notify: Reload systemd
+
+    - name: Flush handlers
+      ansible.builtin.meta: flush_handlers
+
+    - name: Enable collector timer
+      ansible.builtin.systemd_service:
+        name: ingress-wg-metrics.timer
+        state: started
+        enabled: true
+
+    - name: Create alloy directories
+      ansible.builtin.file:
+        path: "{{ item }}"
+        state: directory
+        mode: '0755'
+      loop:
+        - /etc/alloy
+        - /opt/database
+
+    - name: Create apt keyring directory
+      ansible.builtin.file:
+        path: /etc/apt/keyrings
+        state: directory
+        mode: '0755'
+
+    - name: Add grafana apt key
+      ansible.builtin.get_url:
+        url: https://apt.grafana.com/gpg.key
+        dest: /etc/apt/keyrings/grafana.asc
+        mode: '0644'
+
+    - name: Add grafana apt repository
+      ansible.builtin.apt_repository:
+        repo: "deb [signed-by=/etc/apt/keyrings/grafana.asc] https://apt.grafana.com stable main"
+        filename: grafana
+        state: present
+
+    - name: Install alloy
+      ansible.builtin.apt:
+        name: alloy
+        state: present
+        update_cache: true
+
+    - name: Create alloy data directory
+      ansible.builtin.file:
+        path: /var/lib/alloy/data
+        state: directory
+        owner: alloy
+        group: alloy
+        mode: '0755'
+      notify: Restart alloy
+
+    - name: Check the geoip database
+      ansible.builtin.stat:
+        path: /opt/database/GeoLite2-City.mmdb
+      register: geoip_db
+
+    - name: Check when geoip was last downloaded
+      ansible.builtin.stat:
+        path: /opt/.geoip_downloaded
+      register: geoip_stamp
+
+    - name: Decide whether the geoip database needs refreshing
+      ansible.builtin.set_fact:
+        geoip_stale: "{{ not geoip_db.stat.exists or not geoip_stamp.stat.exists or (ansible_facts.date_time.epoch | int - geoip_stamp.stat.mtime | int) > geoip_max_age_seconds | int }}"
+
+    - name: Report the geoip database age
+      ansible.builtin.debug:
+        msg: >-
+          geoip
+          {{ 'not present, downloading' if not (geoip_db.stat.exists and geoip_stamp.stat.exists)
+             else 'downloaded ' + (((ansible_facts.date_time.epoch | int - geoip_stamp.stat.mtime | int) / 3600) | round(1) | string) + 'h ago, ' + ('refreshing' if geoip_stale else 'reusing') }}
+
+    - name: Download geoip database
+      ansible.builtin.get_url:
+        url: "https://download.maxmind.com/geoip/databases/GeoLite2-City/download?suffix=tar.gz"
+        dest: /opt/geoip.tar.gz
+        username: "{{ geoip_username }}"
+        password: "{{ geoip_license }}"
+        mode: '0644'
+        force: true
+      when: geoip_stale
+
+    - name: Extract geoip database
+      ansible.builtin.unarchive:
+        remote_src: true
+        src: /opt/geoip.tar.gz
+        dest: /opt/database
+        extra_opts: ['--strip-components=1', '--show-stored-names']
+      when: geoip_stale
+      notify: Restart alloy
+
+    - name: Stamp the geoip download time
+      ansible.builtin.file:
+        path: /opt/.geoip_downloaded
+        state: touch
+        mode: '0644'
+      when: geoip_stale
+
+    - name: Copy alloy config
+      ansible.builtin.template:
+        src: config.alloy.j2
+        dest: /etc/alloy/config.alloy
+        mode: '0644'
+      notify: Restart alloy
+
+    - name: Configure alloy
+      ansible.builtin.copy:
+        dest: /etc/default/alloy
+        content: |
+          CONFIG_FILE="/etc/alloy/config.alloy"
+          CUSTOM_ARGS="--server.http.listen-addr={{ wireguard_ingress_ip }}:12345 --stability.level=experimental"
+          RESTART_ON_UPGRADE=true
+        mode: '0644'
+      notify: Restart alloy
+
+    - name: Allow alloy to read logs
+      ansible.builtin.user:
+        name: alloy
+        groups:
+          - adm
+          - systemd-journal
+        append: true
+      notify: Restart alloy
+
+    - name: Enable services
+      ansible.builtin.service:
+        name: "{{ item }}"
+        state: started
+        enabled: true
+      loop:
+        - prometheus-node-exporter
+        - prometheus-blackbox-exporter
+        - alloy
+
+    - name: Run collector once
+      changed_when: true
+      ansible.builtin.command:
+        cmd: /usr/local/bin/wg_metrics.sh
+
+    - name: Debug
+      ansible.builtin.debug:
+        msg:
+          - "Node metrics: curl -s {{ wireguard_ingress_ip }}:9100/metrics | head"
+          - "Blackbox probe: curl -s '{{ wireguard_ingress_ip }}:9115/probe?target={{ wireguard_beepi_ip }}:443&module=tcp_connect'"
+          - "Alloy logs: journalctl -u alloy -n 50"
+      tags:
+        - always
+
+  handlers:
+    - name: Apply sysctl
+      changed_when: true
+      ansible.builtin.command:
+        cmd: sysctl --system
+
+    - name: Reload systemd
+      ansible.builtin.systemd_service:
+        daemon_reload: true
+
+    - name: Restart node exporter
+      ansible.builtin.service:
+        name: prometheus-node-exporter
+        state: restarted
+
+    - name: Restart blackbox exporter
+      ansible.builtin.service:
+        name: prometheus-blackbox-exporter
+        state: restarted
+
+    - name: Restart alloy
+      ansible.builtin.service:
+        name: alloy
+        state: restarted

playbooks/monitoring/blackbox.yml.j2

@@ -0,0 +1,70 @@
+modules:
+  http_2xx:
+    prober: http
+    timeout: 10s
+    http:
+      valid_status_codes: [200]
+      valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
+      preferred_ip_protocol: ip4
+      ip_protocol_fallback: false
+
+  http_2xx_any:
+    prober: http
+    timeout: 10s
+    http:
+      valid_status_codes: [200, 204, 301, 302, 307, 308, 401, 403]
+      preferred_ip_protocol: ip4
+      ip_protocol_fallback: false
+
+  https_vhost:
+    prober: http
+    timeout: 10s
+    http:
+      valid_status_codes: [200, 204, 301, 302, 307, 308, 401, 403]
+      preferred_ip_protocol: ip4
+      ip_protocol_fallback: false
+      fail_if_not_ssl: true
+      tls_config:
+        insecure_skip_verify: false
+
+  tcp_connect:
+    prober: tcp
+    timeout: 10s
+    tcp:
+      preferred_ip_protocol: ip4
+      ip_protocol_fallback: false
+
+  smtp_starttls:
+    prober: tcp
+    timeout: 10s
+    tcp:
+      preferred_ip_protocol: ip4
+      ip_protocol_fallback: false
+      query_response:
+        - expect: "^220"
+        - send: "EHLO prober"
+        - expect: "^250-STARTTLS"
+        - send: "QUIT"
+
+  imaps_banner:
+    prober: tcp
+    timeout: 10s
+    tcp:
+      preferred_ip_protocol: ip4
+      ip_protocol_fallback: false
+      tls: true
+
+  ssh_banner:
+    prober: tcp
+    timeout: 10s
+    tcp:
+      preferred_ip_protocol: ip4
+      ip_protocol_fallback: false
+      query_response:
+        - expect: "^SSH-2.0-"
+
+  icmp:
+    prober: icmp
+    timeout: 5s
+    icmp:
+      preferred_ip_protocol: ip4

playbooks/monitoring/config.alloy.j2

@@ -10,8 +10,17 @@ local.file_match "fail2ban" {
   path_targets = [{"__path__" = "/var/log/fail2ban.log", job = "fail2ban", host = "beepi"}]
 }
 
+local.file_match "backup" {
+  path_targets = [{"__path__" = "/var/log/beepi-backup.log", job = "backup", host = "beepi"}]
+}
+
 local.file_match "varlog" {
-  path_targets = [{"__path__" = "/var/log/*.log", job = "varlog", "__path_exclude__" = "/var/log/fail2ban.log", host = "beepi"}]
+  path_targets = [{
+    "__path__" = "/var/log/*.log",
+    job = "varlog",
+    "__path_exclude__" = "/var/log/{fail2ban,beepi-backup}.log",
+    host = "beepi",
+  }]
 }
 
 loki.source.file "varlog" {
@@ -19,6 +28,34 @@ loki.source.file "varlog" {
   forward_to = [loki.write.default.receiver]
 }
 
+loki.source.file "backup" {
+  targets    = local.file_match.backup.targets
+  forward_to = [loki.process.backup.receiver]
+}
+
+loki.process "backup" {
+    forward_to = [loki.write.default.receiver]
+
+    stage.regex {
+        expression = `^\[(?P<ts>[^\]]+)\]\s(?P<backup_job>\w+):\s(?P<message>.*)$`
+    }
+
+    stage.labels {
+        values = {
+            backup_job = "",
+        }
+    }
+
+    stage.match {
+        selector = `{job="backup"} |~ "(?i)(FAILED|verify failed|refusing to sync|error)"`
+        stage.static_labels {
+            values = {
+                level = "error",
+            }
+        }
+    }
+}
+
 loki.source.file "fail2ban_source" {
   targets    = local.file_match.fail2ban.targets
   forward_to = [loki.process.geoip.receiver]
@@ -58,4 +95,4 @@ loki.write "default" {
   endpoint {
     url = "http://loki:3100/loki/api/v1/push"
   }
-}
+}

playbooks/monitoring/docker-compose.yml.j2

@@ -12,6 +12,14 @@ services:
     volumes:
       - prometheus-data:/prometheus
       - /opt/beeserver/monitoring/prometheus.yml:/etc/prometheus/prometheus.yml
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+      - --storage.tsdb.path=/prometheus
+      - --storage.tsdb.retention.time=90d
+      - --web.enable-lifecycle
+{% if prometheus_admin_api | default(false) %}
+      - --web.enable-admin-api
+{% endif %}
     networks:
       - monitoring
 
@@ -23,12 +31,17 @@ services:
       - '127.0.0.1:4000:4000'
     volumes:
       - grafana-storage:/var/lib/grafana
+      - /opt/beeserver/monitoring/grafana/provisioning:/etc/grafana/provisioning:ro
+      - /opt/beeserver/monitoring/grafana/dashboards:/var/lib/grafana/dashboards:ro
+      - /opt/beeserver/monitoring/grafana/dashboards-general:/var/lib/grafana/dashboards-general:ro
     environment:
       - GF_SECURITY_SECRET_KEY={{ grafana_secret }}
       - GF_SERVER_DOMAIN=grafana.secretbee.buzz
       - GF_SERVER_ROOT_URL=https://grafana.secretbee.buzz/
       - GF_SERVER_HTTP_PORT=4000
       - GF_SERVER_PROTOCOL=HTTP
+      - GF_INSTALL_PLUGINS=yesoreyeram-infinity-datasource
+      - GF_UNIFIED_ALERTING_ENABLED=true
     networks:
       - monitoring
 
@@ -37,6 +50,7 @@ services:
     container_name: loki
     ports:
       - '127.0.0.1:3100:3100'
+      - '{{ wireguard_beepi_ip }}:3100:3100'
       - '127.0.0.1:9096:9096'
     volumes:
       - loki-data:/loki
@@ -87,8 +101,42 @@ services:
     networks:
       - monitoring
 
+  blackbox:
+    image: docker.io/prom/blackbox-exporter
+    container_name: blackbox
+    restart: always
+    network_mode: host
+    volumes:
+      - /opt/beeserver/monitoring/blackbox.yml:/etc/blackbox_exporter/config.yml
+    command:
+      - --config.file=/etc/blackbox_exporter/config.yml
+      - --web.listen-address=0.0.0.0:9115
+    extra_hosts:
+{% for service in monitored_services %}
+{% if service.vhost is defined %}
+      - '{{ service.vhost }}:127.0.0.1'
+{% endif %}
+{% endfor %}
+
+  topology:
+    image: docker.io/library/python:3-alpine
+    container_name: topology
+    restart: always
+    ports:
+      - '127.0.0.1:8088:8088'
+    volumes:
+      - /opt/beeserver/monitoring/topology:/app:ro
+    command:
+      - python
+      - /app/topology.py
+    environment:
+      - PROMETHEUS_URL=http://prometheus:9090
+      - LISTEN_PORT=8088
+    networks:
+      - monitoring
+
 volumes:
   prometheus-data:
   grafana-storage:
   loki-data:
-  alloy-data:
+  alloy-data:

playbooks/monitoring/grafana/dashboards-general/logging.json

@@ -0,0 +1,713 @@
+{
+  "apiVersion": "dashboard.grafana.app/v2",
+  "kind": "Dashboard",
+  "metadata": {
+    "name": "admltw2"
+  },
+  "spec": {
+    "annotations": [
+      {
+        "kind": "AnnotationQuery",
+        "spec": {
+          "builtIn": true,
+          "enable": true,
+          "hide": true,
+          "iconColor": "rgba(0, 211, 255, 1)",
+          "name": "Annotations & Alerts",
+          "query": {
+            "datasource": {
+              "name": "-- Grafana --"
+            },
+            "group": "grafana",
+            "kind": "DataQuery",
+            "spec": {},
+            "version": "v0"
+          }
+        }
+      }
+    ],
+    "cursorSync": "Off",
+    "editable": true,
+    "elements": {
+      "panel-1": {
+        "kind": "Panel",
+        "spec": {
+          "data": {
+            "kind": "QueryGroup",
+            "spec": {
+              "queries": [
+                {
+                  "kind": "PanelQuery",
+                  "spec": {
+                    "hidden": false,
+                    "query": {
+                      "datasource": {
+                        "name": "bfkfq6kdpznr4e"
+                      },
+                      "group": "loki",
+                      "kind": "DataQuery",
+                      "spec": {
+                        "direction": "backward",
+                        "editorMode": "builder",
+                        "expr": "{host=\"beepi\", filename=\"/var/log/ufw.log\"} |= `[UFW BLOCK]` | detected_level != `info`",
+                        "queryType": "range"
+                      },
+                      "version": "v0"
+                    },
+                    "refId": "A"
+                  }
+                }
+              ],
+              "queryOptions": {},
+              "transformations": []
+            }
+          },
+          "description": "",
+          "id": 1,
+          "links": [],
+          "title": "UFW Blocks",
+          "vizConfig": {
+            "group": "logs",
+            "kind": "VizConfig",
+            "spec": {
+              "fieldConfig": {
+                "defaults": {},
+                "overrides": []
+              },
+              "options": {
+                "dedupStrategy": "numbers",
+                "enableInfiniteScrolling": false,
+                "enableLogDetails": true,
+                "prettifyLogMessage": true,
+                "showControls": false,
+                "showFieldSelector": false,
+                "showLabels": false,
+                "showLevel": true,
+                "showTime": true,
+                "sortOrder": "Descending",
+                "syntaxHighlighting": true,
+                "timestampResolution": "ms",
+                "unwrappedColumns": false,
+                "wrapLogMessage": false
+              }
+            },
+            "version": "13.0.1"
+          }
+        }
+      },
+      "panel-2": {
+        "kind": "Panel",
+        "spec": {
+          "data": {
+            "kind": "QueryGroup",
+            "spec": {
+              "queries": [
+                {
+                  "kind": "PanelQuery",
+                  "spec": {
+                    "hidden": false,
+                    "query": {
+                      "datasource": {
+                        "name": "bfkfq6kdpznr4e"
+                      },
+                      "group": "loki",
+                      "kind": "DataQuery",
+                      "spec": {
+                        "direction": "backward",
+                        "editorMode": "builder",
+                        "expr": "{filename=\"/var/log/fail2ban.log\"} |= `NOTICE`",
+                        "queryType": "range"
+                      },
+                      "version": "v0"
+                    },
+                    "refId": "A"
+                  }
+                }
+              ],
+              "queryOptions": {},
+              "transformations": []
+            }
+          },
+          "description": "",
+          "id": 2,
+          "links": [],
+          "title": "Fail2Ban bans",
+          "vizConfig": {
+            "group": "logs",
+            "kind": "VizConfig",
+            "spec": {
+              "fieldConfig": {
+                "defaults": {},
+                "overrides": []
+              },
+              "options": {
+                "dedupStrategy": "none",
+                "enableInfiniteScrolling": false,
+                "enableLogDetails": true,
+                "prettifyLogMessage": true,
+                "showControls": false,
+                "showFieldSelector": false,
+                "showLabels": false,
+                "showLevel": true,
+                "showTime": false,
+                "sortOrder": "Descending",
+                "timestampResolution": "ms",
+                "unwrappedColumns": false,
+                "wrapLogMessage": false
+              }
+            },
+            "version": "13.0.1"
+          }
+        }
+      },
+      "panel-3": {
+        "kind": "Panel",
+        "spec": {
+          "data": {
+            "kind": "QueryGroup",
+            "spec": {
+              "queries": [
+                {
+                  "kind": "PanelQuery",
+                  "spec": {
+                    "hidden": false,
+                    "query": {
+                      "datasource": {
+                        "name": "bfkfq6kdpznr4e"
+                      },
+                      "group": "loki",
+                      "kind": "DataQuery",
+                      "spec": {
+                        "direction": "backward",
+                        "editorMode": "builder",
+                        "expr": "count_over_time({filename=\"/var/log/fail2ban.log\"} |= `NOTICE` |= `Ban` != `Restore` [$__auto])",
+                        "legendFormat": "",
+                        "queryType": "range",
+                        "step": "10m"
+                      },
+                      "version": "v0"
+                    },
+                    "refId": "A"
+                  }
+                }
+              ],
+              "queryOptions": {},
+              "transformations": []
+            }
+          },
+          "description": "",
+          "id": 3,
+          "links": [],
+          "title": "Fail2Ban Over Time",
+          "vizConfig": {
+            "group": "barchart",
+            "kind": "VizConfig",
+            "spec": {
+              "fieldConfig": {
+                "defaults": {
+                  "color": {
+                    "mode": "thresholds"
+                  },
+                  "custom": {
+                    "axisBorderShow": false,
+                    "axisCenteredZero": false,
+                    "axisColorMode": "text",
+                    "axisLabel": "",
+                    "axisPlacement": "auto",
+                    "fillOpacity": 80,
+                    "gradientMode": "none",
+                    "hideFrom": {
+                      "legend": false,
+                      "tooltip": false,
+                      "viz": false
+                    },
+                    "lineWidth": 1,
+                    "scaleDistribution": {
+                      "type": "linear"
+                    },
+                    "thresholdsStyle": {
+                      "mode": "off"
+                    }
+                  },
+                  "decimals": 0,
+                  "min": 0,
+                  "thresholds": {
+                    "mode": "absolute",
+                    "steps": [
+                      {
+                        "color": "green",
+                        "value": 0
+                      }
+                    ]
+                  },
+                  "unit": "none"
+                },
+                "overrides": []
+              },
+              "options": {
+                "barRadius": 0,
+                "barWidth": 0.97,
+                "fullHighlight": false,
+                "groupWidth": 0.7,
+                "legend": {
+                  "calcs": [],
+                  "displayMode": "list",
+                  "placement": "bottom",
+                  "showLegend": false
+                },
+                "orientation": "auto",
+                "showValue": "auto",
+                "stacking": "none",
+                "tooltip": {
+                  "hideZeros": false,
+                  "mode": "single",
+                  "sort": "none"
+                },
+                "xTickLabelRotation": 0,
+                "xTickLabelSpacing": 0
+              }
+            },
+            "version": "13.0.1"
+          }
+        }
+      },
+      "panel-4": {
+        "kind": "Panel",
+        "spec": {
+          "data": {
+            "kind": "QueryGroup",
+            "spec": {
+              "queries": [
+                {
+                  "kind": "PanelQuery",
+                  "spec": {
+                    "hidden": false,
+                    "query": {
+                      "datasource": {
+                        "name": "bfkfq6kdpznr4e"
+                      },
+                      "group": "loki",
+                      "kind": "DataQuery",
+                      "spec": {
+                        "direction": "backward",
+                        "editorMode": "builder",
+                        "expr": "{type=\"Ban\"}",
+                        "queryType": "range"
+                      },
+                      "version": "v0"
+                    },
+                    "refId": "A"
+                  }
+                }
+              ],
+              "queryOptions": {},
+              "transformations": [
+                {
+                  "group": "extractFields",
+                  "kind": "Transformation",
+                  "spec": {
+                    "options": {
+                      "delimiter": ",",
+                      "format": "json",
+                      "keepTime": false,
+                      "replace": false,
+                      "source": "labels"
+                    }
+                  }
+                },
+                {
+                  "group": "convertFieldType",
+                  "kind": "Transformation",
+                  "spec": {
+                    "options": {
+                      "conversions": [
+                        {
+                          "destinationType": "number",
+                          "targetField": "geoip_location_latitude"
+                        },
+                        {
+                          "destinationType": "number",
+                          "targetField": "geoip_location_longitude"
+                        }
+                      ],
+                      "fields": {}
+                    }
+                  }
+                }
+              ]
+            }
+          },
+          "description": "",
+          "id": 4,
+          "links": [],
+          "title": "Fail2Ban Worldmap",
+          "vizConfig": {
+            "group": "geomap",
+            "kind": "VizConfig",
+            "spec": {
+              "fieldConfig": {
+                "defaults": {
+                  "color": {
+                    "mode": "thresholds"
+                  },
+                  "custom": {
+                    "hideFrom": {
+                      "legend": false,
+                      "tooltip": false,
+                      "viz": false
+                    }
+                  },
+                  "thresholds": {
+                    "mode": "absolute",
+                    "steps": [
+                      {
+                        "color": "green",
+                        "value": 0
+                      }
+                    ]
+                  }
+                },
+                "overrides": [
+                  {
+                    "matcher": {
+                      "id": "byName",
+                      "options": "geoip_location_latitude",
+                      "scope": "series"
+                    },
+                    "properties": [
+                      {
+                        "id": "decimals",
+                        "value": 4
+                      }
+                    ]
+                  },
+                  {
+                    "matcher": {
+                      "id": "byName",
+                      "options": "geoip_location_longitude",
+                      "scope": "series"
+                    },
+                    "properties": [
+                      {
+                        "id": "decimals",
+                        "value": 4
+                      }
+                    ]
+                  },
+                  {
+                    "matcher": {
+                      "id": "byName",
+                      "options": "__error__",
+                      "scope": "series"
+                    },
+                    "properties": [
+                      {
+                        "id": "custom.hideFrom",
+                        "value": {
+                          "legend": true,
+                          "tooltip": true,
+                          "viz": true
+                        }
+                      }
+                    ]
+                  },
+                  {
+                    "matcher": {
+                      "id": "byName",
+                      "options": "__error_details__",
+                      "scope": "series"
+                    },
+                    "properties": [
+                      {
+                        "id": "custom.hideFrom",
+                        "value": {
+                          "legend": true,
+                          "tooltip": true,
+                          "viz": true
+                        }
+                      }
+                    ]
+                  },
+                  {
+                    "matcher": {
+                      "id": "byName",
+                      "options": "labels",
+                      "scope": "series"
+                    },
+                    "properties": [
+                      {
+                        "id": "custom.hideFrom",
+                        "value": {
+                          "legend": true,
+                          "tooltip": true,
+                          "viz": true
+                        }
+                      }
+                    ]
+                  },
+                  {
+                    "matcher": {
+                      "id": "byName",
+                      "options": "labelTypes",
+                      "scope": "series"
+                    },
+                    "properties": [
+                      {
+                        "id": "custom.hideFrom",
+                        "value": {
+                          "legend": true,
+                          "tooltip": true,
+                          "viz": true
+                        }
+                      }
+                    ]
+                  }
+                ]
+              },
+              "options": {
+                "basemap": {
+                  "config": {},
+                  "name": "Layer 0",
+                  "noRepeat": false,
+                  "type": "default"
+                },
+                "controls": {
+                  "mouseWheelZoom": true,
+                  "showAttribution": true,
+                  "showDebug": false,
+                  "showMeasure": false,
+                  "showScale": false,
+                  "showZoom": true
+                },
+                "layers": [
+                  {
+                    "config": {
+                      "blur": 17,
+                      "radius": 9,
+                      "weight": {
+                        "fixed": 0.5,
+                        "max": 1,
+                        "min": 0
+                      }
+                    },
+                    "layer-tooltip": false,
+                    "location": {
+                      "latitude": "geoip_location_latitude",
+                      "longitude": "geoip_location_longitude",
+                      "mode": "coords"
+                    },
+                    "name": "Bans",
+                    "opacity": 1,
+                    "tooltip": true,
+                    "type": "heatmap"
+                  }
+                ],
+                "tooltip": {
+                  "mode": "details"
+                },
+                "view": {
+                  "allLayers": true,
+                  "dashboardVariable": false,
+                  "id": "zero",
+                  "lat": 0,
+                  "lon": 0,
+                  "noRepeat": false,
+                  "zoom": 1
+                }
+              }
+            },
+            "version": "13.0.1"
+          }
+        }
+      },
+      "panel-5": {
+        "kind": "Panel",
+        "spec": {
+          "data": {
+            "kind": "QueryGroup",
+            "spec": {
+              "queries": [
+                {
+                  "kind": "PanelQuery",
+                  "spec": {
+                    "hidden": false,
+                    "query": {
+                      "datasource": {
+                        "name": "bfkfq6kdpznr4e"
+                      },
+                      "group": "loki",
+                      "kind": "DataQuery",
+                      "spec": {
+                        "direction": "backward",
+                        "editorMode": "builder",
+                        "expr": "{job=\"varlog\"} |= `ERROR` or `WARN` or `CRITICAL` != `queryType` or `query_hash` | logfmt",
+                        "queryType": "range"
+                      },
+                      "version": "v0"
+                    },
+                    "refId": "A"
+                  }
+                }
+              ],
+              "queryOptions": {},
+              "transformations": []
+            }
+          },
+          "description": "",
+          "id": 5,
+          "links": [],
+          "title": "New panel",
+          "vizConfig": {
+            "group": "logs",
+            "kind": "VizConfig",
+            "spec": {
+              "fieldConfig": {
+                "defaults": {},
+                "overrides": []
+              },
+              "options": {
+                "dedupStrategy": "none",
+                "enableInfiniteScrolling": false,
+                "enableLogDetails": true,
+                "prettifyLogMessage": true,
+                "showControls": false,
+                "showFieldSelector": false,
+                "showLevel": true,
+                "showTime": false,
+                "sortOrder": "Descending",
+                "timestampResolution": "ms",
+                "unwrappedColumns": false,
+                "wrapLogMessage": false
+              }
+            },
+            "version": "13.0.1"
+          }
+        }
+      }
+    },
+    "layout": {
+      "kind": "GridLayout",
+      "spec": {
+        "items": [
+          {
+            "kind": "GridLayoutItem",
+            "spec": {
+              "element": {
+                "kind": "ElementReference",
+                "name": "panel-5"
+              },
+              "height": 8,
+              "width": 24,
+              "x": 0,
+              "y": 0
+            }
+          },
+          {
+            "kind": "GridLayoutItem",
+            "spec": {
+              "element": {
+                "kind": "ElementReference",
+                "name": "panel-4"
+              },
+              "height": 8,
+              "width": 12,
+              "x": 0,
+              "y": 8
+            }
+          },
+          {
+            "kind": "GridLayoutItem",
+            "spec": {
+              "element": {
+                "kind": "ElementReference",
+                "name": "panel-3"
+              },
+              "height": 8,
+              "width": 12,
+              "x": 12,
+              "y": 8
+            }
+          },
+          {
+            "kind": "GridLayoutItem",
+            "spec": {
+              "element": {
+                "kind": "ElementReference",
+                "name": "panel-2"
+              },
+              "height": 7,
+              "width": 24,
+              "x": 0,
+              "y": 16
+            }
+          },
+          {
+            "kind": "GridLayoutItem",
+            "spec": {
+              "element": {
+                "kind": "ElementReference",
+                "name": "panel-1"
+              },
+              "height": 8,
+              "width": 24,
+              "x": 0,
+              "y": 23
+            }
+          }
+        ]
+      }
+    },
+    "links": [],
+    "liveNow": false,
+    "preferences": {
+      "layout": {
+        "kind": "GridLayout",
+        "spec": {
+          "items": []
+        }
+      }
+    },
+    "preload": false,
+    "tags": [],
+    "timeSettings": {
+      "autoRefresh": "",
+      "autoRefreshIntervals": [
+        "5s",
+        "10s",
+        "30s",
+        "1m",
+        "5m",
+        "15m",
+        "30m",
+        "1h",
+        "2h",
+        "1d"
+      ],
+      "fiscalYearStartMonth": 0,
+      "from": "now-6h",
+      "hideTimepicker": false,
+      "timezone": "browser",
+      "to": "now"
+    },
+    "title": "Logging",
+    "variables": [
+      {
+        "datasource": {
+          "name": "bfkfq6kdpznr4e"
+        },
+        "group": "loki",
+        "kind": "AdhocVariable",
+        "spec": {
+          "allowCustomValue": true,
+          "baseFilters": [],
+          "defaultKeys": [],
+          "enableGroupBy": false,
+          "filters": [],
+          "hide": "dontHide",
+          "name": "Filters",
+          "skipUrlSync": false
+        }
+      }
+    ]
+  }
+}

playbooks/monitoring/grafana/dashboards/backups.json.j2

@@ -0,0 +1,215 @@
+{
+  "id": null,
+  "uid": "beepi-backups",
+  "title": "Backups",
+  "tags": ["beepi", "backup"],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "version": 1,
+  "refresh": "1m",
+  "time": {"from": "now-7d", "to": "now"},
+  "panels": [
+    {
+      "type": "table",
+      "title": "Backup jobs",
+      "id": 1,
+      "gridPos": {"h": 7, "w": 24, "x": 0, "y": 0},
+      "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+      "targets": [
+        {
+          "refId": "age",
+          "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+          "expr": "time() - max by (backup_job) (backup_last_success_timestamp_seconds{backup_job!=\"\"})",
+          "instant": true,
+          "format": "table"
+        },
+        {
+          "refId": "status",
+          "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+          "expr": "max by (backup_job) (backup_last_status{backup_job!=\"\"})",
+          "instant": true,
+          "format": "table"
+        },
+        {
+          "refId": "duration",
+          "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+          "expr": "max by (backup_job) (backup_last_duration_seconds{backup_job!=\"\"})",
+          "instant": true,
+          "format": "table"
+        },
+        {
+          "refId": "size",
+          "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+          "expr": "max by (backup_job) (backup_last_archive_bytes{backup_job!=\"\"})",
+          "instant": true,
+          "format": "table"
+        }
+      ],
+      "transformations": [
+        {"id": "joinByField", "options": {"byField": "backup_job", "mode": "outer"}},
+        {"id": "organize", "options": {
+          "excludeByName": {"Time": true, "Time 1": true, "Time 2": true, "Time 3": true, "Time 4": true},
+          "renameByName": {
+            "backup_job": "Job",
+            "Value #age": "Age",
+            "Value #status": "Status",
+            "Value #duration": "Duration",
+            "Value #size": "Archive"
+          }
+        }}
+      ],
+      "fieldConfig": {
+        "defaults": {"custom": {"align": "auto", "cellOptions": {"type": "auto"}}},
+        "overrides": [
+          {"matcher": {"id": "byName", "options": "Age"}, "properties": [
+            {"id": "unit", "value": "s"},
+            {"id": "custom.cellOptions", "value": {"type": "color-background"}},
+            {"id": "thresholds", "value": {"mode": "absolute", "steps": [
+              {"color": "green", "value": null},
+              {"color": "orange", "value": 90000},
+              {"color": "red", "value": 172800}
+            ]}}
+          ]},
+          {"matcher": {"id": "byName", "options": "Duration"}, "properties": [{"id": "unit", "value": "s"}]},
+          {"matcher": {"id": "byName", "options": "Archive"}, "properties": [{"id": "unit", "value": "bytes"}]},
+          {"matcher": {"id": "byName", "options": "Status"}, "properties": [
+            {"id": "custom.cellOptions", "value": {"type": "color-background"}},
+            {"id": "mappings", "value": [{"type": "value", "options": {
+              "0": {"text": "FAILED", "color": "red", "index": 0},
+              "1": {"text": "ok", "color": "green", "index": 1}
+            }}]},
+            {"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}}
+          ]}
+        ]
+      },
+      "options": {"showHeader": true}
+    },
+    {
+      "type": "timeseries",
+      "title": "Archive size",
+      "id": 2,
+      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 7},
+      "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+          "expr": "backup_archive_newest_bytes",
+          "legendFormat": "newest {{ '{{' }}kind{{ '}}' }}"
+        },
+        {
+          "refId": "B",
+          "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+          "expr": "backup_archive_total_bytes",
+          "legendFormat": "total {{ '{{' }}kind{{ '}}' }}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {"unit": "bytes", "custom": {"lineWidth": 2, "fillOpacity": 8}},
+        "overrides": []
+      },
+      "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi"}}
+    },
+    {
+      "type": "timeseries",
+      "title": "Backup duration",
+      "id": 3,
+      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 7},
+      "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+          "expr": "backup_last_duration_seconds{backup_job!=\"\"}",
+          "legendFormat": "{{ '{{' }}backup_job{{ '}}' }}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {"unit": "s", "custom": {"lineWidth": 2, "fillOpacity": 8}},
+        "overrides": []
+      },
+      "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi"}}
+    },
+    {
+      "type": "stat",
+      "title": "Retained archives",
+      "id": 4,
+      "gridPos": {"h": 5, "w": 6, "x": 0, "y": 15},
+      "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+          "expr": "backup_archive_count",
+          "legendFormat": "{{ '{{' }}kind{{ '}}' }}",
+          "instant": true
+        }
+      ],
+      "fieldConfig": {"defaults": {"unit": "short", "color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 2}]}}, "overrides": []},
+      "options": {"colorMode": "background", "graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}}
+    },
+    {
+      "type": "stat",
+      "title": "RAID mounted",
+      "id": 5,
+      "gridPos": {"h": 5, "w": 6, "x": 6, "y": 15},
+      "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+          "expr": "max(backup_raid_mounted) or vector(0)",
+          "instant": true
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "mappings": [{"type": "value", "options": {"0": {"text": "NOT MOUNTED", "color": "red", "index": 0}, "1": {"text": "mounted", "color": "green", "index": 1}}}],
+          "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}
+        },
+        "overrides": []
+      },
+      "options": {"colorMode": "background", "graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}}
+    },
+    {
+      "type": "timeseries",
+      "title": "RAID free space",
+      "id": 6,
+      "gridPos": {"h": 5, "w": 12, "x": 12, "y": 15},
+      "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+          "expr": "node_filesystem_avail_bytes{mountpoint=\"{{ backup_raid_mount }}\"}",
+          "legendFormat": "free"
+        }
+      ],
+      "fieldConfig": {"defaults": {"unit": "bytes", "custom": {"lineWidth": 2, "fillOpacity": 8}}, "overrides": []},
+      "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "single"}}
+    },
+    {
+      "type": "logs",
+      "title": "Backup log",
+      "id": 7,
+      "gridPos": {"h": 12, "w": 24, "x": 0, "y": 20},
+      "datasource": {"type": "loki", "uid": "{{ grafana_loki_uid }}"},
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {"type": "loki", "uid": "{{ grafana_loki_uid }}"},
+          "expr": "{job=\"backup\"}",
+          "queryType": "range"
+        }
+      ],
+      "options": {
+        "showTime": true,
+        "sortOrder": "Descending",
+        "wrapLogMessage": true,
+        "enableLogDetails": true,
+        "dedupStrategy": "none",
+        "prettifyLogMessage": false
+      }
+    }
+  ]
+}

playbooks/monitoring/grafana/dashboards/infrastructure-map.json.j2

@@ -0,0 +1,364 @@
+{
+  "id": null,
+  "uid": "beepi-map",
+  "title": "Infrastructure Map",
+  "tags": ["beepi", "overview"],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "version": 1,
+  "refresh": "30s",
+  "time": {"from": "now-6h", "to": "now"},
+  "panels": [
+    {
+      "type": "stat",
+      "title": "Services down",
+      "id": 1,
+      "gridPos": {"h": 4, "w": 4, "x": 0, "y": 0},
+      "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+          "expr": "count(min by (service) (probe_success{vantage=\"local\"}) == 0 and on (service) (max by (service) (service_enabled) == 1)) or vector(0)",
+          "instant": true
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {"color": "green", "value": null},
+              {"color": "red", "value": 1}
+            ]
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "auto",
+        "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}
+      }
+    },
+    {
+      "type": "stat",
+      "title": "Last daily backup",
+      "id": 2,
+      "gridPos": {"h": 4, "w": 4, "x": 4, "y": 0},
+      "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+          "expr": "time() - max(backup_last_success_timestamp_seconds{backup_job=\"daily\"})",
+          "instant": true
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {"color": "green", "value": null},
+              {"color": "orange", "value": 90000},
+              {"color": "red", "value": 172800}
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "auto",
+        "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}
+      }
+    },
+    {
+      "type": "stat",
+      "title": "Last offsite sync",
+      "id": 3,
+      "gridPos": {"h": 4, "w": 4, "x": 8, "y": 0},
+      "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+          "expr": "time() - max(backup_last_success_timestamp_seconds{backup_job=\"offsite\"})",
+          "instant": true
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {"color": "green", "value": null},
+              {"color": "orange", "value": 90000},
+              {"color": "red", "value": 172800}
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "auto",
+        "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}
+      }
+    },
+    {
+      "type": "stat",
+      "title": "Tunnel handshake",
+      "id": 4,
+      "gridPos": {"h": 4, "w": 4, "x": 12, "y": 0},
+      "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+          "expr": "max(wireguard_peer_handshake_age_seconds{peer=\"beepi\"})",
+          "instant": true
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {"color": "green", "value": null},
+              {"color": "orange", "value": 180},
+              {"color": "red", "value": 300}
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "auto",
+        "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}
+      }
+    },
+    {
+      "type": "stat",
+      "title": "Ingress",
+      "id": 5,
+      "gridPos": {"h": 4, "w": 4, "x": 16, "y": 0},
+      "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+          "expr": "min(up{job=\"node\",host=\"ingress\"}) or vector(0)",
+          "instant": true
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "mappings": [
+            {"type": "value", "options": {"0": {"text": "DOWN", "color": "red", "index": 0}, "1": {"text": "UP", "color": "green", "index": 1}}}
+          ],
+          "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "auto",
+        "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}
+      }
+    },
+    {
+      "type": "stat",
+      "title": "RAID free",
+      "id": 6,
+      "gridPos": {"h": 4, "w": 4, "x": 20, "y": 0},
+      "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+          "expr": "min(node_filesystem_avail_bytes{mountpoint=\"{{ backup_raid_mount }}\"})",
+          "instant": true
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes",
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {"color": "red", "value": null},
+              {"color": "orange", "value": 100000000000},
+              {"color": "green", "value": 200000000000}
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "auto",
+        "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}
+      }
+    },
+    {
+      "type": "nodeGraph",
+      "title": "Infrastructure",
+      "id": 10,
+      "gridPos": {"h": 18, "w": 18, "x": 0, "y": 4},
+      "datasource": {"type": "yesoreyeram-infinity-datasource", "uid": "{{ grafana_infinity_uid }}"},
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {"type": "yesoreyeram-infinity-datasource", "uid": "{{ grafana_infinity_uid }}"},
+          "type": "json",
+          "source": "url",
+          "parser": "backend",
+          "format": "table",
+          "url": "http://topology:8088/topology.json",
+          "url_options": {"method": "GET", "data": ""},
+          "root_selector": "nodes",
+          "columns": [
+            {"selector": "id", "text": "id", "type": "string"},
+            {"selector": "title", "text": "title", "type": "string"},
+            {"selector": "subtitle", "text": "subtitle", "type": "string"},
+            {"selector": "mainstat", "text": "mainstat", "type": "string"},
+            {"selector": "arc__ok", "text": "arc__ok", "type": "number"},
+            {"selector": "arc__fail", "text": "arc__fail", "type": "number"},
+            {"selector": "arc__disabled", "text": "arc__disabled", "type": "number"},
+            {"selector": "arc__unknown", "text": "arc__unknown", "type": "number"},
+            {"selector": "arc__used", "text": "arc__used", "type": "number"},
+            {"selector": "arc__free", "text": "arc__free", "type": "number"},
+            {"selector": "detail__status", "text": "detail__status", "type": "string"},
+            {"selector": "detail__group", "text": "detail__group", "type": "string"},
+            {"selector": "detail__used", "text": "detail__used", "type": "string"},
+            {"selector": "dashboard", "text": "dashboard", "type": "string"}
+          ]
+        },
+        {
+          "refId": "B",
+          "datasource": {"type": "yesoreyeram-infinity-datasource", "uid": "{{ grafana_infinity_uid }}"},
+          "type": "json",
+          "source": "url",
+          "parser": "backend",
+          "format": "table",
+          "url": "http://topology:8088/topology.json",
+          "url_options": {"method": "GET", "data": ""},
+          "root_selector": "edges",
+          "columns": [
+            {"selector": "id", "text": "id", "type": "string"},
+            {"selector": "source", "text": "source", "type": "string"},
+            {"selector": "target", "text": "target", "type": "string"},
+            {"selector": "mainstat", "text": "mainstat", "type": "string"},
+            {"selector": "color", "text": "color", "type": "string"},
+            {"selector": "thickness", "text": "thickness", "type": "number"}
+          ]
+        }
+      ],
+      "transformations": [
+        {
+          "id": "convertFieldType",
+          "options": {
+            "conversions": [
+              {"targetField": "arc__ok", "destinationType": "number"},
+              {"targetField": "arc__fail", "destinationType": "number"},
+              {"targetField": "arc__disabled", "destinationType": "number"},
+              {"targetField": "arc__unknown", "destinationType": "number"},
+              {"targetField": "arc__used", "destinationType": "number"},
+              {"targetField": "arc__free", "destinationType": "number"}
+            ]
+          }
+        }
+      ],
+      "options": {
+        "nodes": {
+          "mainStatUnit": "",
+          "secondaryStatUnit": "",
+          "arcs": [
+            {"field": "arc__ok", "color": "green"},
+            {"field": "arc__fail", "color": "red"},
+            {"field": "arc__disabled", "color": "text"},
+            {"field": "arc__unknown", "color": "orange"},
+            {"field": "arc__used", "color": "semi-dark-orange"},
+            {"field": "arc__free", "color": "semi-dark-green"}
+          ]
+        },
+        "edges": {"mainStatUnit": ""},
+        "zoomMode": "cooperative"
+      },
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": [
+          {
+            "matcher": {"id": "byName", "options": "arc__ok"},
+            "properties": [{"id": "color", "value": {"mode": "fixed", "fixedColor": "green"}}]
+          },
+          {
+            "matcher": {"id": "byName", "options": "arc__fail"},
+            "properties": [{"id": "color", "value": {"mode": "fixed", "fixedColor": "red"}}]
+          },
+          {
+            "matcher": {"id": "byName", "options": "arc__disabled"},
+            "properties": [{"id": "color", "value": {"mode": "fixed", "fixedColor": "text"}}]
+          },
+          {
+            "matcher": {"id": "byName", "options": "arc__unknown"},
+            "properties": [{"id": "color", "value": {"mode": "fixed", "fixedColor": "orange"}}]
+          },
+          {
+            "matcher": {"id": "byName", "options": "arc__used"},
+            "properties": [{"id": "color", "value": {"mode": "fixed", "fixedColor": "semi-dark-orange"}}]
+          },
+          {
+            "matcher": {"id": "byName", "options": "arc__free"},
+            "properties": [{"id": "color", "value": {"mode": "fixed", "fixedColor": "semi-dark-green"}}]
+          },
+          {
+            "matcher": {"id": "byName", "options": "id"},
+            "properties": [
+              {
+                "id": "links",
+                "value": [
+                  {
+                    "title": "Open related dashboard",
+                    "url": "${__data.fields.dashboard}",
+                    "targetBlank": false
+                  }
+                ]
+              }
+            ]
+          }
+        ]
+      }
+    },
+    {
+      "type": "alertlist",
+      "title": "Firing alerts",
+      "id": 11,
+      "gridPos": {"h": 18, "w": 6, "x": 18, "y": 4},
+      "options": {
+        "alertInstanceLabelFilter": "",
+        "alertName": "",
+        "dashboardAlerts": false,
+        "groupBy": [],
+        "groupMode": "default",
+        "maxItems": 30,
+        "sortOrder": 3,
+        "stateFilter": {"error": true, "firing": true, "noData": true, "normal": false, "pending": true},
+        "viewMode": "list"
+      }
+    }
+  ]
+}

playbooks/monitoring/grafana/dashboards/ingress.json.j2

@@ -0,0 +1,260 @@
+{
+  "id": null,
+  "uid": "beepi-ingress",
+  "title": "Ingress",
+  "tags": ["beepi", "ingress"],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "version": 1,
+  "refresh": "30s",
+  "time": {"from": "now-24h", "to": "now"},
+  "panels": [
+    {
+      "type": "stat",
+      "title": "Ingress uptime",
+      "id": 1,
+      "gridPos": {"h": 4, "w": 5, "x": 0, "y": 0},
+      "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+          "expr": "time() - max(node_boot_time_seconds{host=\"ingress\"})",
+          "instant": true
+        }
+      ],
+      "fieldConfig": {"defaults": {"unit": "s", "color": {"mode": "fixed", "fixedColor": "blue"}}, "overrides": []},
+      "options": {"colorMode": "value", "graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}}
+    },
+    {
+      "type": "stat",
+      "title": "Tunnel RTT",
+      "id": 2,
+      "gridPos": {"h": 4, "w": 5, "x": 5, "y": 0},
+      "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+          "expr": "max(probe_duration_seconds{probe=\"tunnel-rtt\"})",
+          "instant": true
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {"unit": "s", "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 0.1}, {"color": "red", "value": 0.3}]}},
+        "overrides": []
+      },
+      "options": {"colorMode": "background", "graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}}
+    },
+    {
+      "type": "stat",
+      "title": "Banned hosts (24h)",
+      "id": 3,
+      "gridPos": {"h": 4, "w": 5, "x": 10, "y": 0},
+      "datasource": {"type": "loki", "uid": "{{ grafana_loki_uid }}"},
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {"type": "loki", "uid": "{{ grafana_loki_uid }}"},
+          "expr": "sum(count_over_time({job=\"fail2ban\", host=\"ingress\"} |= \"Ban\" [24h]))",
+          "queryType": "instant"
+        }
+      ],
+      "fieldConfig": {"defaults": {"unit": "short", "color": {"mode": "fixed", "fixedColor": "orange"}}, "overrides": []},
+      "options": {"colorMode": "value", "graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}}
+    },
+    {
+      "type": "stat",
+      "title": "Blocked packets (24h)",
+      "id": 4,
+      "gridPos": {"h": 4, "w": 9, "x": 15, "y": 0},
+      "datasource": {"type": "loki", "uid": "{{ grafana_loki_uid }}"},
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {"type": "loki", "uid": "{{ grafana_loki_uid }}"},
+          "expr": "sum(count_over_time({job=\"ufw\", host=\"ingress\"} [24h]))",
+          "queryType": "instant"
+        }
+      ],
+      "fieldConfig": {"defaults": {"unit": "short", "color": {"mode": "fixed", "fixedColor": "purple"}}, "overrides": []},
+      "options": {"colorMode": "value", "graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}}
+    },
+    {
+      "type": "table",
+      "title": "WireGuard peers",
+      "id": 5,
+      "gridPos": {"h": 7, "w": 12, "x": 0, "y": 4},
+      "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+      "targets": [
+        {
+          "refId": "age",
+          "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+          "expr": "max by (peer) (wireguard_peer_handshake_age_seconds)",
+          "instant": true,
+          "format": "table"
+        },
+        {
+          "refId": "rx",
+          "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+          "expr": "max by (peer) (wireguard_peer_receive_bytes_total)",
+          "instant": true,
+          "format": "table"
+        },
+        {
+          "refId": "tx",
+          "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+          "expr": "max by (peer) (wireguard_peer_transmit_bytes_total)",
+          "instant": true,
+          "format": "table"
+        }
+      ],
+      "transformations": [
+        {"id": "joinByField", "options": {"byField": "peer", "mode": "outer"}},
+        {"id": "organize", "options": {
+          "excludeByName": {"Time": true, "Time 1": true, "Time 2": true, "Time 3": true},
+          "renameByName": {"peer": "Peer", "Value #age": "Last handshake", "Value #rx": "Received", "Value #tx": "Sent"}
+        }}
+      ],
+      "fieldConfig": {
+        "defaults": {"custom": {"align": "auto"}},
+        "overrides": [
+          {"matcher": {"id": "byName", "options": "Last handshake"}, "properties": [
+            {"id": "unit", "value": "s"},
+            {"id": "custom.cellOptions", "value": {"type": "color-background"}},
+            {"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 300}, {"color": "red", "value": 3600}]}}
+          ]},
+          {"matcher": {"id": "byRegexp", "options": "Received|Sent"}, "properties": [{"id": "unit", "value": "bytes"}]}
+        ]
+      },
+      "options": {"showHeader": true}
+    },
+    {
+      "type": "state-timeline",
+      "title": "Tunnel path probes",
+      "id": 6,
+      "gridPos": {"h": 7, "w": 12, "x": 12, "y": 4},
+      "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+          "expr": "min by (probe) (probe_success{vantage=\"ingress\"})",
+          "legendFormat": "{{ '{{' }}probe{{ '}}' }}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {"fillOpacity": 90, "lineWidth": 0},
+          "mappings": [{"type": "value", "options": {"0": {"text": "down", "color": "red", "index": 0}, "1": {"text": "up", "color": "green", "index": 1}}}],
+          "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}
+        },
+        "overrides": []
+      },
+      "options": {"mergeValues": true, "showValue": "never", "legend": {"displayMode": "list", "placement": "bottom", "showLegend": false}}
+    },
+    {
+      "type": "timeseries",
+      "title": "Ingress host",
+      "id": 7,
+      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 11},
+      "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+          "expr": "100 - (avg(rate(node_cpu_seconds_total{host=\"ingress\",mode=\"idle\"}[5m])) * 100)",
+          "legendFormat": "cpu"
+        },
+        {
+          "refId": "B",
+          "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+          "expr": "(1 - (node_memory_MemAvailable_bytes{host=\"ingress\"} / node_memory_MemTotal_bytes{host=\"ingress\"})) * 100",
+          "legendFormat": "memory"
+        },
+        {
+          "refId": "C",
+          "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+          "expr": "(1 - (node_filesystem_avail_bytes{host=\"ingress\",mountpoint=\"/\"} / node_filesystem_size_bytes{host=\"ingress\",mountpoint=\"/\"})) * 100",
+          "legendFormat": "disk"
+        }
+      ],
+      "fieldConfig": {"defaults": {"unit": "percent", "max": 100, "min": 0, "custom": {"lineWidth": 2, "fillOpacity": 8}}, "overrides": []},
+      "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi"}}
+    },
+    {
+      "type": "timeseries",
+      "title": "Ingress network",
+      "id": 8,
+      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 11},
+      "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+          "expr": "rate(node_network_receive_bytes_total{host=\"ingress\",device!~\"lo\"}[5m]) * 8",
+          "legendFormat": "in {{ '{{' }}device{{ '}}' }}"
+        },
+        {
+          "refId": "B",
+          "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+          "expr": "rate(node_network_transmit_bytes_total{host=\"ingress\",device!~\"lo\"}[5m]) * 8",
+          "legendFormat": "out {{ '{{' }}device{{ '}}' }}"
+        }
+      ],
+      "fieldConfig": {"defaults": {"unit": "bps", "custom": {"lineWidth": 2, "fillOpacity": 8}}, "overrides": []},
+      "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi"}}
+    },
+    {
+      "type": "geomap",
+      "title": "Ban origins",
+      "id": 9,
+      "gridPos": {"h": 11, "w": 12, "x": 0, "y": 19},
+      "datasource": {"type": "loki", "uid": "{{ grafana_loki_uid }}"},
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {"type": "loki", "uid": "{{ grafana_loki_uid }}"},
+          "expr": "sum by (geoip_location_latitude, geoip_location_longitude, geoip_country_name) (count_over_time({job=\"fail2ban\"} |= \"Ban\" [$__range]))",
+          "queryType": "instant"
+        }
+      ],
+      "options": {
+        "basemap": {"type": "default", "name": "Basemap"},
+        "layers": [
+          {
+            "type": "markers",
+            "name": "bans",
+            "location": {"mode": "coords", "latitude": "geoip_location_latitude", "longitude": "geoip_location_longitude"},
+            "config": {"style": {"color": {"fixed": "red"}, "size": {"field": "Value", "min": 4, "max": 20}, "opacity": 0.6}}
+          }
+        ],
+        "view": {"id": "zero", "lat": 25, "lon": 10, "zoom": 1.6}
+      },
+      "fieldConfig": {"defaults": {}, "overrides": []}
+    },
+    {
+      "type": "logs",
+      "title": "Ingress journal and firewall",
+      "id": 10,
+      "gridPos": {"h": 11, "w": 12, "x": 12, "y": 19},
+      "datasource": {"type": "loki", "uid": "{{ grafana_loki_uid }}"},
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {"type": "loki", "uid": "{{ grafana_loki_uid }}"},
+          "expr": "{host=\"ingress\"}",
+          "queryType": "range"
+        }
+      ],
+      "options": {
+        "showTime": true,
+        "sortOrder": "Descending",
+        "wrapLogMessage": true,
+        "enableLogDetails": true,
+        "dedupStrategy": "none",
+        "prettifyLogMessage": false
+      }
+    }
+  ]
+}

playbooks/monitoring/grafana/dashboards/services.json.j2

@@ -0,0 +1,195 @@
+{
+  "id": null,
+  "uid": "beepi-services",
+  "title": "Services",
+  "tags": ["beepi", "services"],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "version": 1,
+  "refresh": "30s",
+  "time": {"from": "now-24h", "to": "now"},
+  "panels": [
+    {
+      "type": "table",
+      "title": "Service health",
+      "id": 1,
+      "gridPos": {"h": 10, "w": 24, "x": 0, "y": 0},
+      "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+      "targets": [
+        {
+          "refId": "enabled",
+          "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+          "expr": "max by (service) (service_enabled)",
+          "instant": true,
+          "format": "table"
+        },
+        {
+          "refId": "units",
+          "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+          "expr": "max by (service) (service_units_active) / max by (service) (service_units_total)",
+          "instant": true,
+          "format": "table"
+        },
+        {
+          "refId": "local",
+          "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+          "expr": "min by (service) (probe_success{vantage=\"local\"})",
+          "instant": true,
+          "format": "table"
+        },
+        {
+          "refId": "vhost",
+          "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+          "expr": "min by (service) (probe_success{vantage=\"nginx\"})",
+          "instant": true,
+          "format": "table"
+        },
+        {
+          "refId": "latency",
+          "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+          "expr": "max by (service) (probe_duration_seconds{vantage=\"nginx\"})",
+          "instant": true,
+          "format": "table"
+        }
+      ],
+      "transformations": [
+        {"id": "joinByField", "options": {"byField": "service", "mode": "outer"}},
+        {"id": "organize", "options": {
+          "excludeByName": {"Time": true, "Time 1": true, "Time 2": true, "Time 3": true, "Time 4": true, "Time 5": true},
+          "renameByName": {
+            "service": "Service",
+            "Value #enabled": "Expected",
+            "Value #units": "Units",
+            "Value #local": "Local probe",
+            "Value #vhost": "Public vhost",
+            "Value #latency": "Latency"
+          }
+        }}
+      ],
+      "fieldConfig": {
+        "defaults": {"custom": {"align": "auto", "cellOptions": {"type": "auto"}}},
+        "overrides": [
+          {"matcher": {"id": "byName", "options": "Expected"}, "properties": [
+            {"id": "mappings", "value": [{"type": "value", "options": {"0": {"text": "disabled", "color": "text", "index": 0}, "1": {"text": "yes", "color": "green", "index": 1}}}]},
+            {"id": "custom.cellOptions", "value": {"type": "color-text"}}
+          ]},
+          {"matcher": {"id": "byName", "options": "Units"}, "properties": [
+            {"id": "unit", "value": "percentunit"},
+            {"id": "custom.cellOptions", "value": {"type": "color-background"}},
+            {"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "orange", "value": 0.5}, {"color": "green", "value": 1}]}}
+          ]},
+          {"matcher": {"id": "byName", "options": "Local probe"}, "properties": [
+            {"id": "custom.cellOptions", "value": {"type": "color-background"}},
+            {"id": "mappings", "value": [
+              {"type": "value", "options": {"0": {"text": "DOWN", "color": "red", "index": 0}, "1": {"text": "up", "color": "green", "index": 1}}},
+              {"type": "special", "options": {"match": "null", "result": {"text": "no check", "color": "#3a3f47", "index": 2}}}
+            ]},
+            {"id": "noValue", "value": "no check"},
+            {"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}}
+          ]},
+          {"matcher": {"id": "byName", "options": "Public vhost"}, "properties": [
+            {"id": "custom.cellOptions", "value": {"type": "color-background"}},
+            {"id": "mappings", "value": [
+              {"type": "value", "options": {"0": {"text": "DOWN", "color": "red", "index": 0}, "1": {"text": "up", "color": "green", "index": 1}}},
+              {"type": "special", "options": {"match": "null", "result": {"text": "internal only", "color": "#3a3f47", "index": 2}}}
+            ]},
+            {"id": "noValue", "value": "internal only"},
+            {"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}}
+          ]},
+          {"matcher": {"id": "byName", "options": "Latency"}, "properties": [{"id": "unit", "value": "s"}]}
+        ]
+      },
+      "options": {"showHeader": true, "sortBy": [{"displayName": "Service", "desc": false}]}
+    },
+    {
+      "type": "state-timeline",
+      "title": "Probe history",
+      "id": 2,
+      "gridPos": {"h": 9, "w": 24, "x": 0, "y": 10},
+      "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+          "expr": "min by (service) (probe_success{vantage=\"local\"})",
+          "legendFormat": "{{ '{{' }}service{{ '}}' }}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "custom": {"fillOpacity": 90, "lineWidth": 0},
+          "mappings": [{"type": "value", "options": {"0": {"text": "down", "color": "red", "index": 0}, "1": {"text": "up", "color": "green", "index": 1}}}],
+          "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}
+        },
+        "overrides": []
+      },
+      "options": {"mergeValues": true, "showValue": "never", "legend": {"displayMode": "list", "placement": "bottom", "showLegend": false}}
+    },
+    {
+      "type": "table",
+      "title": "Units and containers",
+      "id": 3,
+      "gridPos": {"h": 10, "w": 12, "x": 0, "y": 19},
+      "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+          "expr": "service_unit_active",
+          "instant": true,
+          "format": "table"
+        }
+      ],
+      "transformations": [
+        {"id": "organize", "options": {
+          "excludeByName": {"Time": true, "__name__": true, "instance": true, "job": true, "host": true},
+          "renameByName": {"service": "Service", "unit": "Unit", "kind": "Kind", "Value": "Active"}
+        }}
+      ],
+      "fieldConfig": {
+        "defaults": {"custom": {"align": "auto"}},
+        "overrides": [
+          {"matcher": {"id": "byName", "options": "Active"}, "properties": [
+            {"id": "custom.cellOptions", "value": {"type": "color-background"}},
+            {"id": "mappings", "value": [{"type": "value", "options": {"0": {"text": "stopped", "color": "red", "index": 0}, "1": {"text": "running", "color": "green", "index": 1}}}]},
+            {"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}}
+          ]}
+        ]
+      },
+      "options": {"showHeader": true}
+    },
+    {
+      "type": "table",
+      "title": "TLS certificates",
+      "id": 4,
+      "gridPos": {"h": 10, "w": 12, "x": 12, "y": 19},
+      "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
+          "expr": "min by (service, instance) (probe_ssl_earliest_cert_expiry - time())",
+          "instant": true,
+          "format": "table"
+        }
+      ],
+      "transformations": [
+        {"id": "organize", "options": {
+          "excludeByName": {"Time": true},
+          "renameByName": {"service": "Service", "instance": "Endpoint", "Value": "Expires in"}
+        }}
+      ],
+      "fieldConfig": {
+        "defaults": {"custom": {"align": "auto"}},
+        "overrides": [
+          {"matcher": {"id": "byName", "options": "Expires in"}, "properties": [
+            {"id": "unit", "value": "s"},
+            {"id": "custom.cellOptions", "value": {"type": "color-background"}},
+            {"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "orange", "value": 1209600}, {"color": "green", "value": 2592000}]}}
+          ]}
+        ]
+      },
+      "options": {"showHeader": true}
+    }
+  ]
+}

playbooks/monitoring/grafana/dashboards/uptimerobot.json.j2

@@ -0,0 +1,149 @@
+{
+  "id": null,
+  "uid": "beepi-uptimerobot",
+  "title": "External Checks",
+  "tags": ["beepi", "uptimerobot"],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "version": 1,
+  "refresh": "5m",
+  "time": {"from": "now-24h", "to": "now"},
+  "panels": [
+    {
+      "type": "stat",
+      "title": "Monitors down",
+      "id": 1,
+      "gridPos": {"h": 5, "w": 6, "x": 0, "y": 0},
+      "datasource": {"type": "yesoreyeram-infinity-datasource", "uid": "{{ grafana_uptimerobot_uid }}"},
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {"type": "yesoreyeram-infinity-datasource", "uid": "{{ grafana_uptimerobot_uid }}"},
+          "type": "json",
+          "source": "url",
+          "parser": "backend",
+          "format": "table",
+          "url": "https://api.uptimerobot.com/v2/getMonitors?format=json&custom_uptime_ratios=1-7-30",
+          "url_options": {"method": "POST", "data": ""},
+          "root_selector": "monitors",
+          "columns": [
+            {"selector": "friendly_name", "text": "Monitor", "type": "string"},
+            {"selector": "status", "text": "Status", "type": "number"}
+          ],
+          "filterExpression": "status == 8 || status == 9"
+        }
+      ],
+      "transformations": [{"id": "reduce", "options": {"reducers": ["count"]}}],
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "red", "value": 1}]},
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "textMode": "value",
+        "reduceOptions": {"calcs": ["lastNotNull"], "fields": "/^Status$/", "values": false}
+      }
+    },
+    {
+      "type": "table",
+      "title": "UptimeRobot monitors",
+      "id": 2,
+      "gridPos": {"h": 12, "w": 18, "x": 6, "y": 0},
+      "datasource": {"type": "yesoreyeram-infinity-datasource", "uid": "{{ grafana_uptimerobot_uid }}"},
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {"type": "yesoreyeram-infinity-datasource", "uid": "{{ grafana_uptimerobot_uid }}"},
+          "type": "json",
+          "source": "url",
+          "parser": "backend",
+          "format": "table",
+          "url": "https://api.uptimerobot.com/v2/getMonitors?format=json&custom_uptime_ratios=1-7-30",
+          "url_options": {"method": "POST", "data": ""},
+          "root_selector": "monitors",
+          "columns": [
+            {"selector": "friendly_name", "text": "Monitor", "type": "string"},
+            {"selector": "status", "text": "Status", "type": "number"},
+            {"selector": "custom_uptime_ratio", "text": "Uptime 1d-7d-30d", "type": "string"},
+            {"selector": "interval", "text": "Interval", "type": "number"},
+            {"selector": "url", "text": "URL", "type": "string"}
+          ]
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {"custom": {"align": "auto", "cellOptions": {"type": "auto"}}},
+        "overrides": [
+          {
+            "matcher": {"id": "byName", "options": "Status"},
+            "properties": [
+              {"id": "custom.cellOptions", "value": {"type": "color-background"}},
+              {"id": "mappings", "value": [{"type": "value", "options": {
+                "0": {"text": "paused", "color": "text", "index": 0},
+                "1": {"text": "not checked", "color": "blue", "index": 1},
+                "2": {"text": "UP", "color": "green", "index": 2},
+                "8": {"text": "SEEMS DOWN", "color": "orange", "index": 3},
+                "9": {"text": "DOWN", "color": "red", "index": 4}
+              }}]},
+              {"id": "custom.width", "value": 130}
+            ]
+          },
+          {
+            "matcher": {"id": "byName", "options": "Interval"},
+            "properties": [{"id": "unit", "value": "s"}, {"id": "custom.width", "value": 100}]
+          },
+          {
+            "matcher": {"id": "byName", "options": "Uptime 1d-7d-30d"},
+            "properties": [{"id": "custom.width", "value": 190}]
+          }
+        ]
+      },
+      "options": {"showHeader": true, "sortBy": [{"displayName": "Status", "desc": true}]}
+    },
+    {
+      "type": "stat",
+      "title": "Monitors configured",
+      "id": 3,
+      "gridPos": {"h": 5, "w": 6, "x": 0, "y": 5},
+      "datasource": {"type": "yesoreyeram-infinity-datasource", "uid": "{{ grafana_uptimerobot_uid }}"},
+      "targets": [
+        {
+          "refId": "A",
+          "datasource": {"type": "yesoreyeram-infinity-datasource", "uid": "{{ grafana_uptimerobot_uid }}"},
+          "type": "json",
+          "source": "url",
+          "parser": "backend",
+          "format": "table",
+          "url": "https://api.uptimerobot.com/v2/getMonitors?format=json",
+          "url_options": {"method": "POST", "data": ""},
+          "root_selector": "monitors",
+          "columns": [{"selector": "status", "text": "Status", "type": "number"}]
+        }
+      ],
+      "transformations": [{"id": "reduce", "options": {"reducers": ["count"]}}],
+      "fieldConfig": {
+        "defaults": {"color": {"mode": "fixed", "fixedColor": "blue"}, "unit": "short"},
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "textMode": "value",
+        "reduceOptions": {"calcs": ["lastNotNull"], "fields": "/^Status$/", "values": false}
+      }
+    },
+    {
+      "type": "text",
+      "title": "About",
+      "id": 4,
+      "gridPos": {"h": 4, "w": 24, "x": 0, "y": 12},
+      "options": {
+        "mode": "markdown",
+        "content": "External reachability is checked by UptimeRobot from outside the network, which is the one thing this stack cannot do for itself: everything else runs on beepi, so it cannot report that beepi is down.\n\nThe free plan allows 10 API requests per minute. This dashboard refreshes every 5 minutes."
+      }
+    }
+  ]
+}

playbooks/monitoring/grafana/provisioning/alerting/contact-points.yml.j2

@@ -0,0 +1,34 @@
+apiVersion: 1
+
+contactPoints:
+  - orgId: 1
+    name: discord
+    receivers:
+      - uid: discord-beepi
+        type: discord
+        settings:
+          url: {{ discord_webhook }}
+          title: '{{ '{{' }} template "default.title" . {{ '}}' }}'
+          message: |
+            {{ '{{' }} range .Alerts {{ '}}' }}**{{ '{{' }} .Labels.alertname {{ '}}' }}** {{ '{{' }} if .Labels.service {{ '}}' }}({{ '{{' }} .Labels.service {{ '}}' }}){{ '{{' }} end {{ '}}' }}
+            {{ '{{' }} .Annotations.summary {{ '}}' }}
+            {{ '{{' }} end {{ '}}' }}
+          use_discord_username: true
+        disableResolveMessage: false
+
+policies:
+  - orgId: 1
+    receiver: discord
+    group_by:
+      - alertname
+      - service
+      - job
+    group_wait: 30s
+    group_interval: 5m
+    repeat_interval: 12h
+    routes:
+      - receiver: discord
+        object_matchers:
+          - ['severity', '=', 'critical']
+        group_wait: 10s
+        repeat_interval: 4h

playbooks/monitoring/grafana/provisioning/alerting/rules.yml.j2

@@ -0,0 +1,57 @@
+apiVersion: 1
+
+groups:
+{% for group in alert_groups %}
+  - orgId: 1
+    name: {{ group.name }}
+    folder: {{ grafana_folder }}
+    interval: {{ group.interval }}
+    rules:
+{% for rule in group.rules %}
+      - uid: {{ rule.uid }}
+        title: {{ rule.title }}
+        condition: C
+        for: {{ rule['for'] }}
+        noDataState: {{ rule.no_data | default('NoData') }}
+        execErrState: Alerting
+        labels:
+          severity: {{ rule.severity }}
+        annotations:
+          summary: {{ rule.summary }}
+        data:
+          - refId: A
+            relativeTimeRange:
+              from: 600
+              to: 0
+            datasourceUid: {{ grafana_loki_uid if rule.source | default('prometheus') == 'loki' else grafana_prometheus_uid }}
+            model:
+              refId: A
+              expr: '{{ rule.expr }}'
+              instant: true
+{% if rule.source | default('prometheus') == 'loki' %}
+              queryType: instant
+{% else %}
+              range: false
+{% endif %}
+          - refId: B
+            datasourceUid: __expr__
+            model:
+              refId: B
+              type: reduce
+              expression: A
+              reducer: last
+              settings:
+                mode: dropNN
+          - refId: C
+            datasourceUid: __expr__
+            model:
+              refId: C
+              type: threshold
+              expression: B
+              conditions:
+                - evaluator:
+                    type: {{ rule.op }}
+                    params:
+                      - {{ rule.threshold }}
+{% endfor %}
+{% endfor %}

playbooks/monitoring/grafana/provisioning/dashboards/dashboards.yml.j2

@@ -0,0 +1,26 @@
+apiVersion: 1
+
+providers:
+  - name: beepi
+    orgId: 1
+    folder: {{ grafana_folder }}
+    folderUid: {{ grafana_folder_uid }}
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 30
+    allowUiUpdates: false
+    options:
+      path: /var/lib/grafana/dashboards
+      foldersFromFilesStructure: false
+
+  - name: general
+    orgId: 1
+    folder: {{ grafana_general_folder }}
+    folderUid: {{ grafana_general_folder_uid }}
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 30
+    allowUiUpdates: false
+    options:
+      path: /var/lib/grafana/dashboards-general
+      foldersFromFilesStructure: false

playbooks/monitoring/grafana/provisioning/datasources/datasources.yml.j2

@@ -0,0 +1,49 @@
+apiVersion: 1
+
+datasources:
+  - name: prometheus
+    uid: {{ grafana_prometheus_uid }}
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    editable: false
+    jsonData:
+      timeInterval: 15s
+      httpMethod: POST
+
+  - name: loki
+    uid: {{ grafana_loki_uid }}
+    type: loki
+    access: proxy
+    url: http://loki:3100
+    editable: false
+    jsonData:
+      maxLines: 2000
+
+  - name: topology
+    uid: {{ grafana_infinity_uid }}
+    type: yesoreyeram-infinity-datasource
+    access: proxy
+    editable: false
+    jsonData:
+      allowedHosts:
+        - http://topology:8088
+      global_queries: []
+{% if uptimerobot_enabled %}
+
+  - name: uptimerobot
+    uid: {{ grafana_uptimerobot_uid }}
+    type: yesoreyeram-infinity-datasource
+    access: proxy
+    editable: false
+    jsonData:
+      auth_method: apiKey
+      apiKeyKey: api_key
+      apiKeyType: query
+      allowedHosts:
+        - https://api.uptimerobot.com
+      global_queries: []
+    secureJsonData:
+      apiKeyValue: {{ uptimerobot_api_key }}
+{% endif %}

playbooks/monitoring/local-config.yml.j2

@@ -34,9 +34,6 @@ schema_config:
         prefix: index_
         period: 24h
 
-ruler:
-  alertmanager_url: http://localhost:9093
-
 ingester:
   wal:
     enabled: true

playbooks/monitoring/monitoring.yml

@@ -1,6 +1,8 @@
 - name: Prometheus
   hosts: beepi
   become: true
+  vars_files:
+    - vars/alert_rules.yml
   vars:
     grafana_secret: !vault |
           $ANSIBLE_VAULT;1.1;AES256
@@ -25,19 +27,21 @@
               36376463313964333834373435653564346333396538306337333762613737643363636364353139
               3032383564333638613566633363666430356230326231633636
   tasks:
-    - name: Install podman
+    - name: Determine whether alert delivery is configured
-      ansible.builtin.apt:
+      ansible.builtin.set_fact:
-        name: podman
+        alerting_enabled: "{{ discord_webhook is defined and 'REPLACE_ME' not in discord_webhook }}"
-        state: present
 
-    - name: Install podman-compose
+    - name: Determine whether uptimerobot is configured
-      ansible.builtin.apt:
+      ansible.builtin.set_fact:
-        name: podman-compose
+        uptimerobot_enabled: "{{ uptimerobot_api_key is defined and 'REPLACE_ME' not in uptimerobot_api_key }}"
-        state: present
 
-    - name: Install prometheus-node-exporter
+    - name: Install packages
       ansible.builtin.apt:
-        name: prometheus-node-exporter
+        name:
+          - podman
+          - podman-compose
+          - prometheus-node-exporter
+          - wireguard-tools
         state: present
 
     - name: Create monitoring config directory
@@ -46,6 +50,154 @@
         state: directory
         mode: '0755'
 
+    - name: Create grafana provisioning directories
+      ansible.builtin.file:
+        path: "/opt/beeserver/monitoring/grafana/{{ item }}"
+        state: directory
+        mode: '0755'
+      loop:
+        - provisioning/datasources
+        - provisioning/dashboards
+        - provisioning/alerting
+        - dashboards
+        - dashboards-general
+
+    - name: Create topology directory
+      ansible.builtin.file:
+        path: /opt/beeserver/monitoring/topology
+        state: directory
+        mode: '0755'
+
+    - name: Create textfile directory
+      ansible.builtin.file:
+        path: "{{ node_exporter_textfile_dir }}"
+        state: directory
+        mode: '0755'
+
+    - name: Configure node exporter
+      ansible.builtin.copy:
+        dest: /etc/default/prometheus-node-exporter
+        content: |
+          ARGS="--collector.textfile.directory={{ node_exporter_textfile_dir }} --collector.systemd --collector.processes"
+        mode: '0644'
+      notify: Restart node exporter
+
+    - name: Copy textfile collectors
+      ansible.builtin.template:
+        src: "textfile/{{ item }}.sh.j2"
+        dest: "/usr/local/bin/{{ item }}.sh"
+        mode: '0755'
+      loop:
+        - service_metrics
+        - wg_metrics
+
+    - name: Copy uptimerobot collector
+      ansible.builtin.template:
+        src: textfile/uptimerobot_metrics.sh.j2
+        dest: /usr/local/bin/uptimerobot_metrics.sh
+        mode: '0700'
+      when: uptimerobot_enabled
+
+    - name: Install uptimerobot timer
+      ansible.builtin.copy:
+        dest: "/etc/systemd/system/beepi-uptimerobot-metrics.{{ item.kind }}"
+        content: "{{ item.content }}"
+        mode: '0644'
+      loop:
+        - kind: service
+          content: |
+            [Unit]
+            Description=Collect uptimerobot metrics
+
+            [Service]
+            Type=oneshot
+            ExecStart=/usr/local/bin/uptimerobot_metrics.sh
+        - kind: timer
+          content: |
+            [Unit]
+            Description=Collect uptimerobot metrics
+
+            [Timer]
+            OnBootSec=3min
+            OnUnitActiveSec=5min
+            AccuracySec=30s
+
+            [Install]
+            WantedBy=timers.target
+      when: uptimerobot_enabled
+      notify: Reload systemd
+
+    - name: Install collector timers
+      ansible.builtin.copy:
+        dest: "/etc/systemd/system/{{ item.name }}.{{ item.kind }}"
+        content: "{{ item.content }}"
+        mode: '0644'
+      loop:
+        - name: beepi-service-metrics
+          kind: service
+          content: |
+            [Unit]
+            Description=Collect service unit metrics
+
+            [Service]
+            Type=oneshot
+            ExecStart=/usr/local/bin/service_metrics.sh
+        - name: beepi-service-metrics
+          kind: timer
+          content: |
+            [Unit]
+            Description=Collect service unit metrics
+
+            [Timer]
+            OnBootSec=1min
+            OnUnitActiveSec=30s
+            AccuracySec=5s
+
+            [Install]
+            WantedBy=timers.target
+        - name: beepi-wg-metrics
+          kind: service
+          content: |
+            [Unit]
+            Description=Collect wireguard peer metrics
+
+            [Service]
+            Type=oneshot
+            ExecStart=/usr/local/bin/wg_metrics.sh
+        - name: beepi-wg-metrics
+          kind: timer
+          content: |
+            [Unit]
+            Description=Collect wireguard peer metrics
+
+            [Timer]
+            OnBootSec=1min
+            OnUnitActiveSec=30s
+            AccuracySec=5s
+
+            [Install]
+            WantedBy=timers.target
+      notify: Reload systemd
+
+    - name: Flush handlers
+      ansible.builtin.meta: flush_handlers
+
+    - name: Enable collector timers
+      ansible.builtin.systemd_service:
+        name: "{{ item }}.timer"
+        state: started
+        enabled: true
+      loop:
+        - beepi-service-metrics
+        - beepi-wg-metrics
+
+    - name: Enable uptimerobot timer
+      ansible.builtin.systemd_service:
+        name: beepi-uptimerobot-metrics.timer
+        state: started
+        enabled: true
+      when: uptimerobot_enabled
+
     - name: Compose down
       changed_when: true
       failed_when: false
@@ -60,11 +212,17 @@
         mode: '0600'
 
     - name: Copy prometheus config
-      ansible.builtin.copy:
+      ansible.builtin.template:
-        src: prometheus_config.yml
+        src: prometheus.yml.j2
         dest: /opt/beeserver/monitoring/prometheus.yml
         mode: '0644'
 
+    - name: Copy blackbox config
+      ansible.builtin.template:
+        src: blackbox.yml.j2
+        dest: /opt/beeserver/monitoring/blackbox.yml
+        mode: '0644'
+
     - name: Copy loki config
       ansible.builtin.template:
         src: local-config.yml.j2
@@ -77,17 +235,104 @@
         dest: /opt/beeserver/monitoring/config.alloy
         mode: '0644'
 
+    - name: Copy topology service
+      ansible.builtin.copy:
+        src: topology/topology.py
+        dest: /opt/beeserver/monitoring/topology/topology.py
+        mode: '0644'
+
+    - name: Copy topology spec
+      ansible.builtin.template:
+        src: topology/topology-spec.json.j2
+        dest: /opt/beeserver/monitoring/topology/topology-spec.json
+        mode: '0644'
+
+    - name: Copy grafana datasources
+      ansible.builtin.template:
+        src: grafana/provisioning/datasources/datasources.yml.j2
+        dest: /opt/beeserver/monitoring/grafana/provisioning/datasources/datasources.yml
+        owner: '472'
+        group: '472'
+        mode: '0600'
+
+    - name: Copy grafana dashboard provider
+      ansible.builtin.template:
+        src: grafana/provisioning/dashboards/dashboards.yml.j2
+        dest: /opt/beeserver/monitoring/grafana/provisioning/dashboards/dashboards.yml
+        mode: '0644'
+
+    - name: Copy hand made dashboards
+      ansible.builtin.copy:
+        src: grafana/dashboards-general/
+        dest: /opt/beeserver/monitoring/grafana/dashboards-general/
+        mode: '0644'
+
+    - name: Copy grafana contact points
+      ansible.builtin.template:
+        src: grafana/provisioning/alerting/contact-points.yml.j2
+        dest: /opt/beeserver/monitoring/grafana/provisioning/alerting/contact-points.yml
+        owner: '472'
+        group: '472'
+        mode: '0600'
+      when: alerting_enabled
+
+    - name: Copy grafana alert rules
+      ansible.builtin.template:
+        src: grafana/provisioning/alerting/rules.yml.j2
+        dest: /opt/beeserver/monitoring/grafana/provisioning/alerting/rules.yml
+        mode: '0644'
+
+    - name: Copy grafana dashboards
+      ansible.builtin.template:
+        src: "grafana/dashboards/{{ item }}.json.j2"
+        dest: "/opt/beeserver/monitoring/grafana/dashboards/{{ item }}.json"
+        mode: '0644'
+      loop:
+        - infrastructure-map
+        - backups
+        - services
+        - ingress
+
+    - name: Copy uptimerobot dashboard
+      ansible.builtin.template:
+        src: grafana/dashboards/uptimerobot.json.j2
+        dest: /opt/beeserver/monitoring/grafana/dashboards/uptimerobot.json
+        mode: '0644'
+      when: uptimerobot_enabled
+
     - name: Enable podman.socket
       ansible.builtin.service:
         name: podman.socket
         state: started
         enabled: true
 
+    - name: Check the geoip database
+      ansible.builtin.stat:
+        path: /opt/beeserver/monitoring/database/GeoLite2-City.mmdb
+      register: geoip_db
+
+    - name: Check when geoip was last downloaded
+      ansible.builtin.stat:
+        path: /opt/beeserver/monitoring/.geoip_downloaded
+      register: geoip_stamp
+
+    - name: Decide whether the geoip database needs refreshing
+      ansible.builtin.set_fact:
+        geoip_stale: "{{ not geoip_db.stat.exists or not geoip_stamp.stat.exists or (ansible_facts.date_time.epoch | int - geoip_stamp.stat.mtime | int) > geoip_max_age_seconds | int }}"
+
+    - name: Report the geoip database age
+      ansible.builtin.debug:
+        msg: >-
+          geoip
+          {{ 'not present, downloading' if not (geoip_db.stat.exists and geoip_stamp.stat.exists)
+             else 'downloaded ' + (((ansible_facts.date_time.epoch | int - geoip_stamp.stat.mtime | int) / 3600) | round(1) | string) + 'h ago, ' + ('refreshing' if geoip_stale else 'reusing') }}
+
     - name: Delete database directory
       ansible.builtin.file:
         path: /opt/beeserver/monitoring/database
         state: absent
         mode: '0755'
+      when: geoip_stale
 
     - name: Create database directory
       ansible.builtin.file:
@@ -95,6 +340,12 @@
         state: directory
         mode: '0755'
 
+    - name: Remove stale geoip archives
+      ansible.builtin.shell:
+        cmd: rm -f /opt/beeserver/monitoring/GeoLite2-City_*.tar.gz
+      changed_when: true
+      when: geoip_stale
+
     - name: Download geoip database
       ansible.builtin.get_url:
         url: "https://download.maxmind.com/geoip/databases/GeoLite2-City/download?suffix=tar.gz"
@@ -102,6 +353,7 @@
         username: "{{ geoip_username }}"
         password: "{{ geoip_license }}"
         mode: '0644'
+      when: geoip_stale
 
     - name: Get geoip tar file
       ansible.builtin.find:
@@ -111,6 +363,7 @@
         use_regex: true
         limit: 1
       register: found
+      when: geoip_stale
 
     - name: Extract geopip database
       ansible.builtin.unarchive:
@@ -118,9 +371,50 @@
         src: "{{ found.files[0].path }}"
         dest: /opt/beeserver/monitoring/database
         extra_opts: ['--strip-components=1', '--show-stored-names']
+      when: geoip_stale and found.files | length > 0
+
+    - name: Stamp the geoip download time
+      ansible.builtin.file:
+        path: /opt/beeserver/monitoring/.geoip_downloaded
+        state: touch
+        mode: '0644'
+      when: geoip_stale and found.files | length > 0
 
     - name: Compose up
       changed_when: true
       ansible.builtin.command:
         cmd: podman-compose up -d
         chdir: /opt/beeserver/monitoring
+
+    - name: Run collectors once
+      changed_when: true
+      ansible.builtin.command:
+        cmd: "/usr/local/bin/{{ item }}.sh"
+      loop:
+        - service_metrics
+        - wg_metrics
+
+    - name: Run uptimerobot collector once
+      changed_when: true
+      ansible.builtin.command:
+        cmd: /usr/local/bin/uptimerobot_metrics.sh
+      when: uptimerobot_enabled
+
+    - name: Debug
+      ansible.builtin.debug:
+        msg:
+          - "Grafana: https://grafana.secretbee.buzz/"
+          - "Topology: curl -s localhost:8088/topology.json | head"
+          - "Prometheus targets: curl -s localhost:9090/api/v1/targets | jq '.data.activeTargets[] | {job: .labels.job, health}'"
+      tags:
+        - always
+
+  handlers:
+    - name: Reload systemd
+      ansible.builtin.systemd_service:
+        daemon_reload: true
+
+    - name: Restart node exporter
+      ansible.builtin.service:
+        name: prometheus-node-exporter
+        state: restarted

playbooks/monitoring/prometheus.yml.j2

@@ -0,0 +1,154 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+  external_labels:
+    monitor: beepi
+
+scrape_configs:
+  - job_name: prometheus
+    static_configs:
+      - targets: ['localhost:9090']
+
+  - job_name: node
+    static_configs:
+      - targets: ['host.containers.internal:9100']
+        labels:
+          host: beepi
+      - targets: ['{{ wireguard_ingress_ip }}:9100']
+        labels:
+          host: ingress
+      - targets: ['192.168.178.51:9100']
+        labels:
+          host: beeserver
+
+  - job_name: podman
+    static_configs:
+      - targets: ['podman-exporter:9882']
+        labels:
+          host: beepi
+
+  - job_name: alloy
+    static_configs:
+      - targets: ['alloy:12345']
+        labels:
+          host: beepi
+      - targets: ['{{ wireguard_ingress_ip }}:12345']
+        labels:
+          host: ingress
+
+  - job_name: loki
+    static_configs:
+      - targets: ['loki:3100']
+
+  - job_name: grafana
+    static_configs:
+      - targets: ['grafana:4000']
+
+  - job_name: blackbox
+    static_configs:
+      - targets: ['host.containers.internal:9115']
+
+  - job_name: probe-app
+    metrics_path: /probe
+    params:
+      module: [http_2xx_any]
+    static_configs:
+{% for service in monitored_services %}
+{% if service.probe is defined %}
+      - targets: ['{{ service.probe }}']
+        labels:
+          service: {{ service.name }}
+          group: {{ service.group }}
+          vantage: local
+{% endif %}
+{% endfor %}
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: host.containers.internal:9115
+
+  - job_name: probe-vhost
+    metrics_path: /probe
+    params:
+      module: [https_vhost]
+    static_configs:
+{% for service in monitored_services %}
+{% if service.vhost is defined %}
+      - targets: ['https://{{ service.vhost }}']
+        labels:
+          service: {{ service.name }}
+          group: {{ service.group }}
+          vantage: nginx
+{% endif %}
+{% endfor %}
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: host.containers.internal:9115
+
+  - job_name: probe-tcp
+    metrics_path: /probe
+    params:
+      module: [tcp_connect]
+    static_configs:
+{% for service in monitored_services %}
+{% for probe in service.tcp | default([]) %}
+      - targets: ['{{ probe.target }}']
+        labels:
+          service: {{ service.name }}
+          group: {{ service.group }}
+          probe: {{ probe.name }}
+          vantage: local
+{% endfor %}
+{% endfor %}
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: host.containers.internal:9115
+
+  - job_name: probe-tunnel
+    scrape_interval: 30s
+    metrics_path: /probe
+    params:
+      module: [tcp_connect]
+    static_configs:
+{% for probe in ingress_probe_targets %}
+      - targets: ['{{ probe.target }}']
+        labels:
+          probe: {{ probe.name }}
+          vantage: ingress
+{% endfor %}
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: '{{ wireguard_ingress_ip }}:9115'
+
+  - job_name: probe-tunnel-icmp
+    scrape_interval: 30s
+    metrics_path: /probe
+    params:
+      module: [icmp]
+    static_configs:
+      - targets: ['{{ wireguard_beepi_ip }}']
+        labels:
+          probe: tunnel-rtt
+          vantage: ingress
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: '{{ wireguard_ingress_ip }}:9115'

playbooks/monitoring/prometheus_config.yml

@@ -1,16 +0,0 @@
-global:
-  scrape_interval:     15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
-  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
-
-scrape_configs:
-  - job_name: 'prometheus'
-    static_configs:
-      - targets: ['localhost:9090']
-  - job_name: node
-    static_configs:
-      - targets: 
-        - 'host.containers.internal:9100'
-        - '192.168.178.51:9100'
-  - job_name: podman
-    static_configs:
-      - targets: ['podman-exporter:9882']

playbooks/monitoring/textfile/service_metrics.sh.j2

@@ -0,0 +1,105 @@
+#!/bin/bash
+
+set -uo pipefail
+
+textfile_dir={{ node_exporter_textfile_dir }}
+file=${textfile_dir}/services.prom
+tmp=${file}.$$
+
+mkdir -p ${textfile_dir}
+
+declare -A active_count
+declare -A unit_count
+units=""
+
+record() {
+    local service=$1
+    local unit=$2
+    local kind=$3
+    local state=$4
+
+    units+="service_unit_active{service=\"${service}\",unit=\"${unit}\",kind=\"${kind}\"} ${state}"$'\n'
+    unit_count[${service}]=$(( ${unit_count[${service}]:-0} + 1 ))
+    active_count[${service}]=$(( ${active_count[${service}]:-0} + state ))
+}
+
+check_user_unit() {
+    local service=$1
+    local owner=$2
+    local unit=$3
+    local state=0
+
+    if systemctl --user --machine ${owner}@.host is-active --quiet ${unit} 2>/dev/null
+    then
+        state=1
+    fi
+
+    record ${service} ${unit} rootless ${state}
+}
+
+check_system_unit() {
+    local service=$1
+    local unit=$2
+    local state=0
+
+    if systemctl is-active --quiet ${unit} 2>/dev/null
+    then
+        state=1
+    fi
+
+    record ${service} ${unit} system ${state}
+}
+
+check_container() {
+    local service=$1
+    local container=$2
+    local state=0
+
+    if [ "$(podman container inspect -f '{% raw %}{{.State.Running}}{% endraw %}' ${container} 2>/dev/null)" = "true" ]
+    then
+        state=1
+    fi
+
+    record ${service} ${container} container ${state}
+}
+
+{% for service in monitored_services %}
+{% for unit in service.units | default([]) %}
+check_user_unit {{ service.name }} {{ service.owner }} {{ unit }}
+{% endfor %}
+{% for unit in service.system_units | default([]) %}
+check_system_unit {{ service.name }} {{ unit }}
+{% endfor %}
+{% for container in service.containers | default([]) %}
+check_container {{ service.name }} {{ container }}
+{% endfor %}
+{% endfor %}
+
+{
+    echo "# HELP service_unit_active Whether a unit or container backing a service is running."
+    echo "# TYPE service_unit_active gauge"
+    printf '%s' "${units}"
+
+    echo "# HELP service_enabled Whether a service is expected to be running."
+    echo "# TYPE service_enabled gauge"
+{% for service in monitored_services %}
+    echo "service_enabled{service=\"{{ service.name }}\",group=\"{{ service.group }}\"} {{ 1 if service.enabled else 0 }}"
+{% endfor %}
+
+    echo "# HELP service_units_total Number of units or containers backing a service."
+    echo "# TYPE service_units_total gauge"
+    for service in "${!unit_count[@]}"
+    do
+        echo "service_units_total{service=\"${service}\"} ${unit_count[${service}]}"
+    done
+
+    echo "# HELP service_units_active Number of running units or containers backing a service."
+    echo "# TYPE service_units_active gauge"
+    for service in "${!active_count[@]}"
+    do
+        echo "service_units_active{service=\"${service}\"} ${active_count[${service}]}"
+    done
+} > "${tmp}"
+
+chmod 0644 "${tmp}"
+mv "${tmp}" "${file}"

playbooks/monitoring/textfile/uptimerobot_metrics.sh.j2

@@ -0,0 +1,71 @@
+#!/bin/bash
+
+set -uo pipefail
+
+textfile_dir={{ node_exporter_textfile_dir }}
+file=${textfile_dir}/uptimerobot.prom
+tmp=${file}.$$
+
+mkdir -p ${textfile_dir}
+
+response=$(curl -sS -m 25 -X POST \
+    "https://api.uptimerobot.com/v2/getMonitors?api_key={{ uptimerobot_api_key }}&format=json&custom_uptime_ratios=1-7-30" \
+    2>/dev/null)
+
+{% raw %}
+printf '%s' "${response}" | python3 -c '
+import json
+import sys
+
+print("# HELP uptimerobot_api_ok Whether the uptimerobot api answered successfully.")
+print("# TYPE uptimerobot_api_ok gauge")
+print("# HELP uptimerobot_monitor_up Whether an external monitor reports the target as up.")
+print("# TYPE uptimerobot_monitor_up gauge")
+print("# HELP uptimerobot_monitor_enabled Whether an external monitor is active rather than paused.")
+print("# TYPE uptimerobot_monitor_enabled gauge")
+print("# HELP uptimerobot_monitor_status Raw uptimerobot status, 0 paused 1 unchecked 2 up 8 seems down 9 down.")
+print("# TYPE uptimerobot_monitor_status gauge")
+print("# HELP uptimerobot_uptime_ratio Uptime percentage over the given window.")
+print("# TYPE uptimerobot_uptime_ratio gauge")
+
+try:
+    payload = json.load(sys.stdin)
+except ValueError:
+    print("uptimerobot_api_ok 0")
+    sys.exit(0)
+
+if payload.get("stat") != "ok":
+    print("uptimerobot_api_ok 0")
+    sys.exit(0)
+
+print("uptimerobot_api_ok 1")
+
+def escape(value):
+    return value.replace("\\", "\\\\").replace("\"", "\\\"")
+
+for monitor in payload.get("monitors", []):
+    name = escape(str(monitor.get("friendly_name", "unknown")))
+    status = int(monitor.get("status", 1))
+    labels = "monitor=\"%s\"" % name
+
+    print("uptimerobot_monitor_status{%s} %d" % (labels, status))
+    print("uptimerobot_monitor_enabled{%s} %d" % (labels, 0 if status == 0 else 1))
+    print("uptimerobot_monitor_up{%s} %d" % (labels, 1 if status == 2 else 0))
+
+    ratios = str(monitor.get("custom_uptime_ratio", "")).split("-")
+    for window, ratio in zip(("1d", "7d", "30d"), ratios):
+        try:
+            print("uptimerobot_uptime_ratio{%s,window=\"%s\"} %s" % (labels, window, float(ratio)))
+        except ValueError:
+            pass
+' > "${tmp}"
+{% endraw %}
+
+if [ ! -s "${tmp}" ]
+then
+    rm -f "${tmp}"
+    exit 1
+fi
+
+chmod 0644 "${tmp}"
+mv "${tmp}" "${file}"

playbooks/monitoring/textfile/wg_metrics.sh.j2

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+set -uo pipefail
+
+textfile_dir={{ node_exporter_textfile_dir }}
+file=${textfile_dir}/wireguard.prom
+tmp=${file}.$$
+now=$(date +%s)
+
+mkdir -p ${textfile_dir}
+
+peer_name() {
+    case "$1" in
+{% for key, name in wireguard_peer_names.items() %}
+        "{{ key }}") echo "{{ name }}" ;;
+{% endfor %}
+        *) echo "unknown" ;;
+    esac
+}
+
+{
+    echo "# HELP wireguard_peer_last_handshake_seconds Unix time of the last handshake with a peer."
+    echo "# TYPE wireguard_peer_last_handshake_seconds gauge"
+    echo "# HELP wireguard_peer_handshake_age_seconds Seconds since the last handshake with a peer."
+    echo "# TYPE wireguard_peer_handshake_age_seconds gauge"
+    echo "# HELP wireguard_peer_up Whether a peer handshaked within the last five minutes."
+    echo "# TYPE wireguard_peer_up gauge"
+    echo "# HELP wireguard_peer_receive_bytes_total Bytes received from a peer."
+    echo "# TYPE wireguard_peer_receive_bytes_total counter"
+    echo "# HELP wireguard_peer_transmit_bytes_total Bytes sent to a peer."
+    echo "# TYPE wireguard_peer_transmit_bytes_total counter"
+
+    wg show all dump 2>/dev/null | awk 'NF >= 8' | while read -r interface key psk endpoint allowed handshake rx tx keepalive
+    do
+        name=$(peer_name "${key}")
+        labels="interface=\"${interface}\",peer=\"${name}\""
+
+        age=$((now - handshake))
+        up=0
+
+        if [ "${handshake}" -gt 0 ] && [ "${age}" -lt 300 ]
+        then
+            up=1
+        fi
+
+        if [ "${handshake}" -eq 0 ]
+        then
+            age=-1
+        fi
+
+        echo "wireguard_peer_last_handshake_seconds{${labels}} ${handshake}"
+        echo "wireguard_peer_handshake_age_seconds{${labels}} ${age}"
+        echo "wireguard_peer_up{${labels}} ${up}"
+        echo "wireguard_peer_receive_bytes_total{${labels}} ${rx}"
+        echo "wireguard_peer_transmit_bytes_total{${labels}} ${tx}"
+    done
+} > "${tmp}"
+
+chmod 0644 "${tmp}"
+mv "${tmp}" "${file}"

playbooks/monitoring/topology/topology-spec.json.j2

@@ -0,0 +1,130 @@
+{
+  "nodes": [
+    {
+      "id": "internet",
+      "title": "Internet",
+      "subtitle": "public",
+      "group": "edge"
+    },
+    {
+      "id": "ingress",
+      "title": "Ingress",
+      "subtitle": "animeistrash",
+      "group": "edge",
+      "up": "up{job=\"node\",host=\"ingress\"}",
+      "stat": "100 - (avg(rate(node_cpu_seconds_total{job=\"node\",host=\"ingress\",mode=\"idle\"}[5m])) * 100)",
+      "stat_unit": "pct"
+    },
+    {
+      "id": "tunnel",
+      "title": "WireGuard",
+      "subtitle": "{{ wireguard_ingress_ip }} to {{ wireguard_beepi_ip }}",
+      "group": "edge",
+      "up": "max(wireguard_peer_up{peer=\"beepi\"})",
+      "stat": "max(wireguard_peer_handshake_age_seconds{peer=\"beepi\"})",
+      "stat_unit": "h"
+    },
+    {
+      "id": "beepi",
+      "title": "beepi",
+      "subtitle": "raspberry pi",
+      "group": "host",
+      "up": "up{job=\"node\",host=\"beepi\"}",
+      "stat": "100 - (avg(rate(node_cpu_seconds_total{job=\"node\",host=\"beepi\",mode=\"idle\"}[5m])) * 100)",
+      "stat_unit": "pct"
+    },
+{% for service in monitored_services %}
+    {
+      "id": "{{ service.name }}",
+      "title": "{{ service.title }}",
+      "subtitle": "{{ service.vhost | default(service.group) }}",
+      "group": "{{ service.group }}",
+      "enabled": "max(service_enabled{service=\"{{ service.name }}\"})",
+{% if service.probe is defined or service.tcp is defined %}
+      "up": "min(probe_success{service=\"{{ service.name }}\",vantage=\"local\"})",
+      "stat": "max(probe_duration_seconds{service=\"{{ service.name }}\",vantage=\"local\"})",
+      "stat_unit": "s"
+{% elif service.vhost is defined %}
+      "up": "min(probe_success{service=\"{{ service.name }}\",vantage=\"nginx\"})",
+      "stat": "max(probe_duration_seconds{service=\"{{ service.name }}\",vantage=\"nginx\"})",
+      "stat_unit": "s"
+{% else %}
+      "up": "min(service_units_active{service=\"{{ service.name }}\"} / service_units_total{service=\"{{ service.name }}\"})"
+{% endif %}
+    },
+{% endfor %}
+    {
+      "id": "raid",
+      "title": "RAID",
+      "subtitle": "{{ backup_raid_mount }}",
+      "group": "storage",
+      "up": "max(backup_raid_mounted)",
+      "stat": "max(node_filesystem_avail_bytes{mountpoint=\"{{ backup_raid_mount }}\"})",
+      "stat_unit": "bytes",
+      "fill": "1 - (max(node_filesystem_avail_bytes{mountpoint=\"{{ backup_raid_mount }}\"}) / max(node_filesystem_size_bytes{mountpoint=\"{{ backup_raid_mount }}\"}))"
+    },
+{% for job in backup_jobs %}
+{% if job.name != 'offsite' %}
+    {
+      "id": "backup-{{ job.name }}",
+      "title": "{{ job.title }}",
+      "subtitle": "backup job",
+      "group": "backup",
+      "up": "(max(backup_last_status{backup_job=\"{{ job.name }}\"}) == 1) and ((time() - max(backup_last_success_timestamp_seconds{backup_job=\"{{ job.name }}\"})) < {{ job.max_age_hours * 3600 }})",
+      "stat": "time() - max(backup_last_success_timestamp_seconds{backup_job=\"{{ job.name }}\"})",
+      "stat_unit": "h"
+    },
+{% endif %}
+{% endfor %}
+    {
+      "id": "offsite",
+      "title": "Offsite sync",
+      "subtitle": "daily 07:00",
+      "group": "storage",
+      "up": "(time() - max(backup_last_success_timestamp_seconds{backup_job=\"offsite\"})) < {{ 26 * 3600 }}",
+      "stat": "time() - max(backup_last_success_timestamp_seconds{backup_job=\"offsite\"})",
+      "stat_unit": "h"
+    },
+    {
+      "id": "hetzner",
+      "title": "Hetzner box",
+      "subtitle": "{{ hetzner_host | default('storage box') }}",
+      "group": "storage",
+      "up": "(max(hetzner_reachable) == 1) and (max(hetzner_remote_files{path=\"beepi\"}) > 0)",
+      "stat": "max(hetzner_remote_bytes{path=\"beepi\"})",
+      "stat_unit": "bytes",
+      "fill": "sum(hetzner_remote_bytes) / {{ hetzner_box_bytes }}"
+    },
+    {
+      "id": "uptimerobot",
+      "title": "External checks",
+      "subtitle": "uptimerobot",
+      "group": "edge",
+      "dashboard": "/d/beepi-uptimerobot",
+      "enabled": "max(uptimerobot_api_ok)",
+      "up": "min(uptimerobot_monitor_up and on (monitor) (uptimerobot_monitor_enabled == 1))",
+      "stat": "count(uptimerobot_monitor_enabled == 1)"
+    }
+  ],
+  "edges": [
+    {"source": "uptimerobot", "target": "internet", "label": "probes"},
+    {"source": "internet", "target": "ingress", "label": "public"},
+    {"source": "ingress", "target": "tunnel", "label": "dnat"},
+    {"source": "tunnel", "target": "beepi", "label": "wg0"},
+{% for service in monitored_services %}
+{% if service.vhost is defined and service.name != 'nginx' %}
+    {"source": "nginx", "target": "{{ service.name }}", "label": "proxy"},
+{% endif %}
+{% if service.vhost is not defined %}
+    {"source": "beepi", "target": "{{ service.name }}", "label": "{{ service.group }}"},
+{% endif %}
+{% endfor %}
+    {"source": "beepi", "target": "nginx", "label": "tls"},
+    {"source": "beepi", "target": "backup-daily", "label": "03:00"},
+    {"source": "backup-daily", "target": "backup-weekly", "label": "sun 05:00"},
+    {"source": "backup-daily", "target": "backup-restoretest", "label": "sun 06:00"},
+    {"source": "backup-daily", "target": "raid", "label": "archive"},
+    {"source": "raid", "target": "offsite", "label": "rsync"},
+    {"source": "offsite", "target": "hetzner", "label": "ssh 23"}
+  ]
+}

playbooks/monitoring/topology/topology.py

@@ -0,0 +1,218 @@
+import json
+import os
+import threading
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+PROMETHEUS_URL = os.environ.get("PROMETHEUS_URL", "http://prometheus:9090")
+LISTEN_PORT = int(os.environ.get("LISTEN_PORT", "8088"))
+SPEC_PATH = os.environ.get("SPEC_PATH", "/app/topology-spec.json")
+CACHE_SECONDS = int(os.environ.get("CACHE_SECONDS", "10"))
+
+STATUS_OK = "ok"
+STATUS_FAIL = "fail"
+STATUS_DISABLED = "disabled"
+STATUS_UNKNOWN = "unknown"
+
+ARCS = {
+    STATUS_OK: "arc__ok",
+    STATUS_FAIL: "arc__fail",
+    STATUS_DISABLED: "arc__disabled",
+    STATUS_UNKNOWN: "arc__unknown",
+}
+
+ARC_FIELDS = list(ARCS.values()) + ["arc__used", "arc__free"]
+
+COLORS = {
+    STATUS_OK: "green",
+    STATUS_FAIL: "red",
+    STATUS_DISABLED: "#6e7079",
+    STATUS_UNKNOWN: "orange",
+}
+
+DASHBOARDS = {
+    "apps": "/d/beepi-services",
+    "platform": "/d/beepi-services",
+    "backup": "/d/beepi-backups",
+    "storage": "/d/beepi-backups",
+    "edge": "/d/beepi-ingress",
+    "host": "/d/beepi-ingress",
+}
+
+_lock = threading.Lock()
+_cache = {"at": 0.0, "body": None}
+
+
+def query(expr):
+    url = PROMETHEUS_URL + "/api/v1/query?" + urllib.parse.urlencode({"query": expr})
+
+    try:
+        with urllib.request.urlopen(url, timeout=8) as response:
+            payload = json.load(response)
+    except (urllib.error.URLError, OSError, ValueError):
+        return None
+
+    if payload.get("status") != "success":
+        return None
+
+    result = payload.get("data", {}).get("result", [])
+
+    if not result:
+        return None
+
+    try:
+        return float(result[0]["value"][1])
+    except (KeyError, IndexError, TypeError, ValueError):
+        return None
+
+
+def format_stat(value, unit):
+    if value is None:
+        return ""
+
+    if unit == "s":
+        return "%.0f ms" % (value * 1000) if value < 1 else "%.1f s" % value
+
+    if unit == "h":
+        return "%.1f h" % (value / 3600.0)
+
+    if unit == "d":
+        return "%.1f d" % (value / 86400.0)
+
+    if unit == "bytes":
+        size = float(value)
+        for suffix in ("B", "KiB", "MiB", "GiB", "TiB"):
+            if size < 1024 or suffix == "TiB":
+                return "%.1f %s" % (size, suffix)
+            size /= 1024.0
+
+    if unit == "pct":
+        return "%.0f%%" % value
+
+    return "%.0f" % value
+
+
+def evaluate(node):
+    enabled = node.get("enabled")
+
+    if enabled:
+        value = query(enabled)
+
+        if value is not None and value < 1:
+            return STATUS_DISABLED
+
+    up = node.get("up")
+
+    if not up:
+        return STATUS_OK
+
+    value = query(up)
+
+    if value is None:
+        return STATUS_UNKNOWN
+
+    return STATUS_OK if value >= 1 else STATUS_FAIL
+
+
+def build():
+    with open(SPEC_PATH) as handle:
+        spec = json.load(handle)
+
+    nodes = []
+    statuses = {}
+
+    for node in spec["nodes"]:
+        status = evaluate(node)
+        statuses[node["id"]] = status
+
+        entry = {
+            "id": node["id"],
+            "title": node["title"],
+            "subtitle": node.get("subtitle", ""),
+            "mainstat": format_stat(query(node["stat"]), node.get("stat_unit", "")) if node.get("stat") else "",
+            "detail__group": node.get("group", ""),
+            "detail__status": status,
+            "detail__used": "",
+            "dashboard": node.get("dashboard") or DASHBOARDS.get(node.get("group", ""), "/d/beepi-map"),
+        }
+
+        for name in ARC_FIELDS:
+            entry[name] = 0.0
+
+        used = query(node["fill"]) if node.get("fill") and status == STATUS_OK else None
+
+        if used is None:
+            entry[ARCS[status]] = 1.0
+        else:
+            used = min(max(used, 0.0), 1.0)
+            entry["arc__used"] = used
+            entry["arc__free"] = 1.0 - used
+            entry["detail__used"] = "%.0f%% used" % (used * 100)
+
+        nodes.append(entry)
+
+    edges = []
+
+    for index, edge in enumerate(spec["edges"]):
+        source = statuses.get(edge["source"], STATUS_UNKNOWN)
+        target = statuses.get(edge["target"], STATUS_UNKNOWN)
+        degraded = STATUS_FAIL in (source, target)
+        idle = STATUS_DISABLED in (source, target)
+
+        if degraded:
+            color = COLORS[STATUS_FAIL]
+        elif idle:
+            color = COLORS[STATUS_DISABLED]
+        else:
+            color = COLORS[STATUS_OK]
+
+        edges.append({
+            "id": str(index),
+            "source": edge["source"],
+            "target": edge["target"],
+            "mainstat": edge.get("label", ""),
+            "color": color,
+            "thickness": 3 if degraded else 1,
+        })
+
+    return {"nodes": nodes, "edges": edges}
+
+
+def cached():
+    with _lock:
+        now = time.time()
+
+        if _cache["body"] is None or now - _cache["at"] > CACHE_SECONDS:
+            _cache["body"] = json.dumps(build()).encode()
+            _cache["at"] = now
+
+        return _cache["body"]
+
+
+class Handler(BaseHTTPRequestHandler):
+    def do_GET(self):
+        if self.path.rstrip("/") not in ("", "/topology.json", "/topology"):
+            self.send_error(404)
+            return
+
+        try:
+            body = cached()
+        except Exception:
+            self.send_error(500)
+            return
+
+        self.send_response(200)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(body)))
+        self.end_headers()
+        self.wfile.write(body)
+
+    def log_message(self, fmt, *args):
+        pass
+
+
+if __name__ == "__main__":
+    ThreadingHTTPServer(("0.0.0.0", LISTEN_PORT), Handler).serve_forever()

playbooks/monitoring/vars/alert_rules.yml

@@ -0,0 +1,242 @@
+alert_groups:
+  - name: backup
+    interval: 5m
+    rules:
+      - uid: backup-daily-stale
+        no_data: Alerting
+        title: Daily backup stale
+        expr: '(time() - max(backup_last_success_timestamp_seconds{backup_job="daily"})) / 3600'
+        op: gt
+        threshold: 26
+        for: 15m
+        severity: critical
+        summary: No successful daily backup for more than 26 hours.
+
+      - uid: backup-weekly-stale
+        no_data: Alerting
+        title: Weekly backup stale
+        expr: '(time() - max(backup_last_success_timestamp_seconds{backup_job="weekly"})) / 3600'
+        op: gt
+        threshold: 192
+        for: 30m
+        severity: critical
+        summary: No successful weekly backup for more than 8 days.
+
+      - uid: backup-restoretest-stale
+        no_data: Alerting
+        title: Restore test stale
+        expr: '(time() - max(backup_last_success_timestamp_seconds{backup_job="restoretest"})) / 3600'
+        op: gt
+        threshold: 192
+        for: 30m
+        severity: warning
+        summary: The vaultwarden restore test has not passed in more than 8 days, backups are unverified.
+
+      - uid: backup-offsite-stale
+        no_data: Alerting
+        title: Offsite sync stale
+        expr: '(time() - max(backup_last_success_timestamp_seconds{backup_job="offsite"})) / 3600'
+        op: gt
+        threshold: 26
+        for: 15m
+        severity: critical
+        summary: Backups have not reached the Hetzner storage box in more than 26 hours.
+
+      - uid: backup-job-failed
+        no_data: Alerting
+        title: Backup job failed
+        expr: 'min by (backup_job) (backup_last_status)'
+        op: lt
+        threshold: 1
+        for: 5m
+        severity: critical
+        summary: The last run of this backup job exited with a failure.
+
+      - uid: backup-archive-shrunk
+        title: Backup archive shrunk
+        expr: 'backup_archive_newest_bytes{kind="daily"} / avg_over_time(backup_archive_newest_bytes{kind="daily"}[7d])'
+        op: lt
+        threshold: 0.5
+        for: 1h
+        severity: warning
+        summary: The newest daily archive is less than half the size of the weekly average, data may be missing.
+
+      - uid: backup-raid-unmounted
+        no_data: Alerting
+        title: Backup RAID unmounted
+        expr: 'min(backup_raid_mounted)'
+        op: lt
+        threshold: 1
+        for: 5m
+        severity: critical
+        summary: The backup RAID is not mounted, backups cannot be written.
+
+      - uid: backup-raid-space
+        title: Backup RAID low on space
+        expr: 'min(node_filesystem_avail_bytes{mountpoint="/home/beeshare/raid"} / node_filesystem_size_bytes{mountpoint="/home/beeshare/raid"}) * 100'
+        op: lt
+        threshold: 15
+        for: 30m
+        severity: warning
+        summary: Less than 15 percent free space left on the backup RAID.
+
+      - uid: offsite-empty
+        no_data: Alerting
+        title: Offsite storage empty
+        expr: 'max(hetzner_remote_files{path="beepi"})'
+        op: lt
+        threshold: 1
+        for: 30m
+        severity: critical
+        summary: The Hetzner storage box holds no completed backup files, offsite recovery is not possible.
+
+      - uid: offsite-unreachable
+        no_data: Alerting
+        title: Offsite storage unreachable
+        expr: 'max(hetzner_reachable)'
+        op: lt
+        threshold: 1
+        for: 2h
+        severity: warning
+        summary: The Hetzner storage box could not be listed, check external reachability on the box.
+
+      - uid: offsite-content-stale
+        title: Offsite content stale
+        expr: 'min(hetzner_remote_newest_age_seconds{path="beepi"}) / 3600'
+        op: gt
+        threshold: 48
+        for: 1h
+        severity: critical
+        summary: The newest backup on the Hetzner storage box is more than 48 hours old.
+
+      - uid: backup-log-errors
+        title: Backup log errors
+        no_data: OK
+        source: loki
+        expr: 'sum(count_over_time({job="backup"} |~ "(?i)(FAILED|verify failed|refusing to sync)" [15m]))'
+        op: gt
+        threshold: 0
+        for: 0s
+        severity: critical
+        summary: The backup log reported a failure in the last 15 minutes.
+
+  - name: services
+    interval: 1m
+    rules:
+      - uid: service-probe-down
+        title: Service probe failing
+        expr: 'min by (service) (probe_success{vantage="local"}) and on (service) (max by (service) (service_enabled) == 1)'
+        op: lt
+        threshold: 1
+        for: 5m
+        severity: critical
+        summary: A service that should be running is not answering its health probe.
+
+      - uid: service-unit-down
+        title: Service unit not running
+        expr: 'min by (service) (service_units_active / service_units_total) and on (service) (max by (service) (service_enabled) == 1)'
+        op: lt
+        threshold: 1
+        for: 5m
+        severity: critical
+        summary: A unit or container backing an enabled service is not running.
+
+      - uid: service-vhost-down
+        title: Public vhost failing
+        expr: 'min by (service) (probe_success{vantage="nginx"}) and on (service) (max by (service) (service_enabled) == 1)'
+        op: lt
+        threshold: 1
+        for: 5m
+        severity: critical
+        summary: An enabled service is not reachable through nginx and TLS.
+
+      - uid: external-check-down
+        title: External check failing
+        expr: 'min by (monitor) (uptimerobot_monitor_up) and on (monitor) (max by (monitor) (uptimerobot_monitor_enabled) == 1)'
+        op: lt
+        threshold: 1
+        for: 5m
+        severity: critical
+        summary: UptimeRobot reports this endpoint as unreachable from outside the network.
+
+      - uid: service-cert-expiry
+        title: TLS certificate expiring
+        expr: 'min by (instance) (probe_ssl_earliest_cert_expiry - time()) / 86400'
+        op: lt
+        threshold: 14
+        for: 1h
+        severity: warning
+        summary: A TLS certificate expires in less than 14 days.
+
+      - uid: scrape-target-down
+        title: Scrape target down
+        expr: 'min by (job, instance) (up{host!~"{{ alert_excluded_hosts }}"})'
+        op: lt
+        threshold: 1
+        for: 10m
+        severity: warning
+        summary: Prometheus cannot scrape this target.
+
+  - name: infra
+    interval: 1m
+    rules:
+      - uid: ingress-down
+        title: Ingress host down
+        expr: 'min(up{job="node",host="ingress"})'
+        op: lt
+        threshold: 1
+        for: 5m
+        severity: critical
+        summary: The ingress VPS is not answering scrapes, inbound traffic is likely broken.
+
+      - uid: tunnel-peer-down
+        title: WireGuard tunnel down
+        expr: 'min by (peer) (wireguard_peer_up{peer=~"beepi|ingress"})'
+        op: lt
+        threshold: 1
+        for: 5m
+        severity: critical
+        summary: The site to site WireGuard peer has not handshaked in the last five minutes.
+
+      - uid: tunnel-probe-down
+        title: Tunnel path failing
+        expr: 'min by (probe) (probe_success{vantage="ingress"})'
+        op: lt
+        threshold: 1
+        for: 5m
+        severity: critical
+        summary: The ingress host cannot reach this port on beepi through the tunnel.
+
+      - uid: host-disk-space
+        title: Host low on disk
+        expr: >-
+          min by (host, device)
+          (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|ramfs",host!~"{{ alert_excluded_hosts }}"}
+          / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|ramfs",host!~"{{ alert_excluded_hosts }}"}) * 100
+        op: lt
+        threshold: 10
+        for: 30m
+        severity: warning
+        summary: Less than 10 percent free space left on this filesystem.
+
+      - uid: host-memory
+        title: Host low on memory
+        expr: >-
+          min by (host) (node_memory_MemAvailable_bytes{host!~"{{ alert_excluded_hosts }}"}
+          / node_memory_MemTotal_bytes{host!~"{{ alert_excluded_hosts }}"}) * 100
+        op: lt
+        threshold: 10
+        for: 15m
+        severity: warning
+        summary: Less than 10 percent available memory on this host.
+
+      - uid: host-systemd-failed
+        title: Failed systemd units
+        expr: >-
+          sum by (host) (node_systemd_unit_state{state="failed",
+          host!~"{{ alert_excluded_hosts }}", name!~"{{ alert_ignored_units }}"})
+        op: gt
+        threshold: 0
+        for: 15m
+        severity: warning
+        summary: A systemd unit outside the known-broken list is in a failed state on this host.

playbooks/nginx/sites-enabled/lounge.secretbee.buzz

@@ -1,36 +0,0 @@
-server {
-  listen          80;
-  server_name     lounge.secretbee.buzz;
-
-  location / {
-    return 301 https://$host$request_uri;
-  }
-}
-
-server {
-  listen         443 ssl;
-  server_name    lounge.secretbee.buzz;
-
-  limit_req zone=mylimit burst=20;
-
-  add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
-
-  ssl_certificate     /etc/letsencrypt/live/lounge.secretbee.buzz/fullchain.pem;
-  ssl_certificate_key /etc/letsencrypt/live/lounge.secretbee.buzz/privkey.pem;
-  include /etc/nginx/ssl.conf;
-
-  location / {
-	proxy_pass http://127.0.0.1:9000/;
-	proxy_http_version 1.1;
-	proxy_set_header Connection "upgrade";
-	proxy_set_header Upgrade $http_upgrade;
-	proxy_set_header X-Forwarded-For $remote_addr;
-	proxy_set_header X-Forwarded-Proto $scheme;
-	proxy_read_timeout 1d;
-  }
-
-  location /folder/ {
-	proxy_pass http://127.0.0.1:9000/uploads/;
-	proxy_set_header X-Forwarded-For $remote_addr;
-  }
-}

playbooks/ufw-ingress/ufw-ingress.yml

@@ -41,6 +41,18 @@
         - "80"
         - "443"
 
+    - name: UFW - Allow monitoring scrapes from beepi
+      community.general.ufw:
+        rule: allow
+        interface_in: wg0
+        proto: tcp
+        from_ip: "{{ wireguard_beepi_ip }}"
+        to_port: '{{ item }}'
+      loop:
+        - "9100"  # node exporter
+        - "9115"  # blackbox exporter
+        - "12345" # alloy
+
     - name: UFW - Allow wireguard outgoing
       community.general.ufw:
         route: true

playbooks/ufw/ufw.yml

@@ -111,6 +111,13 @@
         interface_out: eth0
         route: true
 
+    - name: UFW - Allow podman to wireguard
+      community.general.ufw:
+        rule: allow
+        interface_in: podman+
+        interface_out: wg0
+        route: true
+
     - name: UFW - Allow node exporter scraping
       community.general.ufw:
         rule: allow
@@ -125,6 +132,31 @@
         proto: "tcp"
         to_port: 9882
 
+    - name: UFW - Allow blackbox exporter scraping
+      community.general.ufw:
+        rule: allow
+        interface_in: podman+
+        proto: "tcp"
+        to_port: 9115
+
+    - name: UFW - Allow loki push from ingress
+      community.general.ufw:
+        rule: allow
+        interface_in: wg0
+        proto: tcp
+        from_ip: "{{ wireguard_ingress_ip }}"
+        to_port: "3100"
+
+    - name: UFW - Allow loki push routing from ingress
+      community.general.ufw:
+        rule: allow
+        interface_in: wg0
+        interface_out: podman+
+        route: true
+        proto: tcp
+        from_ip: "{{ wireguard_ingress_ip }}"
+        to_port: "3100"
+
     - name: UFW - Enable and deny by default
       community.general.ufw:
         state: enabled

vaultShow.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import re
+import subprocess
+import sys
+import textwrap
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent
+KEY = re.compile(r"^(?P<indent>[ \t]*)(?:-\s+)?(?P<name>[\w.-]+):\s*!vault\s*\|-?\s*$")
+HEADER = "$ANSIBLE_VAULT"
+
+
+def password_file():
+    override = os.environ.get("ANSIBLE_VAULT_PASSWORD_FILE")
+    path = Path(override) if override else ROOT / ".vault-password"
+    if not path.is_file():
+        sys.exit(f"vault password file not found: {path}")
+    return str(path)
+
+
+def decrypt(blob, pw):
+    result = subprocess.run(
+        ["ansible-vault", "decrypt", "--vault-password-file", pw, "--output", "-"],
+        input=blob,
+        capture_output=True,
+        text=True,
+    )
+    if result.returncode != 0:
+        return None, result.stderr.strip()
+    return result.stdout, None
+
+
+def extract(text):
+    lines = text.splitlines()
+    found = []
+    i = 0
+    while i < len(lines):
+        match = KEY.match(lines[i])
+        if not match:
+            i += 1
+            continue
+        indent = len(match.group("indent").expandtabs(8))
+        body = []
+        i += 1
+        while i < len(lines):
+            line = lines[i]
+            if not line.strip():
+                body.append("")
+                i += 1
+                continue
+            width = len(line.expandtabs(8)) - len(line.expandtabs(8).lstrip())
+            if width <= indent:
+                break
+            body.append(line)
+            i += 1
+        blob = textwrap.dedent("\n".join(body)).strip()
+        if blob.startswith(HEADER):
+            found.append((match.group("name"), blob))
+    return found
+
+
+def show(name, value):
+    value = value.rstrip("\n")
+    if "\n" in value:
+        print(f"{name}:")
+        print(textwrap.indent(value, "  "))
+    else:
+        print(f"{name}: {value}")
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Decrypt inline !vault variables in a playbook or vars file."
+    )
+    parser.add_argument("file", help="path to the file, or - to read a blob from stdin")
+    parser.add_argument("names", nargs="*", help="only show these variable names")
+    args = parser.parse_args()
+
+    pw = password_file()
+
+    if args.file == "-":
+        blob = textwrap.dedent(sys.stdin.read()).strip()
+        if not blob.startswith(HEADER):
+            sys.exit("stdin does not contain a vault blob")
+        value, error = decrypt(blob, pw)
+        if error:
+            sys.exit(error)
+        print(value.rstrip("\n"))
+        return
+
+    path = Path(args.file)
+    text = path.read_text()
+
+    if text.lstrip().startswith(HEADER):
+        value, error = decrypt(text.strip(), pw)
+        if error:
+            sys.exit(error)
+        sys.stdout.write(value)
+        return
+
+    variables = extract(text)
+    if args.names:
+        variables = [(n, b) for n, b in variables if n in args.names]
+    if not variables:
+        sys.exit(f"no inline vault variables found in {path}")
+
+    failures = 0
+    for name, blob in variables:
+        value, error = decrypt(blob, pw)
+        if error:
+            print(f"{name}: <failed: {error}>", file=sys.stderr)
+            failures += 1
+            continue
+        show(name, value)
+
+    sys.exit(1 if failures else 0)
+
+
+if __name__ == "__main__":
+    main()