lots of stuff
This commit is contained in:
@@ -41,12 +41,72 @@
|
||||
dest: /opt/backup/restoreTest.sh
|
||||
mode: '0700'
|
||||
|
||||
- name: Copy metrics library
|
||||
ansible.builtin.template:
|
||||
src: metrics.sh.j2
|
||||
dest: /opt/backup/metrics.sh
|
||||
mode: '0700'
|
||||
|
||||
- name: Copy inventory collector
|
||||
ansible.builtin.template:
|
||||
src: backupInventory.sh.j2
|
||||
dest: /opt/backup/backupInventory.sh
|
||||
mode: '0700'
|
||||
|
||||
- name: Copy passphrase
|
||||
ansible.builtin.template:
|
||||
src: passphrase.txt.j2
|
||||
dest: /opt/backup/passphrase.txt
|
||||
mode: '0600'
|
||||
|
||||
- name: Create textfile directory
|
||||
ansible.builtin.file:
|
||||
path: "{{ node_exporter_textfile_dir }}"
|
||||
state: directory
|
||||
mode: '0755'
|
||||
|
||||
- name: Install inventory timer
|
||||
ansible.builtin.copy:
|
||||
dest: "/etc/systemd/system/beepi-backup-inventory.{{ item.kind }}"
|
||||
content: "{{ item.content }}"
|
||||
mode: '0644'
|
||||
loop:
|
||||
- kind: service
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Collect backup inventory metrics
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/opt/backup/backupInventory.sh
|
||||
- kind: timer
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Collect backup inventory metrics
|
||||
|
||||
[Timer]
|
||||
OnBootSec=2min
|
||||
OnUnitActiveSec=15min
|
||||
AccuracySec=1min
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
notify: Reload systemd
|
||||
|
||||
- name: Flush handlers
|
||||
ansible.builtin.meta: flush_handlers
|
||||
|
||||
- name: Enable inventory timer
|
||||
ansible.builtin.systemd_service:
|
||||
name: beepi-backup-inventory.timer
|
||||
state: started
|
||||
enabled: true
|
||||
|
||||
- name: Run inventory collector once
|
||||
changed_when: true
|
||||
ansible.builtin.command:
|
||||
cmd: /opt/backup/backupInventory.sh
|
||||
|
||||
- name: Set path
|
||||
ansible.builtin.cron:
|
||||
name: PATH
|
||||
@@ -78,3 +138,8 @@
|
||||
hour: '6'
|
||||
weekday: '0'
|
||||
job: /opt/backup/restoreTest.sh >> /var/log/beepi-backup.log 2>&1
|
||||
|
||||
handlers:
|
||||
- name: Reload systemd
|
||||
ansible.builtin.systemd_service:
|
||||
daemon_reload: true
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
source /opt/backup/metrics.sh
|
||||
|
||||
now=$(date +"%Y%m%d%H%M")
|
||||
|
||||
declare -a arr=("mail" "matrix" "monitoring")
|
||||
@@ -12,6 +14,8 @@ declare -a immich=("systemd-immich-pgdata" "systemd-immich-upload")
|
||||
gitea_uid=$(id -u gitea)
|
||||
immich_uid=$(id -u immich)
|
||||
|
||||
metric_init daily
|
||||
|
||||
restore() {
|
||||
for i in "${arr[@]}"; do
|
||||
cd /opt/beeserver/${i}
|
||||
@@ -32,10 +36,21 @@ restore() {
|
||||
rm -f /home/immich/systemd-immich-*.tar
|
||||
}
|
||||
|
||||
trap restore EXIT
|
||||
finish() {
|
||||
local raw=$?
|
||||
local status
|
||||
|
||||
status=$(metric_status ${raw})
|
||||
restore
|
||||
metric_finish ${status}
|
||||
exit ${status}
|
||||
}
|
||||
|
||||
trap finish EXIT
|
||||
|
||||
for i in "${arr[@]}"
|
||||
do
|
||||
log "staging ${i}"
|
||||
cd /opt/beeserver/${i}
|
||||
podman-compose down -t 60
|
||||
if [[ "$i" == "monitoring" ]]
|
||||
@@ -57,10 +72,12 @@ do
|
||||
podman-compose up -d
|
||||
done
|
||||
|
||||
log "staging gitea"
|
||||
systemctl --user --machine gitea@.host stop gitea-server.service gitea-db.service
|
||||
rsync -aH /home/gitea/data /opt/backup/beeserver/gitea
|
||||
systemctl --user --machine gitea@.host start gitea-db.service gitea-server.service
|
||||
|
||||
log "staging immich"
|
||||
systemctl --user --machine immich@.host stop immich-server.service immich-db.service immich-redis.service
|
||||
for v in "${immich[@]}"
|
||||
do
|
||||
@@ -70,17 +87,22 @@ systemctl --user --machine immich@.host start immich-db.service immich-redis.ser
|
||||
mkdir -p /opt/backup/beeserver/immich/
|
||||
mv /home/immich/systemd-immich-*.tar /opt/backup/beeserver/immich/
|
||||
|
||||
log "staging vaultwarden"
|
||||
systemctl --user --machine vaultwarden@.host stop vaultwarden.service
|
||||
rsync -aH /home/vaultwarden/data /opt/backup/beeserver/vaultwarden
|
||||
systemctl --user --machine vaultwarden@.host start vaultwarden.service
|
||||
|
||||
archive=/home/beeshare/raid/serverBackup/beeserver_daily_${now}.tar.gz.gpg
|
||||
|
||||
log "encrypting ${archive}"
|
||||
tar -czf - /opt/backup/beeserver | gpg --batch --passphrase-file /opt/backup/passphrase.txt -c -o ${archive}
|
||||
|
||||
log "verifying ${archive}"
|
||||
if ! gpg --batch --passphrase-file /opt/backup/passphrase.txt -d ${archive} | tar -tzf - >/dev/null
|
||||
then
|
||||
echo "[${now}] verify failed, discarding ${archive}"
|
||||
log "verify failed, discarding ${archive}"
|
||||
rm -f ${archive}
|
||||
exit 1
|
||||
fi
|
||||
|
||||
metric_archive ${archive}
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
textfile_dir={{ node_exporter_textfile_dir }}
|
||||
archive_dir={{ backup_archive_dir }}
|
||||
raid_mount={{ backup_raid_mount }}
|
||||
|
||||
file=${textfile_dir}/backup_inventory.prom
|
||||
tmp=${file}.$$
|
||||
now=$(date +%s)
|
||||
|
||||
mkdir -p ${textfile_dir}
|
||||
|
||||
mounted=0
|
||||
if mountpoint -q ${raid_mount}
|
||||
then
|
||||
mounted=1
|
||||
fi
|
||||
|
||||
emit_kind() {
|
||||
local kind=$1
|
||||
local newest
|
||||
local count=0
|
||||
local age=0
|
||||
local bytes=0
|
||||
local total=0
|
||||
|
||||
if [ ${mounted} -eq 1 ]
|
||||
then
|
||||
count=$(find ${archive_dir} -maxdepth 1 -name "beeserver_${kind}_*.tar.gz.gpg" -type f 2>/dev/null | wc -l)
|
||||
newest=$(find ${archive_dir} -maxdepth 1 -name "beeserver_${kind}_*.tar.gz.gpg" -type f -printf '%T@ %p\n' 2>/dev/null | sort -rn | head -1 | cut -d' ' -f2-)
|
||||
total=$(find ${archive_dir} -maxdepth 1 -name "beeserver_${kind}_*.tar.gz.gpg" -type f -printf '%s\n' 2>/dev/null | awk '{ s += $1 } END { print s + 0 }')
|
||||
|
||||
if [ -n "${newest:-}" ]
|
||||
then
|
||||
bytes=$(stat -c %s "${newest}")
|
||||
age=$((now - $(stat -c %Y "${newest}")))
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "backup_archive_count{kind=\"${kind}\"} ${count}"
|
||||
echo "backup_archive_newest_age_seconds{kind=\"${kind}\"} ${age}"
|
||||
echo "backup_archive_newest_bytes{kind=\"${kind}\"} ${bytes}"
|
||||
echo "backup_archive_total_bytes{kind=\"${kind}\"} ${total}"
|
||||
}
|
||||
|
||||
{
|
||||
echo "# HELP backup_raid_mounted Whether the backup raid is mounted."
|
||||
echo "# TYPE backup_raid_mounted gauge"
|
||||
echo "backup_raid_mounted ${mounted}"
|
||||
echo "# HELP backup_archive_count Number of retained archives."
|
||||
echo "# TYPE backup_archive_count gauge"
|
||||
echo "# HELP backup_archive_newest_age_seconds Age of the newest archive."
|
||||
echo "# TYPE backup_archive_newest_age_seconds gauge"
|
||||
echo "# HELP backup_archive_newest_bytes Size of the newest archive."
|
||||
echo "# TYPE backup_archive_newest_bytes gauge"
|
||||
echo "# HELP backup_archive_total_bytes Size of all retained archives."
|
||||
echo "# TYPE backup_archive_total_bytes gauge"
|
||||
emit_kind daily
|
||||
emit_kind weekly
|
||||
} > "${tmp}"
|
||||
|
||||
chmod 0644 "${tmp}"
|
||||
mv "${tmp}" "${file}"
|
||||
@@ -2,20 +2,30 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
source /opt/backup/metrics.sh
|
||||
|
||||
now=$(date +"%Y%m%d%H%M")
|
||||
|
||||
archive=/home/beeshare/raid/serverBackup/beeserver_weekly_${now}.tar.gz.gpg
|
||||
|
||||
metric_init weekly
|
||||
trap metric_guard EXIT
|
||||
|
||||
log "encrypting ${archive}"
|
||||
tar -czf - /opt/backup/beeserver | gpg --batch --passphrase-file /opt/backup/passphrase.txt -c -o ${archive}
|
||||
|
||||
log "verifying ${archive}"
|
||||
if ! gpg --batch --passphrase-file /opt/backup/passphrase.txt -d ${archive} | tar -tzf - >/dev/null
|
||||
then
|
||||
echo "[${now}] verify failed, discarding ${archive} and keeping staging"
|
||||
log "verify failed, discarding ${archive} and keeping staging"
|
||||
rm -f ${archive}
|
||||
exit 1
|
||||
fi
|
||||
|
||||
metric_archive ${archive}
|
||||
|
||||
rm -rf /opt/backup/beeserver
|
||||
|
||||
log "pruning expired archives"
|
||||
find /home/beeshare/raid/serverBackup -maxdepth 1 -name 'beeserver_weekly_*.tar.gz.gpg' -mtime +30 -delete
|
||||
find /home/beeshare/raid/serverBackup -maxdepth 1 -name 'beeserver_daily_*.tar.gz.gpg' -mtime +7 -delete
|
||||
find /home/beeshare/raid/serverBackup -maxdepth 1 -name 'beeserver_daily_*.tar.gz.gpg' -mtime +7 -delete
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
#!/bin/bash
|
||||
|
||||
textfile_dir={{ node_exporter_textfile_dir }}
|
||||
|
||||
metric_job=""
|
||||
metric_started=0
|
||||
metric_bytes=0
|
||||
metric_signalled=""
|
||||
|
||||
metric_on_signal() {
|
||||
metric_signalled=$1
|
||||
exit $((128 + $1))
|
||||
}
|
||||
|
||||
metric_install_traps() {
|
||||
trap 'metric_on_signal 1' HUP
|
||||
trap 'metric_on_signal 2' INT
|
||||
trap 'metric_on_signal 15' TERM
|
||||
}
|
||||
|
||||
metric_status() {
|
||||
if [ -n "${metric_signalled}" ]
|
||||
then
|
||||
echo $((128 + metric_signalled))
|
||||
else
|
||||
echo "$1"
|
||||
fi
|
||||
}
|
||||
|
||||
log() {
|
||||
echo "[$(date --rfc-3339=seconds)] ${metric_job}: $*"
|
||||
}
|
||||
|
||||
metric_previous_success() {
|
||||
local file=${textfile_dir}/backup_${metric_job}.prom
|
||||
|
||||
if [ -r "${file}" ]
|
||||
then
|
||||
grep "^backup_last_success_timestamp_seconds{backup_job=\"${metric_job}\"} " "${file}" | awk '{ print $2 }'
|
||||
fi
|
||||
}
|
||||
|
||||
metric_init() {
|
||||
metric_job=$1
|
||||
metric_started=$(date +%s)
|
||||
mkdir -p ${textfile_dir}
|
||||
metric_install_traps
|
||||
log "start"
|
||||
}
|
||||
|
||||
metric_archive() {
|
||||
metric_bytes=$(stat -c %s "$1" 2>/dev/null || echo 0)
|
||||
}
|
||||
|
||||
metric_finish() {
|
||||
local status=$1
|
||||
local now
|
||||
local success
|
||||
local previous
|
||||
local file=${textfile_dir}/backup_${metric_job}.prom
|
||||
local tmp=${file}.$$
|
||||
|
||||
now=$(date +%s)
|
||||
previous=$(metric_previous_success)
|
||||
|
||||
if [ "${status}" -eq 0 ]
|
||||
then
|
||||
success=${now}
|
||||
log "done in $((now - metric_started))s"
|
||||
else
|
||||
success=${previous:-0}
|
||||
log "FAILED with status ${status} after $((now - metric_started))s"
|
||||
fi
|
||||
|
||||
cat > "${tmp}" <<EOF
|
||||
# HELP backup_last_run_timestamp_seconds Unix time of the last run of this backup job.
|
||||
# TYPE backup_last_run_timestamp_seconds gauge
|
||||
backup_last_run_timestamp_seconds{backup_job="${metric_job}"} ${now}
|
||||
# HELP backup_last_success_timestamp_seconds Unix time of the last successful run of this backup job.
|
||||
# TYPE backup_last_success_timestamp_seconds gauge
|
||||
backup_last_success_timestamp_seconds{backup_job="${metric_job}"} ${success}
|
||||
# HELP backup_last_duration_seconds Duration of the last run of this backup job.
|
||||
# TYPE backup_last_duration_seconds gauge
|
||||
backup_last_duration_seconds{backup_job="${metric_job}"} $((now - metric_started))
|
||||
# HELP backup_last_status Exit status of the last run of this backup job, 1 is success.
|
||||
# TYPE backup_last_status gauge
|
||||
backup_last_status{backup_job="${metric_job}"} $([ "${status}" -eq 0 ] && echo 1 || echo 0)
|
||||
# HELP backup_last_archive_bytes Size of the archive produced by the last run of this backup job.
|
||||
# TYPE backup_last_archive_bytes gauge
|
||||
backup_last_archive_bytes{backup_job="${metric_job}"} ${metric_bytes}
|
||||
EOF
|
||||
|
||||
chmod 0644 "${tmp}"
|
||||
mv "${tmp}" "${file}"
|
||||
}
|
||||
|
||||
metric_guard() {
|
||||
local raw=$?
|
||||
local status
|
||||
|
||||
status=$(metric_status ${raw})
|
||||
metric_finish ${status}
|
||||
exit ${status}
|
||||
}
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
source /opt/backup/metrics.sh
|
||||
|
||||
now=$(date +"%Y%m%d%H%M")
|
||||
backup_dir=/home/beeshare/raid/serverBackup
|
||||
image=docker.io/vaultwarden/server:1.37.1
|
||||
@@ -9,15 +11,23 @@ container=vaultwarden-restoretest
|
||||
port=18222
|
||||
work=$(mktemp -d)
|
||||
|
||||
metric_init restoretest
|
||||
|
||||
cleanup() {
|
||||
local raw=$?
|
||||
local status
|
||||
|
||||
status=$(metric_status ${raw})
|
||||
podman rm -f ${container} >/dev/null 2>&1 || true
|
||||
rm -rf "${work}"
|
||||
metric_finish ${status}
|
||||
exit ${status}
|
||||
}
|
||||
|
||||
trap cleanup EXIT
|
||||
|
||||
latest=$(ls -1t ${backup_dir}/beeserver_*.tar.gz.gpg | head -1)
|
||||
echo "[${now}] restore test using ${latest}"
|
||||
log "restore test using ${latest}"
|
||||
|
||||
gpg --batch --passphrase-file /opt/backup/passphrase.txt -d "${latest}" \
|
||||
| tar -xzf - -C "${work}" opt/backup/beeserver/vaultwarden
|
||||
@@ -26,15 +36,15 @@ data="${work}/opt/backup/beeserver/vaultwarden/data"
|
||||
|
||||
[[ -s "${data}/db.sqlite3" ]]
|
||||
[[ -s "${data}/rsa_key.pem" ]]
|
||||
echo " db and rsa_key present"
|
||||
log "db and rsa_key present"
|
||||
|
||||
integrity=$(sqlite3 "${data}/db.sqlite3" "PRAGMA integrity_check;")
|
||||
[[ "${integrity}" == "ok" ]]
|
||||
echo " integrity_check ok"
|
||||
log "integrity_check ok"
|
||||
|
||||
users=$(sqlite3 "${data}/db.sqlite3" "select count(*) from users;")
|
||||
[[ "${users}" -ge 1 ]]
|
||||
echo " users ${users}"
|
||||
log "users ${users}"
|
||||
|
||||
email=$(sqlite3 "${data}/db.sqlite3" "select email from users limit 1;")
|
||||
|
||||
@@ -51,12 +61,14 @@ done
|
||||
|
||||
config=$(curl -sf http://127.0.0.1:${port}/api/config)
|
||||
[[ "${config}" == *'"version"'* ]]
|
||||
echo " api/config ok"
|
||||
log "api/config ok"
|
||||
|
||||
prelogin=$(curl -sf -X POST http://127.0.0.1:${port}/identity/accounts/prelogin \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "{\"email\":\"${email}\"}")
|
||||
[[ "${prelogin}" == *[Kk]df* ]]
|
||||
echo " prelogin ok for ${email}"
|
||||
log "prelogin ok for ${email}"
|
||||
|
||||
echo "[${now}] restore test PASSED"
|
||||
metric_archive "${latest}"
|
||||
|
||||
log "restore test PASSED"
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
- name: Dashboards
|
||||
hosts: beepi
|
||||
become: true
|
||||
vars:
|
||||
dashboard_dir: /opt/beeserver/monitoring/grafana/dashboards
|
||||
dashboard_general_dir: /opt/beeserver/monitoring/grafana/dashboards-general
|
||||
provisioning_dir: /opt/beeserver/monitoring/grafana/provisioning/dashboards
|
||||
tasks:
|
||||
- name: Determine whether uptimerobot is configured
|
||||
ansible.builtin.set_fact:
|
||||
uptimerobot_enabled: "{{ uptimerobot_api_key is defined and 'REPLACE_ME' not in uptimerobot_api_key }}"
|
||||
|
||||
- name: Check that grafana is running
|
||||
ansible.builtin.command:
|
||||
cmd: podman container inspect -f '{{ '{{' }}.State.Running{{ '}}' }}' grafana
|
||||
register: grafana_running
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
|
||||
- name: Require a running grafana
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- grafana_running.stdout | trim == 'true'
|
||||
fail_msg: >-
|
||||
grafana is not running, so provisioned changes cannot be picked up.
|
||||
Run make deploy-monitoring first.
|
||||
|
||||
- name: Create dashboard directories
|
||||
ansible.builtin.file:
|
||||
path: "{{ item }}"
|
||||
state: directory
|
||||
mode: '0755'
|
||||
loop:
|
||||
- "{{ dashboard_dir }}"
|
||||
- "{{ dashboard_general_dir }}"
|
||||
- "{{ provisioning_dir }}"
|
||||
|
||||
- name: Copy dashboard provider
|
||||
ansible.builtin.template:
|
||||
src: ../monitoring/grafana/provisioning/dashboards/dashboards.yml.j2
|
||||
dest: "{{ provisioning_dir }}/dashboards.yml"
|
||||
mode: '0644'
|
||||
notify: Restart grafana
|
||||
|
||||
- name: Copy generated dashboards
|
||||
ansible.builtin.template:
|
||||
src: "../monitoring/grafana/dashboards/{{ item }}.json.j2"
|
||||
dest: "{{ dashboard_dir }}/{{ item }}.json"
|
||||
mode: '0644'
|
||||
validate: "python3 -c \"import json; json.load(open('%s'))\""
|
||||
loop:
|
||||
- infrastructure-map
|
||||
- backups
|
||||
- services
|
||||
- ingress
|
||||
|
||||
- name: Copy uptimerobot dashboard
|
||||
ansible.builtin.template:
|
||||
src: ../monitoring/grafana/dashboards/uptimerobot.json.j2
|
||||
dest: "{{ dashboard_dir }}/uptimerobot.json"
|
||||
mode: '0644'
|
||||
validate: "python3 -c \"import json; json.load(open('%s'))\""
|
||||
when: uptimerobot_enabled
|
||||
|
||||
- name: Copy hand made dashboards
|
||||
ansible.builtin.copy:
|
||||
src: ../monitoring/grafana/dashboards-general/
|
||||
dest: "{{ dashboard_general_dir }}/"
|
||||
mode: '0644'
|
||||
|
||||
- name: Debug
|
||||
ansible.builtin.debug:
|
||||
msg:
|
||||
- "Dashboards are polled by grafana every 30s, no restart needed."
|
||||
- "Grafana: https://grafana.secretbee.buzz/"
|
||||
- "Errors: sudo podman logs --tail 40 grafana | grep -i provision"
|
||||
tags:
|
||||
- always
|
||||
|
||||
handlers:
|
||||
- name: Restart grafana
|
||||
changed_when: true
|
||||
ansible.builtin.command:
|
||||
cmd: podman restart grafana
|
||||
@@ -6,7 +6,6 @@
|
||||
user_home: /home/hetzner
|
||||
|
||||
hetzner_raid_mount: /home/beeshare/raid
|
||||
hetzner_port: 23
|
||||
|
||||
hetzner_paths:
|
||||
- src: /home/beeshare/raid/serverBackup
|
||||
@@ -14,10 +13,6 @@
|
||||
- src: /home/beeshare/raid/foundry/backup
|
||||
dest: ./foundry
|
||||
|
||||
# Fill in after creating the storage box sub-account, then re-run
|
||||
hetzner_host: "u651287.your-storagebox.de"
|
||||
hetzner_user: "u651287"
|
||||
|
||||
tasks:
|
||||
- name: Install rsync
|
||||
ansible.builtin.apt:
|
||||
@@ -88,6 +83,58 @@
|
||||
job: /opt/backup/hetznerSync.sh >> /var/log/beepi-backup.log 2>&1
|
||||
when: hetzner_host | length > 0
|
||||
|
||||
- name: Copy metrics collector
|
||||
ansible.builtin.template:
|
||||
src: hetznerMetrics.sh.j2
|
||||
dest: /opt/backup/hetznerMetrics.sh
|
||||
mode: '0700'
|
||||
when: hetzner_host | length > 0
|
||||
|
||||
- name: Install metrics timer
|
||||
ansible.builtin.copy:
|
||||
dest: "/etc/systemd/system/beepi-hetzner-metrics.{{ item.kind }}"
|
||||
content: "{{ item.content }}"
|
||||
mode: '0644'
|
||||
loop:
|
||||
- kind: service
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Collect hetzner storage box metrics
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/opt/backup/hetznerMetrics.sh
|
||||
- kind: timer
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Collect hetzner storage box metrics
|
||||
|
||||
[Timer]
|
||||
OnBootSec=5min
|
||||
OnUnitActiveSec=1h
|
||||
AccuracySec=5min
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
when: hetzner_host | length > 0
|
||||
notify: Reload systemd
|
||||
|
||||
- name: Flush handlers
|
||||
ansible.builtin.meta: flush_handlers
|
||||
|
||||
- name: Enable metrics timer
|
||||
ansible.builtin.systemd_service:
|
||||
name: beepi-hetzner-metrics.timer
|
||||
state: started
|
||||
enabled: true
|
||||
when: hetzner_host | length > 0
|
||||
|
||||
- name: Run metrics collector once
|
||||
changed_when: true
|
||||
ansible.builtin.command:
|
||||
cmd: /opt/backup/hetznerMetrics.sh
|
||||
when: hetzner_host | length > 0
|
||||
|
||||
- name: Debug
|
||||
ansible.builtin.debug:
|
||||
msg:
|
||||
@@ -95,3 +142,8 @@
|
||||
- "Read logs: sudo tail -50 /var/log/beepi-backup.log"
|
||||
tags:
|
||||
- always
|
||||
|
||||
handlers:
|
||||
- name: Reload systemd
|
||||
ansible.builtin.systemd_service:
|
||||
daemon_reload: true
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
textfile_dir={{ node_exporter_textfile_dir }}
|
||||
file=${textfile_dir}/hetzner.prom
|
||||
tmp=${file}.$$
|
||||
now=$(date +%s)
|
||||
|
||||
ssh_opts="-p {{ hetzner_port }} -i {{ user_home }}/.ssh/id_ed25519 -o UserKnownHostsFile={{ user_home }}/.ssh/known_hosts -o StrictHostKeyChecking=yes -o BatchMode=yes -o ConnectTimeout=15"
|
||||
|
||||
mkdir -p ${textfile_dir}
|
||||
|
||||
reachable=0
|
||||
body=""
|
||||
|
||||
emit_path() {
|
||||
local label=$1
|
||||
local remote=$2
|
||||
local listing
|
||||
local files
|
||||
local bytes
|
||||
local pending
|
||||
local newest
|
||||
local age=-1
|
||||
|
||||
if ! listing=$(sudo -u {{ user }} rsync --list-only -e "ssh ${ssh_opts}" {{ hetzner_user }}@{{ hetzner_host }}:${remote}/ 2>/dev/null)
|
||||
then
|
||||
body+="hetzner_remote_listing_ok{path=\"${label}\"} 0"$'\n'
|
||||
return
|
||||
fi
|
||||
|
||||
reachable=1
|
||||
|
||||
files=$(echo "${listing}" | awk '$1 ~ /^-/ && substr($5,1,1) != "." { c++ } END { print c + 0 }')
|
||||
bytes=$(echo "${listing}" | awk '$1 ~ /^-/ && substr($5,1,1) != "." { gsub(",", "", $2); s += $2 } END { printf "%.0f", s + 0 }')
|
||||
pending=$(echo "${listing}" | awk '$1 ~ /^-/ && substr($5,1,1) == "." { gsub(",", "", $2); s += $2 } END { printf "%.0f", s + 0 }')
|
||||
newest=$(echo "${listing}" | awk '$1 ~ /^-/ && substr($5,1,1) != "." { print $3" "$4 }' | sort -r | head -1)
|
||||
|
||||
if [ -n "${newest}" ]
|
||||
then
|
||||
age=$((now - $(date -d "$(echo ${newest} | tr '/' '-')" +%s)))
|
||||
fi
|
||||
|
||||
body+="hetzner_remote_listing_ok{path=\"${label}\"} 1"$'\n'
|
||||
body+="hetzner_remote_files{path=\"${label}\"} ${files}"$'\n'
|
||||
body+="hetzner_remote_bytes{path=\"${label}\"} ${bytes}"$'\n'
|
||||
body+="hetzner_remote_pending_bytes{path=\"${label}\"} ${pending}"$'\n'
|
||||
body+="hetzner_remote_newest_age_seconds{path=\"${label}\"} ${age}"$'\n'
|
||||
}
|
||||
|
||||
{% for path in hetzner_paths %}
|
||||
emit_path "{{ path.dest | replace('./', '') }}" "{{ path.dest }}"
|
||||
{% endfor %}
|
||||
|
||||
{
|
||||
echo "# HELP hetzner_reachable Whether the storage box answered at least one listing."
|
||||
echo "# TYPE hetzner_reachable gauge"
|
||||
echo "hetzner_reachable ${reachable}"
|
||||
echo "# HELP hetzner_last_check_timestamp_seconds Unix time of the last storage box check."
|
||||
echo "# TYPE hetzner_last_check_timestamp_seconds gauge"
|
||||
echo "hetzner_last_check_timestamp_seconds ${now}"
|
||||
echo "# HELP hetzner_remote_listing_ok Whether this remote path could be listed."
|
||||
echo "# TYPE hetzner_remote_listing_ok gauge"
|
||||
echo "# HELP hetzner_remote_files Number of completed files stored at this remote path."
|
||||
echo "# TYPE hetzner_remote_files gauge"
|
||||
echo "# HELP hetzner_remote_bytes Bytes of completed files stored at this remote path."
|
||||
echo "# TYPE hetzner_remote_bytes gauge"
|
||||
echo "# HELP hetzner_remote_pending_bytes Bytes of in flight partial transfers at this remote path."
|
||||
echo "# TYPE hetzner_remote_pending_bytes gauge"
|
||||
echo "# HELP hetzner_remote_newest_age_seconds Age of the newest completed file at this remote path."
|
||||
echo "# TYPE hetzner_remote_newest_age_seconds gauge"
|
||||
printf '%s' "${body}"
|
||||
} > "${tmp}"
|
||||
|
||||
chmod 0644 "${tmp}"
|
||||
mv "${tmp}" "${file}"
|
||||
@@ -2,11 +2,22 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
now=$(date +"%Y%m%d%H%M")
|
||||
exec 9>/var/lock/hetznerSync.lock
|
||||
|
||||
if ! flock -n 9
|
||||
then
|
||||
echo "[$(date --rfc-3339=seconds)] offsite: another sync is still running, skipping this run"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
source /opt/backup/metrics.sh
|
||||
|
||||
metric_init offsite
|
||||
trap metric_guard EXIT
|
||||
|
||||
if ! mountpoint -q {{ hetzner_raid_mount }}
|
||||
then
|
||||
echo "[${now}] {{ hetzner_raid_mount }} not mounted, refusing to sync"
|
||||
log "{{ hetzner_raid_mount }} not mounted, refusing to sync"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -16,19 +27,17 @@ sync_path() {
|
||||
|
||||
if [ -z "$(ls -A ${src} 2>/dev/null)" ]
|
||||
then
|
||||
echo "[${now}] ${src} empty, refusing to sync"
|
||||
log "${src} empty, refusing to sync"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[${now}] syncing ${src} to ${dest}"
|
||||
log "syncing ${src} to ${dest}"
|
||||
sudo -u {{ user }} rsync -a --delete --partial \
|
||||
-e "ssh -p {{ hetzner_port }} -i {{ user_home }}/.ssh/id_ed25519 -o UserKnownHostsFile={{ user_home }}/.ssh/known_hosts -o StrictHostKeyChecking=yes" \
|
||||
${src}/ \
|
||||
{{ hetzner_user }}@{{ hetzner_host }}:${dest}/
|
||||
}
|
||||
|
||||
echo "[${now}] offsite sync start"
|
||||
{% for path in hetzner_paths %}
|
||||
sync_path {{ path.src }} {{ path.dest }}
|
||||
{% endfor %}
|
||||
echo "[${now}] offsite sync done"
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
loki.relabel "journal" {
|
||||
forward_to = []
|
||||
|
||||
rule {
|
||||
source_labels = ["__journal__systemd_unit"]
|
||||
target_label = "unit"
|
||||
}
|
||||
|
||||
rule {
|
||||
source_labels = ["__journal_priority_keyword"]
|
||||
target_label = "level"
|
||||
}
|
||||
}
|
||||
|
||||
loki.source.journal "system" {
|
||||
forward_to = [loki.process.journal.receiver]
|
||||
relabel_rules = loki.relabel.journal.rules
|
||||
labels = {
|
||||
job = "systemd-journal",
|
||||
host = "ingress",
|
||||
}
|
||||
}
|
||||
|
||||
loki.process "journal" {
|
||||
forward_to = [loki.write.default.receiver]
|
||||
|
||||
stage.match {
|
||||
selector = `{job="systemd-journal"} |~ "\\[UFW AUDIT\\]"`
|
||||
action = "drop"
|
||||
}
|
||||
|
||||
stage.match {
|
||||
selector = `{job="systemd-journal"} |~ "\\[UFW BLOCK\\]"`
|
||||
|
||||
stage.regex {
|
||||
expression = `\[UFW\s(?P<action>\w+)\].*?SRC=(?P<ip>[0-9.]+).*?DST=(?P<dst>[0-9.]+).*?PROTO=(?P<proto>\w+)(?:\sSPT=(?P<spt>\d+))?(?:\sDPT=(?P<dpt>\d+))?`
|
||||
}
|
||||
|
||||
stage.geoip {
|
||||
db = "/opt/database/GeoLite2-City.mmdb"
|
||||
source = "ip"
|
||||
db_type = "city"
|
||||
}
|
||||
|
||||
stage.labels {
|
||||
values = {
|
||||
action = "",
|
||||
proto = "",
|
||||
dpt = "",
|
||||
geoip_country_name = "",
|
||||
geoip_country_code = "",
|
||||
geoip_location_latitude = "",
|
||||
geoip_location_longitude = "",
|
||||
}
|
||||
}
|
||||
|
||||
stage.static_labels {
|
||||
values = {
|
||||
job = "ufw",
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
local.file_match "fail2ban" {
|
||||
path_targets = [{"__path__" = "/var/log/fail2ban.log", job = "fail2ban", host = "ingress"}]
|
||||
}
|
||||
|
||||
loki.source.file "fail2ban_source" {
|
||||
targets = local.file_match.fail2ban.targets
|
||||
forward_to = [loki.process.fail2ban.receiver]
|
||||
}
|
||||
|
||||
loki.process "fail2ban" {
|
||||
forward_to = [loki.write.default.receiver]
|
||||
|
||||
stage.regex {
|
||||
expression = `^.*:\s(?<level>\w*)\s*\[(?<service>\w*)\]\s(?<type>\w*)\s(?<ip>(?:(?:2(?:[0-4][0-9]|5[0-5])|[0-1]?[0-9]?[0-9])\.){3}(?:(?:2([0-4][0-9]|5[0-5])|[0-1]?[0-9]?[0-9]))).*$`
|
||||
labels_from_groups = true
|
||||
}
|
||||
|
||||
stage.geoip {
|
||||
db = "/opt/database/GeoLite2-City.mmdb"
|
||||
source = "ip"
|
||||
db_type = "city"
|
||||
}
|
||||
|
||||
stage.labels {
|
||||
values = {
|
||||
geoip_country_name = "",
|
||||
geoip_country_code = "",
|
||||
geoip_location_latitude = "",
|
||||
geoip_location_longitude = "",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
loki.write "default" {
|
||||
endpoint {
|
||||
url = "http://{{ wireguard_beepi_ip }}:3100/loki/api/v1/push"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,297 @@
|
||||
- name: Monitoring-ingress
|
||||
hosts: animeistrash
|
||||
become: true
|
||||
vars:
|
||||
geoip_username: !vault |
|
||||
$ANSIBLE_VAULT;1.1;AES256
|
||||
30656365643463396265633130356339386433633331613130633134336633643637653761623962
|
||||
3633336466326564616432373031303530376166353436610a323033353164313436613363613836
|
||||
30303931323234306162343361643162313238313234383837323530343936343134326365666166
|
||||
3438643238626266320a333732363461613834613833303436363534393031323333613534333534
|
||||
3438
|
||||
geoip_license: !vault |
|
||||
$ANSIBLE_VAULT;1.1;AES256
|
||||
32336437653435396231663666643631373638616533653538333934356339666661313439343566
|
||||
6462303439666233316162653430363239393031633362630a323533393233633164613031646633
|
||||
62366362383162306563383235373262326262323337303333313163653838613436316337633063
|
||||
3630356262666338320a333137333063303534386437343762616665383034336365663732343566
|
||||
36376463313964333834373435653564346333396538306337333762613737643363636364353139
|
||||
3032383564333638613566633363666430356230326231633636
|
||||
tasks:
|
||||
- name: Install exporters
|
||||
ansible.builtin.apt:
|
||||
name:
|
||||
- prometheus-node-exporter
|
||||
- prometheus-blackbox-exporter
|
||||
- wireguard-tools
|
||||
state: present
|
||||
update_cache: true
|
||||
|
||||
- name: Create textfile directory
|
||||
ansible.builtin.file:
|
||||
path: "{{ node_exporter_textfile_dir }}"
|
||||
state: directory
|
||||
mode: '0755'
|
||||
|
||||
- name: Configure node exporter
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/default/prometheus-node-exporter
|
||||
content: |
|
||||
ARGS="--web.listen-address={{ wireguard_ingress_ip }}:9100 --collector.textfile.directory={{ node_exporter_textfile_dir }} --collector.systemd"
|
||||
mode: '0644'
|
||||
notify: Restart node exporter
|
||||
|
||||
- name: Configure blackbox exporter
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/default/prometheus-blackbox-exporter
|
||||
content: |
|
||||
ARGS="--config.file=/etc/prometheus/blackbox.yml --web.listen-address={{ wireguard_ingress_ip }}:9115"
|
||||
mode: '0644'
|
||||
notify: Restart blackbox exporter
|
||||
|
||||
- name: Copy blackbox config
|
||||
ansible.builtin.copy:
|
||||
src: ../monitoring/blackbox.yml
|
||||
dest: /etc/prometheus/blackbox.yml
|
||||
mode: '0644'
|
||||
notify: Restart blackbox exporter
|
||||
|
||||
- name: Look up the prometheus group
|
||||
ansible.builtin.getent:
|
||||
database: group
|
||||
key: prometheus
|
||||
|
||||
- name: Allow the prometheus group to send icmp
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/sysctl.d/99-blackbox-icmp.conf
|
||||
content: |
|
||||
net.ipv4.ping_group_range = {{ prometheus_gid }} {{ prometheus_gid }}
|
||||
mode: '0644'
|
||||
vars:
|
||||
prometheus_gid: "{{ ansible_facts.getent_group.prometheus[1] }}"
|
||||
notify: Apply sysctl
|
||||
|
||||
- name: Create blackbox override directory
|
||||
ansible.builtin.file:
|
||||
path: /etc/systemd/system/prometheus-blackbox-exporter.service.d
|
||||
state: directory
|
||||
mode: '0755'
|
||||
|
||||
- name: Allow blackbox to send icmp
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/systemd/system/prometheus-blackbox-exporter.service.d/capabilities.conf
|
||||
content: |
|
||||
[Service]
|
||||
AmbientCapabilities=CAP_NET_RAW
|
||||
CapabilityBoundingSet=CAP_NET_RAW
|
||||
mode: '0644'
|
||||
notify:
|
||||
- Reload systemd
|
||||
- Restart blackbox exporter
|
||||
|
||||
- name: Copy wireguard collector
|
||||
ansible.builtin.template:
|
||||
src: ../monitoring/textfile/wg_metrics.sh.j2
|
||||
dest: /usr/local/bin/wg_metrics.sh
|
||||
mode: '0755'
|
||||
|
||||
- name: Install collector timer
|
||||
ansible.builtin.copy:
|
||||
dest: "/etc/systemd/system/ingress-wg-metrics.{{ item.kind }}"
|
||||
content: "{{ item.content }}"
|
||||
mode: '0644'
|
||||
loop:
|
||||
- kind: service
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Collect wireguard peer metrics
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/bin/wg_metrics.sh
|
||||
- kind: timer
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Collect wireguard peer metrics
|
||||
|
||||
[Timer]
|
||||
OnBootSec=1min
|
||||
OnUnitActiveSec=30s
|
||||
AccuracySec=5s
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
notify: Reload systemd
|
||||
|
||||
- name: Flush handlers
|
||||
ansible.builtin.meta: flush_handlers
|
||||
|
||||
- name: Enable collector timer
|
||||
ansible.builtin.systemd_service:
|
||||
name: ingress-wg-metrics.timer
|
||||
state: started
|
||||
enabled: true
|
||||
|
||||
- name: Create alloy directories
|
||||
ansible.builtin.file:
|
||||
path: "{{ item }}"
|
||||
state: directory
|
||||
mode: '0755'
|
||||
loop:
|
||||
- /etc/alloy
|
||||
- /opt/database
|
||||
|
||||
- name: Create apt keyring directory
|
||||
ansible.builtin.file:
|
||||
path: /etc/apt/keyrings
|
||||
state: directory
|
||||
mode: '0755'
|
||||
|
||||
- name: Add grafana apt key
|
||||
ansible.builtin.get_url:
|
||||
url: https://apt.grafana.com/gpg.key
|
||||
dest: /etc/apt/keyrings/grafana.asc
|
||||
mode: '0644'
|
||||
|
||||
- name: Add grafana apt repository
|
||||
ansible.builtin.apt_repository:
|
||||
repo: "deb [signed-by=/etc/apt/keyrings/grafana.asc] https://apt.grafana.com stable main"
|
||||
filename: grafana
|
||||
state: present
|
||||
|
||||
- name: Install alloy
|
||||
ansible.builtin.apt:
|
||||
name: alloy
|
||||
state: present
|
||||
update_cache: true
|
||||
|
||||
- name: Create alloy data directory
|
||||
ansible.builtin.file:
|
||||
path: /var/lib/alloy/data
|
||||
state: directory
|
||||
owner: alloy
|
||||
group: alloy
|
||||
mode: '0755'
|
||||
notify: Restart alloy
|
||||
|
||||
- name: Check the geoip database
|
||||
ansible.builtin.stat:
|
||||
path: /opt/database/GeoLite2-City.mmdb
|
||||
register: geoip_db
|
||||
|
||||
- name: Check when geoip was last downloaded
|
||||
ansible.builtin.stat:
|
||||
path: /opt/.geoip_downloaded
|
||||
register: geoip_stamp
|
||||
|
||||
- name: Decide whether the geoip database needs refreshing
|
||||
ansible.builtin.set_fact:
|
||||
geoip_stale: "{{ not geoip_db.stat.exists or not geoip_stamp.stat.exists or (ansible_facts.date_time.epoch | int - geoip_stamp.stat.mtime | int) > geoip_max_age_seconds | int }}"
|
||||
|
||||
- name: Report the geoip database age
|
||||
ansible.builtin.debug:
|
||||
msg: >-
|
||||
geoip
|
||||
{{ 'not present, downloading' if not (geoip_db.stat.exists and geoip_stamp.stat.exists)
|
||||
else 'downloaded ' + (((ansible_facts.date_time.epoch | int - geoip_stamp.stat.mtime | int) / 3600) | round(1) | string) + 'h ago, ' + ('refreshing' if geoip_stale else 'reusing') }}
|
||||
|
||||
- name: Download geoip database
|
||||
ansible.builtin.get_url:
|
||||
url: "https://download.maxmind.com/geoip/databases/GeoLite2-City/download?suffix=tar.gz"
|
||||
dest: /opt/geoip.tar.gz
|
||||
username: "{{ geoip_username }}"
|
||||
password: "{{ geoip_license }}"
|
||||
mode: '0644'
|
||||
force: true
|
||||
when: geoip_stale
|
||||
|
||||
- name: Extract geoip database
|
||||
ansible.builtin.unarchive:
|
||||
remote_src: true
|
||||
src: /opt/geoip.tar.gz
|
||||
dest: /opt/database
|
||||
extra_opts: ['--strip-components=1', '--show-stored-names']
|
||||
when: geoip_stale
|
||||
notify: Restart alloy
|
||||
|
||||
- name: Stamp the geoip download time
|
||||
ansible.builtin.file:
|
||||
path: /opt/.geoip_downloaded
|
||||
state: touch
|
||||
mode: '0644'
|
||||
when: geoip_stale
|
||||
|
||||
- name: Copy alloy config
|
||||
ansible.builtin.template:
|
||||
src: config.alloy.j2
|
||||
dest: /etc/alloy/config.alloy
|
||||
mode: '0644'
|
||||
notify: Restart alloy
|
||||
|
||||
- name: Configure alloy
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/default/alloy
|
||||
content: |
|
||||
CONFIG_FILE="/etc/alloy/config.alloy"
|
||||
CUSTOM_ARGS="--server.http.listen-addr={{ wireguard_ingress_ip }}:12345 --stability.level=experimental"
|
||||
RESTART_ON_UPGRADE=true
|
||||
mode: '0644'
|
||||
notify: Restart alloy
|
||||
|
||||
- name: Allow alloy to read logs
|
||||
ansible.builtin.user:
|
||||
name: alloy
|
||||
groups:
|
||||
- adm
|
||||
- systemd-journal
|
||||
append: true
|
||||
notify: Restart alloy
|
||||
|
||||
- name: Enable services
|
||||
ansible.builtin.service:
|
||||
name: "{{ item }}"
|
||||
state: started
|
||||
enabled: true
|
||||
loop:
|
||||
- prometheus-node-exporter
|
||||
- prometheus-blackbox-exporter
|
||||
- alloy
|
||||
|
||||
- name: Run collector once
|
||||
changed_when: true
|
||||
ansible.builtin.command:
|
||||
cmd: /usr/local/bin/wg_metrics.sh
|
||||
|
||||
- name: Debug
|
||||
ansible.builtin.debug:
|
||||
msg:
|
||||
- "Node metrics: curl -s {{ wireguard_ingress_ip }}:9100/metrics | head"
|
||||
- "Blackbox probe: curl -s '{{ wireguard_ingress_ip }}:9115/probe?target={{ wireguard_beepi_ip }}:443&module=tcp_connect'"
|
||||
- "Alloy logs: journalctl -u alloy -n 50"
|
||||
tags:
|
||||
- always
|
||||
|
||||
handlers:
|
||||
- name: Apply sysctl
|
||||
changed_when: true
|
||||
ansible.builtin.command:
|
||||
cmd: sysctl --system
|
||||
|
||||
- name: Reload systemd
|
||||
ansible.builtin.systemd_service:
|
||||
daemon_reload: true
|
||||
|
||||
- name: Restart node exporter
|
||||
ansible.builtin.service:
|
||||
name: prometheus-node-exporter
|
||||
state: restarted
|
||||
|
||||
- name: Restart blackbox exporter
|
||||
ansible.builtin.service:
|
||||
name: prometheus-blackbox-exporter
|
||||
state: restarted
|
||||
|
||||
- name: Restart alloy
|
||||
ansible.builtin.service:
|
||||
name: alloy
|
||||
state: restarted
|
||||
@@ -0,0 +1,70 @@
|
||||
modules:
|
||||
http_2xx:
|
||||
prober: http
|
||||
timeout: 10s
|
||||
http:
|
||||
valid_status_codes: [200]
|
||||
valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
|
||||
preferred_ip_protocol: ip4
|
||||
ip_protocol_fallback: false
|
||||
|
||||
http_2xx_any:
|
||||
prober: http
|
||||
timeout: 10s
|
||||
http:
|
||||
valid_status_codes: [200, 204, 301, 302, 307, 308, 401, 403]
|
||||
preferred_ip_protocol: ip4
|
||||
ip_protocol_fallback: false
|
||||
|
||||
https_vhost:
|
||||
prober: http
|
||||
timeout: 10s
|
||||
http:
|
||||
valid_status_codes: [200, 204, 301, 302, 307, 308, 401, 403]
|
||||
preferred_ip_protocol: ip4
|
||||
ip_protocol_fallback: false
|
||||
fail_if_not_ssl: true
|
||||
tls_config:
|
||||
insecure_skip_verify: false
|
||||
|
||||
tcp_connect:
|
||||
prober: tcp
|
||||
timeout: 10s
|
||||
tcp:
|
||||
preferred_ip_protocol: ip4
|
||||
ip_protocol_fallback: false
|
||||
|
||||
smtp_starttls:
|
||||
prober: tcp
|
||||
timeout: 10s
|
||||
tcp:
|
||||
preferred_ip_protocol: ip4
|
||||
ip_protocol_fallback: false
|
||||
query_response:
|
||||
- expect: "^220"
|
||||
- send: "EHLO prober"
|
||||
- expect: "^250-STARTTLS"
|
||||
- send: "QUIT"
|
||||
|
||||
imaps_banner:
|
||||
prober: tcp
|
||||
timeout: 10s
|
||||
tcp:
|
||||
preferred_ip_protocol: ip4
|
||||
ip_protocol_fallback: false
|
||||
tls: true
|
||||
|
||||
ssh_banner:
|
||||
prober: tcp
|
||||
timeout: 10s
|
||||
tcp:
|
||||
preferred_ip_protocol: ip4
|
||||
ip_protocol_fallback: false
|
||||
query_response:
|
||||
- expect: "^SSH-2.0-"
|
||||
|
||||
icmp:
|
||||
prober: icmp
|
||||
timeout: 5s
|
||||
icmp:
|
||||
preferred_ip_protocol: ip4
|
||||
@@ -10,8 +10,17 @@ local.file_match "fail2ban" {
|
||||
path_targets = [{"__path__" = "/var/log/fail2ban.log", job = "fail2ban", host = "beepi"}]
|
||||
}
|
||||
|
||||
local.file_match "backup" {
|
||||
path_targets = [{"__path__" = "/var/log/beepi-backup.log", job = "backup", host = "beepi"}]
|
||||
}
|
||||
|
||||
local.file_match "varlog" {
|
||||
path_targets = [{"__path__" = "/var/log/*.log", job = "varlog", "__path_exclude__" = "/var/log/fail2ban.log", host = "beepi"}]
|
||||
path_targets = [{
|
||||
"__path__" = "/var/log/*.log",
|
||||
job = "varlog",
|
||||
"__path_exclude__" = "/var/log/{fail2ban,beepi-backup}.log",
|
||||
host = "beepi",
|
||||
}]
|
||||
}
|
||||
|
||||
loki.source.file "varlog" {
|
||||
@@ -19,6 +28,34 @@ loki.source.file "varlog" {
|
||||
forward_to = [loki.write.default.receiver]
|
||||
}
|
||||
|
||||
loki.source.file "backup" {
|
||||
targets = local.file_match.backup.targets
|
||||
forward_to = [loki.process.backup.receiver]
|
||||
}
|
||||
|
||||
loki.process "backup" {
|
||||
forward_to = [loki.write.default.receiver]
|
||||
|
||||
stage.regex {
|
||||
expression = `^\[(?P<ts>[^\]]+)\]\s(?P<backup_job>\w+):\s(?P<message>.*)$`
|
||||
}
|
||||
|
||||
stage.labels {
|
||||
values = {
|
||||
backup_job = "",
|
||||
}
|
||||
}
|
||||
|
||||
stage.match {
|
||||
selector = `{job="backup"} |~ "(?i)(FAILED|verify failed|refusing to sync|error)"`
|
||||
stage.static_labels {
|
||||
values = {
|
||||
level = "error",
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
loki.source.file "fail2ban_source" {
|
||||
targets = local.file_match.fail2ban.targets
|
||||
forward_to = [loki.process.geoip.receiver]
|
||||
@@ -58,4 +95,4 @@ loki.write "default" {
|
||||
endpoint {
|
||||
url = "http://loki:3100/loki/api/v1/push"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,14 @@ services:
|
||||
volumes:
|
||||
- prometheus-data:/prometheus
|
||||
- /opt/beeserver/monitoring/prometheus.yml:/etc/prometheus/prometheus.yml
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time=90d
|
||||
- --web.enable-lifecycle
|
||||
{% if prometheus_admin_api | default(false) %}
|
||||
- --web.enable-admin-api
|
||||
{% endif %}
|
||||
networks:
|
||||
- monitoring
|
||||
|
||||
@@ -23,12 +31,17 @@ services:
|
||||
- '127.0.0.1:4000:4000'
|
||||
volumes:
|
||||
- grafana-storage:/var/lib/grafana
|
||||
- /opt/beeserver/monitoring/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- /opt/beeserver/monitoring/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
- /opt/beeserver/monitoring/grafana/dashboards-general:/var/lib/grafana/dashboards-general:ro
|
||||
environment:
|
||||
- GF_SECURITY_SECRET_KEY={{ grafana_secret }}
|
||||
- GF_SERVER_DOMAIN=grafana.secretbee.buzz
|
||||
- GF_SERVER_ROOT_URL=https://grafana.secretbee.buzz/
|
||||
- GF_SERVER_HTTP_PORT=4000
|
||||
- GF_SERVER_PROTOCOL=HTTP
|
||||
- GF_INSTALL_PLUGINS=yesoreyeram-infinity-datasource
|
||||
- GF_UNIFIED_ALERTING_ENABLED=true
|
||||
networks:
|
||||
- monitoring
|
||||
|
||||
@@ -37,6 +50,7 @@ services:
|
||||
container_name: loki
|
||||
ports:
|
||||
- '127.0.0.1:3100:3100'
|
||||
- '{{ wireguard_beepi_ip }}:3100:3100'
|
||||
- '127.0.0.1:9096:9096'
|
||||
volumes:
|
||||
- loki-data:/loki
|
||||
@@ -87,8 +101,42 @@ services:
|
||||
networks:
|
||||
- monitoring
|
||||
|
||||
blackbox:
|
||||
image: docker.io/prom/blackbox-exporter
|
||||
container_name: blackbox
|
||||
restart: always
|
||||
network_mode: host
|
||||
volumes:
|
||||
- /opt/beeserver/monitoring/blackbox.yml:/etc/blackbox_exporter/config.yml
|
||||
command:
|
||||
- --config.file=/etc/blackbox_exporter/config.yml
|
||||
- --web.listen-address=0.0.0.0:9115
|
||||
extra_hosts:
|
||||
{% for service in monitored_services %}
|
||||
{% if service.vhost is defined %}
|
||||
- '{{ service.vhost }}:127.0.0.1'
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
topology:
|
||||
image: docker.io/library/python:3-alpine
|
||||
container_name: topology
|
||||
restart: always
|
||||
ports:
|
||||
- '127.0.0.1:8088:8088'
|
||||
volumes:
|
||||
- /opt/beeserver/monitoring/topology:/app:ro
|
||||
command:
|
||||
- python
|
||||
- /app/topology.py
|
||||
environment:
|
||||
- PROMETHEUS_URL=http://prometheus:9090
|
||||
- LISTEN_PORT=8088
|
||||
networks:
|
||||
- monitoring
|
||||
|
||||
volumes:
|
||||
prometheus-data:
|
||||
grafana-storage:
|
||||
loki-data:
|
||||
alloy-data:
|
||||
alloy-data:
|
||||
|
||||
@@ -0,0 +1,713 @@
|
||||
{
|
||||
"apiVersion": "dashboard.grafana.app/v2",
|
||||
"kind": "Dashboard",
|
||||
"metadata": {
|
||||
"name": "admltw2"
|
||||
},
|
||||
"spec": {
|
||||
"annotations": [
|
||||
{
|
||||
"kind": "AnnotationQuery",
|
||||
"spec": {
|
||||
"builtIn": true,
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"query": {
|
||||
"datasource": {
|
||||
"name": "-- Grafana --"
|
||||
},
|
||||
"group": "grafana",
|
||||
"kind": "DataQuery",
|
||||
"spec": {},
|
||||
"version": "v0"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"cursorSync": "Off",
|
||||
"editable": true,
|
||||
"elements": {
|
||||
"panel-1": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"data": {
|
||||
"kind": "QueryGroup",
|
||||
"spec": {
|
||||
"queries": [
|
||||
{
|
||||
"kind": "PanelQuery",
|
||||
"spec": {
|
||||
"hidden": false,
|
||||
"query": {
|
||||
"datasource": {
|
||||
"name": "bfkfq6kdpznr4e"
|
||||
},
|
||||
"group": "loki",
|
||||
"kind": "DataQuery",
|
||||
"spec": {
|
||||
"direction": "backward",
|
||||
"editorMode": "builder",
|
||||
"expr": "{host=\"beepi\", filename=\"/var/log/ufw.log\"} |= `[UFW BLOCK]` | detected_level != `info`",
|
||||
"queryType": "range"
|
||||
},
|
||||
"version": "v0"
|
||||
},
|
||||
"refId": "A"
|
||||
}
|
||||
}
|
||||
],
|
||||
"queryOptions": {},
|
||||
"transformations": []
|
||||
}
|
||||
},
|
||||
"description": "",
|
||||
"id": 1,
|
||||
"links": [],
|
||||
"title": "UFW Blocks",
|
||||
"vizConfig": {
|
||||
"group": "logs",
|
||||
"kind": "VizConfig",
|
||||
"spec": {
|
||||
"fieldConfig": {
|
||||
"defaults": {},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"dedupStrategy": "numbers",
|
||||
"enableInfiniteScrolling": false,
|
||||
"enableLogDetails": true,
|
||||
"prettifyLogMessage": true,
|
||||
"showControls": false,
|
||||
"showFieldSelector": false,
|
||||
"showLabels": false,
|
||||
"showLevel": true,
|
||||
"showTime": true,
|
||||
"sortOrder": "Descending",
|
||||
"syntaxHighlighting": true,
|
||||
"timestampResolution": "ms",
|
||||
"unwrappedColumns": false,
|
||||
"wrapLogMessage": false
|
||||
}
|
||||
},
|
||||
"version": "13.0.1"
|
||||
}
|
||||
}
|
||||
},
|
||||
"panel-2": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"data": {
|
||||
"kind": "QueryGroup",
|
||||
"spec": {
|
||||
"queries": [
|
||||
{
|
||||
"kind": "PanelQuery",
|
||||
"spec": {
|
||||
"hidden": false,
|
||||
"query": {
|
||||
"datasource": {
|
||||
"name": "bfkfq6kdpznr4e"
|
||||
},
|
||||
"group": "loki",
|
||||
"kind": "DataQuery",
|
||||
"spec": {
|
||||
"direction": "backward",
|
||||
"editorMode": "builder",
|
||||
"expr": "{filename=\"/var/log/fail2ban.log\"} |= `NOTICE`",
|
||||
"queryType": "range"
|
||||
},
|
||||
"version": "v0"
|
||||
},
|
||||
"refId": "A"
|
||||
}
|
||||
}
|
||||
],
|
||||
"queryOptions": {},
|
||||
"transformations": []
|
||||
}
|
||||
},
|
||||
"description": "",
|
||||
"id": 2,
|
||||
"links": [],
|
||||
"title": "Fail2Ban bans",
|
||||
"vizConfig": {
|
||||
"group": "logs",
|
||||
"kind": "VizConfig",
|
||||
"spec": {
|
||||
"fieldConfig": {
|
||||
"defaults": {},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"dedupStrategy": "none",
|
||||
"enableInfiniteScrolling": false,
|
||||
"enableLogDetails": true,
|
||||
"prettifyLogMessage": true,
|
||||
"showControls": false,
|
||||
"showFieldSelector": false,
|
||||
"showLabels": false,
|
||||
"showLevel": true,
|
||||
"showTime": false,
|
||||
"sortOrder": "Descending",
|
||||
"timestampResolution": "ms",
|
||||
"unwrappedColumns": false,
|
||||
"wrapLogMessage": false
|
||||
}
|
||||
},
|
||||
"version": "13.0.1"
|
||||
}
|
||||
}
|
||||
},
|
||||
"panel-3": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"data": {
|
||||
"kind": "QueryGroup",
|
||||
"spec": {
|
||||
"queries": [
|
||||
{
|
||||
"kind": "PanelQuery",
|
||||
"spec": {
|
||||
"hidden": false,
|
||||
"query": {
|
||||
"datasource": {
|
||||
"name": "bfkfq6kdpznr4e"
|
||||
},
|
||||
"group": "loki",
|
||||
"kind": "DataQuery",
|
||||
"spec": {
|
||||
"direction": "backward",
|
||||
"editorMode": "builder",
|
||||
"expr": "count_over_time({filename=\"/var/log/fail2ban.log\"} |= `NOTICE` |= `Ban` != `Restore` [$__auto])",
|
||||
"legendFormat": "",
|
||||
"queryType": "range",
|
||||
"step": "10m"
|
||||
},
|
||||
"version": "v0"
|
||||
},
|
||||
"refId": "A"
|
||||
}
|
||||
}
|
||||
],
|
||||
"queryOptions": {},
|
||||
"transformations": []
|
||||
}
|
||||
},
|
||||
"description": "",
|
||||
"id": 3,
|
||||
"links": [],
|
||||
"title": "Fail2Ban Over Time",
|
||||
"vizConfig": {
|
||||
"group": "barchart",
|
||||
"kind": "VizConfig",
|
||||
"spec": {
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"fillOpacity": 80,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"lineWidth": 1,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"decimals": 0,
|
||||
"min": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "none"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"barRadius": 0,
|
||||
"barWidth": 0.97,
|
||||
"fullHighlight": false,
|
||||
"groupWidth": 0.7,
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": false
|
||||
},
|
||||
"orientation": "auto",
|
||||
"showValue": "auto",
|
||||
"stacking": "none",
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
},
|
||||
"xTickLabelRotation": 0,
|
||||
"xTickLabelSpacing": 0
|
||||
}
|
||||
},
|
||||
"version": "13.0.1"
|
||||
}
|
||||
}
|
||||
},
|
||||
"panel-4": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"data": {
|
||||
"kind": "QueryGroup",
|
||||
"spec": {
|
||||
"queries": [
|
||||
{
|
||||
"kind": "PanelQuery",
|
||||
"spec": {
|
||||
"hidden": false,
|
||||
"query": {
|
||||
"datasource": {
|
||||
"name": "bfkfq6kdpznr4e"
|
||||
},
|
||||
"group": "loki",
|
||||
"kind": "DataQuery",
|
||||
"spec": {
|
||||
"direction": "backward",
|
||||
"editorMode": "builder",
|
||||
"expr": "{type=\"Ban\"}",
|
||||
"queryType": "range"
|
||||
},
|
||||
"version": "v0"
|
||||
},
|
||||
"refId": "A"
|
||||
}
|
||||
}
|
||||
],
|
||||
"queryOptions": {},
|
||||
"transformations": [
|
||||
{
|
||||
"group": "extractFields",
|
||||
"kind": "Transformation",
|
||||
"spec": {
|
||||
"options": {
|
||||
"delimiter": ",",
|
||||
"format": "json",
|
||||
"keepTime": false,
|
||||
"replace": false,
|
||||
"source": "labels"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"group": "convertFieldType",
|
||||
"kind": "Transformation",
|
||||
"spec": {
|
||||
"options": {
|
||||
"conversions": [
|
||||
{
|
||||
"destinationType": "number",
|
||||
"targetField": "geoip_location_latitude"
|
||||
},
|
||||
{
|
||||
"destinationType": "number",
|
||||
"targetField": "geoip_location_longitude"
|
||||
}
|
||||
],
|
||||
"fields": {}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"description": "",
|
||||
"id": 4,
|
||||
"links": [],
|
||||
"title": "Fail2Ban Worldmap",
|
||||
"vizConfig": {
|
||||
"group": "geomap",
|
||||
"kind": "VizConfig",
|
||||
"spec": {
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"custom": {
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
}
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "geoip_location_latitude",
|
||||
"scope": "series"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "decimals",
|
||||
"value": 4
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "geoip_location_longitude",
|
||||
"scope": "series"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "decimals",
|
||||
"value": 4
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "__error__",
|
||||
"scope": "series"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "custom.hideFrom",
|
||||
"value": {
|
||||
"legend": true,
|
||||
"tooltip": true,
|
||||
"viz": true
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "__error_details__",
|
||||
"scope": "series"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "custom.hideFrom",
|
||||
"value": {
|
||||
"legend": true,
|
||||
"tooltip": true,
|
||||
"viz": true
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "labels",
|
||||
"scope": "series"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "custom.hideFrom",
|
||||
"value": {
|
||||
"legend": true,
|
||||
"tooltip": true,
|
||||
"viz": true
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "labelTypes",
|
||||
"scope": "series"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "custom.hideFrom",
|
||||
"value": {
|
||||
"legend": true,
|
||||
"tooltip": true,
|
||||
"viz": true
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"basemap": {
|
||||
"config": {},
|
||||
"name": "Layer 0",
|
||||
"noRepeat": false,
|
||||
"type": "default"
|
||||
},
|
||||
"controls": {
|
||||
"mouseWheelZoom": true,
|
||||
"showAttribution": true,
|
||||
"showDebug": false,
|
||||
"showMeasure": false,
|
||||
"showScale": false,
|
||||
"showZoom": true
|
||||
},
|
||||
"layers": [
|
||||
{
|
||||
"config": {
|
||||
"blur": 17,
|
||||
"radius": 9,
|
||||
"weight": {
|
||||
"fixed": 0.5,
|
||||
"max": 1,
|
||||
"min": 0
|
||||
}
|
||||
},
|
||||
"layer-tooltip": false,
|
||||
"location": {
|
||||
"latitude": "geoip_location_latitude",
|
||||
"longitude": "geoip_location_longitude",
|
||||
"mode": "coords"
|
||||
},
|
||||
"name": "Bans",
|
||||
"opacity": 1,
|
||||
"tooltip": true,
|
||||
"type": "heatmap"
|
||||
}
|
||||
],
|
||||
"tooltip": {
|
||||
"mode": "details"
|
||||
},
|
||||
"view": {
|
||||
"allLayers": true,
|
||||
"dashboardVariable": false,
|
||||
"id": "zero",
|
||||
"lat": 0,
|
||||
"lon": 0,
|
||||
"noRepeat": false,
|
||||
"zoom": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"version": "13.0.1"
|
||||
}
|
||||
}
|
||||
},
|
||||
"panel-5": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"data": {
|
||||
"kind": "QueryGroup",
|
||||
"spec": {
|
||||
"queries": [
|
||||
{
|
||||
"kind": "PanelQuery",
|
||||
"spec": {
|
||||
"hidden": false,
|
||||
"query": {
|
||||
"datasource": {
|
||||
"name": "bfkfq6kdpznr4e"
|
||||
},
|
||||
"group": "loki",
|
||||
"kind": "DataQuery",
|
||||
"spec": {
|
||||
"direction": "backward",
|
||||
"editorMode": "builder",
|
||||
"expr": "{job=\"varlog\"} |= `ERROR` or `WARN` or `CRITICAL` != `queryType` or `query_hash` | logfmt",
|
||||
"queryType": "range"
|
||||
},
|
||||
"version": "v0"
|
||||
},
|
||||
"refId": "A"
|
||||
}
|
||||
}
|
||||
],
|
||||
"queryOptions": {},
|
||||
"transformations": []
|
||||
}
|
||||
},
|
||||
"description": "",
|
||||
"id": 5,
|
||||
"links": [],
|
||||
"title": "New panel",
|
||||
"vizConfig": {
|
||||
"group": "logs",
|
||||
"kind": "VizConfig",
|
||||
"spec": {
|
||||
"fieldConfig": {
|
||||
"defaults": {},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"dedupStrategy": "none",
|
||||
"enableInfiniteScrolling": false,
|
||||
"enableLogDetails": true,
|
||||
"prettifyLogMessage": true,
|
||||
"showControls": false,
|
||||
"showFieldSelector": false,
|
||||
"showLevel": true,
|
||||
"showTime": false,
|
||||
"sortOrder": "Descending",
|
||||
"timestampResolution": "ms",
|
||||
"unwrappedColumns": false,
|
||||
"wrapLogMessage": false
|
||||
}
|
||||
},
|
||||
"version": "13.0.1"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"layout": {
|
||||
"kind": "GridLayout",
|
||||
"spec": {
|
||||
"items": [
|
||||
{
|
||||
"kind": "GridLayoutItem",
|
||||
"spec": {
|
||||
"element": {
|
||||
"kind": "ElementReference",
|
||||
"name": "panel-5"
|
||||
},
|
||||
"height": 8,
|
||||
"width": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "GridLayoutItem",
|
||||
"spec": {
|
||||
"element": {
|
||||
"kind": "ElementReference",
|
||||
"name": "panel-4"
|
||||
},
|
||||
"height": 8,
|
||||
"width": 12,
|
||||
"x": 0,
|
||||
"y": 8
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "GridLayoutItem",
|
||||
"spec": {
|
||||
"element": {
|
||||
"kind": "ElementReference",
|
||||
"name": "panel-3"
|
||||
},
|
||||
"height": 8,
|
||||
"width": 12,
|
||||
"x": 12,
|
||||
"y": 8
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "GridLayoutItem",
|
||||
"spec": {
|
||||
"element": {
|
||||
"kind": "ElementReference",
|
||||
"name": "panel-2"
|
||||
},
|
||||
"height": 7,
|
||||
"width": 24,
|
||||
"x": 0,
|
||||
"y": 16
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "GridLayoutItem",
|
||||
"spec": {
|
||||
"element": {
|
||||
"kind": "ElementReference",
|
||||
"name": "panel-1"
|
||||
},
|
||||
"height": 8,
|
||||
"width": 24,
|
||||
"x": 0,
|
||||
"y": 23
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"links": [],
|
||||
"liveNow": false,
|
||||
"preferences": {
|
||||
"layout": {
|
||||
"kind": "GridLayout",
|
||||
"spec": {
|
||||
"items": []
|
||||
}
|
||||
}
|
||||
},
|
||||
"preload": false,
|
||||
"tags": [],
|
||||
"timeSettings": {
|
||||
"autoRefresh": "",
|
||||
"autoRefreshIntervals": [
|
||||
"5s",
|
||||
"10s",
|
||||
"30s",
|
||||
"1m",
|
||||
"5m",
|
||||
"15m",
|
||||
"30m",
|
||||
"1h",
|
||||
"2h",
|
||||
"1d"
|
||||
],
|
||||
"fiscalYearStartMonth": 0,
|
||||
"from": "now-6h",
|
||||
"hideTimepicker": false,
|
||||
"timezone": "browser",
|
||||
"to": "now"
|
||||
},
|
||||
"title": "Logging",
|
||||
"variables": [
|
||||
{
|
||||
"datasource": {
|
||||
"name": "bfkfq6kdpznr4e"
|
||||
},
|
||||
"group": "loki",
|
||||
"kind": "AdhocVariable",
|
||||
"spec": {
|
||||
"allowCustomValue": true,
|
||||
"baseFilters": [],
|
||||
"defaultKeys": [],
|
||||
"enableGroupBy": false,
|
||||
"filters": [],
|
||||
"hide": "dontHide",
|
||||
"name": "Filters",
|
||||
"skipUrlSync": false
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,215 @@
|
||||
{
|
||||
"id": null,
|
||||
"uid": "beepi-backups",
|
||||
"title": "Backups",
|
||||
"tags": ["beepi", "backup"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"refresh": "1m",
|
||||
"time": {"from": "now-7d", "to": "now"},
|
||||
"panels": [
|
||||
{
|
||||
"type": "table",
|
||||
"title": "Backup jobs",
|
||||
"id": 1,
|
||||
"gridPos": {"h": 7, "w": 24, "x": 0, "y": 0},
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "age",
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"expr": "time() - max by (backup_job) (backup_last_success_timestamp_seconds{backup_job!=\"\"})",
|
||||
"instant": true,
|
||||
"format": "table"
|
||||
},
|
||||
{
|
||||
"refId": "status",
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"expr": "max by (backup_job) (backup_last_status{backup_job!=\"\"})",
|
||||
"instant": true,
|
||||
"format": "table"
|
||||
},
|
||||
{
|
||||
"refId": "duration",
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"expr": "max by (backup_job) (backup_last_duration_seconds{backup_job!=\"\"})",
|
||||
"instant": true,
|
||||
"format": "table"
|
||||
},
|
||||
{
|
||||
"refId": "size",
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"expr": "max by (backup_job) (backup_last_archive_bytes{backup_job!=\"\"})",
|
||||
"instant": true,
|
||||
"format": "table"
|
||||
}
|
||||
],
|
||||
"transformations": [
|
||||
{"id": "joinByField", "options": {"byField": "backup_job", "mode": "outer"}},
|
||||
{"id": "organize", "options": {
|
||||
"excludeByName": {"Time": true, "Time 1": true, "Time 2": true, "Time 3": true, "Time 4": true},
|
||||
"renameByName": {
|
||||
"backup_job": "Job",
|
||||
"Value #age": "Age",
|
||||
"Value #status": "Status",
|
||||
"Value #duration": "Duration",
|
||||
"Value #size": "Archive"
|
||||
}
|
||||
}}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {"custom": {"align": "auto", "cellOptions": {"type": "auto"}}},
|
||||
"overrides": [
|
||||
{"matcher": {"id": "byName", "options": "Age"}, "properties": [
|
||||
{"id": "unit", "value": "s"},
|
||||
{"id": "custom.cellOptions", "value": {"type": "color-background"}},
|
||||
{"id": "thresholds", "value": {"mode": "absolute", "steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "orange", "value": 90000},
|
||||
{"color": "red", "value": 172800}
|
||||
]}}
|
||||
]},
|
||||
{"matcher": {"id": "byName", "options": "Duration"}, "properties": [{"id": "unit", "value": "s"}]},
|
||||
{"matcher": {"id": "byName", "options": "Archive"}, "properties": [{"id": "unit", "value": "bytes"}]},
|
||||
{"matcher": {"id": "byName", "options": "Status"}, "properties": [
|
||||
{"id": "custom.cellOptions", "value": {"type": "color-background"}},
|
||||
{"id": "mappings", "value": [{"type": "value", "options": {
|
||||
"0": {"text": "FAILED", "color": "red", "index": 0},
|
||||
"1": {"text": "ok", "color": "green", "index": 1}
|
||||
}}]},
|
||||
{"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}}
|
||||
]}
|
||||
]
|
||||
},
|
||||
"options": {"showHeader": true}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Archive size",
|
||||
"id": 2,
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 7},
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"expr": "backup_archive_newest_bytes",
|
||||
"legendFormat": "newest {{ '{{' }}kind{{ '}}' }}"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"expr": "backup_archive_total_bytes",
|
||||
"legendFormat": "total {{ '{{' }}kind{{ '}}' }}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {"unit": "bytes", "custom": {"lineWidth": 2, "fillOpacity": 8}},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi"}}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Backup duration",
|
||||
"id": 3,
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 7},
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"expr": "backup_last_duration_seconds{backup_job!=\"\"}",
|
||||
"legendFormat": "{{ '{{' }}backup_job{{ '}}' }}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {"unit": "s", "custom": {"lineWidth": 2, "fillOpacity": 8}},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi"}}
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Retained archives",
|
||||
"id": 4,
|
||||
"gridPos": {"h": 5, "w": 6, "x": 0, "y": 15},
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"expr": "backup_archive_count",
|
||||
"legendFormat": "{{ '{{' }}kind{{ '}}' }}",
|
||||
"instant": true
|
||||
}
|
||||
],
|
||||
"fieldConfig": {"defaults": {"unit": "short", "color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 2}]}}, "overrides": []},
|
||||
"options": {"colorMode": "background", "graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}}
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "RAID mounted",
|
||||
"id": 5,
|
||||
"gridPos": {"h": 5, "w": 6, "x": 6, "y": 15},
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"expr": "max(backup_raid_mounted) or vector(0)",
|
||||
"instant": true
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"mappings": [{"type": "value", "options": {"0": {"text": "NOT MOUNTED", "color": "red", "index": 0}, "1": {"text": "mounted", "color": "green", "index": 1}}}],
|
||||
"thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {"colorMode": "background", "graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "RAID free space",
|
||||
"id": 6,
|
||||
"gridPos": {"h": 5, "w": 12, "x": 12, "y": 15},
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"expr": "node_filesystem_avail_bytes{mountpoint=\"{{ backup_raid_mount }}\"}",
|
||||
"legendFormat": "free"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {"defaults": {"unit": "bytes", "custom": {"lineWidth": 2, "fillOpacity": 8}}, "overrides": []},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "single"}}
|
||||
},
|
||||
{
|
||||
"type": "logs",
|
||||
"title": "Backup log",
|
||||
"id": 7,
|
||||
"gridPos": {"h": 12, "w": 24, "x": 0, "y": 20},
|
||||
"datasource": {"type": "loki", "uid": "{{ grafana_loki_uid }}"},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "loki", "uid": "{{ grafana_loki_uid }}"},
|
||||
"expr": "{job=\"backup\"}",
|
||||
"queryType": "range"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"showTime": true,
|
||||
"sortOrder": "Descending",
|
||||
"wrapLogMessage": true,
|
||||
"enableLogDetails": true,
|
||||
"dedupStrategy": "none",
|
||||
"prettifyLogMessage": false
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,364 @@
|
||||
{
|
||||
"id": null,
|
||||
"uid": "beepi-map",
|
||||
"title": "Infrastructure Map",
|
||||
"tags": ["beepi", "overview"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"refresh": "30s",
|
||||
"time": {"from": "now-6h", "to": "now"},
|
||||
"panels": [
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Services down",
|
||||
"id": 1,
|
||||
"gridPos": {"h": 4, "w": 4, "x": 0, "y": 0},
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"expr": "count(min by (service) (probe_success{vantage=\"local\"}) == 0 and on (service) (max by (service) (service_enabled) == 1)) or vector(0)",
|
||||
"instant": true
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "red", "value": 1}
|
||||
]
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"textMode": "auto",
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Last daily backup",
|
||||
"id": 2,
|
||||
"gridPos": {"h": 4, "w": 4, "x": 4, "y": 0},
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"expr": "time() - max(backup_last_success_timestamp_seconds{backup_job=\"daily\"})",
|
||||
"instant": true
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "orange", "value": 90000},
|
||||
{"color": "red", "value": 172800}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"textMode": "auto",
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Last offsite sync",
|
||||
"id": 3,
|
||||
"gridPos": {"h": 4, "w": 4, "x": 8, "y": 0},
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"expr": "time() - max(backup_last_success_timestamp_seconds{backup_job=\"offsite\"})",
|
||||
"instant": true
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "orange", "value": 90000},
|
||||
{"color": "red", "value": 172800}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"textMode": "auto",
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Tunnel handshake",
|
||||
"id": 4,
|
||||
"gridPos": {"h": 4, "w": 4, "x": 12, "y": 0},
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"expr": "max(wireguard_peer_handshake_age_seconds{peer=\"beepi\"})",
|
||||
"instant": true
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "green", "value": null},
|
||||
{"color": "orange", "value": 180},
|
||||
{"color": "red", "value": 300}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"textMode": "auto",
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Ingress",
|
||||
"id": 5,
|
||||
"gridPos": {"h": 4, "w": 4, "x": 16, "y": 0},
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"expr": "min(up{job=\"node\",host=\"ingress\"}) or vector(0)",
|
||||
"instant": true
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"mappings": [
|
||||
{"type": "value", "options": {"0": {"text": "DOWN", "color": "red", "index": 0}, "1": {"text": "UP", "color": "green", "index": 1}}}
|
||||
],
|
||||
"thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"textMode": "auto",
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "RAID free",
|
||||
"id": 6,
|
||||
"gridPos": {"h": 4, "w": 4, "x": 20, "y": 0},
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"expr": "min(node_filesystem_avail_bytes{mountpoint=\"{{ backup_raid_mount }}\"})",
|
||||
"instant": true
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{"color": "red", "value": null},
|
||||
{"color": "orange", "value": 100000000000},
|
||||
{"color": "green", "value": 200000000000}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"textMode": "auto",
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "nodeGraph",
|
||||
"title": "Infrastructure",
|
||||
"id": 10,
|
||||
"gridPos": {"h": 18, "w": 18, "x": 0, "y": 4},
|
||||
"datasource": {"type": "yesoreyeram-infinity-datasource", "uid": "{{ grafana_infinity_uid }}"},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "yesoreyeram-infinity-datasource", "uid": "{{ grafana_infinity_uid }}"},
|
||||
"type": "json",
|
||||
"source": "url",
|
||||
"parser": "backend",
|
||||
"format": "table",
|
||||
"url": "http://topology:8088/topology.json",
|
||||
"url_options": {"method": "GET", "data": ""},
|
||||
"root_selector": "nodes",
|
||||
"columns": [
|
||||
{"selector": "id", "text": "id", "type": "string"},
|
||||
{"selector": "title", "text": "title", "type": "string"},
|
||||
{"selector": "subtitle", "text": "subtitle", "type": "string"},
|
||||
{"selector": "mainstat", "text": "mainstat", "type": "string"},
|
||||
{"selector": "arc__ok", "text": "arc__ok", "type": "number"},
|
||||
{"selector": "arc__fail", "text": "arc__fail", "type": "number"},
|
||||
{"selector": "arc__disabled", "text": "arc__disabled", "type": "number"},
|
||||
{"selector": "arc__unknown", "text": "arc__unknown", "type": "number"},
|
||||
{"selector": "arc__used", "text": "arc__used", "type": "number"},
|
||||
{"selector": "arc__free", "text": "arc__free", "type": "number"},
|
||||
{"selector": "detail__status", "text": "detail__status", "type": "string"},
|
||||
{"selector": "detail__group", "text": "detail__group", "type": "string"},
|
||||
{"selector": "detail__used", "text": "detail__used", "type": "string"},
|
||||
{"selector": "dashboard", "text": "dashboard", "type": "string"}
|
||||
]
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": {"type": "yesoreyeram-infinity-datasource", "uid": "{{ grafana_infinity_uid }}"},
|
||||
"type": "json",
|
||||
"source": "url",
|
||||
"parser": "backend",
|
||||
"format": "table",
|
||||
"url": "http://topology:8088/topology.json",
|
||||
"url_options": {"method": "GET", "data": ""},
|
||||
"root_selector": "edges",
|
||||
"columns": [
|
||||
{"selector": "id", "text": "id", "type": "string"},
|
||||
{"selector": "source", "text": "source", "type": "string"},
|
||||
{"selector": "target", "text": "target", "type": "string"},
|
||||
{"selector": "mainstat", "text": "mainstat", "type": "string"},
|
||||
{"selector": "color", "text": "color", "type": "string"},
|
||||
{"selector": "thickness", "text": "thickness", "type": "number"}
|
||||
]
|
||||
}
|
||||
],
|
||||
"transformations": [
|
||||
{
|
||||
"id": "convertFieldType",
|
||||
"options": {
|
||||
"conversions": [
|
||||
{"targetField": "arc__ok", "destinationType": "number"},
|
||||
{"targetField": "arc__fail", "destinationType": "number"},
|
||||
{"targetField": "arc__disabled", "destinationType": "number"},
|
||||
{"targetField": "arc__unknown", "destinationType": "number"},
|
||||
{"targetField": "arc__used", "destinationType": "number"},
|
||||
{"targetField": "arc__free", "destinationType": "number"}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"nodes": {
|
||||
"mainStatUnit": "",
|
||||
"secondaryStatUnit": "",
|
||||
"arcs": [
|
||||
{"field": "arc__ok", "color": "green"},
|
||||
{"field": "arc__fail", "color": "red"},
|
||||
{"field": "arc__disabled", "color": "text"},
|
||||
{"field": "arc__unknown", "color": "orange"},
|
||||
{"field": "arc__used", "color": "semi-dark-orange"},
|
||||
{"field": "arc__free", "color": "semi-dark-green"}
|
||||
]
|
||||
},
|
||||
"edges": {"mainStatUnit": ""},
|
||||
"zoomMode": "cooperative"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {"id": "byName", "options": "arc__ok"},
|
||||
"properties": [{"id": "color", "value": {"mode": "fixed", "fixedColor": "green"}}]
|
||||
},
|
||||
{
|
||||
"matcher": {"id": "byName", "options": "arc__fail"},
|
||||
"properties": [{"id": "color", "value": {"mode": "fixed", "fixedColor": "red"}}]
|
||||
},
|
||||
{
|
||||
"matcher": {"id": "byName", "options": "arc__disabled"},
|
||||
"properties": [{"id": "color", "value": {"mode": "fixed", "fixedColor": "text"}}]
|
||||
},
|
||||
{
|
||||
"matcher": {"id": "byName", "options": "arc__unknown"},
|
||||
"properties": [{"id": "color", "value": {"mode": "fixed", "fixedColor": "orange"}}]
|
||||
},
|
||||
{
|
||||
"matcher": {"id": "byName", "options": "arc__used"},
|
||||
"properties": [{"id": "color", "value": {"mode": "fixed", "fixedColor": "semi-dark-orange"}}]
|
||||
},
|
||||
{
|
||||
"matcher": {"id": "byName", "options": "arc__free"},
|
||||
"properties": [{"id": "color", "value": {"mode": "fixed", "fixedColor": "semi-dark-green"}}]
|
||||
},
|
||||
{
|
||||
"matcher": {"id": "byName", "options": "id"},
|
||||
"properties": [
|
||||
{
|
||||
"id": "links",
|
||||
"value": [
|
||||
{
|
||||
"title": "Open related dashboard",
|
||||
"url": "${__data.fields.dashboard}",
|
||||
"targetBlank": false
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "alertlist",
|
||||
"title": "Firing alerts",
|
||||
"id": 11,
|
||||
"gridPos": {"h": 18, "w": 6, "x": 18, "y": 4},
|
||||
"options": {
|
||||
"alertInstanceLabelFilter": "",
|
||||
"alertName": "",
|
||||
"dashboardAlerts": false,
|
||||
"groupBy": [],
|
||||
"groupMode": "default",
|
||||
"maxItems": 30,
|
||||
"sortOrder": 3,
|
||||
"stateFilter": {"error": true, "firing": true, "noData": true, "normal": false, "pending": true},
|
||||
"viewMode": "list"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,260 @@
|
||||
{
|
||||
"id": null,
|
||||
"uid": "beepi-ingress",
|
||||
"title": "Ingress",
|
||||
"tags": ["beepi", "ingress"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"refresh": "30s",
|
||||
"time": {"from": "now-24h", "to": "now"},
|
||||
"panels": [
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Ingress uptime",
|
||||
"id": 1,
|
||||
"gridPos": {"h": 4, "w": 5, "x": 0, "y": 0},
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"expr": "time() - max(node_boot_time_seconds{host=\"ingress\"})",
|
||||
"instant": true
|
||||
}
|
||||
],
|
||||
"fieldConfig": {"defaults": {"unit": "s", "color": {"mode": "fixed", "fixedColor": "blue"}}, "overrides": []},
|
||||
"options": {"colorMode": "value", "graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}}
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Tunnel RTT",
|
||||
"id": 2,
|
||||
"gridPos": {"h": 4, "w": 5, "x": 5, "y": 0},
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"expr": "max(probe_duration_seconds{probe=\"tunnel-rtt\"})",
|
||||
"instant": true
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {"unit": "s", "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 0.1}, {"color": "red", "value": 0.3}]}},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {"colorMode": "background", "graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}}
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Banned hosts (24h)",
|
||||
"id": 3,
|
||||
"gridPos": {"h": 4, "w": 5, "x": 10, "y": 0},
|
||||
"datasource": {"type": "loki", "uid": "{{ grafana_loki_uid }}"},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "loki", "uid": "{{ grafana_loki_uid }}"},
|
||||
"expr": "sum(count_over_time({job=\"fail2ban\", host=\"ingress\"} |= \"Ban\" [24h]))",
|
||||
"queryType": "instant"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {"defaults": {"unit": "short", "color": {"mode": "fixed", "fixedColor": "orange"}}, "overrides": []},
|
||||
"options": {"colorMode": "value", "graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}}
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Blocked packets (24h)",
|
||||
"id": 4,
|
||||
"gridPos": {"h": 4, "w": 9, "x": 15, "y": 0},
|
||||
"datasource": {"type": "loki", "uid": "{{ grafana_loki_uid }}"},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "loki", "uid": "{{ grafana_loki_uid }}"},
|
||||
"expr": "sum(count_over_time({job=\"ufw\", host=\"ingress\"} [24h]))",
|
||||
"queryType": "instant"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {"defaults": {"unit": "short", "color": {"mode": "fixed", "fixedColor": "purple"}}, "overrides": []},
|
||||
"options": {"colorMode": "value", "graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}}
|
||||
},
|
||||
{
|
||||
"type": "table",
|
||||
"title": "WireGuard peers",
|
||||
"id": 5,
|
||||
"gridPos": {"h": 7, "w": 12, "x": 0, "y": 4},
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "age",
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"expr": "max by (peer) (wireguard_peer_handshake_age_seconds)",
|
||||
"instant": true,
|
||||
"format": "table"
|
||||
},
|
||||
{
|
||||
"refId": "rx",
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"expr": "max by (peer) (wireguard_peer_receive_bytes_total)",
|
||||
"instant": true,
|
||||
"format": "table"
|
||||
},
|
||||
{
|
||||
"refId": "tx",
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"expr": "max by (peer) (wireguard_peer_transmit_bytes_total)",
|
||||
"instant": true,
|
||||
"format": "table"
|
||||
}
|
||||
],
|
||||
"transformations": [
|
||||
{"id": "joinByField", "options": {"byField": "peer", "mode": "outer"}},
|
||||
{"id": "organize", "options": {
|
||||
"excludeByName": {"Time": true, "Time 1": true, "Time 2": true, "Time 3": true},
|
||||
"renameByName": {"peer": "Peer", "Value #age": "Last handshake", "Value #rx": "Received", "Value #tx": "Sent"}
|
||||
}}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {"custom": {"align": "auto"}},
|
||||
"overrides": [
|
||||
{"matcher": {"id": "byName", "options": "Last handshake"}, "properties": [
|
||||
{"id": "unit", "value": "s"},
|
||||
{"id": "custom.cellOptions", "value": {"type": "color-background"}},
|
||||
{"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 300}, {"color": "red", "value": 3600}]}}
|
||||
]},
|
||||
{"matcher": {"id": "byRegexp", "options": "Received|Sent"}, "properties": [{"id": "unit", "value": "bytes"}]}
|
||||
]
|
||||
},
|
||||
"options": {"showHeader": true}
|
||||
},
|
||||
{
|
||||
"type": "state-timeline",
|
||||
"title": "Tunnel path probes",
|
||||
"id": 6,
|
||||
"gridPos": {"h": 7, "w": 12, "x": 12, "y": 4},
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"expr": "min by (probe) (probe_success{vantage=\"ingress\"})",
|
||||
"legendFormat": "{{ '{{' }}probe{{ '}}' }}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {"fillOpacity": 90, "lineWidth": 0},
|
||||
"mappings": [{"type": "value", "options": {"0": {"text": "down", "color": "red", "index": 0}, "1": {"text": "up", "color": "green", "index": 1}}}],
|
||||
"thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {"mergeValues": true, "showValue": "never", "legend": {"displayMode": "list", "placement": "bottom", "showLegend": false}}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Ingress host",
|
||||
"id": 7,
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 11},
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"expr": "100 - (avg(rate(node_cpu_seconds_total{host=\"ingress\",mode=\"idle\"}[5m])) * 100)",
|
||||
"legendFormat": "cpu"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"expr": "(1 - (node_memory_MemAvailable_bytes{host=\"ingress\"} / node_memory_MemTotal_bytes{host=\"ingress\"})) * 100",
|
||||
"legendFormat": "memory"
|
||||
},
|
||||
{
|
||||
"refId": "C",
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"expr": "(1 - (node_filesystem_avail_bytes{host=\"ingress\",mountpoint=\"/\"} / node_filesystem_size_bytes{host=\"ingress\",mountpoint=\"/\"})) * 100",
|
||||
"legendFormat": "disk"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {"defaults": {"unit": "percent", "max": 100, "min": 0, "custom": {"lineWidth": 2, "fillOpacity": 8}}, "overrides": []},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi"}}
|
||||
},
|
||||
{
|
||||
"type": "timeseries",
|
||||
"title": "Ingress network",
|
||||
"id": 8,
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 11},
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"expr": "rate(node_network_receive_bytes_total{host=\"ingress\",device!~\"lo\"}[5m]) * 8",
|
||||
"legendFormat": "in {{ '{{' }}device{{ '}}' }}"
|
||||
},
|
||||
{
|
||||
"refId": "B",
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"expr": "rate(node_network_transmit_bytes_total{host=\"ingress\",device!~\"lo\"}[5m]) * 8",
|
||||
"legendFormat": "out {{ '{{' }}device{{ '}}' }}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {"defaults": {"unit": "bps", "custom": {"lineWidth": 2, "fillOpacity": 8}}, "overrides": []},
|
||||
"options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi"}}
|
||||
},
|
||||
{
|
||||
"type": "geomap",
|
||||
"title": "Ban origins",
|
||||
"id": 9,
|
||||
"gridPos": {"h": 11, "w": 12, "x": 0, "y": 19},
|
||||
"datasource": {"type": "loki", "uid": "{{ grafana_loki_uid }}"},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "loki", "uid": "{{ grafana_loki_uid }}"},
|
||||
"expr": "sum by (geoip_location_latitude, geoip_location_longitude, geoip_country_name) (count_over_time({job=\"fail2ban\"} |= \"Ban\" [$__range]))",
|
||||
"queryType": "instant"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"basemap": {"type": "default", "name": "Basemap"},
|
||||
"layers": [
|
||||
{
|
||||
"type": "markers",
|
||||
"name": "bans",
|
||||
"location": {"mode": "coords", "latitude": "geoip_location_latitude", "longitude": "geoip_location_longitude"},
|
||||
"config": {"style": {"color": {"fixed": "red"}, "size": {"field": "Value", "min": 4, "max": 20}, "opacity": 0.6}}
|
||||
}
|
||||
],
|
||||
"view": {"id": "zero", "lat": 25, "lon": 10, "zoom": 1.6}
|
||||
},
|
||||
"fieldConfig": {"defaults": {}, "overrides": []}
|
||||
},
|
||||
{
|
||||
"type": "logs",
|
||||
"title": "Ingress journal and firewall",
|
||||
"id": 10,
|
||||
"gridPos": {"h": 11, "w": 12, "x": 12, "y": 19},
|
||||
"datasource": {"type": "loki", "uid": "{{ grafana_loki_uid }}"},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "loki", "uid": "{{ grafana_loki_uid }}"},
|
||||
"expr": "{host=\"ingress\"}",
|
||||
"queryType": "range"
|
||||
}
|
||||
],
|
||||
"options": {
|
||||
"showTime": true,
|
||||
"sortOrder": "Descending",
|
||||
"wrapLogMessage": true,
|
||||
"enableLogDetails": true,
|
||||
"dedupStrategy": "none",
|
||||
"prettifyLogMessage": false
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
{
|
||||
"id": null,
|
||||
"uid": "beepi-services",
|
||||
"title": "Services",
|
||||
"tags": ["beepi", "services"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"refresh": "30s",
|
||||
"time": {"from": "now-24h", "to": "now"},
|
||||
"panels": [
|
||||
{
|
||||
"type": "table",
|
||||
"title": "Service health",
|
||||
"id": 1,
|
||||
"gridPos": {"h": 10, "w": 24, "x": 0, "y": 0},
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "enabled",
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"expr": "max by (service) (service_enabled)",
|
||||
"instant": true,
|
||||
"format": "table"
|
||||
},
|
||||
{
|
||||
"refId": "units",
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"expr": "max by (service) (service_units_active) / max by (service) (service_units_total)",
|
||||
"instant": true,
|
||||
"format": "table"
|
||||
},
|
||||
{
|
||||
"refId": "local",
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"expr": "min by (service) (probe_success{vantage=\"local\"})",
|
||||
"instant": true,
|
||||
"format": "table"
|
||||
},
|
||||
{
|
||||
"refId": "vhost",
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"expr": "min by (service) (probe_success{vantage=\"nginx\"})",
|
||||
"instant": true,
|
||||
"format": "table"
|
||||
},
|
||||
{
|
||||
"refId": "latency",
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"expr": "max by (service) (probe_duration_seconds{vantage=\"nginx\"})",
|
||||
"instant": true,
|
||||
"format": "table"
|
||||
}
|
||||
],
|
||||
"transformations": [
|
||||
{"id": "joinByField", "options": {"byField": "service", "mode": "outer"}},
|
||||
{"id": "organize", "options": {
|
||||
"excludeByName": {"Time": true, "Time 1": true, "Time 2": true, "Time 3": true, "Time 4": true, "Time 5": true},
|
||||
"renameByName": {
|
||||
"service": "Service",
|
||||
"Value #enabled": "Expected",
|
||||
"Value #units": "Units",
|
||||
"Value #local": "Local probe",
|
||||
"Value #vhost": "Public vhost",
|
||||
"Value #latency": "Latency"
|
||||
}
|
||||
}}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {"custom": {"align": "auto", "cellOptions": {"type": "auto"}}},
|
||||
"overrides": [
|
||||
{"matcher": {"id": "byName", "options": "Expected"}, "properties": [
|
||||
{"id": "mappings", "value": [{"type": "value", "options": {"0": {"text": "disabled", "color": "text", "index": 0}, "1": {"text": "yes", "color": "green", "index": 1}}}]},
|
||||
{"id": "custom.cellOptions", "value": {"type": "color-text"}}
|
||||
]},
|
||||
{"matcher": {"id": "byName", "options": "Units"}, "properties": [
|
||||
{"id": "unit", "value": "percentunit"},
|
||||
{"id": "custom.cellOptions", "value": {"type": "color-background"}},
|
||||
{"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "orange", "value": 0.5}, {"color": "green", "value": 1}]}}
|
||||
]},
|
||||
{"matcher": {"id": "byName", "options": "Local probe"}, "properties": [
|
||||
{"id": "custom.cellOptions", "value": {"type": "color-background"}},
|
||||
{"id": "mappings", "value": [
|
||||
{"type": "value", "options": {"0": {"text": "DOWN", "color": "red", "index": 0}, "1": {"text": "up", "color": "green", "index": 1}}},
|
||||
{"type": "special", "options": {"match": "null", "result": {"text": "no check", "color": "#3a3f47", "index": 2}}}
|
||||
]},
|
||||
{"id": "noValue", "value": "no check"},
|
||||
{"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}}
|
||||
]},
|
||||
{"matcher": {"id": "byName", "options": "Public vhost"}, "properties": [
|
||||
{"id": "custom.cellOptions", "value": {"type": "color-background"}},
|
||||
{"id": "mappings", "value": [
|
||||
{"type": "value", "options": {"0": {"text": "DOWN", "color": "red", "index": 0}, "1": {"text": "up", "color": "green", "index": 1}}},
|
||||
{"type": "special", "options": {"match": "null", "result": {"text": "internal only", "color": "#3a3f47", "index": 2}}}
|
||||
]},
|
||||
{"id": "noValue", "value": "internal only"},
|
||||
{"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}}
|
||||
]},
|
||||
{"matcher": {"id": "byName", "options": "Latency"}, "properties": [{"id": "unit", "value": "s"}]}
|
||||
]
|
||||
},
|
||||
"options": {"showHeader": true, "sortBy": [{"displayName": "Service", "desc": false}]}
|
||||
},
|
||||
{
|
||||
"type": "state-timeline",
|
||||
"title": "Probe history",
|
||||
"id": 2,
|
||||
"gridPos": {"h": 9, "w": 24, "x": 0, "y": 10},
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"expr": "min by (service) (probe_success{vantage=\"local\"})",
|
||||
"legendFormat": "{{ '{{' }}service{{ '}}' }}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {"fillOpacity": 90, "lineWidth": 0},
|
||||
"mappings": [{"type": "value", "options": {"0": {"text": "down", "color": "red", "index": 0}, "1": {"text": "up", "color": "green", "index": 1}}}],
|
||||
"thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {"mergeValues": true, "showValue": "never", "legend": {"displayMode": "list", "placement": "bottom", "showLegend": false}}
|
||||
},
|
||||
{
|
||||
"type": "table",
|
||||
"title": "Units and containers",
|
||||
"id": 3,
|
||||
"gridPos": {"h": 10, "w": 12, "x": 0, "y": 19},
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"expr": "service_unit_active",
|
||||
"instant": true,
|
||||
"format": "table"
|
||||
}
|
||||
],
|
||||
"transformations": [
|
||||
{"id": "organize", "options": {
|
||||
"excludeByName": {"Time": true, "__name__": true, "instance": true, "job": true, "host": true},
|
||||
"renameByName": {"service": "Service", "unit": "Unit", "kind": "Kind", "Value": "Active"}
|
||||
}}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {"custom": {"align": "auto"}},
|
||||
"overrides": [
|
||||
{"matcher": {"id": "byName", "options": "Active"}, "properties": [
|
||||
{"id": "custom.cellOptions", "value": {"type": "color-background"}},
|
||||
{"id": "mappings", "value": [{"type": "value", "options": {"0": {"text": "stopped", "color": "red", "index": 0}, "1": {"text": "running", "color": "green", "index": 1}}}]},
|
||||
{"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}}
|
||||
]}
|
||||
]
|
||||
},
|
||||
"options": {"showHeader": true}
|
||||
},
|
||||
{
|
||||
"type": "table",
|
||||
"title": "TLS certificates",
|
||||
"id": 4,
|
||||
"gridPos": {"h": 10, "w": 12, "x": 12, "y": 19},
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"},
|
||||
"expr": "min by (service, instance) (probe_ssl_earliest_cert_expiry - time())",
|
||||
"instant": true,
|
||||
"format": "table"
|
||||
}
|
||||
],
|
||||
"transformations": [
|
||||
{"id": "organize", "options": {
|
||||
"excludeByName": {"Time": true},
|
||||
"renameByName": {"service": "Service", "instance": "Endpoint", "Value": "Expires in"}
|
||||
}}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {"custom": {"align": "auto"}},
|
||||
"overrides": [
|
||||
{"matcher": {"id": "byName", "options": "Expires in"}, "properties": [
|
||||
{"id": "unit", "value": "s"},
|
||||
{"id": "custom.cellOptions", "value": {"type": "color-background"}},
|
||||
{"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "orange", "value": 1209600}, {"color": "green", "value": 2592000}]}}
|
||||
]}
|
||||
]
|
||||
},
|
||||
"options": {"showHeader": true}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
{
|
||||
"id": null,
|
||||
"uid": "beepi-uptimerobot",
|
||||
"title": "External Checks",
|
||||
"tags": ["beepi", "uptimerobot"],
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"version": 1,
|
||||
"refresh": "5m",
|
||||
"time": {"from": "now-24h", "to": "now"},
|
||||
"panels": [
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Monitors down",
|
||||
"id": 1,
|
||||
"gridPos": {"h": 5, "w": 6, "x": 0, "y": 0},
|
||||
"datasource": {"type": "yesoreyeram-infinity-datasource", "uid": "{{ grafana_uptimerobot_uid }}"},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "yesoreyeram-infinity-datasource", "uid": "{{ grafana_uptimerobot_uid }}"},
|
||||
"type": "json",
|
||||
"source": "url",
|
||||
"parser": "backend",
|
||||
"format": "table",
|
||||
"url": "https://api.uptimerobot.com/v2/getMonitors?format=json&custom_uptime_ratios=1-7-30",
|
||||
"url_options": {"method": "POST", "data": ""},
|
||||
"root_selector": "monitors",
|
||||
"columns": [
|
||||
{"selector": "friendly_name", "text": "Monitor", "type": "string"},
|
||||
{"selector": "status", "text": "Status", "type": "number"}
|
||||
],
|
||||
"filterExpression": "status == 8 || status == 9"
|
||||
}
|
||||
],
|
||||
"transformations": [{"id": "reduce", "options": {"reducers": ["count"]}}],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "red", "value": 1}]},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"textMode": "value",
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "/^Status$/", "values": false}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "table",
|
||||
"title": "UptimeRobot monitors",
|
||||
"id": 2,
|
||||
"gridPos": {"h": 12, "w": 18, "x": 6, "y": 0},
|
||||
"datasource": {"type": "yesoreyeram-infinity-datasource", "uid": "{{ grafana_uptimerobot_uid }}"},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "yesoreyeram-infinity-datasource", "uid": "{{ grafana_uptimerobot_uid }}"},
|
||||
"type": "json",
|
||||
"source": "url",
|
||||
"parser": "backend",
|
||||
"format": "table",
|
||||
"url": "https://api.uptimerobot.com/v2/getMonitors?format=json&custom_uptime_ratios=1-7-30",
|
||||
"url_options": {"method": "POST", "data": ""},
|
||||
"root_selector": "monitors",
|
||||
"columns": [
|
||||
{"selector": "friendly_name", "text": "Monitor", "type": "string"},
|
||||
{"selector": "status", "text": "Status", "type": "number"},
|
||||
{"selector": "custom_uptime_ratio", "text": "Uptime 1d-7d-30d", "type": "string"},
|
||||
{"selector": "interval", "text": "Interval", "type": "number"},
|
||||
{"selector": "url", "text": "URL", "type": "string"}
|
||||
]
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {"custom": {"align": "auto", "cellOptions": {"type": "auto"}}},
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {"id": "byName", "options": "Status"},
|
||||
"properties": [
|
||||
{"id": "custom.cellOptions", "value": {"type": "color-background"}},
|
||||
{"id": "mappings", "value": [{"type": "value", "options": {
|
||||
"0": {"text": "paused", "color": "text", "index": 0},
|
||||
"1": {"text": "not checked", "color": "blue", "index": 1},
|
||||
"2": {"text": "UP", "color": "green", "index": 2},
|
||||
"8": {"text": "SEEMS DOWN", "color": "orange", "index": 3},
|
||||
"9": {"text": "DOWN", "color": "red", "index": 4}
|
||||
}}]},
|
||||
{"id": "custom.width", "value": 130}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {"id": "byName", "options": "Interval"},
|
||||
"properties": [{"id": "unit", "value": "s"}, {"id": "custom.width", "value": 100}]
|
||||
},
|
||||
{
|
||||
"matcher": {"id": "byName", "options": "Uptime 1d-7d-30d"},
|
||||
"properties": [{"id": "custom.width", "value": 190}]
|
||||
}
|
||||
]
|
||||
},
|
||||
"options": {"showHeader": true, "sortBy": [{"displayName": "Status", "desc": true}]}
|
||||
},
|
||||
{
|
||||
"type": "stat",
|
||||
"title": "Monitors configured",
|
||||
"id": 3,
|
||||
"gridPos": {"h": 5, "w": 6, "x": 0, "y": 5},
|
||||
"datasource": {"type": "yesoreyeram-infinity-datasource", "uid": "{{ grafana_uptimerobot_uid }}"},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"datasource": {"type": "yesoreyeram-infinity-datasource", "uid": "{{ grafana_uptimerobot_uid }}"},
|
||||
"type": "json",
|
||||
"source": "url",
|
||||
"parser": "backend",
|
||||
"format": "table",
|
||||
"url": "https://api.uptimerobot.com/v2/getMonitors?format=json",
|
||||
"url_options": {"method": "POST", "data": ""},
|
||||
"root_selector": "monitors",
|
||||
"columns": [{"selector": "status", "text": "Status", "type": "number"}]
|
||||
}
|
||||
],
|
||||
"transformations": [{"id": "reduce", "options": {"reducers": ["count"]}}],
|
||||
"fieldConfig": {
|
||||
"defaults": {"color": {"mode": "fixed", "fixedColor": "blue"}, "unit": "short"},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "none",
|
||||
"textMode": "value",
|
||||
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "/^Status$/", "values": false}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"title": "About",
|
||||
"id": 4,
|
||||
"gridPos": {"h": 4, "w": 24, "x": 0, "y": 12},
|
||||
"options": {
|
||||
"mode": "markdown",
|
||||
"content": "External reachability is checked by UptimeRobot from outside the network, which is the one thing this stack cannot do for itself: everything else runs on beepi, so it cannot report that beepi is down.\n\nThe free plan allows 10 API requests per minute. This dashboard refreshes every 5 minutes."
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
apiVersion: 1
|
||||
|
||||
contactPoints:
|
||||
- orgId: 1
|
||||
name: discord
|
||||
receivers:
|
||||
- uid: discord-beepi
|
||||
type: discord
|
||||
settings:
|
||||
url: {{ discord_webhook }}
|
||||
title: '{{ '{{' }} template "default.title" . {{ '}}' }}'
|
||||
message: |
|
||||
{{ '{{' }} range .Alerts {{ '}}' }}**{{ '{{' }} .Labels.alertname {{ '}}' }}** {{ '{{' }} if .Labels.service {{ '}}' }}({{ '{{' }} .Labels.service {{ '}}' }}){{ '{{' }} end {{ '}}' }}
|
||||
{{ '{{' }} .Annotations.summary {{ '}}' }}
|
||||
{{ '{{' }} end {{ '}}' }}
|
||||
use_discord_username: true
|
||||
disableResolveMessage: false
|
||||
|
||||
policies:
|
||||
- orgId: 1
|
||||
receiver: discord
|
||||
group_by:
|
||||
- alertname
|
||||
- service
|
||||
- job
|
||||
group_wait: 30s
|
||||
group_interval: 5m
|
||||
repeat_interval: 12h
|
||||
routes:
|
||||
- receiver: discord
|
||||
object_matchers:
|
||||
- ['severity', '=', 'critical']
|
||||
group_wait: 10s
|
||||
repeat_interval: 4h
|
||||
@@ -0,0 +1,57 @@
|
||||
apiVersion: 1
|
||||
|
||||
groups:
|
||||
{% for group in alert_groups %}
|
||||
- orgId: 1
|
||||
name: {{ group.name }}
|
||||
folder: {{ grafana_folder }}
|
||||
interval: {{ group.interval }}
|
||||
rules:
|
||||
{% for rule in group.rules %}
|
||||
- uid: {{ rule.uid }}
|
||||
title: {{ rule.title }}
|
||||
condition: C
|
||||
for: {{ rule['for'] }}
|
||||
noDataState: {{ rule.no_data | default('NoData') }}
|
||||
execErrState: Alerting
|
||||
labels:
|
||||
severity: {{ rule.severity }}
|
||||
annotations:
|
||||
summary: {{ rule.summary }}
|
||||
data:
|
||||
- refId: A
|
||||
relativeTimeRange:
|
||||
from: 600
|
||||
to: 0
|
||||
datasourceUid: {{ grafana_loki_uid if rule.source | default('prometheus') == 'loki' else grafana_prometheus_uid }}
|
||||
model:
|
||||
refId: A
|
||||
expr: '{{ rule.expr }}'
|
||||
instant: true
|
||||
{% if rule.source | default('prometheus') == 'loki' %}
|
||||
queryType: instant
|
||||
{% else %}
|
||||
range: false
|
||||
{% endif %}
|
||||
- refId: B
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
refId: B
|
||||
type: reduce
|
||||
expression: A
|
||||
reducer: last
|
||||
settings:
|
||||
mode: dropNN
|
||||
- refId: C
|
||||
datasourceUid: __expr__
|
||||
model:
|
||||
refId: C
|
||||
type: threshold
|
||||
expression: B
|
||||
conditions:
|
||||
- evaluator:
|
||||
type: {{ rule.op }}
|
||||
params:
|
||||
- {{ rule.threshold }}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
@@ -0,0 +1,26 @@
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: beepi
|
||||
orgId: 1
|
||||
folder: {{ grafana_folder }}
|
||||
folderUid: {{ grafana_folder_uid }}
|
||||
type: file
|
||||
disableDeletion: false
|
||||
updateIntervalSeconds: 30
|
||||
allowUiUpdates: false
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
foldersFromFilesStructure: false
|
||||
|
||||
- name: general
|
||||
orgId: 1
|
||||
folder: {{ grafana_general_folder }}
|
||||
folderUid: {{ grafana_general_folder_uid }}
|
||||
type: file
|
||||
disableDeletion: false
|
||||
updateIntervalSeconds: 30
|
||||
allowUiUpdates: false
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards-general
|
||||
foldersFromFilesStructure: false
|
||||
@@ -0,0 +1,49 @@
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: prometheus
|
||||
uid: {{ grafana_prometheus_uid }}
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
jsonData:
|
||||
timeInterval: 15s
|
||||
httpMethod: POST
|
||||
|
||||
- name: loki
|
||||
uid: {{ grafana_loki_uid }}
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
editable: false
|
||||
jsonData:
|
||||
maxLines: 2000
|
||||
|
||||
- name: topology
|
||||
uid: {{ grafana_infinity_uid }}
|
||||
type: yesoreyeram-infinity-datasource
|
||||
access: proxy
|
||||
editable: false
|
||||
jsonData:
|
||||
allowedHosts:
|
||||
- http://topology:8088
|
||||
global_queries: []
|
||||
{% if uptimerobot_enabled %}
|
||||
|
||||
- name: uptimerobot
|
||||
uid: {{ grafana_uptimerobot_uid }}
|
||||
type: yesoreyeram-infinity-datasource
|
||||
access: proxy
|
||||
editable: false
|
||||
jsonData:
|
||||
auth_method: apiKey
|
||||
apiKeyKey: api_key
|
||||
apiKeyType: query
|
||||
allowedHosts:
|
||||
- https://api.uptimerobot.com
|
||||
global_queries: []
|
||||
secureJsonData:
|
||||
apiKeyValue: {{ uptimerobot_api_key }}
|
||||
{% endif %}
|
||||
@@ -34,9 +34,6 @@ schema_config:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
|
||||
ruler:
|
||||
alertmanager_url: http://localhost:9093
|
||||
|
||||
ingester:
|
||||
wal:
|
||||
enabled: true
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
- name: Prometheus
|
||||
hosts: beepi
|
||||
become: true
|
||||
vars_files:
|
||||
- vars/alert_rules.yml
|
||||
vars:
|
||||
grafana_secret: !vault |
|
||||
$ANSIBLE_VAULT;1.1;AES256
|
||||
@@ -25,19 +27,21 @@
|
||||
36376463313964333834373435653564346333396538306337333762613737643363636364353139
|
||||
3032383564333638613566633363666430356230326231633636
|
||||
tasks:
|
||||
- name: Install podman
|
||||
ansible.builtin.apt:
|
||||
name: podman
|
||||
state: present
|
||||
- name: Determine whether alert delivery is configured
|
||||
ansible.builtin.set_fact:
|
||||
alerting_enabled: "{{ discord_webhook is defined and 'REPLACE_ME' not in discord_webhook }}"
|
||||
|
||||
- name: Install podman-compose
|
||||
ansible.builtin.apt:
|
||||
name: podman-compose
|
||||
state: present
|
||||
- name: Determine whether uptimerobot is configured
|
||||
ansible.builtin.set_fact:
|
||||
uptimerobot_enabled: "{{ uptimerobot_api_key is defined and 'REPLACE_ME' not in uptimerobot_api_key }}"
|
||||
|
||||
- name: Install prometheus-node-exporter
|
||||
- name: Install packages
|
||||
ansible.builtin.apt:
|
||||
name: prometheus-node-exporter
|
||||
name:
|
||||
- podman
|
||||
- podman-compose
|
||||
- prometheus-node-exporter
|
||||
- wireguard-tools
|
||||
state: present
|
||||
|
||||
- name: Create monitoring config directory
|
||||
@@ -46,6 +50,154 @@
|
||||
state: directory
|
||||
mode: '0755'
|
||||
|
||||
- name: Create grafana provisioning directories
|
||||
ansible.builtin.file:
|
||||
path: "/opt/beeserver/monitoring/grafana/{{ item }}"
|
||||
state: directory
|
||||
mode: '0755'
|
||||
loop:
|
||||
- provisioning/datasources
|
||||
- provisioning/dashboards
|
||||
- provisioning/alerting
|
||||
- dashboards
|
||||
- dashboards-general
|
||||
|
||||
- name: Create topology directory
|
||||
ansible.builtin.file:
|
||||
path: /opt/beeserver/monitoring/topology
|
||||
state: directory
|
||||
mode: '0755'
|
||||
|
||||
- name: Create textfile directory
|
||||
ansible.builtin.file:
|
||||
path: "{{ node_exporter_textfile_dir }}"
|
||||
state: directory
|
||||
mode: '0755'
|
||||
|
||||
- name: Configure node exporter
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/default/prometheus-node-exporter
|
||||
content: |
|
||||
ARGS="--collector.textfile.directory={{ node_exporter_textfile_dir }} --collector.systemd --collector.processes"
|
||||
mode: '0644'
|
||||
notify: Restart node exporter
|
||||
|
||||
- name: Copy textfile collectors
|
||||
ansible.builtin.template:
|
||||
src: "textfile/{{ item }}.sh.j2"
|
||||
dest: "/usr/local/bin/{{ item }}.sh"
|
||||
mode: '0755'
|
||||
loop:
|
||||
- service_metrics
|
||||
- wg_metrics
|
||||
|
||||
- name: Copy uptimerobot collector
|
||||
ansible.builtin.template:
|
||||
src: textfile/uptimerobot_metrics.sh.j2
|
||||
dest: /usr/local/bin/uptimerobot_metrics.sh
|
||||
mode: '0700'
|
||||
when: uptimerobot_enabled
|
||||
|
||||
- name: Install uptimerobot timer
|
||||
ansible.builtin.copy:
|
||||
dest: "/etc/systemd/system/beepi-uptimerobot-metrics.{{ item.kind }}"
|
||||
content: "{{ item.content }}"
|
||||
mode: '0644'
|
||||
loop:
|
||||
- kind: service
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Collect uptimerobot metrics
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/bin/uptimerobot_metrics.sh
|
||||
- kind: timer
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Collect uptimerobot metrics
|
||||
|
||||
[Timer]
|
||||
OnBootSec=3min
|
||||
OnUnitActiveSec=5min
|
||||
AccuracySec=30s
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
when: uptimerobot_enabled
|
||||
notify: Reload systemd
|
||||
|
||||
- name: Install collector timers
|
||||
ansible.builtin.copy:
|
||||
dest: "/etc/systemd/system/{{ item.name }}.{{ item.kind }}"
|
||||
content: "{{ item.content }}"
|
||||
mode: '0644'
|
||||
loop:
|
||||
- name: beepi-service-metrics
|
||||
kind: service
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Collect service unit metrics
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/bin/service_metrics.sh
|
||||
- name: beepi-service-metrics
|
||||
kind: timer
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Collect service unit metrics
|
||||
|
||||
[Timer]
|
||||
OnBootSec=1min
|
||||
OnUnitActiveSec=30s
|
||||
AccuracySec=5s
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
- name: beepi-wg-metrics
|
||||
kind: service
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Collect wireguard peer metrics
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/bin/wg_metrics.sh
|
||||
- name: beepi-wg-metrics
|
||||
kind: timer
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Collect wireguard peer metrics
|
||||
|
||||
[Timer]
|
||||
OnBootSec=1min
|
||||
OnUnitActiveSec=30s
|
||||
AccuracySec=5s
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
notify: Reload systemd
|
||||
|
||||
- name: Flush handlers
|
||||
ansible.builtin.meta: flush_handlers
|
||||
|
||||
- name: Enable collector timers
|
||||
ansible.builtin.systemd_service:
|
||||
name: "{{ item }}.timer"
|
||||
state: started
|
||||
enabled: true
|
||||
loop:
|
||||
- beepi-service-metrics
|
||||
- beepi-wg-metrics
|
||||
|
||||
- name: Enable uptimerobot timer
|
||||
ansible.builtin.systemd_service:
|
||||
name: beepi-uptimerobot-metrics.timer
|
||||
state: started
|
||||
enabled: true
|
||||
when: uptimerobot_enabled
|
||||
|
||||
- name: Compose down
|
||||
changed_when: true
|
||||
failed_when: false
|
||||
@@ -60,11 +212,17 @@
|
||||
mode: '0600'
|
||||
|
||||
- name: Copy prometheus config
|
||||
ansible.builtin.copy:
|
||||
src: prometheus_config.yml
|
||||
ansible.builtin.template:
|
||||
src: prometheus.yml.j2
|
||||
dest: /opt/beeserver/monitoring/prometheus.yml
|
||||
mode: '0644'
|
||||
|
||||
- name: Copy blackbox config
|
||||
ansible.builtin.template:
|
||||
src: blackbox.yml.j2
|
||||
dest: /opt/beeserver/monitoring/blackbox.yml
|
||||
mode: '0644'
|
||||
|
||||
- name: Copy loki config
|
||||
ansible.builtin.template:
|
||||
src: local-config.yml.j2
|
||||
@@ -77,17 +235,104 @@
|
||||
dest: /opt/beeserver/monitoring/config.alloy
|
||||
mode: '0644'
|
||||
|
||||
- name: Copy topology service
|
||||
ansible.builtin.copy:
|
||||
src: topology/topology.py
|
||||
dest: /opt/beeserver/monitoring/topology/topology.py
|
||||
mode: '0644'
|
||||
|
||||
- name: Copy topology spec
|
||||
ansible.builtin.template:
|
||||
src: topology/topology-spec.json.j2
|
||||
dest: /opt/beeserver/monitoring/topology/topology-spec.json
|
||||
mode: '0644'
|
||||
|
||||
- name: Copy grafana datasources
|
||||
ansible.builtin.template:
|
||||
src: grafana/provisioning/datasources/datasources.yml.j2
|
||||
dest: /opt/beeserver/monitoring/grafana/provisioning/datasources/datasources.yml
|
||||
owner: '472'
|
||||
group: '472'
|
||||
mode: '0600'
|
||||
|
||||
- name: Copy grafana dashboard provider
|
||||
ansible.builtin.template:
|
||||
src: grafana/provisioning/dashboards/dashboards.yml.j2
|
||||
dest: /opt/beeserver/monitoring/grafana/provisioning/dashboards/dashboards.yml
|
||||
mode: '0644'
|
||||
|
||||
- name: Copy hand made dashboards
|
||||
ansible.builtin.copy:
|
||||
src: grafana/dashboards-general/
|
||||
dest: /opt/beeserver/monitoring/grafana/dashboards-general/
|
||||
mode: '0644'
|
||||
|
||||
- name: Copy grafana contact points
|
||||
ansible.builtin.template:
|
||||
src: grafana/provisioning/alerting/contact-points.yml.j2
|
||||
dest: /opt/beeserver/monitoring/grafana/provisioning/alerting/contact-points.yml
|
||||
owner: '472'
|
||||
group: '472'
|
||||
mode: '0600'
|
||||
when: alerting_enabled
|
||||
|
||||
- name: Copy grafana alert rules
|
||||
ansible.builtin.template:
|
||||
src: grafana/provisioning/alerting/rules.yml.j2
|
||||
dest: /opt/beeserver/monitoring/grafana/provisioning/alerting/rules.yml
|
||||
mode: '0644'
|
||||
|
||||
- name: Copy grafana dashboards
|
||||
ansible.builtin.template:
|
||||
src: "grafana/dashboards/{{ item }}.json.j2"
|
||||
dest: "/opt/beeserver/monitoring/grafana/dashboards/{{ item }}.json"
|
||||
mode: '0644'
|
||||
loop:
|
||||
- infrastructure-map
|
||||
- backups
|
||||
- services
|
||||
- ingress
|
||||
|
||||
- name: Copy uptimerobot dashboard
|
||||
ansible.builtin.template:
|
||||
src: grafana/dashboards/uptimerobot.json.j2
|
||||
dest: /opt/beeserver/monitoring/grafana/dashboards/uptimerobot.json
|
||||
mode: '0644'
|
||||
when: uptimerobot_enabled
|
||||
|
||||
- name: Enable podman.socket
|
||||
ansible.builtin.service:
|
||||
name: podman.socket
|
||||
state: started
|
||||
enabled: true
|
||||
|
||||
- name: Check the geoip database
|
||||
ansible.builtin.stat:
|
||||
path: /opt/beeserver/monitoring/database/GeoLite2-City.mmdb
|
||||
register: geoip_db
|
||||
|
||||
- name: Check when geoip was last downloaded
|
||||
ansible.builtin.stat:
|
||||
path: /opt/beeserver/monitoring/.geoip_downloaded
|
||||
register: geoip_stamp
|
||||
|
||||
- name: Decide whether the geoip database needs refreshing
|
||||
ansible.builtin.set_fact:
|
||||
geoip_stale: "{{ not geoip_db.stat.exists or not geoip_stamp.stat.exists or (ansible_facts.date_time.epoch | int - geoip_stamp.stat.mtime | int) > geoip_max_age_seconds | int }}"
|
||||
|
||||
- name: Report the geoip database age
|
||||
ansible.builtin.debug:
|
||||
msg: >-
|
||||
geoip
|
||||
{{ 'not present, downloading' if not (geoip_db.stat.exists and geoip_stamp.stat.exists)
|
||||
else 'downloaded ' + (((ansible_facts.date_time.epoch | int - geoip_stamp.stat.mtime | int) / 3600) | round(1) | string) + 'h ago, ' + ('refreshing' if geoip_stale else 'reusing') }}
|
||||
|
||||
- name: Delete database directory
|
||||
ansible.builtin.file:
|
||||
path: /opt/beeserver/monitoring/database
|
||||
state: absent
|
||||
mode: '0755'
|
||||
when: geoip_stale
|
||||
|
||||
- name: Create database directory
|
||||
ansible.builtin.file:
|
||||
@@ -95,6 +340,12 @@
|
||||
state: directory
|
||||
mode: '0755'
|
||||
|
||||
- name: Remove stale geoip archives
|
||||
ansible.builtin.shell:
|
||||
cmd: rm -f /opt/beeserver/monitoring/GeoLite2-City_*.tar.gz
|
||||
changed_when: true
|
||||
when: geoip_stale
|
||||
|
||||
- name: Download geoip database
|
||||
ansible.builtin.get_url:
|
||||
url: "https://download.maxmind.com/geoip/databases/GeoLite2-City/download?suffix=tar.gz"
|
||||
@@ -102,6 +353,7 @@
|
||||
username: "{{ geoip_username }}"
|
||||
password: "{{ geoip_license }}"
|
||||
mode: '0644'
|
||||
when: geoip_stale
|
||||
|
||||
- name: Get geoip tar file
|
||||
ansible.builtin.find:
|
||||
@@ -111,6 +363,7 @@
|
||||
use_regex: true
|
||||
limit: 1
|
||||
register: found
|
||||
when: geoip_stale
|
||||
|
||||
- name: Extract geopip database
|
||||
ansible.builtin.unarchive:
|
||||
@@ -118,9 +371,50 @@
|
||||
src: "{{ found.files[0].path }}"
|
||||
dest: /opt/beeserver/monitoring/database
|
||||
extra_opts: ['--strip-components=1', '--show-stored-names']
|
||||
when: geoip_stale and found.files | length > 0
|
||||
|
||||
- name: Stamp the geoip download time
|
||||
ansible.builtin.file:
|
||||
path: /opt/beeserver/monitoring/.geoip_downloaded
|
||||
state: touch
|
||||
mode: '0644'
|
||||
when: geoip_stale and found.files | length > 0
|
||||
|
||||
- name: Compose up
|
||||
changed_when: true
|
||||
ansible.builtin.command:
|
||||
cmd: podman-compose up -d
|
||||
chdir: /opt/beeserver/monitoring
|
||||
|
||||
- name: Run collectors once
|
||||
changed_when: true
|
||||
ansible.builtin.command:
|
||||
cmd: "/usr/local/bin/{{ item }}.sh"
|
||||
loop:
|
||||
- service_metrics
|
||||
- wg_metrics
|
||||
|
||||
- name: Run uptimerobot collector once
|
||||
changed_when: true
|
||||
ansible.builtin.command:
|
||||
cmd: /usr/local/bin/uptimerobot_metrics.sh
|
||||
when: uptimerobot_enabled
|
||||
|
||||
- name: Debug
|
||||
ansible.builtin.debug:
|
||||
msg:
|
||||
- "Grafana: https://grafana.secretbee.buzz/"
|
||||
- "Topology: curl -s localhost:8088/topology.json | head"
|
||||
- "Prometheus targets: curl -s localhost:9090/api/v1/targets | jq '.data.activeTargets[] | {job: .labels.job, health}'"
|
||||
tags:
|
||||
- always
|
||||
|
||||
handlers:
|
||||
- name: Reload systemd
|
||||
ansible.builtin.systemd_service:
|
||||
daemon_reload: true
|
||||
|
||||
- name: Restart node exporter
|
||||
ansible.builtin.service:
|
||||
name: prometheus-node-exporter
|
||||
state: restarted
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
external_labels:
|
||||
monitor: beepi
|
||||
|
||||
scrape_configs:
|
||||
- job_name: prometheus
|
||||
static_configs:
|
||||
- targets: ['localhost:9090']
|
||||
|
||||
- job_name: node
|
||||
static_configs:
|
||||
- targets: ['host.containers.internal:9100']
|
||||
labels:
|
||||
host: beepi
|
||||
- targets: ['{{ wireguard_ingress_ip }}:9100']
|
||||
labels:
|
||||
host: ingress
|
||||
- targets: ['192.168.178.51:9100']
|
||||
labels:
|
||||
host: beeserver
|
||||
|
||||
- job_name: podman
|
||||
static_configs:
|
||||
- targets: ['podman-exporter:9882']
|
||||
labels:
|
||||
host: beepi
|
||||
|
||||
- job_name: alloy
|
||||
static_configs:
|
||||
- targets: ['alloy:12345']
|
||||
labels:
|
||||
host: beepi
|
||||
- targets: ['{{ wireguard_ingress_ip }}:12345']
|
||||
labels:
|
||||
host: ingress
|
||||
|
||||
- job_name: loki
|
||||
static_configs:
|
||||
- targets: ['loki:3100']
|
||||
|
||||
- job_name: grafana
|
||||
static_configs:
|
||||
- targets: ['grafana:4000']
|
||||
|
||||
- job_name: blackbox
|
||||
static_configs:
|
||||
- targets: ['host.containers.internal:9115']
|
||||
|
||||
- job_name: probe-app
|
||||
metrics_path: /probe
|
||||
params:
|
||||
module: [http_2xx_any]
|
||||
static_configs:
|
||||
{% for service in monitored_services %}
|
||||
{% if service.probe is defined %}
|
||||
- targets: ['{{ service.probe }}']
|
||||
labels:
|
||||
service: {{ service.name }}
|
||||
group: {{ service.group }}
|
||||
vantage: local
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- source_labels: [__param_target]
|
||||
target_label: instance
|
||||
- target_label: __address__
|
||||
replacement: host.containers.internal:9115
|
||||
|
||||
- job_name: probe-vhost
|
||||
metrics_path: /probe
|
||||
params:
|
||||
module: [https_vhost]
|
||||
static_configs:
|
||||
{% for service in monitored_services %}
|
||||
{% if service.vhost is defined %}
|
||||
- targets: ['https://{{ service.vhost }}']
|
||||
labels:
|
||||
service: {{ service.name }}
|
||||
group: {{ service.group }}
|
||||
vantage: nginx
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- source_labels: [__param_target]
|
||||
target_label: instance
|
||||
- target_label: __address__
|
||||
replacement: host.containers.internal:9115
|
||||
|
||||
- job_name: probe-tcp
|
||||
metrics_path: /probe
|
||||
params:
|
||||
module: [tcp_connect]
|
||||
static_configs:
|
||||
{% for service in monitored_services %}
|
||||
{% for probe in service.tcp | default([]) %}
|
||||
- targets: ['{{ probe.target }}']
|
||||
labels:
|
||||
service: {{ service.name }}
|
||||
group: {{ service.group }}
|
||||
probe: {{ probe.name }}
|
||||
vantage: local
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- source_labels: [__param_target]
|
||||
target_label: instance
|
||||
- target_label: __address__
|
||||
replacement: host.containers.internal:9115
|
||||
|
||||
- job_name: probe-tunnel
|
||||
scrape_interval: 30s
|
||||
metrics_path: /probe
|
||||
params:
|
||||
module: [tcp_connect]
|
||||
static_configs:
|
||||
{% for probe in ingress_probe_targets %}
|
||||
- targets: ['{{ probe.target }}']
|
||||
labels:
|
||||
probe: {{ probe.name }}
|
||||
vantage: ingress
|
||||
{% endfor %}
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- source_labels: [__param_target]
|
||||
target_label: instance
|
||||
- target_label: __address__
|
||||
replacement: '{{ wireguard_ingress_ip }}:9115'
|
||||
|
||||
- job_name: probe-tunnel-icmp
|
||||
scrape_interval: 30s
|
||||
metrics_path: /probe
|
||||
params:
|
||||
module: [icmp]
|
||||
static_configs:
|
||||
- targets: ['{{ wireguard_beepi_ip }}']
|
||||
labels:
|
||||
probe: tunnel-rtt
|
||||
vantage: ingress
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
target_label: __param_target
|
||||
- source_labels: [__param_target]
|
||||
target_label: instance
|
||||
- target_label: __address__
|
||||
replacement: '{{ wireguard_ingress_ip }}:9115'
|
||||
@@ -1,16 +0,0 @@
|
||||
global:
|
||||
scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
|
||||
evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
|
||||
|
||||
scrape_configs:
|
||||
- job_name: 'prometheus'
|
||||
static_configs:
|
||||
- targets: ['localhost:9090']
|
||||
- job_name: node
|
||||
static_configs:
|
||||
- targets:
|
||||
- 'host.containers.internal:9100'
|
||||
- '192.168.178.51:9100'
|
||||
- job_name: podman
|
||||
static_configs:
|
||||
- targets: ['podman-exporter:9882']
|
||||
@@ -0,0 +1,105 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
textfile_dir={{ node_exporter_textfile_dir }}
|
||||
file=${textfile_dir}/services.prom
|
||||
tmp=${file}.$$
|
||||
|
||||
mkdir -p ${textfile_dir}
|
||||
|
||||
declare -A active_count
|
||||
declare -A unit_count
|
||||
units=""
|
||||
|
||||
record() {
|
||||
local service=$1
|
||||
local unit=$2
|
||||
local kind=$3
|
||||
local state=$4
|
||||
|
||||
units+="service_unit_active{service=\"${service}\",unit=\"${unit}\",kind=\"${kind}\"} ${state}"$'\n'
|
||||
unit_count[${service}]=$(( ${unit_count[${service}]:-0} + 1 ))
|
||||
active_count[${service}]=$(( ${active_count[${service}]:-0} + state ))
|
||||
}
|
||||
|
||||
check_user_unit() {
|
||||
local service=$1
|
||||
local owner=$2
|
||||
local unit=$3
|
||||
local state=0
|
||||
|
||||
if systemctl --user --machine ${owner}@.host is-active --quiet ${unit} 2>/dev/null
|
||||
then
|
||||
state=1
|
||||
fi
|
||||
|
||||
record ${service} ${unit} rootless ${state}
|
||||
}
|
||||
|
||||
check_system_unit() {
|
||||
local service=$1
|
||||
local unit=$2
|
||||
local state=0
|
||||
|
||||
if systemctl is-active --quiet ${unit} 2>/dev/null
|
||||
then
|
||||
state=1
|
||||
fi
|
||||
|
||||
record ${service} ${unit} system ${state}
|
||||
}
|
||||
|
||||
check_container() {
|
||||
local service=$1
|
||||
local container=$2
|
||||
local state=0
|
||||
|
||||
if [ "$(podman container inspect -f '{% raw %}{{.State.Running}}{% endraw %}' ${container} 2>/dev/null)" = "true" ]
|
||||
then
|
||||
state=1
|
||||
fi
|
||||
|
||||
record ${service} ${container} container ${state}
|
||||
}
|
||||
|
||||
{% for service in monitored_services %}
|
||||
{% for unit in service.units | default([]) %}
|
||||
check_user_unit {{ service.name }} {{ service.owner }} {{ unit }}
|
||||
{% endfor %}
|
||||
{% for unit in service.system_units | default([]) %}
|
||||
check_system_unit {{ service.name }} {{ unit }}
|
||||
{% endfor %}
|
||||
{% for container in service.containers | default([]) %}
|
||||
check_container {{ service.name }} {{ container }}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
|
||||
{
|
||||
echo "# HELP service_unit_active Whether a unit or container backing a service is running."
|
||||
echo "# TYPE service_unit_active gauge"
|
||||
printf '%s' "${units}"
|
||||
|
||||
echo "# HELP service_enabled Whether a service is expected to be running."
|
||||
echo "# TYPE service_enabled gauge"
|
||||
{% for service in monitored_services %}
|
||||
echo "service_enabled{service=\"{{ service.name }}\",group=\"{{ service.group }}\"} {{ 1 if service.enabled else 0 }}"
|
||||
{% endfor %}
|
||||
|
||||
echo "# HELP service_units_total Number of units or containers backing a service."
|
||||
echo "# TYPE service_units_total gauge"
|
||||
for service in "${!unit_count[@]}"
|
||||
do
|
||||
echo "service_units_total{service=\"${service}\"} ${unit_count[${service}]}"
|
||||
done
|
||||
|
||||
echo "# HELP service_units_active Number of running units or containers backing a service."
|
||||
echo "# TYPE service_units_active gauge"
|
||||
for service in "${!active_count[@]}"
|
||||
do
|
||||
echo "service_units_active{service=\"${service}\"} ${active_count[${service}]}"
|
||||
done
|
||||
} > "${tmp}"
|
||||
|
||||
chmod 0644 "${tmp}"
|
||||
mv "${tmp}" "${file}"
|
||||
@@ -0,0 +1,71 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
textfile_dir={{ node_exporter_textfile_dir }}
|
||||
file=${textfile_dir}/uptimerobot.prom
|
||||
tmp=${file}.$$
|
||||
|
||||
mkdir -p ${textfile_dir}
|
||||
|
||||
response=$(curl -sS -m 25 -X POST \
|
||||
"https://api.uptimerobot.com/v2/getMonitors?api_key={{ uptimerobot_api_key }}&format=json&custom_uptime_ratios=1-7-30" \
|
||||
2>/dev/null)
|
||||
|
||||
{% raw %}
|
||||
printf '%s' "${response}" | python3 -c '
|
||||
import json
|
||||
import sys
|
||||
|
||||
print("# HELP uptimerobot_api_ok Whether the uptimerobot api answered successfully.")
|
||||
print("# TYPE uptimerobot_api_ok gauge")
|
||||
print("# HELP uptimerobot_monitor_up Whether an external monitor reports the target as up.")
|
||||
print("# TYPE uptimerobot_monitor_up gauge")
|
||||
print("# HELP uptimerobot_monitor_enabled Whether an external monitor is active rather than paused.")
|
||||
print("# TYPE uptimerobot_monitor_enabled gauge")
|
||||
print("# HELP uptimerobot_monitor_status Raw uptimerobot status, 0 paused 1 unchecked 2 up 8 seems down 9 down.")
|
||||
print("# TYPE uptimerobot_monitor_status gauge")
|
||||
print("# HELP uptimerobot_uptime_ratio Uptime percentage over the given window.")
|
||||
print("# TYPE uptimerobot_uptime_ratio gauge")
|
||||
|
||||
try:
|
||||
payload = json.load(sys.stdin)
|
||||
except ValueError:
|
||||
print("uptimerobot_api_ok 0")
|
||||
sys.exit(0)
|
||||
|
||||
if payload.get("stat") != "ok":
|
||||
print("uptimerobot_api_ok 0")
|
||||
sys.exit(0)
|
||||
|
||||
print("uptimerobot_api_ok 1")
|
||||
|
||||
def escape(value):
|
||||
return value.replace("\\", "\\\\").replace("\"", "\\\"")
|
||||
|
||||
for monitor in payload.get("monitors", []):
|
||||
name = escape(str(monitor.get("friendly_name", "unknown")))
|
||||
status = int(monitor.get("status", 1))
|
||||
labels = "monitor=\"%s\"" % name
|
||||
|
||||
print("uptimerobot_monitor_status{%s} %d" % (labels, status))
|
||||
print("uptimerobot_monitor_enabled{%s} %d" % (labels, 0 if status == 0 else 1))
|
||||
print("uptimerobot_monitor_up{%s} %d" % (labels, 1 if status == 2 else 0))
|
||||
|
||||
ratios = str(monitor.get("custom_uptime_ratio", "")).split("-")
|
||||
for window, ratio in zip(("1d", "7d", "30d"), ratios):
|
||||
try:
|
||||
print("uptimerobot_uptime_ratio{%s,window=\"%s\"} %s" % (labels, window, float(ratio)))
|
||||
except ValueError:
|
||||
pass
|
||||
' > "${tmp}"
|
||||
{% endraw %}
|
||||
|
||||
if [ ! -s "${tmp}" ]
|
||||
then
|
||||
rm -f "${tmp}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
chmod 0644 "${tmp}"
|
||||
mv "${tmp}" "${file}"
|
||||
@@ -0,0 +1,60 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
textfile_dir={{ node_exporter_textfile_dir }}
|
||||
file=${textfile_dir}/wireguard.prom
|
||||
tmp=${file}.$$
|
||||
now=$(date +%s)
|
||||
|
||||
mkdir -p ${textfile_dir}
|
||||
|
||||
peer_name() {
|
||||
case "$1" in
|
||||
{% for key, name in wireguard_peer_names.items() %}
|
||||
"{{ key }}") echo "{{ name }}" ;;
|
||||
{% endfor %}
|
||||
*) echo "unknown" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
{
|
||||
echo "# HELP wireguard_peer_last_handshake_seconds Unix time of the last handshake with a peer."
|
||||
echo "# TYPE wireguard_peer_last_handshake_seconds gauge"
|
||||
echo "# HELP wireguard_peer_handshake_age_seconds Seconds since the last handshake with a peer."
|
||||
echo "# TYPE wireguard_peer_handshake_age_seconds gauge"
|
||||
echo "# HELP wireguard_peer_up Whether a peer handshaked within the last five minutes."
|
||||
echo "# TYPE wireguard_peer_up gauge"
|
||||
echo "# HELP wireguard_peer_receive_bytes_total Bytes received from a peer."
|
||||
echo "# TYPE wireguard_peer_receive_bytes_total counter"
|
||||
echo "# HELP wireguard_peer_transmit_bytes_total Bytes sent to a peer."
|
||||
echo "# TYPE wireguard_peer_transmit_bytes_total counter"
|
||||
|
||||
wg show all dump 2>/dev/null | awk 'NF >= 8' | while read -r interface key psk endpoint allowed handshake rx tx keepalive
|
||||
do
|
||||
name=$(peer_name "${key}")
|
||||
labels="interface=\"${interface}\",peer=\"${name}\""
|
||||
|
||||
age=$((now - handshake))
|
||||
up=0
|
||||
|
||||
if [ "${handshake}" -gt 0 ] && [ "${age}" -lt 300 ]
|
||||
then
|
||||
up=1
|
||||
fi
|
||||
|
||||
if [ "${handshake}" -eq 0 ]
|
||||
then
|
||||
age=-1
|
||||
fi
|
||||
|
||||
echo "wireguard_peer_last_handshake_seconds{${labels}} ${handshake}"
|
||||
echo "wireguard_peer_handshake_age_seconds{${labels}} ${age}"
|
||||
echo "wireguard_peer_up{${labels}} ${up}"
|
||||
echo "wireguard_peer_receive_bytes_total{${labels}} ${rx}"
|
||||
echo "wireguard_peer_transmit_bytes_total{${labels}} ${tx}"
|
||||
done
|
||||
} > "${tmp}"
|
||||
|
||||
chmod 0644 "${tmp}"
|
||||
mv "${tmp}" "${file}"
|
||||
@@ -0,0 +1,130 @@
|
||||
{
|
||||
"nodes": [
|
||||
{
|
||||
"id": "internet",
|
||||
"title": "Internet",
|
||||
"subtitle": "public",
|
||||
"group": "edge"
|
||||
},
|
||||
{
|
||||
"id": "ingress",
|
||||
"title": "Ingress",
|
||||
"subtitle": "animeistrash",
|
||||
"group": "edge",
|
||||
"up": "up{job=\"node\",host=\"ingress\"}",
|
||||
"stat": "100 - (avg(rate(node_cpu_seconds_total{job=\"node\",host=\"ingress\",mode=\"idle\"}[5m])) * 100)",
|
||||
"stat_unit": "pct"
|
||||
},
|
||||
{
|
||||
"id": "tunnel",
|
||||
"title": "WireGuard",
|
||||
"subtitle": "{{ wireguard_ingress_ip }} to {{ wireguard_beepi_ip }}",
|
||||
"group": "edge",
|
||||
"up": "max(wireguard_peer_up{peer=\"beepi\"})",
|
||||
"stat": "max(wireguard_peer_handshake_age_seconds{peer=\"beepi\"})",
|
||||
"stat_unit": "h"
|
||||
},
|
||||
{
|
||||
"id": "beepi",
|
||||
"title": "beepi",
|
||||
"subtitle": "raspberry pi",
|
||||
"group": "host",
|
||||
"up": "up{job=\"node\",host=\"beepi\"}",
|
||||
"stat": "100 - (avg(rate(node_cpu_seconds_total{job=\"node\",host=\"beepi\",mode=\"idle\"}[5m])) * 100)",
|
||||
"stat_unit": "pct"
|
||||
},
|
||||
{% for service in monitored_services %}
|
||||
{
|
||||
"id": "{{ service.name }}",
|
||||
"title": "{{ service.title }}",
|
||||
"subtitle": "{{ service.vhost | default(service.group) }}",
|
||||
"group": "{{ service.group }}",
|
||||
"enabled": "max(service_enabled{service=\"{{ service.name }}\"})",
|
||||
{% if service.probe is defined or service.tcp is defined %}
|
||||
"up": "min(probe_success{service=\"{{ service.name }}\",vantage=\"local\"})",
|
||||
"stat": "max(probe_duration_seconds{service=\"{{ service.name }}\",vantage=\"local\"})",
|
||||
"stat_unit": "s"
|
||||
{% elif service.vhost is defined %}
|
||||
"up": "min(probe_success{service=\"{{ service.name }}\",vantage=\"nginx\"})",
|
||||
"stat": "max(probe_duration_seconds{service=\"{{ service.name }}\",vantage=\"nginx\"})",
|
||||
"stat_unit": "s"
|
||||
{% else %}
|
||||
"up": "min(service_units_active{service=\"{{ service.name }}\"} / service_units_total{service=\"{{ service.name }}\"})"
|
||||
{% endif %}
|
||||
},
|
||||
{% endfor %}
|
||||
{
|
||||
"id": "raid",
|
||||
"title": "RAID",
|
||||
"subtitle": "{{ backup_raid_mount }}",
|
||||
"group": "storage",
|
||||
"up": "max(backup_raid_mounted)",
|
||||
"stat": "max(node_filesystem_avail_bytes{mountpoint=\"{{ backup_raid_mount }}\"})",
|
||||
"stat_unit": "bytes",
|
||||
"fill": "1 - (max(node_filesystem_avail_bytes{mountpoint=\"{{ backup_raid_mount }}\"}) / max(node_filesystem_size_bytes{mountpoint=\"{{ backup_raid_mount }}\"}))"
|
||||
},
|
||||
{% for job in backup_jobs %}
|
||||
{% if job.name != 'offsite' %}
|
||||
{
|
||||
"id": "backup-{{ job.name }}",
|
||||
"title": "{{ job.title }}",
|
||||
"subtitle": "backup job",
|
||||
"group": "backup",
|
||||
"up": "(max(backup_last_status{backup_job=\"{{ job.name }}\"}) == 1) and ((time() - max(backup_last_success_timestamp_seconds{backup_job=\"{{ job.name }}\"})) < {{ job.max_age_hours * 3600 }})",
|
||||
"stat": "time() - max(backup_last_success_timestamp_seconds{backup_job=\"{{ job.name }}\"})",
|
||||
"stat_unit": "h"
|
||||
},
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{
|
||||
"id": "offsite",
|
||||
"title": "Offsite sync",
|
||||
"subtitle": "daily 07:00",
|
||||
"group": "storage",
|
||||
"up": "(time() - max(backup_last_success_timestamp_seconds{backup_job=\"offsite\"})) < {{ 26 * 3600 }}",
|
||||
"stat": "time() - max(backup_last_success_timestamp_seconds{backup_job=\"offsite\"})",
|
||||
"stat_unit": "h"
|
||||
},
|
||||
{
|
||||
"id": "hetzner",
|
||||
"title": "Hetzner box",
|
||||
"subtitle": "{{ hetzner_host | default('storage box') }}",
|
||||
"group": "storage",
|
||||
"up": "(max(hetzner_reachable) == 1) and (max(hetzner_remote_files{path=\"beepi\"}) > 0)",
|
||||
"stat": "max(hetzner_remote_bytes{path=\"beepi\"})",
|
||||
"stat_unit": "bytes",
|
||||
"fill": "sum(hetzner_remote_bytes) / {{ hetzner_box_bytes }}"
|
||||
},
|
||||
{
|
||||
"id": "uptimerobot",
|
||||
"title": "External checks",
|
||||
"subtitle": "uptimerobot",
|
||||
"group": "edge",
|
||||
"dashboard": "/d/beepi-uptimerobot",
|
||||
"enabled": "max(uptimerobot_api_ok)",
|
||||
"up": "min(uptimerobot_monitor_up and on (monitor) (uptimerobot_monitor_enabled == 1))",
|
||||
"stat": "count(uptimerobot_monitor_enabled == 1)"
|
||||
}
|
||||
],
|
||||
"edges": [
|
||||
{"source": "uptimerobot", "target": "internet", "label": "probes"},
|
||||
{"source": "internet", "target": "ingress", "label": "public"},
|
||||
{"source": "ingress", "target": "tunnel", "label": "dnat"},
|
||||
{"source": "tunnel", "target": "beepi", "label": "wg0"},
|
||||
{% for service in monitored_services %}
|
||||
{% if service.vhost is defined and service.name != 'nginx' %}
|
||||
{"source": "nginx", "target": "{{ service.name }}", "label": "proxy"},
|
||||
{% endif %}
|
||||
{% if service.vhost is not defined %}
|
||||
{"source": "beepi", "target": "{{ service.name }}", "label": "{{ service.group }}"},
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{"source": "beepi", "target": "nginx", "label": "tls"},
|
||||
{"source": "beepi", "target": "backup-daily", "label": "03:00"},
|
||||
{"source": "backup-daily", "target": "backup-weekly", "label": "sun 05:00"},
|
||||
{"source": "backup-daily", "target": "backup-restoretest", "label": "sun 06:00"},
|
||||
{"source": "backup-daily", "target": "raid", "label": "archive"},
|
||||
{"source": "raid", "target": "offsite", "label": "rsync"},
|
||||
{"source": "offsite", "target": "hetzner", "label": "ssh 23"}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
PROMETHEUS_URL = os.environ.get("PROMETHEUS_URL", "http://prometheus:9090")
|
||||
LISTEN_PORT = int(os.environ.get("LISTEN_PORT", "8088"))
|
||||
SPEC_PATH = os.environ.get("SPEC_PATH", "/app/topology-spec.json")
|
||||
CACHE_SECONDS = int(os.environ.get("CACHE_SECONDS", "10"))
|
||||
|
||||
STATUS_OK = "ok"
|
||||
STATUS_FAIL = "fail"
|
||||
STATUS_DISABLED = "disabled"
|
||||
STATUS_UNKNOWN = "unknown"
|
||||
|
||||
ARCS = {
|
||||
STATUS_OK: "arc__ok",
|
||||
STATUS_FAIL: "arc__fail",
|
||||
STATUS_DISABLED: "arc__disabled",
|
||||
STATUS_UNKNOWN: "arc__unknown",
|
||||
}
|
||||
|
||||
ARC_FIELDS = list(ARCS.values()) + ["arc__used", "arc__free"]
|
||||
|
||||
COLORS = {
|
||||
STATUS_OK: "green",
|
||||
STATUS_FAIL: "red",
|
||||
STATUS_DISABLED: "#6e7079",
|
||||
STATUS_UNKNOWN: "orange",
|
||||
}
|
||||
|
||||
DASHBOARDS = {
|
||||
"apps": "/d/beepi-services",
|
||||
"platform": "/d/beepi-services",
|
||||
"backup": "/d/beepi-backups",
|
||||
"storage": "/d/beepi-backups",
|
||||
"edge": "/d/beepi-ingress",
|
||||
"host": "/d/beepi-ingress",
|
||||
}
|
||||
|
||||
_lock = threading.Lock()
|
||||
_cache = {"at": 0.0, "body": None}
|
||||
|
||||
|
||||
def query(expr):
|
||||
url = PROMETHEUS_URL + "/api/v1/query?" + urllib.parse.urlencode({"query": expr})
|
||||
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=8) as response:
|
||||
payload = json.load(response)
|
||||
except (urllib.error.URLError, OSError, ValueError):
|
||||
return None
|
||||
|
||||
if payload.get("status") != "success":
|
||||
return None
|
||||
|
||||
result = payload.get("data", {}).get("result", [])
|
||||
|
||||
if not result:
|
||||
return None
|
||||
|
||||
try:
|
||||
return float(result[0]["value"][1])
|
||||
except (KeyError, IndexError, TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def format_stat(value, unit):
|
||||
if value is None:
|
||||
return ""
|
||||
|
||||
if unit == "s":
|
||||
return "%.0f ms" % (value * 1000) if value < 1 else "%.1f s" % value
|
||||
|
||||
if unit == "h":
|
||||
return "%.1f h" % (value / 3600.0)
|
||||
|
||||
if unit == "d":
|
||||
return "%.1f d" % (value / 86400.0)
|
||||
|
||||
if unit == "bytes":
|
||||
size = float(value)
|
||||
for suffix in ("B", "KiB", "MiB", "GiB", "TiB"):
|
||||
if size < 1024 or suffix == "TiB":
|
||||
return "%.1f %s" % (size, suffix)
|
||||
size /= 1024.0
|
||||
|
||||
if unit == "pct":
|
||||
return "%.0f%%" % value
|
||||
|
||||
return "%.0f" % value
|
||||
|
||||
|
||||
def evaluate(node):
|
||||
enabled = node.get("enabled")
|
||||
|
||||
if enabled:
|
||||
value = query(enabled)
|
||||
|
||||
if value is not None and value < 1:
|
||||
return STATUS_DISABLED
|
||||
|
||||
up = node.get("up")
|
||||
|
||||
if not up:
|
||||
return STATUS_OK
|
||||
|
||||
value = query(up)
|
||||
|
||||
if value is None:
|
||||
return STATUS_UNKNOWN
|
||||
|
||||
return STATUS_OK if value >= 1 else STATUS_FAIL
|
||||
|
||||
|
||||
def build():
|
||||
with open(SPEC_PATH) as handle:
|
||||
spec = json.load(handle)
|
||||
|
||||
nodes = []
|
||||
statuses = {}
|
||||
|
||||
for node in spec["nodes"]:
|
||||
status = evaluate(node)
|
||||
statuses[node["id"]] = status
|
||||
|
||||
entry = {
|
||||
"id": node["id"],
|
||||
"title": node["title"],
|
||||
"subtitle": node.get("subtitle", ""),
|
||||
"mainstat": format_stat(query(node["stat"]), node.get("stat_unit", "")) if node.get("stat") else "",
|
||||
"detail__group": node.get("group", ""),
|
||||
"detail__status": status,
|
||||
"detail__used": "",
|
||||
"dashboard": node.get("dashboard") or DASHBOARDS.get(node.get("group", ""), "/d/beepi-map"),
|
||||
}
|
||||
|
||||
for name in ARC_FIELDS:
|
||||
entry[name] = 0.0
|
||||
|
||||
used = query(node["fill"]) if node.get("fill") and status == STATUS_OK else None
|
||||
|
||||
if used is None:
|
||||
entry[ARCS[status]] = 1.0
|
||||
else:
|
||||
used = min(max(used, 0.0), 1.0)
|
||||
entry["arc__used"] = used
|
||||
entry["arc__free"] = 1.0 - used
|
||||
entry["detail__used"] = "%.0f%% used" % (used * 100)
|
||||
|
||||
nodes.append(entry)
|
||||
|
||||
edges = []
|
||||
|
||||
for index, edge in enumerate(spec["edges"]):
|
||||
source = statuses.get(edge["source"], STATUS_UNKNOWN)
|
||||
target = statuses.get(edge["target"], STATUS_UNKNOWN)
|
||||
degraded = STATUS_FAIL in (source, target)
|
||||
idle = STATUS_DISABLED in (source, target)
|
||||
|
||||
if degraded:
|
||||
color = COLORS[STATUS_FAIL]
|
||||
elif idle:
|
||||
color = COLORS[STATUS_DISABLED]
|
||||
else:
|
||||
color = COLORS[STATUS_OK]
|
||||
|
||||
edges.append({
|
||||
"id": str(index),
|
||||
"source": edge["source"],
|
||||
"target": edge["target"],
|
||||
"mainstat": edge.get("label", ""),
|
||||
"color": color,
|
||||
"thickness": 3 if degraded else 1,
|
||||
})
|
||||
|
||||
return {"nodes": nodes, "edges": edges}
|
||||
|
||||
|
||||
def cached():
|
||||
with _lock:
|
||||
now = time.time()
|
||||
|
||||
if _cache["body"] is None or now - _cache["at"] > CACHE_SECONDS:
|
||||
_cache["body"] = json.dumps(build()).encode()
|
||||
_cache["at"] = now
|
||||
|
||||
return _cache["body"]
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
if self.path.rstrip("/") not in ("", "/topology.json", "/topology"):
|
||||
self.send_error(404)
|
||||
return
|
||||
|
||||
try:
|
||||
body = cached()
|
||||
except Exception:
|
||||
self.send_error(500)
|
||||
return
|
||||
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, fmt, *args):
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
ThreadingHTTPServer(("0.0.0.0", LISTEN_PORT), Handler).serve_forever()
|
||||
@@ -0,0 +1,242 @@
|
||||
alert_groups:
|
||||
- name: backup
|
||||
interval: 5m
|
||||
rules:
|
||||
- uid: backup-daily-stale
|
||||
no_data: Alerting
|
||||
title: Daily backup stale
|
||||
expr: '(time() - max(backup_last_success_timestamp_seconds{backup_job="daily"})) / 3600'
|
||||
op: gt
|
||||
threshold: 26
|
||||
for: 15m
|
||||
severity: critical
|
||||
summary: No successful daily backup for more than 26 hours.
|
||||
|
||||
- uid: backup-weekly-stale
|
||||
no_data: Alerting
|
||||
title: Weekly backup stale
|
||||
expr: '(time() - max(backup_last_success_timestamp_seconds{backup_job="weekly"})) / 3600'
|
||||
op: gt
|
||||
threshold: 192
|
||||
for: 30m
|
||||
severity: critical
|
||||
summary: No successful weekly backup for more than 8 days.
|
||||
|
||||
- uid: backup-restoretest-stale
|
||||
no_data: Alerting
|
||||
title: Restore test stale
|
||||
expr: '(time() - max(backup_last_success_timestamp_seconds{backup_job="restoretest"})) / 3600'
|
||||
op: gt
|
||||
threshold: 192
|
||||
for: 30m
|
||||
severity: warning
|
||||
summary: The vaultwarden restore test has not passed in more than 8 days, backups are unverified.
|
||||
|
||||
- uid: backup-offsite-stale
|
||||
no_data: Alerting
|
||||
title: Offsite sync stale
|
||||
expr: '(time() - max(backup_last_success_timestamp_seconds{backup_job="offsite"})) / 3600'
|
||||
op: gt
|
||||
threshold: 26
|
||||
for: 15m
|
||||
severity: critical
|
||||
summary: Backups have not reached the Hetzner storage box in more than 26 hours.
|
||||
|
||||
- uid: backup-job-failed
|
||||
no_data: Alerting
|
||||
title: Backup job failed
|
||||
expr: 'min by (backup_job) (backup_last_status)'
|
||||
op: lt
|
||||
threshold: 1
|
||||
for: 5m
|
||||
severity: critical
|
||||
summary: The last run of this backup job exited with a failure.
|
||||
|
||||
- uid: backup-archive-shrunk
|
||||
title: Backup archive shrunk
|
||||
expr: 'backup_archive_newest_bytes{kind="daily"} / avg_over_time(backup_archive_newest_bytes{kind="daily"}[7d])'
|
||||
op: lt
|
||||
threshold: 0.5
|
||||
for: 1h
|
||||
severity: warning
|
||||
summary: The newest daily archive is less than half the size of the weekly average, data may be missing.
|
||||
|
||||
- uid: backup-raid-unmounted
|
||||
no_data: Alerting
|
||||
title: Backup RAID unmounted
|
||||
expr: 'min(backup_raid_mounted)'
|
||||
op: lt
|
||||
threshold: 1
|
||||
for: 5m
|
||||
severity: critical
|
||||
summary: The backup RAID is not mounted, backups cannot be written.
|
||||
|
||||
- uid: backup-raid-space
|
||||
title: Backup RAID low on space
|
||||
expr: 'min(node_filesystem_avail_bytes{mountpoint="/home/beeshare/raid"} / node_filesystem_size_bytes{mountpoint="/home/beeshare/raid"}) * 100'
|
||||
op: lt
|
||||
threshold: 15
|
||||
for: 30m
|
||||
severity: warning
|
||||
summary: Less than 15 percent free space left on the backup RAID.
|
||||
|
||||
- uid: offsite-empty
|
||||
no_data: Alerting
|
||||
title: Offsite storage empty
|
||||
expr: 'max(hetzner_remote_files{path="beepi"})'
|
||||
op: lt
|
||||
threshold: 1
|
||||
for: 30m
|
||||
severity: critical
|
||||
summary: The Hetzner storage box holds no completed backup files, offsite recovery is not possible.
|
||||
|
||||
- uid: offsite-unreachable
|
||||
no_data: Alerting
|
||||
title: Offsite storage unreachable
|
||||
expr: 'max(hetzner_reachable)'
|
||||
op: lt
|
||||
threshold: 1
|
||||
for: 2h
|
||||
severity: warning
|
||||
summary: The Hetzner storage box could not be listed, check external reachability on the box.
|
||||
|
||||
- uid: offsite-content-stale
|
||||
title: Offsite content stale
|
||||
expr: 'min(hetzner_remote_newest_age_seconds{path="beepi"}) / 3600'
|
||||
op: gt
|
||||
threshold: 48
|
||||
for: 1h
|
||||
severity: critical
|
||||
summary: The newest backup on the Hetzner storage box is more than 48 hours old.
|
||||
|
||||
- uid: backup-log-errors
|
||||
title: Backup log errors
|
||||
no_data: OK
|
||||
source: loki
|
||||
expr: 'sum(count_over_time({job="backup"} |~ "(?i)(FAILED|verify failed|refusing to sync)" [15m]))'
|
||||
op: gt
|
||||
threshold: 0
|
||||
for: 0s
|
||||
severity: critical
|
||||
summary: The backup log reported a failure in the last 15 minutes.
|
||||
|
||||
- name: services
|
||||
interval: 1m
|
||||
rules:
|
||||
- uid: service-probe-down
|
||||
title: Service probe failing
|
||||
expr: 'min by (service) (probe_success{vantage="local"}) and on (service) (max by (service) (service_enabled) == 1)'
|
||||
op: lt
|
||||
threshold: 1
|
||||
for: 5m
|
||||
severity: critical
|
||||
summary: A service that should be running is not answering its health probe.
|
||||
|
||||
- uid: service-unit-down
|
||||
title: Service unit not running
|
||||
expr: 'min by (service) (service_units_active / service_units_total) and on (service) (max by (service) (service_enabled) == 1)'
|
||||
op: lt
|
||||
threshold: 1
|
||||
for: 5m
|
||||
severity: critical
|
||||
summary: A unit or container backing an enabled service is not running.
|
||||
|
||||
- uid: service-vhost-down
|
||||
title: Public vhost failing
|
||||
expr: 'min by (service) (probe_success{vantage="nginx"}) and on (service) (max by (service) (service_enabled) == 1)'
|
||||
op: lt
|
||||
threshold: 1
|
||||
for: 5m
|
||||
severity: critical
|
||||
summary: An enabled service is not reachable through nginx and TLS.
|
||||
|
||||
- uid: external-check-down
|
||||
title: External check failing
|
||||
expr: 'min by (monitor) (uptimerobot_monitor_up) and on (monitor) (max by (monitor) (uptimerobot_monitor_enabled) == 1)'
|
||||
op: lt
|
||||
threshold: 1
|
||||
for: 5m
|
||||
severity: critical
|
||||
summary: UptimeRobot reports this endpoint as unreachable from outside the network.
|
||||
|
||||
- uid: service-cert-expiry
|
||||
title: TLS certificate expiring
|
||||
expr: 'min by (instance) (probe_ssl_earliest_cert_expiry - time()) / 86400'
|
||||
op: lt
|
||||
threshold: 14
|
||||
for: 1h
|
||||
severity: warning
|
||||
summary: A TLS certificate expires in less than 14 days.
|
||||
|
||||
- uid: scrape-target-down
|
||||
title: Scrape target down
|
||||
expr: 'min by (job, instance) (up{host!~"{{ alert_excluded_hosts }}"})'
|
||||
op: lt
|
||||
threshold: 1
|
||||
for: 10m
|
||||
severity: warning
|
||||
summary: Prometheus cannot scrape this target.
|
||||
|
||||
- name: infra
|
||||
interval: 1m
|
||||
rules:
|
||||
- uid: ingress-down
|
||||
title: Ingress host down
|
||||
expr: 'min(up{job="node",host="ingress"})'
|
||||
op: lt
|
||||
threshold: 1
|
||||
for: 5m
|
||||
severity: critical
|
||||
summary: The ingress VPS is not answering scrapes, inbound traffic is likely broken.
|
||||
|
||||
- uid: tunnel-peer-down
|
||||
title: WireGuard tunnel down
|
||||
expr: 'min by (peer) (wireguard_peer_up{peer=~"beepi|ingress"})'
|
||||
op: lt
|
||||
threshold: 1
|
||||
for: 5m
|
||||
severity: critical
|
||||
summary: The site to site WireGuard peer has not handshaked in the last five minutes.
|
||||
|
||||
- uid: tunnel-probe-down
|
||||
title: Tunnel path failing
|
||||
expr: 'min by (probe) (probe_success{vantage="ingress"})'
|
||||
op: lt
|
||||
threshold: 1
|
||||
for: 5m
|
||||
severity: critical
|
||||
summary: The ingress host cannot reach this port on beepi through the tunnel.
|
||||
|
||||
- uid: host-disk-space
|
||||
title: Host low on disk
|
||||
expr: >-
|
||||
min by (host, device)
|
||||
(node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|ramfs",host!~"{{ alert_excluded_hosts }}"}
|
||||
/ node_filesystem_size_bytes{fstype!~"tmpfs|overlay|ramfs",host!~"{{ alert_excluded_hosts }}"}) * 100
|
||||
op: lt
|
||||
threshold: 10
|
||||
for: 30m
|
||||
severity: warning
|
||||
summary: Less than 10 percent free space left on this filesystem.
|
||||
|
||||
- uid: host-memory
|
||||
title: Host low on memory
|
||||
expr: >-
|
||||
min by (host) (node_memory_MemAvailable_bytes{host!~"{{ alert_excluded_hosts }}"}
|
||||
/ node_memory_MemTotal_bytes{host!~"{{ alert_excluded_hosts }}"}) * 100
|
||||
op: lt
|
||||
threshold: 10
|
||||
for: 15m
|
||||
severity: warning
|
||||
summary: Less than 10 percent available memory on this host.
|
||||
|
||||
- uid: host-systemd-failed
|
||||
title: Failed systemd units
|
||||
expr: >-
|
||||
sum by (host) (node_systemd_unit_state{state="failed",
|
||||
host!~"{{ alert_excluded_hosts }}", name!~"{{ alert_ignored_units }}"})
|
||||
op: gt
|
||||
threshold: 0
|
||||
for: 15m
|
||||
severity: warning
|
||||
summary: A systemd unit outside the known-broken list is in a failed state on this host.
|
||||
@@ -1,36 +0,0 @@
|
||||
server {
|
||||
listen 80;
|
||||
server_name lounge.secretbee.buzz;
|
||||
|
||||
location / {
|
||||
return 301 https://$host$request_uri;
|
||||
}
|
||||
}
|
||||
|
||||
server {
|
||||
listen 443 ssl;
|
||||
server_name lounge.secretbee.buzz;
|
||||
|
||||
limit_req zone=mylimit burst=20;
|
||||
|
||||
add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
|
||||
|
||||
ssl_certificate /etc/letsencrypt/live/lounge.secretbee.buzz/fullchain.pem;
|
||||
ssl_certificate_key /etc/letsencrypt/live/lounge.secretbee.buzz/privkey.pem;
|
||||
include /etc/nginx/ssl.conf;
|
||||
|
||||
location / {
|
||||
proxy_pass http://127.0.0.1:9000/;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection "upgrade";
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header X-Forwarded-For $remote_addr;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_read_timeout 1d;
|
||||
}
|
||||
|
||||
location /folder/ {
|
||||
proxy_pass http://127.0.0.1:9000/uploads/;
|
||||
proxy_set_header X-Forwarded-For $remote_addr;
|
||||
}
|
||||
}
|
||||
@@ -41,6 +41,18 @@
|
||||
- "80"
|
||||
- "443"
|
||||
|
||||
- name: UFW - Allow monitoring scrapes from beepi
|
||||
community.general.ufw:
|
||||
rule: allow
|
||||
interface_in: wg0
|
||||
proto: tcp
|
||||
from_ip: "{{ wireguard_beepi_ip }}"
|
||||
to_port: '{{ item }}'
|
||||
loop:
|
||||
- "9100" # node exporter
|
||||
- "9115" # blackbox exporter
|
||||
- "12345" # alloy
|
||||
|
||||
- name: UFW - Allow wireguard outgoing
|
||||
community.general.ufw:
|
||||
route: true
|
||||
|
||||
@@ -111,6 +111,13 @@
|
||||
interface_out: eth0
|
||||
route: true
|
||||
|
||||
- name: UFW - Allow podman to wireguard
|
||||
community.general.ufw:
|
||||
rule: allow
|
||||
interface_in: podman+
|
||||
interface_out: wg0
|
||||
route: true
|
||||
|
||||
- name: UFW - Allow node exporter scraping
|
||||
community.general.ufw:
|
||||
rule: allow
|
||||
@@ -125,6 +132,31 @@
|
||||
proto: "tcp"
|
||||
to_port: 9882
|
||||
|
||||
- name: UFW - Allow blackbox exporter scraping
|
||||
community.general.ufw:
|
||||
rule: allow
|
||||
interface_in: podman+
|
||||
proto: "tcp"
|
||||
to_port: 9115
|
||||
|
||||
- name: UFW - Allow loki push from ingress
|
||||
community.general.ufw:
|
||||
rule: allow
|
||||
interface_in: wg0
|
||||
proto: tcp
|
||||
from_ip: "{{ wireguard_ingress_ip }}"
|
||||
to_port: "3100"
|
||||
|
||||
- name: UFW - Allow loki push routing from ingress
|
||||
community.general.ufw:
|
||||
rule: allow
|
||||
interface_in: wg0
|
||||
interface_out: podman+
|
||||
route: true
|
||||
proto: tcp
|
||||
from_ip: "{{ wireguard_ingress_ip }}"
|
||||
to_port: "3100"
|
||||
|
||||
- name: UFW - Enable and deny by default
|
||||
community.general.ufw:
|
||||
state: enabled
|
||||
|
||||
Reference in New Issue
Block a user