diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 58517cd..e032102 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -10,6 +10,36 @@ "command": "ansible-vault encrypt_string ${input:pw} --name ${input:name} --vault-password-file .vault-password", "problemMatcher": [] }, + { + "label": "Show vault secrets in current file", + "type": "shell", + "options": { + "cwd": "${workspaceFolder}" + }, + "command": "python3 vaultShow.py ${relativeFile}", + "presentation": { + "reveal": "always", + "panel": "dedicated", + "echo": false, + "clear": true + }, + "problemMatcher": [] + }, + { + "label": "Show one vault secret in current file", + "type": "shell", + "options": { + "cwd": "${workspaceFolder}" + }, + "command": "python3 vaultShow.py ${relativeFile} ${input:varName}", + "presentation": { + "reveal": "always", + "panel": "dedicated", + "echo": false, + "clear": true + }, + "problemMatcher": [] + }, { "label": "Deploy: nginx", "type": "shell", @@ -234,6 +264,11 @@ "id": "name", "type": "promptString", "description": "Name for the string" + }, + { + "id": "varName", + "type": "promptString", + "description": "Variable name to decrypt" } ] } \ No newline at end of file diff --git a/group_vars/all.yml b/group_vars/all.yml new file mode 100644 index 0000000..4ecccb8 --- /dev/null +++ b/group_vars/all.yml @@ -0,0 +1,198 @@ +wireguard_ingress_ip: 10.10.1.1 +wireguard_beepi_ip: 10.10.1.2 + +node_exporter_textfile_dir: /var/lib/node_exporter/textfile + +geoip_max_age_seconds: 86400 + +alert_excluded_hosts: beeserver + +alert_ignored_units: exim4.service|openipmi.service|smartmontools.service|wg-quick@wg0.service + +discord_webhook: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 66663162633962616438306339616364343634633030663665393934323831303162326632613637 + 3531373835656236326436623036636665356433303762350a613366376231353362353961393566 + 36643136616537666632343463333334636136383232346233353963303334616261613265396632 + 3835316435393935350a373265366239393435373734396538323965326366343832373561366539 + 64613966326233303336653738386161376331373962623764633261353462636231666439643735 + 35333564323831626430373030313861396132363336326639303962326661386533383661333264 + 34663435646434316431343936343131326462646632393833623132666335386134373037323433 + 33373738303162386430616465383766363266333765313533373161376262306532346265303062 + 30393164313431613734323832626637346537656334623232373531633139653064666132373863 + 3762666465303665303666396537323062396139313764353263 + +uptimerobot_api_key: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 62643666393833356630356130373262336130336135393062626130343530363430616532303338 + 6265393131663863323832616334353835386332373037640a333164303034656664643330626532 + 66643238393436653665346538343339346435643934333830636136306565363537336364363038 + 3131653631623961340a373263323362323533353638313361353566383039613662306434366633 + 62386430663466646430373735636132356164393265666634326535333265616233346530663063 + 3935356539346134303038333632373163316438633136396466 + +grafana_prometheus_uid: ffkc0x7oj7g1sc +grafana_loki_uid: bfkfq6kdpznr4e +grafana_infinity_uid: beepitopology +grafana_uptimerobot_uid: beepiuptimerobot +grafana_folder: Beepi +grafana_folder_uid: efva4v6nh301sd +grafana_general_folder: General Dashboards +grafana_general_folder_uid: bfkfzgwdpkqgwc + +wireguard_peer_names: + r0XaxIE3AVud9Z42UPZF3mwoh2R0B0M1EUDY9cvuyDA=: ingress + oI69uWkbkwtpuIYxDNGmzg4nRjY1BIZFVFjZwguIvwE=: beepi + Jq1FhThPekHlc69t5UcugV9e7Z/Es9lpPR9Ttc0GICk=: phone + +monitored_services: + - name: gitea + title: Gitea + group: apps + enabled: true + owner: gitea + units: + - gitea-db.service + - gitea-server.service + vhost: git.secretbee.buzz + probe: http://127.0.0.1:3000/api/healthz + tcp: + - name: gitea-ssh + target: 127.0.0.1:2222 + + - name: immich + title: Immich + group: apps + enabled: true + owner: immich + units: + - immich-db.service + - immich-redis.service + - immich-server.service + vhost: gallery.secretbee.buzz + probe: http://127.0.0.1:2283/api/server/ping + + - name: vaultwarden + title: Vaultwarden + group: apps + enabled: true + owner: vaultwarden + units: + - vaultwarden.service + vhost: vault.secretbee.buzz + probe: http://127.0.0.1:8222/alive + + - name: ott + title: OpenTogetherTube + group: apps + enabled: false + owner: ott + units: + - ott-db.service + - ott-redis.service + - ott-server.service + vhost: ott.secretbee.buzz + probe: http://127.0.0.1:8080/ + + - name: matrix + title: Matrix + group: apps + enabled: true + containers: + - matrix_homeserver_1 + vhost: matrix.secretbee.buzz + probe: http://127.0.0.1:8008/_matrix/client/versions + tcp: + - name: matrix-federation + target: 127.0.0.1:8448 + + - name: mail + title: Mail + group: apps + enabled: true + containers: + - mailserver + tcp: + - name: smtp + target: 127.0.0.1:25 + - name: submission + target: 127.0.0.1:587 + - name: smtps + target: 127.0.0.1:465 + - name: imaps + target: 127.0.0.1:993 + + - name: nginx + title: Nginx + group: platform + enabled: true + system_units: + - nginx.service + vhost: secretbee.buzz + + - name: prometheus + title: Prometheus + group: platform + enabled: true + containers: + - prometheus + probe: http://127.0.0.1:9090/-/healthy + + - name: grafana + title: Grafana + group: platform + enabled: true + containers: + - grafana + vhost: grafana.secretbee.buzz + probe: http://127.0.0.1:4000/api/health + + - name: loki + title: Loki + group: platform + enabled: true + containers: + - loki + probe: http://127.0.0.1:3100/ready + + - name: alloy + title: Alloy + group: platform + enabled: true + containers: + - alloy + probe: http://127.0.0.1:12345/-/ready + +ingress_probe_targets: + - name: nginx-http + target: 10.10.1.2:80 + - name: nginx-https + target: 10.10.1.2:443 + - name: matrix-federation + target: 10.10.1.2:8448 + - name: smtp + target: 10.10.1.2:25 + - name: backup-ssh + target: 10.10.1.2:4444 + +backup_jobs: + - name: daily + title: Daily backup + max_age_hours: 26 + - name: weekly + title: Weekly backup + max_age_hours: 192 + - name: restoretest + title: Restore test + max_age_hours: 192 + - name: offsite + title: Offsite sync + max_age_hours: 26 + +hetzner_host: u651287.your-storagebox.de +hetzner_user: u651287 +hetzner_port: 23 +hetzner_box_bytes: 1099511627776 + +backup_archive_dir: /home/beeshare/raid/serverBackup +backup_raid_mount: /home/beeshare/raid diff --git a/makefile b/makefile index a3dbc14..813c6c2 100644 --- a/makefile +++ b/makefile @@ -8,4 +8,7 @@ push-%: build-% podman push $(REGISTRY)/bee/$*:latest deploy-%: - ansible-playbook -i $(INVENTORY) playbooks/$*/$*.yml --vault-password-file .vault-password $(ARGS) \ No newline at end of file + ansible-playbook -i $(INVENTORY) playbooks/$*/$*.yml --vault-password-file .vault-password $(ARGS) + +secrets-%: + @python3 vaultShow.py playbooks/$*/$*.yml $(ARGS) \ No newline at end of file diff --git a/playbooks/backup/backup.yml b/playbooks/backup/backup.yml index 098948e..c11565f 100644 --- a/playbooks/backup/backup.yml +++ b/playbooks/backup/backup.yml @@ -41,12 +41,72 @@ dest: /opt/backup/restoreTest.sh mode: '0700' + - name: Copy metrics library + ansible.builtin.template: + src: metrics.sh.j2 + dest: /opt/backup/metrics.sh + mode: '0700' + + - name: Copy inventory collector + ansible.builtin.template: + src: backupInventory.sh.j2 + dest: /opt/backup/backupInventory.sh + mode: '0700' + - name: Copy passphrase ansible.builtin.template: src: passphrase.txt.j2 dest: /opt/backup/passphrase.txt mode: '0600' + - name: Create textfile directory + ansible.builtin.file: + path: "{{ node_exporter_textfile_dir }}" + state: directory + mode: '0755' + + - name: Install inventory timer + ansible.builtin.copy: + dest: "/etc/systemd/system/beepi-backup-inventory.{{ item.kind }}" + content: "{{ item.content }}" + mode: '0644' + loop: + - kind: service + content: | + [Unit] + Description=Collect backup inventory metrics + + [Service] + Type=oneshot + ExecStart=/opt/backup/backupInventory.sh + - kind: timer + content: | + [Unit] + Description=Collect backup inventory metrics + + [Timer] + OnBootSec=2min + OnUnitActiveSec=15min + AccuracySec=1min + + [Install] + WantedBy=timers.target + notify: Reload systemd + + - name: Flush handlers + ansible.builtin.meta: flush_handlers + + - name: Enable inventory timer + ansible.builtin.systemd_service: + name: beepi-backup-inventory.timer + state: started + enabled: true + + - name: Run inventory collector once + changed_when: true + ansible.builtin.command: + cmd: /opt/backup/backupInventory.sh + - name: Set path ansible.builtin.cron: name: PATH @@ -78,3 +138,8 @@ hour: '6' weekday: '0' job: /opt/backup/restoreTest.sh >> /var/log/beepi-backup.log 2>&1 + + handlers: + - name: Reload systemd + ansible.builtin.systemd_service: + daemon_reload: true diff --git a/playbooks/backup/backupDaily.sh b/playbooks/backup/backupDaily.sh index eeee364..953909f 100644 --- a/playbooks/backup/backupDaily.sh +++ b/playbooks/backup/backupDaily.sh @@ -2,6 +2,8 @@ set -euo pipefail +source /opt/backup/metrics.sh + now=$(date +"%Y%m%d%H%M") declare -a arr=("mail" "matrix" "monitoring") @@ -12,6 +14,8 @@ declare -a immich=("systemd-immich-pgdata" "systemd-immich-upload") gitea_uid=$(id -u gitea) immich_uid=$(id -u immich) +metric_init daily + restore() { for i in "${arr[@]}"; do cd /opt/beeserver/${i} @@ -32,10 +36,21 @@ restore() { rm -f /home/immich/systemd-immich-*.tar } -trap restore EXIT +finish() { + local raw=$? + local status + + status=$(metric_status ${raw}) + restore + metric_finish ${status} + exit ${status} +} + +trap finish EXIT for i in "${arr[@]}" do + log "staging ${i}" cd /opt/beeserver/${i} podman-compose down -t 60 if [[ "$i" == "monitoring" ]] @@ -57,10 +72,12 @@ do podman-compose up -d done +log "staging gitea" systemctl --user --machine gitea@.host stop gitea-server.service gitea-db.service rsync -aH /home/gitea/data /opt/backup/beeserver/gitea systemctl --user --machine gitea@.host start gitea-db.service gitea-server.service +log "staging immich" systemctl --user --machine immich@.host stop immich-server.service immich-db.service immich-redis.service for v in "${immich[@]}" do @@ -70,17 +87,22 @@ systemctl --user --machine immich@.host start immich-db.service immich-redis.ser mkdir -p /opt/backup/beeserver/immich/ mv /home/immich/systemd-immich-*.tar /opt/backup/beeserver/immich/ +log "staging vaultwarden" systemctl --user --machine vaultwarden@.host stop vaultwarden.service rsync -aH /home/vaultwarden/data /opt/backup/beeserver/vaultwarden systemctl --user --machine vaultwarden@.host start vaultwarden.service archive=/home/beeshare/raid/serverBackup/beeserver_daily_${now}.tar.gz.gpg +log "encrypting ${archive}" tar -czf - /opt/backup/beeserver | gpg --batch --passphrase-file /opt/backup/passphrase.txt -c -o ${archive} +log "verifying ${archive}" if ! gpg --batch --passphrase-file /opt/backup/passphrase.txt -d ${archive} | tar -tzf - >/dev/null then - echo "[${now}] verify failed, discarding ${archive}" + log "verify failed, discarding ${archive}" rm -f ${archive} exit 1 fi + +metric_archive ${archive} diff --git a/playbooks/backup/backupInventory.sh.j2 b/playbooks/backup/backupInventory.sh.j2 new file mode 100644 index 0000000..6a05bb4 --- /dev/null +++ b/playbooks/backup/backupInventory.sh.j2 @@ -0,0 +1,65 @@ +#!/bin/bash + +set -uo pipefail + +textfile_dir={{ node_exporter_textfile_dir }} +archive_dir={{ backup_archive_dir }} +raid_mount={{ backup_raid_mount }} + +file=${textfile_dir}/backup_inventory.prom +tmp=${file}.$$ +now=$(date +%s) + +mkdir -p ${textfile_dir} + +mounted=0 +if mountpoint -q ${raid_mount} +then + mounted=1 +fi + +emit_kind() { + local kind=$1 + local newest + local count=0 + local age=0 + local bytes=0 + local total=0 + + if [ ${mounted} -eq 1 ] + then + count=$(find ${archive_dir} -maxdepth 1 -name "beeserver_${kind}_*.tar.gz.gpg" -type f 2>/dev/null | wc -l) + newest=$(find ${archive_dir} -maxdepth 1 -name "beeserver_${kind}_*.tar.gz.gpg" -type f -printf '%T@ %p\n' 2>/dev/null | sort -rn | head -1 | cut -d' ' -f2-) + total=$(find ${archive_dir} -maxdepth 1 -name "beeserver_${kind}_*.tar.gz.gpg" -type f -printf '%s\n' 2>/dev/null | awk '{ s += $1 } END { print s + 0 }') + + if [ -n "${newest:-}" ] + then + bytes=$(stat -c %s "${newest}") + age=$((now - $(stat -c %Y "${newest}"))) + fi + fi + + echo "backup_archive_count{kind=\"${kind}\"} ${count}" + echo "backup_archive_newest_age_seconds{kind=\"${kind}\"} ${age}" + echo "backup_archive_newest_bytes{kind=\"${kind}\"} ${bytes}" + echo "backup_archive_total_bytes{kind=\"${kind}\"} ${total}" +} + +{ + echo "# HELP backup_raid_mounted Whether the backup raid is mounted." + echo "# TYPE backup_raid_mounted gauge" + echo "backup_raid_mounted ${mounted}" + echo "# HELP backup_archive_count Number of retained archives." + echo "# TYPE backup_archive_count gauge" + echo "# HELP backup_archive_newest_age_seconds Age of the newest archive." + echo "# TYPE backup_archive_newest_age_seconds gauge" + echo "# HELP backup_archive_newest_bytes Size of the newest archive." + echo "# TYPE backup_archive_newest_bytes gauge" + echo "# HELP backup_archive_total_bytes Size of all retained archives." + echo "# TYPE backup_archive_total_bytes gauge" + emit_kind daily + emit_kind weekly +} > "${tmp}" + +chmod 0644 "${tmp}" +mv "${tmp}" "${file}" diff --git a/playbooks/backup/backupWeekly.sh b/playbooks/backup/backupWeekly.sh index 20c7c62..21cdaa4 100644 --- a/playbooks/backup/backupWeekly.sh +++ b/playbooks/backup/backupWeekly.sh @@ -2,20 +2,30 @@ set -euo pipefail +source /opt/backup/metrics.sh + now=$(date +"%Y%m%d%H%M") archive=/home/beeshare/raid/serverBackup/beeserver_weekly_${now}.tar.gz.gpg +metric_init weekly +trap metric_guard EXIT + +log "encrypting ${archive}" tar -czf - /opt/backup/beeserver | gpg --batch --passphrase-file /opt/backup/passphrase.txt -c -o ${archive} +log "verifying ${archive}" if ! gpg --batch --passphrase-file /opt/backup/passphrase.txt -d ${archive} | tar -tzf - >/dev/null then - echo "[${now}] verify failed, discarding ${archive} and keeping staging" + log "verify failed, discarding ${archive} and keeping staging" rm -f ${archive} exit 1 fi +metric_archive ${archive} + rm -rf /opt/backup/beeserver +log "pruning expired archives" find /home/beeshare/raid/serverBackup -maxdepth 1 -name 'beeserver_weekly_*.tar.gz.gpg' -mtime +30 -delete -find /home/beeshare/raid/serverBackup -maxdepth 1 -name 'beeserver_daily_*.tar.gz.gpg' -mtime +7 -delete \ No newline at end of file +find /home/beeshare/raid/serverBackup -maxdepth 1 -name 'beeserver_daily_*.tar.gz.gpg' -mtime +7 -delete diff --git a/playbooks/backup/metrics.sh.j2 b/playbooks/backup/metrics.sh.j2 new file mode 100644 index 0000000..f5af296 --- /dev/null +++ b/playbooks/backup/metrics.sh.j2 @@ -0,0 +1,104 @@ +#!/bin/bash + +textfile_dir={{ node_exporter_textfile_dir }} + +metric_job="" +metric_started=0 +metric_bytes=0 +metric_signalled="" + +metric_on_signal() { + metric_signalled=$1 + exit $((128 + $1)) +} + +metric_install_traps() { + trap 'metric_on_signal 1' HUP + trap 'metric_on_signal 2' INT + trap 'metric_on_signal 15' TERM +} + +metric_status() { + if [ -n "${metric_signalled}" ] + then + echo $((128 + metric_signalled)) + else + echo "$1" + fi +} + +log() { + echo "[$(date --rfc-3339=seconds)] ${metric_job}: $*" +} + +metric_previous_success() { + local file=${textfile_dir}/backup_${metric_job}.prom + + if [ -r "${file}" ] + then + grep "^backup_last_success_timestamp_seconds{backup_job=\"${metric_job}\"} " "${file}" | awk '{ print $2 }' + fi +} + +metric_init() { + metric_job=$1 + metric_started=$(date +%s) + mkdir -p ${textfile_dir} + metric_install_traps + log "start" +} + +metric_archive() { + metric_bytes=$(stat -c %s "$1" 2>/dev/null || echo 0) +} + +metric_finish() { + local status=$1 + local now + local success + local previous + local file=${textfile_dir}/backup_${metric_job}.prom + local tmp=${file}.$$ + + now=$(date +%s) + previous=$(metric_previous_success) + + if [ "${status}" -eq 0 ] + then + success=${now} + log "done in $((now - metric_started))s" + else + success=${previous:-0} + log "FAILED with status ${status} after $((now - metric_started))s" + fi + + cat > "${tmp}" </dev/null 2>&1 || true rm -rf "${work}" + metric_finish ${status} + exit ${status} } trap cleanup EXIT latest=$(ls -1t ${backup_dir}/beeserver_*.tar.gz.gpg | head -1) -echo "[${now}] restore test using ${latest}" +log "restore test using ${latest}" gpg --batch --passphrase-file /opt/backup/passphrase.txt -d "${latest}" \ | tar -xzf - -C "${work}" opt/backup/beeserver/vaultwarden @@ -26,15 +36,15 @@ data="${work}/opt/backup/beeserver/vaultwarden/data" [[ -s "${data}/db.sqlite3" ]] [[ -s "${data}/rsa_key.pem" ]] -echo " db and rsa_key present" +log "db and rsa_key present" integrity=$(sqlite3 "${data}/db.sqlite3" "PRAGMA integrity_check;") [[ "${integrity}" == "ok" ]] -echo " integrity_check ok" +log "integrity_check ok" users=$(sqlite3 "${data}/db.sqlite3" "select count(*) from users;") [[ "${users}" -ge 1 ]] -echo " users ${users}" +log "users ${users}" email=$(sqlite3 "${data}/db.sqlite3" "select email from users limit 1;") @@ -51,12 +61,14 @@ done config=$(curl -sf http://127.0.0.1:${port}/api/config) [[ "${config}" == *'"version"'* ]] -echo " api/config ok" +log "api/config ok" prelogin=$(curl -sf -X POST http://127.0.0.1:${port}/identity/accounts/prelogin \ -H 'Content-Type: application/json' \ -d "{\"email\":\"${email}\"}") [[ "${prelogin}" == *[Kk]df* ]] -echo " prelogin ok for ${email}" +log "prelogin ok for ${email}" -echo "[${now}] restore test PASSED" +metric_archive "${latest}" + +log "restore test PASSED" diff --git a/playbooks/dashboards/dashboards.yml b/playbooks/dashboards/dashboards.yml new file mode 100644 index 0000000..d047617 --- /dev/null +++ b/playbooks/dashboards/dashboards.yml @@ -0,0 +1,84 @@ +- name: Dashboards + hosts: beepi + become: true + vars: + dashboard_dir: /opt/beeserver/monitoring/grafana/dashboards + dashboard_general_dir: /opt/beeserver/monitoring/grafana/dashboards-general + provisioning_dir: /opt/beeserver/monitoring/grafana/provisioning/dashboards + tasks: + - name: Determine whether uptimerobot is configured + ansible.builtin.set_fact: + uptimerobot_enabled: "{{ uptimerobot_api_key is defined and 'REPLACE_ME' not in uptimerobot_api_key }}" + + - name: Check that grafana is running + ansible.builtin.command: + cmd: podman container inspect -f '{{ '{{' }}.State.Running{{ '}}' }}' grafana + register: grafana_running + changed_when: false + failed_when: false + + - name: Require a running grafana + ansible.builtin.assert: + that: + - grafana_running.stdout | trim == 'true' + fail_msg: >- + grafana is not running, so provisioned changes cannot be picked up. + Run make deploy-monitoring first. + + - name: Create dashboard directories + ansible.builtin.file: + path: "{{ item }}" + state: directory + mode: '0755' + loop: + - "{{ dashboard_dir }}" + - "{{ dashboard_general_dir }}" + - "{{ provisioning_dir }}" + + - name: Copy dashboard provider + ansible.builtin.template: + src: ../monitoring/grafana/provisioning/dashboards/dashboards.yml.j2 + dest: "{{ provisioning_dir }}/dashboards.yml" + mode: '0644' + notify: Restart grafana + + - name: Copy generated dashboards + ansible.builtin.template: + src: "../monitoring/grafana/dashboards/{{ item }}.json.j2" + dest: "{{ dashboard_dir }}/{{ item }}.json" + mode: '0644' + validate: "python3 -c \"import json; json.load(open('%s'))\"" + loop: + - infrastructure-map + - backups + - services + - ingress + + - name: Copy uptimerobot dashboard + ansible.builtin.template: + src: ../monitoring/grafana/dashboards/uptimerobot.json.j2 + dest: "{{ dashboard_dir }}/uptimerobot.json" + mode: '0644' + validate: "python3 -c \"import json; json.load(open('%s'))\"" + when: uptimerobot_enabled + + - name: Copy hand made dashboards + ansible.builtin.copy: + src: ../monitoring/grafana/dashboards-general/ + dest: "{{ dashboard_general_dir }}/" + mode: '0644' + + - name: Debug + ansible.builtin.debug: + msg: + - "Dashboards are polled by grafana every 30s, no restart needed." + - "Grafana: https://grafana.secretbee.buzz/" + - "Errors: sudo podman logs --tail 40 grafana | grep -i provision" + tags: + - always + + handlers: + - name: Restart grafana + changed_when: true + ansible.builtin.command: + cmd: podman restart grafana diff --git a/playbooks/hetzner/hetzner.yml b/playbooks/hetzner/hetzner.yml index 09ea6c0..133d4cc 100644 --- a/playbooks/hetzner/hetzner.yml +++ b/playbooks/hetzner/hetzner.yml @@ -6,7 +6,6 @@ user_home: /home/hetzner hetzner_raid_mount: /home/beeshare/raid - hetzner_port: 23 hetzner_paths: - src: /home/beeshare/raid/serverBackup @@ -14,10 +13,6 @@ - src: /home/beeshare/raid/foundry/backup dest: ./foundry - # Fill in after creating the storage box sub-account, then re-run - hetzner_host: "u651287.your-storagebox.de" - hetzner_user: "u651287" - tasks: - name: Install rsync ansible.builtin.apt: @@ -88,6 +83,58 @@ job: /opt/backup/hetznerSync.sh >> /var/log/beepi-backup.log 2>&1 when: hetzner_host | length > 0 + - name: Copy metrics collector + ansible.builtin.template: + src: hetznerMetrics.sh.j2 + dest: /opt/backup/hetznerMetrics.sh + mode: '0700' + when: hetzner_host | length > 0 + + - name: Install metrics timer + ansible.builtin.copy: + dest: "/etc/systemd/system/beepi-hetzner-metrics.{{ item.kind }}" + content: "{{ item.content }}" + mode: '0644' + loop: + - kind: service + content: | + [Unit] + Description=Collect hetzner storage box metrics + + [Service] + Type=oneshot + ExecStart=/opt/backup/hetznerMetrics.sh + - kind: timer + content: | + [Unit] + Description=Collect hetzner storage box metrics + + [Timer] + OnBootSec=5min + OnUnitActiveSec=1h + AccuracySec=5min + + [Install] + WantedBy=timers.target + when: hetzner_host | length > 0 + notify: Reload systemd + + - name: Flush handlers + ansible.builtin.meta: flush_handlers + + - name: Enable metrics timer + ansible.builtin.systemd_service: + name: beepi-hetzner-metrics.timer + state: started + enabled: true + when: hetzner_host | length > 0 + + - name: Run metrics collector once + changed_when: true + ansible.builtin.command: + cmd: /opt/backup/hetznerMetrics.sh + when: hetzner_host | length > 0 + - name: Debug ansible.builtin.debug: msg: @@ -95,3 +142,8 @@ - "Read logs: sudo tail -50 /var/log/beepi-backup.log" tags: - always + + handlers: + - name: Reload systemd + ansible.builtin.systemd_service: + daemon_reload: true diff --git a/playbooks/hetzner/hetznerMetrics.sh.j2 b/playbooks/hetzner/hetznerMetrics.sh.j2 new file mode 100644 index 0000000..ea7a87b --- /dev/null +++ b/playbooks/hetzner/hetznerMetrics.sh.j2 @@ -0,0 +1,77 @@ +#!/bin/bash + +set -uo pipefail + +textfile_dir={{ node_exporter_textfile_dir }} +file=${textfile_dir}/hetzner.prom +tmp=${file}.$$ +now=$(date +%s) + +ssh_opts="-p {{ hetzner_port }} -i {{ user_home }}/.ssh/id_ed25519 -o UserKnownHostsFile={{ user_home }}/.ssh/known_hosts -o StrictHostKeyChecking=yes -o BatchMode=yes -o ConnectTimeout=15" + +mkdir -p ${textfile_dir} + +reachable=0 +body="" + +emit_path() { + local label=$1 + local remote=$2 + local listing + local files + local bytes + local pending + local newest + local age=-1 + + if ! listing=$(sudo -u {{ user }} rsync --list-only -e "ssh ${ssh_opts}" {{ hetzner_user }}@{{ hetzner_host }}:${remote}/ 2>/dev/null) + then + body+="hetzner_remote_listing_ok{path=\"${label}\"} 0"$'\n' + return + fi + + reachable=1 + + files=$(echo "${listing}" | awk '$1 ~ /^-/ && substr($5,1,1) != "." { c++ } END { print c + 0 }') + bytes=$(echo "${listing}" | awk '$1 ~ /^-/ && substr($5,1,1) != "." { gsub(",", "", $2); s += $2 } END { printf "%.0f", s + 0 }') + pending=$(echo "${listing}" | awk '$1 ~ /^-/ && substr($5,1,1) == "." { gsub(",", "", $2); s += $2 } END { printf "%.0f", s + 0 }') + newest=$(echo "${listing}" | awk '$1 ~ /^-/ && substr($5,1,1) != "." { print $3" "$4 }' | sort -r | head -1) + + if [ -n "${newest}" ] + then + age=$((now - $(date -d "$(echo ${newest} | tr '/' '-')" +%s))) + fi + + body+="hetzner_remote_listing_ok{path=\"${label}\"} 1"$'\n' + body+="hetzner_remote_files{path=\"${label}\"} ${files}"$'\n' + body+="hetzner_remote_bytes{path=\"${label}\"} ${bytes}"$'\n' + body+="hetzner_remote_pending_bytes{path=\"${label}\"} ${pending}"$'\n' + body+="hetzner_remote_newest_age_seconds{path=\"${label}\"} ${age}"$'\n' +} + +{% for path in hetzner_paths %} +emit_path "{{ path.dest | replace('./', '') }}" "{{ path.dest }}" +{% endfor %} + +{ + echo "# HELP hetzner_reachable Whether the storage box answered at least one listing." + echo "# TYPE hetzner_reachable gauge" + echo "hetzner_reachable ${reachable}" + echo "# HELP hetzner_last_check_timestamp_seconds Unix time of the last storage box check." + echo "# TYPE hetzner_last_check_timestamp_seconds gauge" + echo "hetzner_last_check_timestamp_seconds ${now}" + echo "# HELP hetzner_remote_listing_ok Whether this remote path could be listed." + echo "# TYPE hetzner_remote_listing_ok gauge" + echo "# HELP hetzner_remote_files Number of completed files stored at this remote path." + echo "# TYPE hetzner_remote_files gauge" + echo "# HELP hetzner_remote_bytes Bytes of completed files stored at this remote path." + echo "# TYPE hetzner_remote_bytes gauge" + echo "# HELP hetzner_remote_pending_bytes Bytes of in flight partial transfers at this remote path." + echo "# TYPE hetzner_remote_pending_bytes gauge" + echo "# HELP hetzner_remote_newest_age_seconds Age of the newest completed file at this remote path." + echo "# TYPE hetzner_remote_newest_age_seconds gauge" + printf '%s' "${body}" +} > "${tmp}" + +chmod 0644 "${tmp}" +mv "${tmp}" "${file}" diff --git a/playbooks/hetzner/hetznerSync.sh.j2 b/playbooks/hetzner/hetznerSync.sh.j2 index a2870f8..c7be75d 100644 --- a/playbooks/hetzner/hetznerSync.sh.j2 +++ b/playbooks/hetzner/hetznerSync.sh.j2 @@ -2,11 +2,22 @@ set -euo pipefail -now=$(date +"%Y%m%d%H%M") +exec 9>/var/lock/hetznerSync.lock + +if ! flock -n 9 +then + echo "[$(date --rfc-3339=seconds)] offsite: another sync is still running, skipping this run" + exit 0 +fi + +source /opt/backup/metrics.sh + +metric_init offsite +trap metric_guard EXIT if ! mountpoint -q {{ hetzner_raid_mount }} then - echo "[${now}] {{ hetzner_raid_mount }} not mounted, refusing to sync" + log "{{ hetzner_raid_mount }} not mounted, refusing to sync" exit 1 fi @@ -16,19 +27,17 @@ sync_path() { if [ -z "$(ls -A ${src} 2>/dev/null)" ] then - echo "[${now}] ${src} empty, refusing to sync" + log "${src} empty, refusing to sync" exit 1 fi - echo "[${now}] syncing ${src} to ${dest}" + log "syncing ${src} to ${dest}" sudo -u {{ user }} rsync -a --delete --partial \ -e "ssh -p {{ hetzner_port }} -i {{ user_home }}/.ssh/id_ed25519 -o UserKnownHostsFile={{ user_home }}/.ssh/known_hosts -o StrictHostKeyChecking=yes" \ ${src}/ \ {{ hetzner_user }}@{{ hetzner_host }}:${dest}/ } -echo "[${now}] offsite sync start" {% for path in hetzner_paths %} sync_path {{ path.src }} {{ path.dest }} {% endfor %} -echo "[${now}] offsite sync done" diff --git a/playbooks/monitoring-ingress/config.alloy.j2 b/playbooks/monitoring-ingress/config.alloy.j2 new file mode 100644 index 0000000..1cf6cdf --- /dev/null +++ b/playbooks/monitoring-ingress/config.alloy.j2 @@ -0,0 +1,102 @@ +loki.relabel "journal" { + forward_to = [] + + rule { + source_labels = ["__journal__systemd_unit"] + target_label = "unit" + } + + rule { + source_labels = ["__journal_priority_keyword"] + target_label = "level" + } +} + +loki.source.journal "system" { + forward_to = [loki.process.journal.receiver] + relabel_rules = loki.relabel.journal.rules + labels = { + job = "systemd-journal", + host = "ingress", + } +} + +loki.process "journal" { + forward_to = [loki.write.default.receiver] + + stage.match { + selector = `{job="systemd-journal"} |~ "\\[UFW AUDIT\\]"` + action = "drop" + } + + stage.match { + selector = `{job="systemd-journal"} |~ "\\[UFW BLOCK\\]"` + + stage.regex { + expression = `\[UFW\s(?P\w+)\].*?SRC=(?P[0-9.]+).*?DST=(?P[0-9.]+).*?PROTO=(?P\w+)(?:\sSPT=(?P\d+))?(?:\sDPT=(?P\d+))?` + } + + stage.geoip { + db = "/opt/database/GeoLite2-City.mmdb" + source = "ip" + db_type = "city" + } + + stage.labels { + values = { + action = "", + proto = "", + dpt = "", + geoip_country_name = "", + geoip_country_code = "", + geoip_location_latitude = "", + geoip_location_longitude = "", + } + } + + stage.static_labels { + values = { + job = "ufw", + } + } + } +} + +local.file_match "fail2ban" { + path_targets = [{"__path__" = "/var/log/fail2ban.log", job = "fail2ban", host = "ingress"}] +} + +loki.source.file "fail2ban_source" { + targets = local.file_match.fail2ban.targets + forward_to = [loki.process.fail2ban.receiver] +} + +loki.process "fail2ban" { + forward_to = [loki.write.default.receiver] + + stage.regex { + expression = `^.*:\s(?\w*)\s*\[(?\w*)\]\s(?\w*)\s(?(?:(?:2(?:[0-4][0-9]|5[0-5])|[0-1]?[0-9]?[0-9])\.){3}(?:(?:2([0-4][0-9]|5[0-5])|[0-1]?[0-9]?[0-9]))).*$` + labels_from_groups = true + } + + stage.geoip { + db = "/opt/database/GeoLite2-City.mmdb" + source = "ip" + db_type = "city" + } + + stage.labels { + values = { + geoip_country_name = "", + geoip_country_code = "", + geoip_location_latitude = "", + geoip_location_longitude = "", + } + } +} + +loki.write "default" { + endpoint { + url = "http://{{ wireguard_beepi_ip }}:3100/loki/api/v1/push" + } +} diff --git a/playbooks/monitoring-ingress/monitoring-ingress.yml b/playbooks/monitoring-ingress/monitoring-ingress.yml new file mode 100644 index 0000000..a991f65 --- /dev/null +++ b/playbooks/monitoring-ingress/monitoring-ingress.yml @@ -0,0 +1,297 @@ +- name: Monitoring-ingress + hosts: animeistrash + become: true + vars: + geoip_username: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 30656365643463396265633130356339386433633331613130633134336633643637653761623962 + 3633336466326564616432373031303530376166353436610a323033353164313436613363613836 + 30303931323234306162343361643162313238313234383837323530343936343134326365666166 + 3438643238626266320a333732363461613834613833303436363534393031323333613534333534 + 3438 + geoip_license: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 32336437653435396231663666643631373638616533653538333934356339666661313439343566 + 6462303439666233316162653430363239393031633362630a323533393233633164613031646633 + 62366362383162306563383235373262326262323337303333313163653838613436316337633063 + 3630356262666338320a333137333063303534386437343762616665383034336365663732343566 + 36376463313964333834373435653564346333396538306337333762613737643363636364353139 + 3032383564333638613566633363666430356230326231633636 + tasks: + - name: Install exporters + ansible.builtin.apt: + name: + - prometheus-node-exporter + - prometheus-blackbox-exporter + - wireguard-tools + state: present + update_cache: true + + - name: Create textfile directory + ansible.builtin.file: + path: "{{ node_exporter_textfile_dir }}" + state: directory + mode: '0755' + + - name: Configure node exporter + ansible.builtin.copy: + dest: /etc/default/prometheus-node-exporter + content: | + ARGS="--web.listen-address={{ wireguard_ingress_ip }}:9100 --collector.textfile.directory={{ node_exporter_textfile_dir }} --collector.systemd" + mode: '0644' + notify: Restart node exporter + + - name: Configure blackbox exporter + ansible.builtin.copy: + dest: /etc/default/prometheus-blackbox-exporter + content: | + ARGS="--config.file=/etc/prometheus/blackbox.yml --web.listen-address={{ wireguard_ingress_ip }}:9115" + mode: '0644' + notify: Restart blackbox exporter + + - name: Copy blackbox config + ansible.builtin.copy: + src: ../monitoring/blackbox.yml + dest: /etc/prometheus/blackbox.yml + mode: '0644' + notify: Restart blackbox exporter + + - name: Look up the prometheus group + ansible.builtin.getent: + database: group + key: prometheus + + - name: Allow the prometheus group to send icmp + ansible.builtin.copy: + dest: /etc/sysctl.d/99-blackbox-icmp.conf + content: | + net.ipv4.ping_group_range = {{ prometheus_gid }} {{ prometheus_gid }} + mode: '0644' + vars: + prometheus_gid: "{{ ansible_facts.getent_group.prometheus[1] }}" + notify: Apply sysctl + + - name: Create blackbox override directory + ansible.builtin.file: + path: /etc/systemd/system/prometheus-blackbox-exporter.service.d + state: directory + mode: '0755' + + - name: Allow blackbox to send icmp + ansible.builtin.copy: + dest: /etc/systemd/system/prometheus-blackbox-exporter.service.d/capabilities.conf + content: | + [Service] + AmbientCapabilities=CAP_NET_RAW + CapabilityBoundingSet=CAP_NET_RAW + mode: '0644' + notify: + - Reload systemd + - Restart blackbox exporter + + - name: Copy wireguard collector + ansible.builtin.template: + src: ../monitoring/textfile/wg_metrics.sh.j2 + dest: /usr/local/bin/wg_metrics.sh + mode: '0755' + + - name: Install collector timer + ansible.builtin.copy: + dest: "/etc/systemd/system/ingress-wg-metrics.{{ item.kind }}" + content: "{{ item.content }}" + mode: '0644' + loop: + - kind: service + content: | + [Unit] + Description=Collect wireguard peer metrics + + [Service] + Type=oneshot + ExecStart=/usr/local/bin/wg_metrics.sh + - kind: timer + content: | + [Unit] + Description=Collect wireguard peer metrics + + [Timer] + OnBootSec=1min + OnUnitActiveSec=30s + AccuracySec=5s + + [Install] + WantedBy=timers.target + notify: Reload systemd + + - name: Flush handlers + ansible.builtin.meta: flush_handlers + + - name: Enable collector timer + ansible.builtin.systemd_service: + name: ingress-wg-metrics.timer + state: started + enabled: true + + - name: Create alloy directories + ansible.builtin.file: + path: "{{ item }}" + state: directory + mode: '0755' + loop: + - /etc/alloy + - /opt/database + + - name: Create apt keyring directory + ansible.builtin.file: + path: /etc/apt/keyrings + state: directory + mode: '0755' + + - name: Add grafana apt key + ansible.builtin.get_url: + url: https://apt.grafana.com/gpg.key + dest: /etc/apt/keyrings/grafana.asc + mode: '0644' + + - name: Add grafana apt repository + ansible.builtin.apt_repository: + repo: "deb [signed-by=/etc/apt/keyrings/grafana.asc] https://apt.grafana.com stable main" + filename: grafana + state: present + + - name: Install alloy + ansible.builtin.apt: + name: alloy + state: present + update_cache: true + + - name: Create alloy data directory + ansible.builtin.file: + path: /var/lib/alloy/data + state: directory + owner: alloy + group: alloy + mode: '0755' + notify: Restart alloy + + - name: Check the geoip database + ansible.builtin.stat: + path: /opt/database/GeoLite2-City.mmdb + register: geoip_db + + - name: Check when geoip was last downloaded + ansible.builtin.stat: + path: /opt/.geoip_downloaded + register: geoip_stamp + + - name: Decide whether the geoip database needs refreshing + ansible.builtin.set_fact: + geoip_stale: "{{ not geoip_db.stat.exists or not geoip_stamp.stat.exists or (ansible_facts.date_time.epoch | int - geoip_stamp.stat.mtime | int) > geoip_max_age_seconds | int }}" + + - name: Report the geoip database age + ansible.builtin.debug: + msg: >- + geoip + {{ 'not present, downloading' if not (geoip_db.stat.exists and geoip_stamp.stat.exists) + else 'downloaded ' + (((ansible_facts.date_time.epoch | int - geoip_stamp.stat.mtime | int) / 3600) | round(1) | string) + 'h ago, ' + ('refreshing' if geoip_stale else 'reusing') }} + + - name: Download geoip database + ansible.builtin.get_url: + url: "https://download.maxmind.com/geoip/databases/GeoLite2-City/download?suffix=tar.gz" + dest: /opt/geoip.tar.gz + username: "{{ geoip_username }}" + password: "{{ geoip_license }}" + mode: '0644' + force: true + when: geoip_stale + + - name: Extract geoip database + ansible.builtin.unarchive: + remote_src: true + src: /opt/geoip.tar.gz + dest: /opt/database + extra_opts: ['--strip-components=1', '--show-stored-names'] + when: geoip_stale + notify: Restart alloy + + - name: Stamp the geoip download time + ansible.builtin.file: + path: /opt/.geoip_downloaded + state: touch + mode: '0644' + when: geoip_stale + + - name: Copy alloy config + ansible.builtin.template: + src: config.alloy.j2 + dest: /etc/alloy/config.alloy + mode: '0644' + notify: Restart alloy + + - name: Configure alloy + ansible.builtin.copy: + dest: /etc/default/alloy + content: | + CONFIG_FILE="/etc/alloy/config.alloy" + CUSTOM_ARGS="--server.http.listen-addr={{ wireguard_ingress_ip }}:12345 --stability.level=experimental" + RESTART_ON_UPGRADE=true + mode: '0644' + notify: Restart alloy + + - name: Allow alloy to read logs + ansible.builtin.user: + name: alloy + groups: + - adm + - systemd-journal + append: true + notify: Restart alloy + + - name: Enable services + ansible.builtin.service: + name: "{{ item }}" + state: started + enabled: true + loop: + - prometheus-node-exporter + - prometheus-blackbox-exporter + - alloy + + - name: Run collector once + changed_when: true + ansible.builtin.command: + cmd: /usr/local/bin/wg_metrics.sh + + - name: Debug + ansible.builtin.debug: + msg: + - "Node metrics: curl -s {{ wireguard_ingress_ip }}:9100/metrics | head" + - "Blackbox probe: curl -s '{{ wireguard_ingress_ip }}:9115/probe?target={{ wireguard_beepi_ip }}:443&module=tcp_connect'" + - "Alloy logs: journalctl -u alloy -n 50" + tags: + - always + + handlers: + - name: Apply sysctl + changed_when: true + ansible.builtin.command: + cmd: sysctl --system + + - name: Reload systemd + ansible.builtin.systemd_service: + daemon_reload: true + + - name: Restart node exporter + ansible.builtin.service: + name: prometheus-node-exporter + state: restarted + + - name: Restart blackbox exporter + ansible.builtin.service: + name: prometheus-blackbox-exporter + state: restarted + + - name: Restart alloy + ansible.builtin.service: + name: alloy + state: restarted diff --git a/playbooks/monitoring/blackbox.yml.j2 b/playbooks/monitoring/blackbox.yml.j2 new file mode 100644 index 0000000..d840739 --- /dev/null +++ b/playbooks/monitoring/blackbox.yml.j2 @@ -0,0 +1,70 @@ +modules: + http_2xx: + prober: http + timeout: 10s + http: + valid_status_codes: [200] + valid_http_versions: ["HTTP/1.1", "HTTP/2.0"] + preferred_ip_protocol: ip4 + ip_protocol_fallback: false + + http_2xx_any: + prober: http + timeout: 10s + http: + valid_status_codes: [200, 204, 301, 302, 307, 308, 401, 403] + preferred_ip_protocol: ip4 + ip_protocol_fallback: false + + https_vhost: + prober: http + timeout: 10s + http: + valid_status_codes: [200, 204, 301, 302, 307, 308, 401, 403] + preferred_ip_protocol: ip4 + ip_protocol_fallback: false + fail_if_not_ssl: true + tls_config: + insecure_skip_verify: false + + tcp_connect: + prober: tcp + timeout: 10s + tcp: + preferred_ip_protocol: ip4 + ip_protocol_fallback: false + + smtp_starttls: + prober: tcp + timeout: 10s + tcp: + preferred_ip_protocol: ip4 + ip_protocol_fallback: false + query_response: + - expect: "^220" + - send: "EHLO prober" + - expect: "^250-STARTTLS" + - send: "QUIT" + + imaps_banner: + prober: tcp + timeout: 10s + tcp: + preferred_ip_protocol: ip4 + ip_protocol_fallback: false + tls: true + + ssh_banner: + prober: tcp + timeout: 10s + tcp: + preferred_ip_protocol: ip4 + ip_protocol_fallback: false + query_response: + - expect: "^SSH-2.0-" + + icmp: + prober: icmp + timeout: 5s + icmp: + preferred_ip_protocol: ip4 diff --git a/playbooks/monitoring/config.alloy.j2 b/playbooks/monitoring/config.alloy.j2 index a00d949..a4b45d3 100644 --- a/playbooks/monitoring/config.alloy.j2 +++ b/playbooks/monitoring/config.alloy.j2 @@ -10,8 +10,17 @@ local.file_match "fail2ban" { path_targets = [{"__path__" = "/var/log/fail2ban.log", job = "fail2ban", host = "beepi"}] } +local.file_match "backup" { + path_targets = [{"__path__" = "/var/log/beepi-backup.log", job = "backup", host = "beepi"}] +} + local.file_match "varlog" { - path_targets = [{"__path__" = "/var/log/*.log", job = "varlog", "__path_exclude__" = "/var/log/fail2ban.log", host = "beepi"}] + path_targets = [{ + "__path__" = "/var/log/*.log", + job = "varlog", + "__path_exclude__" = "/var/log/{fail2ban,beepi-backup}.log", + host = "beepi", + }] } loki.source.file "varlog" { @@ -19,6 +28,34 @@ loki.source.file "varlog" { forward_to = [loki.write.default.receiver] } +loki.source.file "backup" { + targets = local.file_match.backup.targets + forward_to = [loki.process.backup.receiver] +} + +loki.process "backup" { + forward_to = [loki.write.default.receiver] + + stage.regex { + expression = `^\[(?P[^\]]+)\]\s(?P\w+):\s(?P.*)$` + } + + stage.labels { + values = { + backup_job = "", + } + } + + stage.match { + selector = `{job="backup"} |~ "(?i)(FAILED|verify failed|refusing to sync|error)"` + stage.static_labels { + values = { + level = "error", + } + } + } +} + loki.source.file "fail2ban_source" { targets = local.file_match.fail2ban.targets forward_to = [loki.process.geoip.receiver] @@ -58,4 +95,4 @@ loki.write "default" { endpoint { url = "http://loki:3100/loki/api/v1/push" } -} \ No newline at end of file +} diff --git a/playbooks/monitoring/docker-compose.yml.j2 b/playbooks/monitoring/docker-compose.yml.j2 index 3a19a33..5aae1be 100644 --- a/playbooks/monitoring/docker-compose.yml.j2 +++ b/playbooks/monitoring/docker-compose.yml.j2 @@ -12,6 +12,14 @@ services: volumes: - prometheus-data:/prometheus - /opt/beeserver/monitoring/prometheus.yml:/etc/prometheus/prometheus.yml + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=90d + - --web.enable-lifecycle +{% if prometheus_admin_api | default(false) %} + - --web.enable-admin-api +{% endif %} networks: - monitoring @@ -23,12 +31,17 @@ services: - '127.0.0.1:4000:4000' volumes: - grafana-storage:/var/lib/grafana + - /opt/beeserver/monitoring/grafana/provisioning:/etc/grafana/provisioning:ro + - /opt/beeserver/monitoring/grafana/dashboards:/var/lib/grafana/dashboards:ro + - /opt/beeserver/monitoring/grafana/dashboards-general:/var/lib/grafana/dashboards-general:ro environment: - GF_SECURITY_SECRET_KEY={{ grafana_secret }} - GF_SERVER_DOMAIN=grafana.secretbee.buzz - GF_SERVER_ROOT_URL=https://grafana.secretbee.buzz/ - GF_SERVER_HTTP_PORT=4000 - GF_SERVER_PROTOCOL=HTTP + - GF_INSTALL_PLUGINS=yesoreyeram-infinity-datasource + - GF_UNIFIED_ALERTING_ENABLED=true networks: - monitoring @@ -37,6 +50,7 @@ services: container_name: loki ports: - '127.0.0.1:3100:3100' + - '{{ wireguard_beepi_ip }}:3100:3100' - '127.0.0.1:9096:9096' volumes: - loki-data:/loki @@ -87,8 +101,42 @@ services: networks: - monitoring + blackbox: + image: docker.io/prom/blackbox-exporter + container_name: blackbox + restart: always + network_mode: host + volumes: + - /opt/beeserver/monitoring/blackbox.yml:/etc/blackbox_exporter/config.yml + command: + - --config.file=/etc/blackbox_exporter/config.yml + - --web.listen-address=0.0.0.0:9115 + extra_hosts: +{% for service in monitored_services %} +{% if service.vhost is defined %} + - '{{ service.vhost }}:127.0.0.1' +{% endif %} +{% endfor %} + + topology: + image: docker.io/library/python:3-alpine + container_name: topology + restart: always + ports: + - '127.0.0.1:8088:8088' + volumes: + - /opt/beeserver/monitoring/topology:/app:ro + command: + - python + - /app/topology.py + environment: + - PROMETHEUS_URL=http://prometheus:9090 + - LISTEN_PORT=8088 + networks: + - monitoring + volumes: prometheus-data: grafana-storage: loki-data: - alloy-data: \ No newline at end of file + alloy-data: diff --git a/playbooks/monitoring/grafana/dashboards-general/logging.json b/playbooks/monitoring/grafana/dashboards-general/logging.json new file mode 100644 index 0000000..d365a72 --- /dev/null +++ b/playbooks/monitoring/grafana/dashboards-general/logging.json @@ -0,0 +1,713 @@ +{ + "apiVersion": "dashboard.grafana.app/v2", + "kind": "Dashboard", + "metadata": { + "name": "admltw2" + }, + "spec": { + "annotations": [ + { + "kind": "AnnotationQuery", + "spec": { + "builtIn": true, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "query": { + "datasource": { + "name": "-- Grafana --" + }, + "group": "grafana", + "kind": "DataQuery", + "spec": {}, + "version": "v0" + } + } + } + ], + "cursorSync": "Off", + "editable": true, + "elements": { + "panel-1": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "bfkfq6kdpznr4e" + }, + "group": "loki", + "kind": "DataQuery", + "spec": { + "direction": "backward", + "editorMode": "builder", + "expr": "{host=\"beepi\", filename=\"/var/log/ufw.log\"} |= `[UFW BLOCK]` | detected_level != `info`", + "queryType": "range" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 1, + "links": [], + "title": "UFW Blocks", + "vizConfig": { + "group": "logs", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": { + "dedupStrategy": "numbers", + "enableInfiniteScrolling": false, + "enableLogDetails": true, + "prettifyLogMessage": true, + "showControls": false, + "showFieldSelector": false, + "showLabels": false, + "showLevel": true, + "showTime": true, + "sortOrder": "Descending", + "syntaxHighlighting": true, + "timestampResolution": "ms", + "unwrappedColumns": false, + "wrapLogMessage": false + } + }, + "version": "13.0.1" + } + } + }, + "panel-2": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "bfkfq6kdpznr4e" + }, + "group": "loki", + "kind": "DataQuery", + "spec": { + "direction": "backward", + "editorMode": "builder", + "expr": "{filename=\"/var/log/fail2ban.log\"} |= `NOTICE`", + "queryType": "range" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 2, + "links": [], + "title": "Fail2Ban bans", + "vizConfig": { + "group": "logs", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": { + "dedupStrategy": "none", + "enableInfiniteScrolling": false, + "enableLogDetails": true, + "prettifyLogMessage": true, + "showControls": false, + "showFieldSelector": false, + "showLabels": false, + "showLevel": true, + "showTime": false, + "sortOrder": "Descending", + "timestampResolution": "ms", + "unwrappedColumns": false, + "wrapLogMessage": false + } + }, + "version": "13.0.1" + } + } + }, + "panel-3": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "bfkfq6kdpznr4e" + }, + "group": "loki", + "kind": "DataQuery", + "spec": { + "direction": "backward", + "editorMode": "builder", + "expr": "count_over_time({filename=\"/var/log/fail2ban.log\"} |= `NOTICE` |= `Ban` != `Restore` [$__auto])", + "legendFormat": "", + "queryType": "range", + "step": "10m" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 3, + "links": [], + "title": "Fail2Ban Over Time", + "vizConfig": { + "group": "barchart", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + } + }, + "version": "13.0.1" + } + } + }, + "panel-4": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "bfkfq6kdpznr4e" + }, + "group": "loki", + "kind": "DataQuery", + "spec": { + "direction": "backward", + "editorMode": "builder", + "expr": "{type=\"Ban\"}", + "queryType": "range" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [ + { + "group": "extractFields", + "kind": "Transformation", + "spec": { + "options": { + "delimiter": ",", + "format": "json", + "keepTime": false, + "replace": false, + "source": "labels" + } + } + }, + { + "group": "convertFieldType", + "kind": "Transformation", + "spec": { + "options": { + "conversions": [ + { + "destinationType": "number", + "targetField": "geoip_location_latitude" + }, + { + "destinationType": "number", + "targetField": "geoip_location_longitude" + } + ], + "fields": {} + } + } + } + ] + } + }, + "description": "", + "id": 4, + "links": [], + "title": "Fail2Ban Worldmap", + "vizConfig": { + "group": "geomap", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "geoip_location_latitude", + "scope": "series" + }, + "properties": [ + { + "id": "decimals", + "value": 4 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "geoip_location_longitude", + "scope": "series" + }, + "properties": [ + { + "id": "decimals", + "value": 4 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "__error__", + "scope": "series" + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": true + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "__error_details__", + "scope": "series" + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": true + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "labels", + "scope": "series" + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": true + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "labelTypes", + "scope": "series" + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": true + } + } + ] + } + ] + }, + "options": { + "basemap": { + "config": {}, + "name": "Layer 0", + "noRepeat": false, + "type": "default" + }, + "controls": { + "mouseWheelZoom": true, + "showAttribution": true, + "showDebug": false, + "showMeasure": false, + "showScale": false, + "showZoom": true + }, + "layers": [ + { + "config": { + "blur": 17, + "radius": 9, + "weight": { + "fixed": 0.5, + "max": 1, + "min": 0 + } + }, + "layer-tooltip": false, + "location": { + "latitude": "geoip_location_latitude", + "longitude": "geoip_location_longitude", + "mode": "coords" + }, + "name": "Bans", + "opacity": 1, + "tooltip": true, + "type": "heatmap" + } + ], + "tooltip": { + "mode": "details" + }, + "view": { + "allLayers": true, + "dashboardVariable": false, + "id": "zero", + "lat": 0, + "lon": 0, + "noRepeat": false, + "zoom": 1 + } + } + }, + "version": "13.0.1" + } + } + }, + "panel-5": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "bfkfq6kdpznr4e" + }, + "group": "loki", + "kind": "DataQuery", + "spec": { + "direction": "backward", + "editorMode": "builder", + "expr": "{job=\"varlog\"} |= `ERROR` or `WARN` or `CRITICAL` != `queryType` or `query_hash` | logfmt", + "queryType": "range" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 5, + "links": [], + "title": "New panel", + "vizConfig": { + "group": "logs", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": { + "dedupStrategy": "none", + "enableInfiniteScrolling": false, + "enableLogDetails": true, + "prettifyLogMessage": true, + "showControls": false, + "showFieldSelector": false, + "showLevel": true, + "showTime": false, + "sortOrder": "Descending", + "timestampResolution": "ms", + "unwrappedColumns": false, + "wrapLogMessage": false + } + }, + "version": "13.0.1" + } + } + } + }, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-5" + }, + "height": 8, + "width": 24, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-4" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 8 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-3" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 8 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-2" + }, + "height": 7, + "width": 24, + "x": 0, + "y": 16 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-1" + }, + "height": 8, + "width": 24, + "x": 0, + "y": 23 + } + } + ] + } + }, + "links": [], + "liveNow": false, + "preferences": { + "layout": { + "kind": "GridLayout", + "spec": { + "items": [] + } + } + }, + "preload": false, + "tags": [], + "timeSettings": { + "autoRefresh": "", + "autoRefreshIntervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "fiscalYearStartMonth": 0, + "from": "now-6h", + "hideTimepicker": false, + "timezone": "browser", + "to": "now" + }, + "title": "Logging", + "variables": [ + { + "datasource": { + "name": "bfkfq6kdpznr4e" + }, + "group": "loki", + "kind": "AdhocVariable", + "spec": { + "allowCustomValue": true, + "baseFilters": [], + "defaultKeys": [], + "enableGroupBy": false, + "filters": [], + "hide": "dontHide", + "name": "Filters", + "skipUrlSync": false + } + } + ] + } +} diff --git a/playbooks/monitoring/grafana/dashboards-general/node-exporter-full.json b/playbooks/monitoring/grafana/dashboards-general/node-exporter-full.json new file mode 100644 index 0000000..b2b60cd --- /dev/null +++ b/playbooks/monitoring/grafana/dashboards-general/node-exporter-full.json @@ -0,0 +1,21893 @@ +{ + "apiVersion": "dashboard.grafana.app/v2", + "kind": "Dashboard", + "metadata": { + "name": "rYdddlPWk" + }, + "spec": { + "annotations": [ + { + "kind": "AnnotationQuery", + "spec": { + "builtIn": true, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "query": { + "datasource": { + "name": "grafana" + }, + "group": "grafana", + "kind": "DataQuery", + "spec": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "version": "v0" + } + } + } + ], + "cursorSync": "Crosshair", + "editable": true, + "elements": { + "panel-104": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_netstat_TcpExt_ListenOverflows{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "Listen Overflows", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_netstat_TcpExt_ListenDrops{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "Listen Drops", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_netstat_TcpExt_TCPSynRetrans{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "SYN Retransmits", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "C" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_netstat_Tcp_RetransSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "Segment Retransmits", + "range": true + }, + "version": "v0" + }, + "refId": "D" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_netstat_Tcp_InErrs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "Receive Errors", + "range": true + }, + "version": "v0" + }, + "refId": "E" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_netstat_Tcp_OutRsts{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "RST Sent", + "range": true + }, + "version": "v0" + }, + "refId": "F" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_netstat_TcpExt_TCPRcvQDrop{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "Receive Queue Drops", + "range": true + }, + "version": "v0" + }, + "refId": "G" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_netstat_TcpExt_TCPOFOQueue{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "Out-of-order Queued", + "range": true + }, + "version": "v0" + }, + "refId": "H" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_netstat_TcpExt_TCPTimeouts{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "TCP Timeouts", + "range": true + }, + "version": "v0" + }, + "refId": "I" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Tracks various TCP error and congestion-related events, including retransmissions, timeouts, dropped connections, and buffer issues", + "id": 104, + "links": [], + "title": "TCP Errors", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "pps" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-109": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_netstat_Udp_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "UDP Rx in Errors", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_netstat_Udp_NoPorts{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "UDP No Listener", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_netstat_UdpLite_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "UDPLite Rx in Errors", + "range": true + }, + "version": "v0" + }, + "refId": "C" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_netstat_Udp_RcvbufErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "UDP Rx in Buffer Errors", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "D" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_netstat_Udp_SndbufErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "UDP Tx out Buffer Errors", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "E" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of UDP and UDPLite datagram delivery errors, including missing listeners, buffer overflows, and protocol-specific issues", + "id": 109, + "links": [], + "title": "UDP Errors", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-115": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_netstat_Icmp_InMsgs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "ICMP Rx in", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_netstat_Icmp_OutMsgs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "ICMP Tx out", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Number of ICMP messages sent and received per second, including error and control messages", + "id": 115, + "links": [], + "title": "ICMP In / Out", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-124": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_sockstat_UDPLITE_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "UDPLite - In-Use Sockets", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_sockstat_UDP_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "UDP - In-Use Sockets", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Number of UDP and UDPLite sockets currently in use", + "id": 124, + "links": [], + "title": "Sockstat UDP", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-125": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_sockstat_FRAG_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "FRAG - In-Use Sockets", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_sockstat_RAW_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "RAW - In-Use Sockets", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "C" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Number of FRAG and RAW sockets currently in use. RAW sockets are used for custom protocols or tools like ping; FRAG sockets are used internally for IP packet defragmentation", + "id": 125, + "links": [], + "title": "Sockstat FRAG / RAW", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-126": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_sockstat_sockets_used{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "Total sockets", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Total number of sockets currently in use across all protocols (TCP, UDP, UNIX, etc.), as reported by /proc/net/sockstat", + "id": 126, + "links": [], + "title": "Sockstat Used", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-127": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\",device=~\"[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+\"} [$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "{{device}}", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Percentage of time the disk was actively processing I/O operations", + "id": 127, + "links": [], + "title": "Disk I/O Utilization", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-128": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_DirectMap1G_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "DirectMap 1G \u2013 Memory mapped with 1GB pages", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_DirectMap2M_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "DirectMap 2M \u2013 Memory mapped with 2MB pages", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_DirectMap4k_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "DirectMap 4K \u2013 Memory mapped with 4KB pages", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "C" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "How much memory is directly mapped in the kernel using different page sizes (4K, 2M, 1G). Helps monitor large page utilization in the direct map region", + "id": 128, + "links": [], + "title": "Memory DirectMap", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-129": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_AnonHugePages_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "AnonHugePages \u2013 Anonymous memory using HugePages", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_AnonPages_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "AnonPages \u2013 Anonymous memory (non-file-backed)", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Memory used by anonymous pages (not backed by files), including standard and huge page allocations. Includes heap, stack, and memory-mapped anonymous regions", + "id": 129, + "links": [], + "title": "Memory Anonymous", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-130": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_Writeback_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Writeback \u2013 Memory currently being flushed to disk", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_WritebackTmp_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "WritebackTmp \u2013 FUSE temporary writeback buffers", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_Dirty_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Dirty \u2013 Memory marked dirty (pending write to disk)", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "C" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_NFS_Unstable_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "NFS Unstable \u2013 Pages sent to NFS server, awaiting storage commit", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "D" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Memory currently dirty (modified but not yet written to disk), being actively written back, or held by writeback buffers. High dirty or writeback memory may indicate disk I/O pressure or delayed flushing", + "id": 130, + "links": [], + "title": "Memory Writeback and Dirty", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-131": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_SUnreclaim_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "SUnreclaim \u2013 Non-reclaimable slab memory (kernel objects)", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "SReclaimable \u2013 Potentially reclaimable slab memory (e.g., inode cache)", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Kernel slab memory usage, separated into reclaimable and non-reclaimable categories. Reclaimable memory can be freed under memory pressure (e.g., caches), while unreclaimable memory is locked by the kernel for core functions", + "id": 131, + "links": [], + "title": "Memory Slab", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-133": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_disk_reads_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "legendFormat": "{{device}} - Read", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_disk_writes_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "legendFormat": "{{device}} - Write", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Number of read and write requests merged per second that were queued to the device", + "id": 133, + "links": [], + "title": "Disk R/W Merged", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "read (\u2013) / write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "iops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/sda.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-135": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_Committed_AS_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Committed_AS \u2013 Memory promised to processes (not necessarily used)", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_CommitLimit_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "CommitLimit - Max allowable committed memory", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Displays committed memory usage versus the system's commit limit. Exceeding the limit is allowed under Linux overcommit policies but may increase OOM risks under high load", + "id": 135, + "links": [], + "title": "Memory Committed", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*CommitLimit - *./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-136": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "(node_memory_Inactive_bytes{instance=\"$node\",job=\"$job\"}) \n/ \n(node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "legendFormat": "Inactive \u2013 Less recently used memory, more likely to be reclaimed", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "(node_memory_Active_bytes{instance=\"$node\",job=\"$job\"}) \n/ \n(node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"})\n", + "format": "time_series", + "legendFormat": "Active \u2013 Recently used memory, retained unless under pressure", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Proportion of memory pages in the kernel's active and inactive LRU lists relative to total RAM. Active pages have been recently used, while inactive pages are less recently accessed but still resident in memory", + "id": 136, + "links": [], + "title": "Memory LRU Active / Inactive (%)", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Active.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Inactive.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-137": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_Unevictable_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Unevictable \u2013 Kernel-pinned memory (not swappable)", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_Mlocked_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Mlocked \u2013 Application-locked memory via mlock()", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Memory that is locked in RAM and cannot be swapped out. Includes both kernel-unevictable memory and user-level memory locked with mlock()", + "id": 137, + "links": [], + "title": "Memory Unevictable and MLocked", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-138": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_Mapped_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Mapped \u2013 Memory mapped from files (e.g., libraries, mmap)", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_Shmem_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Shmem \u2013 Shared memory used by processes and tmpfs", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_ShmemHugePages_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "ShmemHugePages \u2013 Shared memory (shmem/tmpfs) allocated with HugePages", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "C" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_ShmemPmdMapped_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "PMD Mapped \u2013 Shmem/tmpfs backed by Transparent HugePages (PMD)", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "D" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Memory used for mapped files (such as libraries) and shared memory (shmem and tmpfs), including variants backed by huge pages", + "id": 138, + "links": [], + "title": "Memory Shared and Mapped", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-14": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "exemplar": false, + "expr": "count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu))", + "instant": true, + "legendFormat": "__auto", + "range": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": { + "maxDataPoints": 100 + }, + "transformations": [] + } + }, + "description": "", + "id": 14, + "links": [], + "title": "CPU Cores", + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.0.1" + } + } + }, + "panel-140": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "(node_memory_HugePages_Total{instance=\"$node\",job=\"$job\"} - node_memory_HugePages_Free{instance=\"$node\",job=\"$job\"}) * node_memory_Hugepagesize_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "HugePages Used \u2013 Currently allocated", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_HugePages_Rsvd{instance=\"$node\",job=\"$job\"} * node_memory_Hugepagesize_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "HugePages Reserved \u2013 Promised but unused", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_HugePages_Surp{instance=\"$node\",job=\"$job\"} * node_memory_Hugepagesize_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "HugePages Surplus \u2013 Dynamic pool extension", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "C" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_HugePages_Total{instance=\"$node\",job=\"$job\"} * node_memory_Hugepagesize_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "HugePages Total \u2013 Reserved memory", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "D" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Displays HugePages memory usage in bytes, including allocated, free, reserved, and surplus memory. All values are calculated based on the number of huge pages multiplied by their configured size", + "id": 140, + "links": [], + "title": "Memory HugePages", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-141": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_network_receive_compressed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "{{device}} - Rx in", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_network_transmit_compressed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "{{device}} - Tx out", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of compressed network packets received and transmitted per interface. These are common in low-bandwidth or special interfaces like PPP or SLIP", + "id": 141, + "links": [], + "title": "Network Traffic Compressed", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-142": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_network_receive_errs_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "{{device}} - Rx in", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_network_transmit_errs_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "{{device}} - Tx out", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of packet-level errors for each network interface. Receive errors may indicate physical or driver issues; transmit errors may reflect collisions or hardware faults", + "id": 142, + "links": [], + "title": "Network Traffic Errors", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-143": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_network_receive_drop_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "{{device}} - Rx in", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_network_transmit_drop_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "{{device}} - Tx out", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of dropped packets per network interface. Receive drops can indicate buffer overflow or driver issues; transmit drops may result from outbound congestion or queuing limits", + "id": 143, + "links": [], + "title": "Network Traffic Drop", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-144": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_network_receive_fifo_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "{{device}} - Rx in", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_network_transmit_fifo_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "{{device}} - Tx out", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Tracks FIFO buffer overrun errors on network interfaces. These occur when incoming or outgoing packets are dropped due to queue or buffer overflows, often indicating congestion or hardware limits", + "id": 144, + "links": [], + "title": "Network Traffic Fifo", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-145": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_network_receive_frame_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "{{device}} - Rx in", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of frame errors on received packets, typically caused by physical layer issues such as bad cables, duplex mismatches, or hardware problems", + "id": 145, + "links": [], + "title": "Network Traffic Frame", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "pps" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-146": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_network_receive_multicast_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "{{device}} - Rx in", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of incoming multicast packets received per network interface. Multicast is used by protocols such as mDNS, SSDP, and some streaming or cluster services", + "id": 146, + "links": [], + "title": "Network Traffic Multicast", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "pps" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-148": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_forks_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "Process Forks per second", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of new processes being created on the system (forks/sec).", + "id": 148, + "links": [], + "title": "Processes Forks", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-149": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "process_virtual_memory_bytes{instance=\"$node\",job=\"$job\"}", + "interval": "", + "legendFormat": "Virtual Memory", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "process_virtual_memory_max_bytes{instance=\"$node\",job=\"$job\"}", + "interval": "", + "legendFormat": "Virtual Memory Limit", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Tracks the memory usage of the process exposing this metric (e.g., node_exporter), including current virtual memory and maximum virtual memory limit", + "id": 149, + "links": [], + "title": "Exporter Processes Memory", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Virtual Memory Limit" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + }, + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Virtual Memory" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": true, + "viz": true + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-15": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "exemplar": false, + "expr": "node_time_seconds{instance=\"$node\",job=\"$job\"} - node_boot_time_seconds{instance=\"$node\",job=\"$job\"}", + "instant": true, + "range": false, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": { + "maxDataPoints": 100 + }, + "transformations": [] + } + }, + "description": "", + "id": 15, + "links": [], + "title": "Uptime", + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.0.1" + } + } + }, + "panel-151": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_entropy_available_bits{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Entropy available", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_entropy_pool_size_bits{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Entropy pool max", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Number of bits of entropy currently available to the system's random number generators (e.g., /dev/random). Low values may indicate that random number generation could block or degrade performance of cryptographic operations", + "id": 151, + "links": [], + "title": "Entropy", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbits" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Entropy pool max" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-152": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "((node_filesystem_size_bytes{instance=\"$node\", job=\"$job\", device!~'rootfs'} - node_filesystem_avail_bytes{instance=\"$node\", job=\"$job\", device!~'rootfs'}) / node_filesystem_size_bytes{instance=\"$node\", job=\"$job\", device!~'rootfs'}) * 100", + "format": "time_series", + "legendFormat": "{{mountpoint}}", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Percentage of filesystem space used for each mounted device", + "id": 152, + "links": [], + "title": "Disk Space Used Basic", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line+area" + } + }, + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 50 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "13.0.1" + } + } + }, + "panel-154": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "exemplar": false, + "expr": "(\n (node_filesystem_size_bytes{instance=\"$node\", job=\"$job\", mountpoint=\"/\", fstype!=\"rootfs\"}\n - node_filesystem_avail_bytes{instance=\"$node\", job=\"$job\", mountpoint=\"/\", fstype!=\"rootfs\"})\n / node_filesystem_size_bytes{instance=\"$node\", job=\"$job\", mountpoint=\"/\", fstype!=\"rootfs\"}\n) * 100\n", + "format": "time_series", + "instant": true, + "range": false, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Used Root FS", + "id": 154, + "links": [], + "title": "Root FS Used", + "vizConfig": { + "group": "gauge", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 0 + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "options": { + "barShape": "flat", + "barWidthFactor": 0.5, + "effects": { + "barGlow": false, + "centerGlow": false, + "gradient": false + }, + "endpointMarker": "point", + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "segmentCount": 1, + "segmentSpacing": 0.3, + "shape": "gauge", + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "sparkline": false, + "textMode": "auto" + } + }, + "version": "13.0.1" + } + } + }, + "panel-155": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "exemplar": false, + "expr": "scalar(node_load1{instance=\"$node\",job=\"$job\"}) * 100 / count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu))", + "format": "time_series", + "instant": true, + "range": false, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "System load over all CPU cores together", + "id": 155, + "links": [], + "title": "Sys Load", + "vizConfig": { + "group": "gauge", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 0 + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 85 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 95 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "options": { + "barShape": "flat", + "barWidthFactor": 0.5, + "effects": { + "barGlow": false, + "centerGlow": false, + "gradient": false + }, + "endpointMarker": "point", + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "segmentCount": 1, + "segmentSpacing": 0.3, + "shape": "gauge", + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "sparkline": false, + "textMode": "auto" + } + }, + "version": "13.0.1" + } + } + }, + "panel-156": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'} - node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "legendFormat": "{{mountpoint}}", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Disk usage (used = total - available) per mountpoint", + "id": 156, + "links": [], + "title": "Filesystem Used", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-157": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_scrape_collector_success{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "{{collector}}", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "1 - node_textfile_scrape_error{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "textfile", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Shows whether each Node Exporter collector scraped successfully (1 = success, 0 = failure), and whether the textfile collector returned an error.", + "id": 157, + "links": [], + "title": "Node Exporter Scrape", + "vizConfig": { + "group": "bargauge", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "dark-red", + "value": 0 + }, + { + "color": "green", + "value": 1 + } + ] + }, + "unit": "bool" + }, + "overrides": [] + }, + "options": { + "displayMode": "basic", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + } + }, + "version": "11.6.1" + } + } + }, + "panel-158": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_hwmon_temp_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "{{ chip_name }} {{ sensor }}", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": true, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "node_hwmon_temp_crit_alarm_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "{{ chip_name }} {{ sensor }} Critical Alarm", + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_hwmon_temp_crit_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "{{ chip_name }} {{ sensor }} Critical", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "C" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": true, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "node_hwmon_temp_crit_hyst_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "{{ chip_name }} {{ sensor }} Critical Hysteresis", + "step": 240 + }, + "version": "v0" + }, + "refId": "D" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": true, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "node_hwmon_temp_max_celsius{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "{{ chip_name }} {{ sensor }} Max", + "step": 240 + }, + "version": "v0" + }, + "refId": "E" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Monitors hardware sensor temperatures and critical thresholds as exposed by Linux hwmon. Includes CPU, GPU, and motherboard sensors where available", + "id": 158, + "links": [], + "title": "Hardware Temperature Monitor", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "celsius" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Critical.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-16": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "exemplar": false, + "expr": "clamp_min((1 - (node_memory_MemAvailable_bytes{instance=\"$node\", job=\"$job\"} / node_memory_MemTotal_bytes{instance=\"$node\", job=\"$job\"})) * 100, 0)", + "format": "time_series", + "instant": true, + "range": false, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Real RAM usage excluding cache and reclaimable memory", + "id": 16, + "links": [], + "title": "RAM Used", + "vizConfig": { + "group": "gauge", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 0 + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 80 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "options": { + "barShape": "flat", + "barWidthFactor": 0.5, + "effects": { + "barGlow": false, + "centerGlow": false, + "gradient": false + }, + "endpointMarker": "point", + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "segmentCount": 1, + "segmentSpacing": 0.3, + "shape": "gauge", + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "sparkline": false, + "textMode": "auto" + } + }, + "version": "13.0.1" + } + } + }, + "panel-160": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_KernelStack_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "KernelStack \u2013 Kernel stack memory (per-thread, non-reclaimable)", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_Percpu_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "PerCPU \u2013 Dynamically allocated per-CPU memory (used by kernel modules)", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_Bounce_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "Bounce Memory \u2013 I/O buffer for DMA-limited devices", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "C" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Tracks kernel memory used for CPU-local structures, per-thread stacks, and bounce buffers used for I/O on DMA-limited devices. These areas are typically small but critical for low-level operations", + "id": 160, + "links": [], + "title": "Memory Kernel / CPU / IO", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-168": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_timex_sync_status{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "Sync status (1 = ok)", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_timex_frequency_adjustment_ratio{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "Frequency Adjustment", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": true, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_timex_tick_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "Tick Interval", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "C" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": true, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_timex_tai_offset_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "TAI Offset", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "D" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Shows whether the system clock is synchronized to a reliable time source, and the current frequency correction ratio applied by the kernel to maintain synchronization", + "id": 168, + "links": [], + "title": "Time Synchronized Status", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-175": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_vmstat_pgfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "Pgfault - Page major and minor fault ops", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_vmstat_pgmajfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "Pgmajfault - Major page fault ops", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_vmstat_pgfault{instance=\"$node\",job=\"$job\"}[$__rate_interval]) - rate(node_vmstat_pgmajfault{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "Pgminfault - Minor page fault ops", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "C" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of memory page faults, split into total, major (disk-backed), and derived minor (non-disk) faults. High major fault rates may indicate memory pressure or insufficient RAM", + "id": 175, + "links": [], + "title": "Memory Page Faults", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Pgfault - Page major and minor fault ops" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.stacking", + "value": { + "group": false, + "mode": "none" + } + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-176": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_vmstat_pgpgin{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "Pagesin - Page in ops", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_vmstat_pgpgout{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "Pagesout - Page out ops", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of memory pages being read from or written to disk (page-in and page-out operations). High page-out may indicate memory pressure or swapping activity", + "id": 176, + "links": [], + "title": "Memory Pages In / Out", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-18": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "exemplar": false, + "expr": "node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"}", + "instant": true, + "range": false, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": { + "maxDataPoints": 100 + }, + "transformations": [] + } + }, + "description": "", + "id": 18, + "links": [], + "title": "SWAP Total", + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.0.1" + } + } + }, + "panel-191": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_Inactive_file_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Inactive_file - File-backed memory on inactive LRU list", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_Inactive_anon_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Inactive_anon \u2013 Anonymous memory on inactive LRU (incl. tmpfs & swap cache)", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_Active_file_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Active_file - File-backed memory on active LRU list", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "C" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_Active_anon_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Active_anon \u2013 Anonymous memory on active LRU (incl. tmpfs & swap cache)", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "D" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Breakdown of memory pages in the kernel's active and inactive LRU lists, separated by anonymous (heap, tmpfs) and file-backed (caches, mmap) pages.", + "id": 191, + "links": [], + "title": "Memory LRU Active / Inactive Detail", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-20": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "exemplar": false, + "expr": "100 * (1 - avg(rate(node_cpu_seconds_total{mode=\"idle\",instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "instant": true, + "legendFormat": "", + "range": false, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Overall CPU busy percentage (averaged across all cores)", + "id": 20, + "links": [], + "title": "CPU Busy", + "vizConfig": { + "group": "gauge", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 0 + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 85 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 95 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "options": { + "barShape": "flat", + "barWidthFactor": 0.5, + "effects": { + "barGlow": false, + "centerGlow": false, + "gradient": false + }, + "endpointMarker": "point", + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "segmentCount": 1, + "segmentSpacing": 0.3, + "shape": "gauge", + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "sparkline": false, + "textMode": "auto" + } + }, + "version": "13.0.1" + } + } + }, + "panel-21": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "exemplar": false, + "expr": "(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} > bool 0) * ((node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"}) / (node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"})) * 100", + "instant": true, + "range": false, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Percentage of swap space currently used by the system", + "id": 21, + "links": [], + "title": "SWAP Used", + "vizConfig": { + "group": "gauge", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 0 + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 10 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 25 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "options": { + "barShape": "flat", + "barWidthFactor": 0.5, + "effects": { + "barGlow": false, + "centerGlow": false, + "gradient": false + }, + "endpointMarker": "point", + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "segmentCount": 1, + "segmentSpacing": 0.3, + "shape": "gauge", + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "sparkline": false, + "textMode": "auto" + } + }, + "version": "13.0.1" + } + } + }, + "panel-219": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_filesystem_files{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "legendFormat": "{{mountpoint}}", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Number of file nodes (inodes) available per mounted filesystem. Reflects maximum file capacity regardless of disk size", + "id": 219, + "links": [], + "title": "File Nodes Size", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-22": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_vmstat_pswpin{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "Pswpin - Pages swapped in", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_vmstat_pswpout{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "Pswpout - Pages swapped out", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate at which memory pages are being swapped in from or out to disk. High swap-out activity may indicate memory pressure", + "id": 22, + "links": [], + "title": "Memory Pages Swap In / Out", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-220": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_sockstat_TCP_mem_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "TCP", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_sockstat_UDP_mem_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "UDP", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_sockstat_FRAG_memory{instance=\"$node\",job=\"$job\"}", + "interval": "", + "legendFormat": "Fragmentation", + "range": true + }, + "version": "v0" + }, + "refId": "C" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Kernel memory used by TCP, UDP, and IP fragmentation buffers", + "id": 220, + "links": [], + "title": "Sockstat Memory Size", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-221": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_netstat_IpExt_InOctets{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "IP Rx in", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_netstat_IpExt_OutOctets{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "IP Tx out", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of octets sent and received at the IP layer, as reported by /proc/net/netstat", + "id": 221, + "links": [], + "title": "Netstat IP In / Out Octets", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-229": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\",device=~\"[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+\"}[$__rate_interval])", + "legendFormat": "{{device}} - Read", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\",device=~\"[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+\"}[$__rate_interval])", + "legendFormat": "{{device}} - Write", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Disk I/O operations per second for each device", + "id": 229, + "links": [], + "title": "Disk IOps", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "read (-) / write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "iops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-23": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "exemplar": false, + "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",mountpoint=\"/\",fstype!=\"rootfs\"}", + "format": "time_series", + "instant": true, + "range": false, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": { + "maxDataPoints": 100 + }, + "transformations": [] + } + }, + "description": "", + "id": 23, + "links": [], + "title": "RootFS Total", + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 0 + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 70 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 90 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.0.1" + } + } + }, + "panel-230": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_arp_entries{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "{{ device }} ARP Table", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Number of ARP entries per interface. Useful for detecting excessive ARP traffic or table growth due to scanning or misconfiguration", + "id": 230, + "links": [], + "title": "ARP Entries", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-231": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_network_transmit_carrier_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "{{device}} - Tx out", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of carrier errors during transmission. These typically indicate physical layer issues like faulty cabling or duplex mismatches", + "id": 231, + "links": [], + "title": "Network Traffic Carrier Errors", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "pps" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-232": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_network_transmit_colls_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "{{device}} - Tx out", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of packet collisions detected during transmission. Mostly relevant on half-duplex or legacy Ethernet networks", + "id": 232, + "links": [], + "title": "Network Traffic Collision", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-24": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} - node_memory_Slab_bytes{instance=\"$node\",job=\"$job\"} - node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Apps - Memory used by user-space applications", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_PageTables_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "PageTables - Memory used to map between virtual and physical memory addresses", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_SwapCached_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "SwapCache - Memory that keeps track of pages that have been fetched from swap but not yet been modified", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "C" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_Slab_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Slab - Memory used by the kernel to cache data structures for its own use (caches like inode, dentry, etc)", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "D" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Cache - Parked file data (file content) cache", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "E" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Buffers - Block device (e.g. harddisk) cache", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "F" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Unused - Free memory unassigned", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "G" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "legendFormat": "Swap - Swap space used", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "H" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_HardwareCorrupted_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "I" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Breakdown of physical memory and swap usage. Hardware-detected memory errors are also displayed", + "id": 24, + "links": [], + "title": "Memory", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused - Free memory unassigned" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Hardware Corrupted - *./" + }, + "properties": [ + { + "id": "custom.stacking", + "value": { + "group": false, + "mode": "normal" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-259": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_interrupts_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "{{ type }} - {{ info }}", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Breaks down hardware interrupts by type and device. Useful for diagnosing IRQ load on network, disk, or CPU interfaces. Requires --collector.interrupts to be enabled in node_exporter", + "id": 259, + "links": [], + "title": "IRQ Detail", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-260": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_timex_estimated_error_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "Estimated error", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_timex_offset_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "Offset local vs reference", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_timex_maxerror_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "Maximum error", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "C" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Tracks the system clock's estimated and maximum error, as well as its offset from the reference clock (e.g., via NTP). Useful for detecting synchronization drift", + "id": 260, + "links": [], + "title": "Time Synchronized Drift", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-28": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_filefd_maximum{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Max open files", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_filefd_allocated{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Open files", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Number of file descriptors currently allocated system-wide versus the system limit. Important for detecting descriptor exhaustion risks", + "id": 28, + "links": [], + "title": "File Descriptor", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Max.*/" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-280": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_network_speed_bytes{instance=\"$node\",job=\"$job\"} * 8", + "format": "time_series", + "legendFormat": "{{ device }}", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Maximum speed of each network interface as reported by the operating system. This is a static hardware capability, not current throughput", + "id": 280, + "links": [], + "title": "Speed", + "vizConfig": { + "group": "bargauge", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "fieldMinMax": false, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bps" + }, + "overrides": [] + }, + "options": { + "displayMode": "basic", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 30, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + } + }, + "version": "11.6.1" + } + } + }, + "panel-288": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_network_mtu_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "{{ device }}", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "MTU (Maximum Transmission Unit) in bytes for each network interface. Affects packet size and transmission efficiency", + "id": 288, + "links": [], + "title": "MTU", + "vizConfig": { + "group": "bargauge", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "options": { + "displayMode": "basic", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 30, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "manual", + "valueMode": "color" + } + }, + "version": "11.6.1" + } + } + }, + "panel-290": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_softnet_processed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "CPU {{cpu}} - Processed", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_softnet_dropped_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "CPU {{cpu}} - Dropped", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Packets processed and dropped by the softnet network stack per CPU. Drops may indicate CPU saturation or network driver limitations", + "id": 290, + "links": [], + "title": "Softnet Packets", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "drop (-) / process (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Dropped.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-291": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_timex_loop_time_constant{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "PLL Time Constant", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "NTP phase-locked loop (PLL) time constant used by the kernel to control time adjustments. Lower values mean faster correction but less stability", + "id": 291, + "links": [], + "title": "Time PLL Adjust", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-297": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_systemd_socket_accepted_connections_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "{{ name }}", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of accepted connections per second for each systemd socket", + "id": 297, + "links": [], + "title": "Systemd Sockets Accepted", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "eps" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-298": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"activating\"}", + "format": "time_series", + "interval": "", + "legendFormat": "Activating", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"active\"}", + "format": "time_series", + "interval": "", + "legendFormat": "Active", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"deactivating\"}", + "format": "time_series", + "interval": "", + "legendFormat": "Deactivating", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "C" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"failed\"}", + "format": "time_series", + "interval": "", + "legendFormat": "Failed", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "D" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_systemd_units{instance=\"$node\",job=\"$job\",state=\"inactive\"}", + "format": "time_series", + "interval": "", + "legendFormat": "Inactive", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "E" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Current number of systemd units in each operational state, such as active, failed, inactive, or transitioning", + "id": 298, + "links": [], + "title": "Systemd Units State", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Failed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#73BF69", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Activating" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C8F2C2", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Deactivating" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-299": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_netstat_Tcp_InSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "TCP Rx in", + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_netstat_Tcp_OutSegs{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "TCP Tx out", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of TCP segments sent and received per second, including data and control segments", + "id": 299, + "links": [], + "title": "TCP In / Out", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-3": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "sum(rate(node_cpu_seconds_total{mode=\"system\",instance=\"$node\",job=\"$job\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "interval": "", + "legendFormat": "System - Processes executing in kernel mode", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "sum(rate(node_cpu_seconds_total{mode=\"user\",instance=\"$node\",job=\"$job\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "legendFormat": "User - Normal processes executing in user mode", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "sum(rate(node_cpu_seconds_total{mode=\"nice\",instance=\"$node\",job=\"$job\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "legendFormat": "Nice - Niced processes executing in user mode", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "C" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "sum(rate(node_cpu_seconds_total{mode=\"iowait\",instance=\"$node\",job=\"$job\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "legendFormat": "Iowait - Waiting for I/O to complete", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "D" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "sum(rate(node_cpu_seconds_total{mode=\"irq\",instance=\"$node\",job=\"$job\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "legendFormat": "Irq - Servicing interrupts", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "E" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "sum(rate(node_cpu_seconds_total{mode=\"softirq\",instance=\"$node\",job=\"$job\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "legendFormat": "Softirq - Servicing softirqs", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "F" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "sum(rate(node_cpu_seconds_total{mode=\"steal\",instance=\"$node\",job=\"$job\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "legendFormat": "Steal - Time spent in other operating systems when running in a virtualized environment", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "G" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "sum(rate(node_cpu_seconds_total{mode=\"idle\",instance=\"$node\",job=\"$job\"}[$__rate_interval])) / scalar(count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu)))", + "format": "time_series", + "legendFormat": "Idle - Waiting for something to happen", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "H" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "sum by(instance) (rate(node_cpu_guest_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])) / on(instance) group_left sum by (instance)((rate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]))) > 0", + "format": "time_series", + "legendFormat": "Guest CPU usage", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "I" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "CPU time usage split by state, normalized across all CPU cores", + "id": 3, + "links": [], + "title": "CPU", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 70, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "percent" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Idle - Waiting for something to happen" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Iowait - Waiting for I/O to complete" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Irq - Servicing interrupts" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Nice - Niced processes executing in user mode" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Softirq - Servicing softirqs" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Steal - Time spent in other operating systems when running in a virtualized environment" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCE2DE", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "System - Processes executing in kernel mode" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "User - Normal processes executing in user mode" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#5195CE", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Guest CPU usage" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "custom.stacking", + "value": { + "group": "A", + "mode": "none" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 250 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-300": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "(node_cooling_device_max_state{instance=\"$node\",job=\"$job\"} > bool 0) * (100 * node_cooling_device_cur_state{instance=\"$node\",job=\"$job\"} / node_cooling_device_max_state{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "interval": "", + "legendFormat": "{{name}} - {{type}}", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Shows how hard each cooling device (fan/throttle) is working relative to its maximum capacity", + "id": 300, + "links": [], + "title": "Cooling Device Utilization", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Max.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-301": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_disk_discards_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{device}} - Discards completed", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_disk_discards_merged_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{device}} - Discards merged", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_disk_flush_requests_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{device}} - Flush", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "C" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Per-second rate of discard (TRIM) and flush (write cache) operations. Useful for monitoring low-level disk activity on SSDs and advanced storage", + "id": 301, + "links": [], + "title": "Disk Ops Discards / Flush", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/sda.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-302": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_power_supply_online{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "{{ power_supply }} online", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Shows the online status of power supplies (e.g., AC, battery). A value of 1-Yes indicates the power supply is active/online", + "id": 302, + "links": [], + "title": "Power Supply", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bool_yes_no" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-305": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": true, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_schedstat_running_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "CPU {{ cpu }} - Running", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": true, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_schedstat_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "CPU {{cpu}} - Waiting Queue", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "((rate(node_schedstat_running_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) + rate(node_schedstat_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])) > bool 0) * (rate(node_schedstat_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) / (rate(node_schedstat_running_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) + rate(node_schedstat_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])))", + "format": "time_series", + "interval": "", + "legendFormat": "CPU {{cpu}}", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "C" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Shows CPU saturation per core, calculated as the proportion of time spent waiting to run relative to total time demanded (running + waiting).", + "id": 305, + "links": [], + "title": "CPU Saturation per Core", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*waiting.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-306": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_schedstat_timeslices_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "CPU {{ cpu }}", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of scheduling timeslices executed per CPU. Reflects how frequently the scheduler switches tasks on each core", + "id": 306, + "links": [], + "title": "CPU Schedule Timeslices", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-307": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_vmstat_oom_kill{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "OOM Kills", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of Out-of-Memory (OOM) kill events. A non-zero value indicates the kernel has terminated one or more processes due to memory exhaustion", + "id": 307, + "links": [], + "title": "OOM Killer", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "OOM Kills" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-308": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(process_cpu_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "Process CPU Usage", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of CPU time used by the process exposing this metric (user + system mode)", + "id": 308, + "links": [], + "title": "Exporter Process CPU Usage", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-309": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": true, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_network_up{operstate=\"up\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "{{interface}} - Operational state UP", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_network_carrier{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{device}} - Physical link" + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Operational and physical link status of each network interface. Values are Yes for 'up' or link present, and No for 'down' or no carrier.", + "id": 309, + "links": [], + "title": "Network Operational Status", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bool_yes_no" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-310": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_softnet_times_squeezed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "CPU {{cpu}} - Times Squeezed", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "How often the kernel was unable to process all packets in the softnet queue before time ran out. Frequent squeezes may indicate CPU contention or driver inefficiency", + "id": 310, + "links": [], + "title": "Softnet Out of Quota", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "eps" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-313": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_processes_pids{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "Number of PIDs", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_processes_max_processes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "PIDs limit", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Number of active PIDs on the system and the configured maximum allowed. Useful for detecting PID exhaustion risk. Requires --collector.processes in node_exporter", + "id": 313, + "links": [], + "title": "PIDs Number and Limit", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "PIDs limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-314": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_processes_threads{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "Allocated threads", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_processes_max_threads{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "Threads limit", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Number of active threads on the system and the configured thread limit. Useful for monitoring thread pressure. Requires --collector.processes in node_exporter", + "id": 314, + "links": [], + "title": "Threads Number and Limit", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Threads limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F2495C", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-315": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_processes_state{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "{{ state }}", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Current number of processes in each state (e.g., running, sleeping, zombie). Requires --collector.processes to be enabled in node_exporter", + "id": 315, + "links": [], + "title": "Processes Detailed States", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "D" + }, + "properties": [ + { + "id": "displayName", + "value": "Uninterruptible Sleeping" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "I" + }, + "properties": [ + { + "id": "displayName", + "value": "Idle Kernel Thread" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "R" + }, + "properties": [ + { + "id": "displayName", + "value": "Running" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "S" + }, + "properties": [ + { + "id": "displayName", + "value": "Interruptible Sleeping" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "T" + }, + "properties": [ + { + "id": "displayName", + "value": "Stopped" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "X" + }, + "properties": [ + { + "id": "displayName", + "value": "Dead" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Z" + }, + "properties": [ + { + "id": "displayName", + "value": "Zombie" + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-320": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"established\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "Established", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"fin_wait2\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "FIN_WAIT2", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"listen\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "Listen", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "C" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"time_wait\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "TIME_WAIT", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "D" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"close_wait\", instance=\"$node\", job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "CLOSE_WAIT", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "E" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Number of TCP sockets in key connection states. Requires the --collector.tcpstat flag on node_exporter", + "id": 320, + "links": [], + "title": "TCP Stat Persistent", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-321": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_cpu_scaling_frequency_hertz{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "CPU {{ cpu }}", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "avg(node_cpu_scaling_frequency_max_hertz{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "interval": "", + "legendFormat": "Max", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "avg(node_cpu_scaling_frequency_min_hertz{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "interval": "", + "legendFormat": "Min", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "C" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Real-time CPU frequency scaling per core, including average minimum and maximum allowed scaling frequencies", + "id": 321, + "links": [], + "title": "CPU Frequency Scaling", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "hertz" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Max" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + }, + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": false, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Min" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": false, + "viz": false + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-322": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_pressure_cpu_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "CPU - Some", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "CPU some" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_pressure_memory_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "Memory - Some", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "Memory some" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_pressure_memory_stalled_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "Memory - Full", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "Memory full" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_pressure_io_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "I/O - Some", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "I/O some" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_pressure_io_stalled_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "I/O - Full", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "I/O full" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_pressure_irq_stalled_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "IRQ - Full", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "How often tasks experience CPU, memory, or I/O delays. 'Some' indicates partial slowdown; 'Full' indicates all tasks are stalled. Based on Linux PSI metrics:\nhttps://docs.kernel.org/accounting/psi.html", + "id": 322, + "links": [], + "title": "Pressure Stall Information", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "some (-) / full (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Some.*/" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Some.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-323": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "exemplar": false, + "expr": "rate(node_pressure_cpu_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "instant": true, + "legendFormat": "CPU", + "range": false, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "exemplar": false, + "expr": "rate(node_pressure_memory_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "instant": true, + "legendFormat": "Mem", + "range": false, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "exemplar": false, + "expr": "rate(node_pressure_io_waiting_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "instant": true, + "legendFormat": "I/O", + "range": false, + "step": 240 + }, + "version": "v0" + }, + "refId": "C" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "exemplar": false, + "expr": "rate(node_pressure_irq_stalled_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "instant": true, + "legendFormat": "Irq", + "range": false, + "step": 240 + }, + "version": "v0" + }, + "refId": "D" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Resource pressure via PSI", + "id": 323, + "links": [], + "title": "Pressure", + "vizConfig": { + "group": "bargauge", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "max": 1, + "min": 0, + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "dark-yellow", + "value": 70 + }, + { + "color": "dark-red", + "value": 90 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "displayMode": "basic", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 10, + "minVizWidth": 0, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "text": {}, + "valueMode": "color" + } + }, + "version": "13.0.1" + } + } + }, + "panel-325": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_hwmon_fan_rpm{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "{{ chip_name }} {{ sensor }}", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": true, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_hwmon_fan_min_rpm{instance=\"$node\",job=\"$job\"} * on(chip) group_left(chip_name) node_hwmon_chip_names{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "{{ chip_name }} {{ sensor }} rpm min", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Displays the current fan speeds (RPM) from hardware sensors via the hwmon interface", + "id": 325, + "links": [], + "title": "Hardware Fan Speed", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "rotrpm" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-326": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_disk_discarded_sectors_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{device}}", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Shows how many disk sectors are discarded (TRIMed) per second. Useful for monitoring SSD behavior and storage efficiency", + "id": 326, + "links": [], + "title": "Disk Sectors Discarded Successfully", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/sda.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-327": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_network_receive_nohandler_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "{{device}} - Rx in", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of received packets that could not be processed due to missing protocol or handler in the kernel. May indicate unsupported traffic or misconfiguration", + "id": 327, + "links": [], + "title": "Network Traffic NoHandler", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "pps" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-33": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "{{device}} - Read", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "exemplar": false, + "expr": "rate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "legendFormat": "{{device}} - Write", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Number of bytes read from or written to the device per second", + "id": 33, + "links": [], + "title": "Disk Read/Write Data", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "read (\u2013) / write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/sda.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-330": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_softnet_received_rps_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "CPU {{cpu}} - Processed", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_softnet_flow_limit_count_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "CPU {{cpu}} - Dropped", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Tracks the number of packets processed or dropped by Receive Packet Steering (RPS), a mechanism to distribute packet processing across CPUs", + "id": 330, + "links": [], + "title": "Softnet RPS", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Dropped.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + }, + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-331": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_systemd_socket_current_connections{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "{{ name }}", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Current number of active connections per systemd socket, as reported by the Node Exporter systemd collector", + "id": 331, + "links": [], + "title": "Systemd Sockets Current", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-332": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_systemd_socket_refused_connections_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "{{ name }}", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of systemd socket connection refusals per second, typically due to service unavailability or backlog overflow", + "id": 332, + "links": [], + "title": "Systemd Sockets Refused", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "eps" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-333": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_timex_pps_frequency_hertz{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "PPS Frequency Offset", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_timex_pps_stability_hertz{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "PPS Frequency Stability", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Displays the PPS signal's frequency offset and stability (jitter) in hertz. Useful for monitoring high-precision time sources like GPS or atomic clocks", + "id": 333, + "links": [], + "title": "PPS Frequency / Stability", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "hertz" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-334": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_timex_pps_jitter_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "PPS Jitter", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_timex_pps_shift_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "PPS Shift", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Tracks PPS signal timing jitter and shift compared to system clock", + "id": 334, + "links": [], + "title": "PPS Time Accuracy", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-335": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_timex_pps_calibration_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "PPS Calibrations/sec", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_timex_pps_error_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "PPS Errors/sec", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_timex_pps_stability_exceeded_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "PPS Stability Exceeded/sec", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "C" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_timex_pps_jitter_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "PPS Jitter Events/sec", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "D" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of PPS synchronization diagnostics including calibration events, jitter violations, errors, and frequency stability exceedances", + "id": 335, + "links": [], + "title": "PPS Sync Events", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-336": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_sockstat_TCP_mem{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "TCP", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_sockstat_UDP_mem{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "UDP", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "TCP/UDP socket memory usage in kernel (in pages)", + "id": 336, + "links": [], + "title": "TCP/UDP Kernel Buffer Memory Pages", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-337": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_udp_queues{instance=\"$node\",job=\"$job\",ip=\"v4\",queue=\"rx\"}", + "format": "time_series", + "interval": "", + "legendFormat": "UDP Rx in Queue", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_udp_queues{instance=\"$node\",job=\"$job\",ip=\"v4\",queue=\"tx\"}", + "format": "time_series", + "interval": "", + "legendFormat": "UDP Tx out Queue", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Number of UDP packets currently queued in the receive (RX) and transmit (TX) buffers. A growing queue may indicate a bottleneck", + "id": 337, + "links": [], + "title": "UDP Queue", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-338": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "(node_network_speed_bytes{instance=\"$node\",job=\"$job\"} > bool 0) * (rate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) / node_network_speed_bytes{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "legendFormat": "{{device}} - Rx in", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "(node_network_speed_bytes{instance=\"$node\",job=\"$job\"} > bool 0) * (rate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) / node_network_speed_bytes{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "legendFormat": "{{device}} - Tx out", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Network interface utilization as a percentage of its maximum capacity", + "id": 338, + "links": [], + "title": "Network Saturation", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-339": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "(node_sockstat_TCP_inuse{instance=\"$node\",job=\"$job\"} > bool 0) * (node_sockstat_TCP_mem_bytes{instance=\"$node\",job=\"$job\"} / node_sockstat_TCP_inuse{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "interval": "", + "legendFormat": "TCP", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "(node_sockstat_UDP_inuse{instance=\"$node\",job=\"$job\"} > bool 0) * (node_sockstat_UDP_mem_bytes{instance=\"$node\",job=\"$job\"} / node_sockstat_UDP_inuse{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "interval": "", + "legendFormat": "UDP", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Average memory used per socket (TCP/UDP). Helps tune net.ipv4.tcp_rmem / tcp_wmem", + "id": 339, + "links": [], + "title": "Sockstat Average Socket Memory", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-34": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_disk_io_now{instance=\"$node\",job=\"$job\"}", + "interval": "", + "legendFormat": "{{device}}", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Number of in-progress I/O requests at the time of sampling (active requests in the disk queue)", + "id": 34, + "links": [], + "title": "Instantaneous Queue Size", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/sda.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-340": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"rx_queued_bytes\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "RX Queued (waiting to be read)", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"tx_queued_bytes\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "TX Queued (waiting to be sent)", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "TCP socket queue sizes. High rx_queued_bytes indicates application not reading fast enough. High tx_queued_bytes indicates network congestion or slow receiver", + "id": 340, + "links": [], + "title": "TCP Socket Queue", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-341": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"syn_sent\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "SYN_SENT", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"syn_recv\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "SYN_RECV", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"fin_wait1\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "FIN_WAIT1", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "C" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"close\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "CLOSE", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "D" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"last_ack\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "LAST_ACK", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "E" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_tcp_connection_states{state=\"closing\",instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "CLOSING", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "F" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Transient TCP connection states. These are typically short-lived during connection establishment and teardown", + "id": 341, + "links": [], + "title": "TCP Stat Transient", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-35": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_disk_io_time_weighted_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{device}}", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Average queue length of the requests that were issued to the device", + "id": 35, + "links": [], + "title": "Average Queue Size", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/sda_*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-36": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_disk_io_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{device}} - General IO", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_disk_discard_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{device}} - Discard/TRIM", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_disk_flush_requests_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{device}} - Flush (write cache)", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "C" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Percentage of time the disk spent actively processing I/O operations, including general I/O, discards (TRIM), and write cache flushes", + "id": 36, + "links": [], + "title": "Time Spent Doing I/Os", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/sda.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-37": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "(rate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) > bool 0) * (rate(node_disk_read_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) / rate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "{{device}} - Read", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "(rate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) > bool 0) * (rate(node_disk_write_time_seconds_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]) / rate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "{{device}} - Write", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Average time for requests issued to the device to be served. This includes the time spent by the requests in queue and the time spent servicing them.", + "id": 37, + "links": [], + "title": "Disk Average Wait Time", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "read (\u2013) / write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/sda.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-40": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_scrape_collector_duration_seconds{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "{{collector}}", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Duration of each individual collector executed during a Node Exporter scrape. Useful for identifying slow or failing collectors", + "id": 40, + "links": [], + "title": "Node Exporter Scrape Time", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-41": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_filesystem_files_free{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "legendFormat": "{{mountpoint}}", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Number of free file nodes (inodes) available per mounted filesystem. A low count may prevent file creation even if disk space is available", + "id": 41, + "links": [], + "title": "File Nodes Free", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-42": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_disk_read_bytes_total{instance=\"$node\",job=\"$job\",device=~\"[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "{{device}} - Read", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_disk_written_bytes_total{instance=\"$node\",job=\"$job\",device=~\"[a-z]+|nvme[0-9]+n[0-9]+|mmcblk[0-9]+\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "{{device}} - Write", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Disk I/O throughput per device", + "id": 42, + "links": [], + "title": "Disk Throughput", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "read (-) / write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-43": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_filesystem_avail_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "legendFormat": "{{mountpoint}}", + "metric": "", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": true, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_filesystem_free_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "legendFormat": "{{mountpoint}} - Free", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": true, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_filesystem_size_bytes{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "legendFormat": "{{mountpoint}} - Size", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "C" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Amount of available disk space per mounted filesystem, excluding rootfs. Based on block availability to non-root users", + "id": 43, + "links": [], + "title": "Filesystem Space Available", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-44": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_filesystem_readonly{instance=\"$node\",job=\"$job\",device!~'rootfs'}", + "format": "time_series", + "legendFormat": "{{mountpoint}} - ReadOnly", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_filesystem_device_error{instance=\"$node\",job=\"$job\",device!~'rootfs',fstype!~'tmpfs'}", + "format": "time_series", + "interval": "", + "legendFormat": "{{mountpoint}} - Device error", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Indicates filesystems mounted in read-only mode or reporting device-level I/O errors.", + "id": 44, + "links": [], + "title": "Filesystem in ReadOnly / Error", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bool_yes_no" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-50": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_netstat_Icmp_InErrors{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "ICMP Rx In", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of incoming ICMP messages that contained protocol-specific errors, such as bad checksums or invalid lengths", + "id": 50, + "links": [], + "title": "ICMP Errors", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "pps" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-55": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_netstat_Udp_InDatagrams{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "UDP Rx in", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_netstat_Udp_OutDatagrams{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "UDP Tx out", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of UDP datagrams sent and received per second, based on /proc/net/netstat", + "id": 55, + "links": [], + "title": "UDP In / Out", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-60": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_network_receive_packets_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "{{device}} - Rx in", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_network_transmit_packets_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "{{device}} - Tx out", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Number of network packets received and transmitted per second, by interface.", + "id": 60, + "links": [], + "title": "Network Traffic by Packets", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-61": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_nf_conntrack_entries{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "NF conntrack entries", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_nf_conntrack_entries_limit{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "NF conntrack limit", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Current and maximum connection tracking entries used by Netfilter (nf_conntrack). High usage approaching the limit may cause packet drops or connection issues", + "id": 61, + "links": [], + "title": "NF Conntrack", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "NF conntrack limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-62": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_procs_blocked{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Blocked (I/O Wait)", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_procs_running{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Runnable (Ready for CPU)", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Processes currently in runnable or blocked states. Helps identify CPU contention or I/O wait bottlenecks.", + "id": 62, + "links": [], + "title": "Processes Status", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-63": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_sockstat_TCP_alloc{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "Allocated Sockets", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_sockstat_TCP_inuse{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "In-Use Sockets", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_sockstat_TCP_orphan{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "Orphaned Sockets", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "C" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_sockstat_TCP_tw{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "TIME_WAIT Sockets", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "D" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Tracks TCP socket usage and memory per node", + "id": 63, + "links": [], + "title": "Sockstat TCP", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-64": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "process_max_fds{instance=\"$node\",job=\"$job\"}", + "interval": "", + "legendFormat": "Maximum open file descriptors", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "process_open_fds{instance=\"$node\",job=\"$job\"}", + "interval": "", + "legendFormat": "Open file descriptors", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Number of file descriptors used by the exporter process versus its configured limit", + "id": 64, + "links": [], + "title": "Exporter File Descriptor Usage", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Max.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + }, + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Open file descriptors" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": true, + "viz": true + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-7": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_load1{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Load 1m", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_load5{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Load 5m", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_load15{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Load 15m", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "C" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "count(count(node_cpu_seconds_total{instance=\"$node\",job=\"$job\"}) by (cpu))", + "format": "time_series", + "legendFormat": "CPU Core Count", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "D" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "System load average over 1, 5, and 15 minutes. Reflects the number of active or waiting processes. Values above CPU core count may indicate overload", + "id": 7, + "links": [], + "title": "System Load", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "CPU Core Count" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-70": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_VmallocChunk_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Vmalloc Free Chunk \u2013 Largest available block in vmalloc area", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_VmallocTotal_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Vmalloc Total \u2013 Total size of the vmalloc memory area", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_VmallocUsed_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Vmalloc Used \u2013 Portion of vmalloc area currently in use", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "C" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Usage of the kernel's vmalloc area, which provides virtual memory allocations for kernel modules and drivers. Includes total, used, and largest free block sizes", + "id": 70, + "links": [], + "title": "Memory Vmalloc", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Total.*/" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-74": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", + "format": "time_series", + "legendFormat": "Rx {{device}}", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", + "format": "time_series", + "legendFormat": "Tx {{device}}", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Per-interface network traffic (receive and transmit) in bits per second", + "id": 74, + "links": [], + "title": "Network Traffic Basic", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Tx.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "13.0.1" + } + } + }, + "panel-75": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "exemplar": false, + "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}", + "instant": true, + "range": false, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": { + "maxDataPoints": 100 + }, + "transformations": [] + } + }, + "description": "", + "id": 75, + "links": [], + "title": "RAM Total", + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.0.1" + } + } + }, + "panel-77": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "exemplar": false, + "expr": "avg(rate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"system\"}[$__rate_interval]))", + "format": "time_series", + "instant": false, + "legendFormat": "Busy System", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "avg(rate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"user\"}[$__rate_interval]))", + "format": "time_series", + "legendFormat": "Busy User", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "avg(rate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"iowait\"}[$__rate_interval]))", + "format": "time_series", + "legendFormat": "Busy Iowait", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "C" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "avg(sum without(mode) (rate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=~\".*irq\"}[$__rate_interval])))", + "format": "time_series", + "legendFormat": "Busy IRQs", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "D" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "avg(sum without (mode) (rate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode!='idle',mode!='user',mode!='system',mode!='iowait',mode!='irq',mode!='softirq'}[$__rate_interval])))", + "format": "time_series", + "legendFormat": "Busy Other", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "E" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "avg(rate(node_cpu_seconds_total{instance=\"$node\",job=\"$job\", mode=\"idle\"}[$__rate_interval]))", + "format": "time_series", + "legendFormat": "Idle", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "F" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "CPU time spent busy vs idle, split by activity type", + "id": 77, + "links": [], + "title": "CPU Basic", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "percent" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Busy Iowait" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Idle" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy System" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy User" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Busy Other" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + } + ] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "width": 250 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + } + }, + "version": "13.0.1" + } + } + }, + "panel-78": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Total", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_MemTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"} - (node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} + node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "legendFormat": "Used", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_Cached_bytes{instance=\"$node\",job=\"$job\"} + node_memory_Buffers_bytes{instance=\"$node\",job=\"$job\"} + node_memory_SReclaimable_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Cache + Buffer", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "C" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_memory_MemFree_bytes{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "legendFormat": "Free", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "D" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "(node_memory_SwapTotal_bytes{instance=\"$node\",job=\"$job\"} - node_memory_SwapFree_bytes{instance=\"$node\",job=\"$job\"})", + "format": "time_series", + "legendFormat": "Swap used", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "E" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "RAM and swap usage overview, including caches", + "id": 78, + "links": [], + "title": "Memory Basic", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Swap used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.stacking", + "value": { + "group": false, + "mode": "normal" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache + Buffer" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "13.0.1" + } + } + }, + "panel-8": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_context_switches_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "Context switches", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_intr_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "legendFormat": "Interrupts", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Per-second rate of context switches and hardware interrupts. High values may indicate intense CPU or I/O activity", + "id": 8, + "links": [], + "title": "Context Switches / Interrupts", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-82": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_netstat_Tcp_ActiveOpens{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "Active Opens", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_netstat_Tcp_PassiveOpens{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "Passive Opens", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of TCP connection initiations per second. 'Active' opens are initiated by this host. 'Passive' opens are accepted from incoming connections", + "id": 82, + "links": [], + "title": "TCP Direct Transition", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "eps" + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-84": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_network_receive_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", + "format": "time_series", + "legendFormat": "{{device}} - Rx in", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_network_transmit_bytes_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])*8", + "format": "time_series", + "legendFormat": "{{device}} - Tx out", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Incoming and outgoing network traffic per interface", + "id": 84, + "links": [], + "title": "Network Traffic", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 40, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-85": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_netstat_Tcp_CurrEstab{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "Current Connections", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "node_netstat_Tcp_MaxConn{instance=\"$node\",job=\"$job\"}", + "format": "time_series", + "interval": "", + "legendFormat": "Max Connections", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Number of currently established TCP connections and the system's max supported limit. On Linux, MaxConn may return -1 to indicate a dynamic/unlimited configuration", + "id": 85, + "links": [], + "title": "TCP Connections", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Max.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-9": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_disk_reads_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "legendFormat": "{{device}} - Read", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_disk_writes_completed_total{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "legendFormat": "{{device}} - Write", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Number of I/O operations completed per second for the device (after merges), including both reads and writes", + "id": 9, + "links": [], + "title": "Disk Read/Write IOps", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "read (\u2013) / write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "iops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/sda.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + }, + "panel-91": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_netstat_TcpExt_SyncookiesFailed{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "SYN Cookies Failed", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_netstat_TcpExt_SyncookiesRecv{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "SYN Cookies Validated", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "rate(node_netstat_TcpExt_SyncookiesSent{instance=\"$node\",job=\"$job\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "legendFormat": "SYN Cookies Sent", + "range": true, + "step": 240 + }, + "version": "v0" + }, + "refId": "C" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of TCP SYN cookies sent, validated, and failed. These are used to protect against SYN flood attacks and manage TCP handshake resources under load", + "id": 91, + "links": [], + "title": "TCP SynCookie", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "eps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Failed.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "11.6.1" + } + } + } + }, + "layout": { + "kind": "RowsLayout", + "spec": { + "rows": [ + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-323" + }, + "height": 4, + "width": 3, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-20" + }, + "height": 4, + "width": 3, + "x": 3, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-155" + }, + "height": 4, + "width": 3, + "x": 6, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-16" + }, + "height": 4, + "width": 3, + "x": 9, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-21" + }, + "height": 4, + "width": 3, + "x": 12, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-154" + }, + "height": 4, + "width": 3, + "x": 15, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-14" + }, + "height": 2, + "width": 2, + "x": 18, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-75" + }, + "height": 2, + "width": 2, + "x": 20, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-18" + }, + "height": 2, + "width": 2, + "x": 22, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-23" + }, + "height": 2, + "width": 2, + "x": 18, + "y": 2 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-15" + }, + "height": 2, + "width": 4, + "x": 20, + "y": 2 + } + } + ] + } + }, + "title": "Quick CPU / Mem / Disk" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-77" + }, + "height": 7, + "width": 12, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-78" + }, + "height": 7, + "width": 12, + "x": 12, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-74" + }, + "height": 7, + "width": 12, + "x": 0, + "y": 7 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-152" + }, + "height": 7, + "width": 12, + "x": 12, + "y": 7 + } + } + ] + } + }, + "title": "Basic CPU / Mem / Net / Disk" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": true, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-3" + }, + "height": 12, + "width": 12, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-24" + }, + "height": 12, + "width": 12, + "x": 12, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-84" + }, + "height": 12, + "width": 12, + "x": 0, + "y": 412 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-338" + }, + "height": 12, + "width": 12, + "x": 12, + "y": 412 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-229" + }, + "height": 12, + "width": 12, + "x": 0, + "y": 424 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-42" + }, + "height": 12, + "width": 12, + "x": 12, + "y": 424 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-43" + }, + "height": 12, + "width": 12, + "x": 0, + "y": 436 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-156" + }, + "height": 12, + "width": 12, + "x": 12, + "y": 436 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-127" + }, + "height": 12, + "width": 12, + "x": 0, + "y": 448 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-322" + }, + "height": 12, + "width": 12, + "x": 12, + "y": 448 + } + } + ] + } + }, + "title": "CPU / Memory / Net / Disk" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": true, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-135" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-130" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-131" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 200 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-138" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 200 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-136" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 210 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-191" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 210 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-160" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 220 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-70" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 220 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-129" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 230 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-137" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 230 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-128" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 240 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-140" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 240 + } + } + ] + } + }, + "title": "Memory Meminfo" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": true, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-176" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-22" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-175" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 180 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-307" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 180 + } + } + ] + } + }, + "title": "Memory Vmstat" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": true, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-260" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-291" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-168" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 150 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-333" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 150 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-334" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 160 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-335" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 160 + } + } + ] + } + }, + "title": "System Timesync" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": true, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-62" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-315" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-148" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 30 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-305" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 30 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-313" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 40 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-314" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 40 + } + } + ] + } + }, + "title": "System Processes" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": true, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-8" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-7" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-321" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 10 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-306" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 10 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-259" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 20 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-151" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 20 + } + } + ] + } + }, + "title": "System Misc" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": true, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-158" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-300" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-302" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 10 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-325" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 10 + } + } + ] + } + }, + "title": "Hardware Misc" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": true, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-298" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-331" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-297" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 10 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-332" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 10 + } + } + ] + } + }, + "title": "Systemd" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": true, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-9" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-33" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-37" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 360 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-35" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 360 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-133" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 370 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-36" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 370 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-301" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 380 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-326" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 380 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-34" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 390 + } + } + ] + } + }, + "title": "Storage Disk" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": true, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-28" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-41" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-44" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 340 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-219" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 340 + } + } + ] + } + }, + "title": "Storage Filesystem" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": true, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-60" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-142" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-143" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 220 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-141" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 220 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-146" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 230 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-327" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 230 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-145" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 240 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-144" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 240 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-232" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 250 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-231" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 250 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-230" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 260 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-61" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 260 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-309" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 270 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-280" + }, + "height": 10, + "width": 6, + "x": 12, + "y": 270 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-288" + }, + "height": 10, + "width": 6, + "x": 18, + "y": 270 + } + } + ] + } + }, + "title": "Network Traffic" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": true, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-63" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-124" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-126" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 10 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-125" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 10 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-220" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 20 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-339" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 20 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-336" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 30 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-290" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 30 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-310" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 40 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-330" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 40 + } + } + ] + } + }, + "title": "Network Sockstat" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": true, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-221" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-299" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-55" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 30 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-115" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 30 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-104" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 40 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-109" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 40 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-50" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 50 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-91" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 50 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-85" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 60 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-337" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 60 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-82" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 70 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-320" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 70 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-341" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 80 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-340" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 80 + } + } + ] + } + }, + "title": "Network Netstat" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": true, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-40" + }, + "height": 10, + "width": 12, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-308" + }, + "height": 10, + "width": 12, + "x": 12, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-149" + }, + "height": 10, + "width": 10, + "x": 0, + "y": 10 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-64" + }, + "height": 10, + "width": 10, + "x": 10, + "y": 10 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-157" + }, + "height": 10, + "width": 4, + "x": 20, + "y": 10 + } + } + ] + } + }, + "title": "Node Exporter" + } + } + ] + } + }, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "GitHub", + "tooltip": "", + "type": "link", + "url": "https://github.com/rfmoz/grafana-dashboards" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": true, + "title": "Grafana", + "tooltip": "", + "type": "link", + "url": "https://grafana.com/grafana/dashboards/1860" + } + ], + "liveNow": false, + "preload": false, + "tags": [ + "linux" + ], + "timeSettings": { + "autoRefresh": "1m", + "autoRefreshIntervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "fiscalYearStartMonth": 0, + "from": "now-24h", + "hideTimepicker": false, + "timezone": "browser", + "to": "now" + }, + "title": "Node Exporter Full", + "variables": [ + { + "kind": "DatasourceVariable", + "spec": { + "allowCustomValue": true, + "current": { + "text": "prometheus", + "value": "ffkc0x7oj7g1sc" + }, + "hide": "dontHide", + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "ds_prometheus", + "options": [], + "pluginId": "prometheus", + "refresh": "onDashboardLoad", + "regex": "", + "skipUrlSync": false + } + }, + { + "kind": "QueryVariable", + "spec": { + "allowCustomValue": true, + "current": { + "text": "node", + "value": "node" + }, + "definition": "", + "hide": "dontHide", + "includeAll": false, + "label": "Job", + "multi": false, + "name": "job", + "options": [], + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "query": "label_values(node_uname_info, job)", + "refId": "Prometheus-job-Variable-Query" + }, + "version": "v0" + }, + "refresh": "onDashboardLoad", + "regex": "", + "regexApplyTo": "value", + "skipUrlSync": false, + "sort": "alphabeticalAsc" + } + }, + { + "kind": "QueryVariable", + "spec": { + "allowCustomValue": true, + "current": { + "text": "beepi", + "value": "beepi" + }, + "definition": "label_values(node_uname_info{job=\"$job\"}, nodename)", + "hide": "dontHide", + "includeAll": false, + "label": "Nodename", + "multi": false, + "name": "nodename", + "options": [], + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "query": "label_values(node_uname_info{job=\"$job\"}, nodename)", + "refId": "Prometheus-nodename-Variable-Query" + }, + "version": "v0" + }, + "refresh": "onDashboardLoad", + "regex": "", + "regexApplyTo": "value", + "skipUrlSync": false, + "sort": "alphabeticalAsc" + } + }, + { + "kind": "QueryVariable", + "spec": { + "allowCustomValue": true, + "current": { + "text": "host.containers.internal:9100", + "value": "host.containers.internal:9100" + }, + "definition": "label_values(node_uname_info{job=\"$job\", nodename=\"$nodename\"}, instance)", + "hide": "dontHide", + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "node", + "options": [], + "query": { + "datasource": { + "name": "${ds_prometheus}" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "query": "label_values(node_uname_info{job=\"$job\", nodename=\"$nodename\"}, instance)", + "refId": "Prometheus-node-Variable-Query" + }, + "version": "v0" + }, + "refresh": "onDashboardLoad", + "regex": "", + "regexApplyTo": "value", + "skipUrlSync": false, + "sort": "alphabeticalAsc" + } + } + ] + } +} diff --git a/playbooks/monitoring/grafana/dashboards-general/podman-exporter-dashboard.json b/playbooks/monitoring/grafana/dashboards-general/podman-exporter-dashboard.json new file mode 100644 index 0000000..b6cb130 --- /dev/null +++ b/playbooks/monitoring/grafana/dashboards-general/podman-exporter-dashboard.json @@ -0,0 +1,1317 @@ +{ + "apiVersion": "dashboard.grafana.app/v2", + "kind": "Dashboard", + "metadata": { + "name": "fdsbn3zzogqv4f" + }, + "spec": { + "annotations": [ + { + "kind": "AnnotationQuery", + "spec": { + "builtIn": true, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "query": { + "datasource": { + "name": "grafana" + }, + "group": "grafana", + "kind": "DataQuery", + "spec": {}, + "version": "v0" + } + } + } + ], + "cursorSync": "Off", + "description": "This dashboard leverages Prometheus podman exporter as the data source and utilizes various visualization panels like time series and tables, providing critical insights into the operational metrics of Podman containers.", + "editable": true, + "elements": { + "panel-14": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "ffkc0x7oj7g1sc" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "sum by(name) (podman_container_info{instance=~\"$instance\", name=~\"$name\"} * on(id) group_right(name) rate(podman_container_net_input_total[1m]) / 1024)", + "interval": "", + "legendFormat": "{{name}} - Read", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "ffkc0x7oj7g1sc" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(rate(podman_container_blkio_device_write_bytes_total{instance=~\"$host\",name=~\"$container\"}[5m])) by (name)", + "interval": "", + "legendFormat": "{{name}} - Write" + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 14, + "links": [], + "title": "Recieved Network Traffic", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Bytes/sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "13.0.1" + } + } + }, + "panel-17": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "ffkc0x7oj7g1sc" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "sum by(name) (podman_container_info{instance=~\"$instance\", name=~\"$name\"} * on(id) group_right(name) rate(podman_container_net_output_total[1m]) / 1024)", + "interval": "", + "legendFormat": "{{name}} - Read", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "ffkc0x7oj7g1sc" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(rate(podman_container_blkio_device_write_bytes_total{instance=~\"$host\",name=~\"$container\"}[5m])) by (name)", + "interval": "", + "legendFormat": "{{name}} - Write" + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 17, + "links": [], + "title": "Sent Network Traffic", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Bytes/sec", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "13.0.1" + } + } + }, + "panel-20": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "ffkc0x7oj7g1sc" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by (name) (podman_container_info{instance=~'$instance', name=~\"$name\"} * on(id) group_right (name) rate(podman_container_cpu_seconds_total[1m])\n)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "interval": "", + "legendFormat": "__auto", + "range": true, + "useBackend": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 20, + "links": [], + "title": "CPU Usage", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "13.0.1" + } + } + }, + "panel-21": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "ffkc0x7oj7g1sc" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "exemplar": false, + "expr": "podman_container_info{instance=~\"$instance\", name=~\"$name\"}", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false + }, + "version": "v0" + }, + "refId": "info" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "ffkc0x7oj7g1sc" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "exemplar": false, + "expr": "(podman_container_info{instance=~\"$instance\", name=~\"$name\"} * on (id) group_right (name) (podman_container_state))", + "format": "table", + "instant": true, + "legendFormat": "\u72b6\u6001", + "range": false + }, + "version": "v0" + }, + "refId": "state" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "ffkc0x7oj7g1sc" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "exemplar": false, + "expr": "(podman_container_info{instance=~\"$instance\", name=~\"$name\"} * on (id) group_right (name) (podman_container_mem_usage_bytes))", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false + }, + "version": "v0" + }, + "refId": "momery" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "ffkc0x7oj7g1sc" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "exemplar": false, + "expr": "(podman_container_info{instance=~\"$instance\", name=~\"$name\"} * on (id) group_right (name) (podman_container_exit_code))", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false + }, + "version": "v0" + }, + "refId": "exit_code" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "ffkc0x7oj7g1sc" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "exemplar": false, + "expr": "(podman_container_info{instance=~\"$instance\", name=~\"$name\"} * on (id) group_right (name) (time() - podman_container_started_seconds))", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false + }, + "version": "v0" + }, + "refId": "uptime" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "ffkc0x7oj7g1sc" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "exemplar": false, + "expr": "(podman_container_info{instance=~\"$instance\", name=~\"$name\"} * on (id) group_right (name) (podman_container_pids))", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false + }, + "version": "v0" + }, + "refId": "PID" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "ffkc0x7oj7g1sc" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "exemplar": false, + "expr": "(podman_container_info{instance=~\"$instance\", name=~\"$name\"} * on (id) group_right (name) (podman_container_mem_usage_bytes / (podman_container_mem_limit_bytes)))", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false + }, + "version": "v0" + }, + "refId": "mem_percentage" + } + } + ], + "queryOptions": {}, + "transformations": [ + { + "group": "filterFieldsByName", + "kind": "Transformation", + "spec": { + "options": { + "include": { + "names": [ + "id", + "image", + "instance", + "name", + "ports", + "Value #state", + "Value #momery", + "Value #exit_code", + "Value #uptime", + "Value #PID", + "Value #mem_percentage" + ] + } + } + } + }, + { + "group": "merge", + "kind": "Transformation", + "spec": { + "options": {} + } + }, + { + "group": "organize", + "kind": "Transformation", + "spec": { + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "Value #A": true, + "__name__": true, + "job": true + }, + "indexByName": { + "Value #PID": 8, + "Value #exit_code": 10, + "Value #mem_percentage": 4, + "Value #momery": 3, + "Value #state": 7, + "Value #uptime": 9, + "id": 1, + "image": 5, + "instance": 2, + "name": 0, + "ports": 6 + }, + "renameByName": { + "Value": "", + "Value #B": "\u72b6\u6001", + "Value #C": "\u9000\u51fa\u4ee3\u7801", + "Value #PID": "PID", + "Value #exit_code": "Exit code", + "Value #ext_code": "\u9519\u8bef\u4ee3\u7801", + "Value #mem_max": "\u5185\u5b58\u4f7f\u7528\u7387", + "Value #mem_percentage": "Mem Usage", + "Value #momery": "Memory", + "Value #state": "Status", + "Value #uptime": "Uptime", + "id": "ID", + "image": "Image", + "instance": "Host", + "name": "name", + "ports": "Port", + "state": "\u72b6\u6001" + } + } + } + } + ] + } + }, + "description": "", + "id": 21, + "links": [], + "title": "Container Overview", + "vizConfig": { + "group": "table", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "center", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "inspect": false + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Memory" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + }, + { + "id": "custom.width", + "value": 90 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "-1": { + "color": "red", + "index": 0, + "text": "unknown" + }, + "0": { + "color": "#c0c0c0", + "index": 1, + "text": "created" + }, + "1": { + "index": 2, + "text": "initialized" + }, + "2": { + "color": "green", + "index": 3, + "text": "running" + }, + "3": { + "index": 4, + "text": "stopped" + }, + "4": { + "color": "#808081", + "index": 5, + "text": "paused" + }, + "5": { + "color": "red", + "index": 6, + "text": "exited" + }, + "6": { + "color": "yellow", + "index": 7, + "text": "removing" + }, + "7": { + "color": "yellow", + "index": 8, + "text": "stopping" + } + }, + "type": "value" + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "custom.width", + "value": 75 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Exit code" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "color": "#808080", + "index": 0, + "text": "none" + } + }, + "type": "value" + }, + { + "options": { + "from": 1, + "result": { + "color": "red", + "index": 1 + }, + "to": 255 + }, + "type": "range" + } + ] + }, + { + "id": "custom.width", + "value": 75 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Uptime" + }, + "properties": [ + { + "id": "unit", + "value": "s" + }, + { + "id": "custom.width", + "value": 90 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PID" + }, + "properties": [ + { + "id": "custom.width", + "value": 50 + }, + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Host" + }, + "properties": [ + { + "id": "custom.width", + "value": 297 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ID" + }, + "properties": [ + { + "id": "custom.width", + "value": 121 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mem Usage" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + }, + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "color", + "value": { + "mode": "continuous-GrYlRd" + } + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + }, + { + "id": "decimals", + "value": 2 + } + ] + } + ] + }, + "options": { + "cellHeight": "sm", + "frameIndex": 1, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Status" + } + ] + } + }, + "version": "13.0.1" + } + } + }, + "panel-9": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "ffkc0x7oj7g1sc" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "editorMode": "code", + "expr": "sum by (name) (\n podman_container_info{instance=~\"$instance\", name=~\"$name\"}\n * on (id) group_right (name)\n (podman_container_mem_usage_bytes / 1024 / 1024)\n)", + "interval": "", + "legendFormat": "{{name}}", + "range": true + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 9, + "links": [], + "title": "Memory Usage", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "13.0.1" + } + } + } + }, + "layout": { + "kind": "RowsLayout", + "spec": { + "rows": [ + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-20" + }, + "height": 7, + "width": 24, + "x": 0, + "y": 0 + } + } + ] + } + }, + "title": "CPU" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-9" + }, + "height": 9, + "width": 24, + "x": 0, + "y": 0 + } + } + ] + } + }, + "title": "Memory" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-14" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-17" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-21" + }, + "height": 9, + "width": 24, + "x": 0, + "y": 8 + } + } + ] + } + }, + "title": "Network" + } + } + ] + } + }, + "links": [], + "liveNow": false, + "preload": false, + "tags": [ + "podman", + "prometheus-podman-exporter" + ], + "timeSettings": { + "autoRefresh": "auto", + "autoRefreshIntervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "fiscalYearStartMonth": 0, + "from": "now-30m", + "hideTimepicker": false, + "timezone": "browser", + "to": "now" + }, + "title": "Podman Exporter Dashboard", + "variables": [ + { + "kind": "QueryVariable", + "spec": { + "allValue": ".*", + "allowCustomValue": true, + "current": { + "text": "", + "value": "" + }, + "definition": "label_values(podman_container_info{instance=~\"$instance\"},instance)", + "hide": "dontHide", + "includeAll": true, + "label": "Host", + "multi": false, + "name": "instance", + "options": [], + "query": { + "datasource": { + "name": "ffkc0x7oj7g1sc" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "qryType": 1, + "query": "label_values(podman_container_info{instance=~\"$instance\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "version": "v0" + }, + "refresh": "onDashboardLoad", + "regex": "", + "regexApplyTo": "value", + "skipUrlSync": false, + "sort": "disabled" + } + }, + { + "kind": "QueryVariable", + "spec": { + "allValue": ".*", + "allowCustomValue": true, + "current": { + "text": "", + "value": "" + }, + "definition": "label_values(podman_container_info{name=~\"$name\", instance=~\"$instance\"},name)", + "hide": "dontHide", + "includeAll": true, + "label": "Container", + "multi": false, + "name": "name", + "options": [], + "query": { + "datasource": { + "name": "ffkc0x7oj7g1sc" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "qryType": 1, + "query": "label_values(podman_container_info{name=~\"$name\", instance=~\"$instance\"},name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "version": "v0" + }, + "refresh": "onDashboardLoad", + "regex": "", + "regexApplyTo": "value", + "skipUrlSync": false, + "sort": "disabled" + } + } + ] + } +} diff --git a/playbooks/monitoring/grafana/dashboards-general/prometheus-2-0-overview.json b/playbooks/monitoring/grafana/dashboards-general/prometheus-2-0-overview.json new file mode 100644 index 0000000..f3f575f --- /dev/null +++ b/playbooks/monitoring/grafana/dashboards-general/prometheus-2-0-overview.json @@ -0,0 +1,4479 @@ +{ + "apiVersion": "dashboard.grafana.app/v2", + "kind": "Dashboard", + "metadata": { + "name": "adt6r6s" + }, + "spec": { + "annotations": [ + { + "kind": "AnnotationQuery", + "spec": { + "builtIn": true, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "query": { + "datasource": { + "name": "-- Grafana --" + }, + "group": "grafana", + "kind": "DataQuery", + "spec": {}, + "version": "v0" + } + } + }, + { + "kind": "AnnotationQuery", + "spec": { + "enable": true, + "hide": false, + "iconColor": "rgb(0, 96, 19)", + "legacyOptions": { + "expr": "sum(changes(prometheus_config_last_reload_success_timestamp_seconds{instance=~\"$instance\"}[10m])) by (instance)", + "limit": 100, + "showIn": 0, + "step": "5m" + }, + "name": "reloads", + "query": { + "group": "prometheus", + "kind": "DataQuery", + "spec": {}, + "version": "v0" + } + } + }, + { + "kind": "AnnotationQuery", + "spec": { + "enable": true, + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "legacyOptions": { + "expr": "count(sum(up{instance=\"$instance\"}) by (instance) < 1)", + "limit": 100, + "showIn": 0, + "step": "5m" + }, + "name": "down", + "query": { + "group": "prometheus", + "kind": "DataQuery", + "spec": {}, + "version": "v0" + } + } + } + ], + "cursorSync": "Off", + "description": "Get started faster with Grafana Cloud then easily build these dashboards. https://grafana.com/products/cloud/\nOverview of metrics from Prometheus 2.0. \nUseful for using prometheus to monitor your prometheus.\nRevisions welcome!", + "editable": true, + "elements": { + "panel-1": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "up{instance=~\"$instance\",job=~\"$job\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "step": 2 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 1, + "links": [], + "title": "Upness (stacked)", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.0.1" + } + } + }, + "panel-10": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "1000 * rate(prometheus_evaluator_duration_seconds_sum{job=~\"$job\", instance=~\"$instance\"}[5m]) / rate(prometheus_evaluator_duration_seconds_count{job=~\"$job\", instance=~\"$instance\"}[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "step": 4 + }, + "version": "v0" + }, + "refId": "E" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "The duration of rule group evaluations", + "id": 10, + "links": [], + "title": "Average Rule Evaluation Duration", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": {} + }, + "version": "" + } + } + }, + "panel-11": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(rate(http_request_duration_microseconds_count{job=~\"$job\",instance=~\"$instance\"}[1m])) by (handler) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{handler}}", + "step": 4 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 11, + "links": [], + "title": "HTTP Request Duration", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": {} + }, + "version": "" + } + } + }, + "panel-12": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(prometheus_tsdb_head_series{job=~\"$job\",instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "step": 40 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": { + "maxDataPoints": 100 + }, + "transformations": [] + } + }, + "description": "Total number of time series in prometheus", + "id": 12, + "links": [], + "title": "Total Series", + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 0 + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 1000000 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 2000000 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.0.1" + } + } + }, + "panel-13": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(sum_over_time(prometheus_tsdb_reloads_failures_total{job=~\"$job\",instance=~\"$instance\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "step": 40 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": { + "maxDataPoints": 100 + }, + "transformations": [] + } + }, + "description": "Number of times the database failed to reload block data from disk.", + "id": 13, + "links": [], + "title": "Reload Failures [$interval]", + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 0 + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 1 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 10 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.0.1" + } + } + }, + "panel-14": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(prometheus_tsdb_head_chunks{job=~\"$job\",instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "step": 40 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": { + "maxDataPoints": 100 + }, + "transformations": [] + } + }, + "description": "", + "id": 14, + "links": [], + "title": "Memory Chunks", + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgb(31, 120, 193)", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.0.1" + } + } + }, + "panel-15": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(prometheus_engine_query_duration_seconds_sum{job=~\"$job\",instance=~\"$instance\"}) by (slice)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{slice}}", + "step": 4 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 15, + "links": [], + "title": "Prometheus Engine Query Duration Seconds", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": {} + }, + "version": "" + } + } + }, + "panel-16": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(sum_over_time(prometheus_evaluator_iterations_missed_total{job=~\"$job\",instance=~\"$instance\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "step": 40 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": { + "maxDataPoints": 100 + }, + "transformations": [] + } + }, + "description": "The total number of rule group evaluations missed due to slow rule group evaluation.", + "id": 16, + "links": [], + "title": "Missed Iterations [$interval]", + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 0 + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 1 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 10 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.0.1" + } + } + }, + "panel-18": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(sum_over_time(prometheus_evaluator_iterations_skipped_total{job=~\"$job\",instance=~\"$instance\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "step": 40 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": { + "maxDataPoints": 100 + }, + "transformations": [] + } + }, + "description": "The total number of rule group evaluations skipped due to throttled metric storage.", + "id": 18, + "links": [], + "title": "Skipped Iterations [$interval]", + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 0 + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 1 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 10 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.0.1" + } + } + }, + "panel-19": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(sum_over_time(prometheus_target_scrapes_exceeded_sample_limit_total{job=~\"$job\",instance=~\"$instance\"}[$interval]))", + "format": "time_series", + "intervalFactor": 2, + "step": 40 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": { + "maxDataPoints": 100 + }, + "transformations": [] + } + }, + "description": "Total number of scrapes that hit the sample limit and were rejected.", + "id": 19, + "links": [], + "title": "Tardy Scrapes [$interval]", + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 0 + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 1 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 10 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.0.1" + } + } + }, + "panel-2": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "avg(avg_over_time(up{instance=~\"$instance\",job=~\"$job\"}[$interval]) * 100)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "step": 40 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": { + "maxDataPoints": 100 + }, + "transformations": [] + } + }, + "description": "Percentage of uptime during the most recent $interval period. Change the period with the 'interval' dropdown above.", + "id": 2, + "links": [], + "title": "Uptime [$interval]", + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "decimals": 3, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 0 + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 90 + }, + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 99 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.0.1" + } + } + }, + "panel-20": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(sum_over_time(prometheus_target_scrapes_exceeded_sample_limit_total{job=~\"$job\",instance=~\"$instance\"}[$interval])) + \nsum(sum_over_time(prometheus_target_scrapes_sample_duplicate_timestamp_total{job=~\"$job\",instance=~\"$instance\"}[$interval])) + \nsum(sum_over_time(prometheus_target_scrapes_sample_out_of_bounds_total{job=~\"$job\",instance=~\"$instance\"}[$interval])) + \nsum(sum_over_time(prometheus_target_scrapes_sample_out_of_order_total{job=~\"$job\",instance=~\"$instance\"}[$interval])) ", + "format": "time_series", + "intervalFactor": 2, + "step": 40 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": { + "maxDataPoints": 100 + }, + "transformations": [] + } + }, + "description": "Sum of all skipped scrapes", + "id": 20, + "links": [], + "title": "Skipped Scrapes [$interval]", + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)", + "value": 0 + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 1 + }, + { + "color": "rgba(245, 54, 54, 0.9)", + "value": 10 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.0.1" + } + } + }, + "panel-21": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(rate(prometheus_target_sync_length_seconds_sum{job=~\"$job\",instance=~\"$instance\"}[2m])) by (scrape_job) * 1000", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{scrape_job}}", + "step": 4 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Actual interval to sync the scrape pool.", + "id": 21, + "links": [], + "title": "Target Sync", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.0.1" + } + } + }, + "panel-22": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "rate(prometheus_notifications_sent_total[5m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "step": 2 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 22, + "links": [], + "title": "Notifications Sent", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": {} + }, + "version": "" + } + } + }, + "panel-23": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "(time() - prometheus_config_last_reload_success_timestamp_seconds{job=~\"$job\",instance=~\"$instance\"}) / 60", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "step": 4 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 23, + "links": [], + "title": "Minutes Since Successful Config Reload", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": {} + }, + "version": "" + } + } + }, + "panel-24": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "prometheus_config_last_reload_successful{job=~\"$job\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "step": 4 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 24, + "links": [], + "title": "Successful Config Reload", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": {} + }, + "version": "" + } + } + }, + "panel-25": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "up{instance=~\"$instance\",job=~\"$job\"} < 1", + "format": "table", + "intervalFactor": 2, + "step": 2 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": { + "hideTimeOverride": true, + "timeFrom": "1s" + }, + "transformations": [ + { + "group": "merge", + "kind": "Transformation", + "spec": { + "options": { + "reducers": [] + } + } + } + ] + } + }, + "description": "Servers which are DOWN RIGHT NOW! \nFIX THEM!!", + "id": 25, + "links": [], + "title": "Currently Down", + "vizConfig": { + "group": "table", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "inspect": false + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "displayName", + "value": "Time" + }, + { + "id": "custom.hideFrom.viz", + "value": true + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/__name__|job|Value/" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.hideFrom.viz", + "value": true + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "displayName", + "value": " " + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "custom.align" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(255, 0, 0, 0.9)", + "value": 0 + }, + { + "color": "rgba(237, 129, 40, 0.89)" + }, + { + "color": "rgba(255, 0, 0, 0.97)" + }, + {} + ] + } + } + ] + } + ] + }, + "options": { + "cellHeight": "sm", + "showHeader": true + } + }, + "version": "13.0.1" + } + } + }, + "panel-26": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": true, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(go_memstats_alloc_bytes_total{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "alloc_bytes_total", + "step": 10 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(go_memstats_alloc_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "allocated", + "step": 10 + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(go_memstats_buck_hash_sys_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "profiling bucket hash table", + "step": 10 + }, + "version": "v0" + }, + "refId": "C" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(go_memstats_gc_sys_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "GC metadata", + "step": 10 + }, + "version": "v0" + }, + "refId": "D" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(go_memstats_heap_alloc_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "heap in-use", + "step": 10 + }, + "version": "v0" + }, + "refId": "E" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(go_memstats_heap_idle_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "heap idle", + "step": 10 + }, + "version": "v0" + }, + "refId": "F" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(go_memstats_heap_inuse_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "heap in use", + "step": 10 + }, + "version": "v0" + }, + "refId": "G" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(go_memstats_heap_released_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "heap released", + "step": 10 + }, + "version": "v0" + }, + "refId": "H" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(go_memstats_heap_sys_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "heap system", + "step": 10 + }, + "version": "v0" + }, + "refId": "I" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(go_memstats_mcache_inuse_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mcache in use", + "step": 10 + }, + "version": "v0" + }, + "refId": "J" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(go_memstats_mcache_sys_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mcache sys", + "step": 10 + }, + "version": "v0" + }, + "refId": "K" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(go_memstats_mspan_inuse_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mspan in use", + "step": 10 + }, + "version": "v0" + }, + "refId": "L" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(go_memstats_mspan_sys_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "mspan sys", + "step": 10 + }, + "version": "v0" + }, + "refId": "M" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(go_memstats_next_gc_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "heap next gc", + "step": 10 + }, + "version": "v0" + }, + "refId": "N" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(go_memstats_other_sys_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "other sys", + "step": 10 + }, + "version": "v0" + }, + "refId": "O" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(go_memstats_stack_inuse_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "stack in use", + "step": 10 + }, + "version": "v0" + }, + "refId": "P" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(go_memstats_stack_sys_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "stack sys", + "step": 10 + }, + "version": "v0" + }, + "refId": "Q" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(go_memstats_sys_bytes{job=~\"$job\", instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "sys", + "step": 10 + }, + "version": "v0" + }, + "refId": "R" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "This is probably wrong! Please help.", + "id": 26, + "links": [], + "title": "Go Memory Usage (FIXME)", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": {} + }, + "version": "" + } + } + }, + "panel-28": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(rate(go_gc_duration_seconds_sum{instance=~\"$instance\",job=~\"$job\"}[2m])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "step": 2 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "GC invocation durations", + "id": 28, + "links": [], + "title": "GC Rate / 2m", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": {} + }, + "version": "" + } + } + }, + "panel-29": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "scrape_duration_seconds{instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "step": 4 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 29, + "links": [], + "title": "Scrape Duration", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.0.1" + } + } + }, + "panel-3": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "prometheus_tsdb_head_series{job=~\"$job\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "step": 4 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 3, + "links": [], + "title": "Series Count", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.0.1" + } + } + }, + "panel-30": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(prometheus_target_scrapes_exceeded_sample_limit_total{job=~\"$job\",instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "exceeded sample limit", + "step": 4 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(prometheus_target_scrapes_sample_duplicate_timestamp_total{job=~\"$job\",instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "duplicate timestamp", + "step": 4 + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(prometheus_target_scrapes_sample_out_of_bounds_total{job=~\"$job\",instance=~\"$instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "out of bounds", + "step": 4 + }, + "version": "v0" + }, + "refId": "C" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(prometheus_target_scrapes_sample_out_of_order_total{job=~\"$job\",instance=~\"$instance\"}) ", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "out of order", + "step": 4 + }, + "version": "v0" + }, + "refId": "D" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Total number of rejected scrapes", + "id": 30, + "links": [], + "title": "Rejected Scrapes", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.0.1" + } + } + }, + "panel-31": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(rate(prometheus_evaluator_iterations_total{job=~\"$job\", instance=~\"$instance\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Total", + "step": 4 + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(rate(prometheus_evaluator_iterations_missed_total{job=~\"$job\", instance=~\"$instance\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Missed", + "step": 4 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(rate(prometheus_evaluator_iterations_skipped_total{job=~\"$job\", instance=~\"$instance\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Skipped", + "step": 4 + }, + "version": "v0" + }, + "refId": "C" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rule-group evaluations \n - total\n - missed due to slow rule group evaluation\n - skipped due to throttled metric storage", + "id": 31, + "links": [], + "title": "Rule Evaluator Iterations", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": {} + }, + "version": "" + } + } + }, + "panel-32": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum( increase(prometheus_tsdb_head_series_created_total{instance=~\"$instance\"}[5m]) )", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "created", + "step": 4 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum( increase(prometheus_tsdb_head_series_removed_total{instance=~\"$instance\"}[5m]) )", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "removed", + "step": 4 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 32, + "links": [], + "title": "Series Created / Removed", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.0.1" + } + } + }, + "panel-33": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(increase(net_conntrack_dialer_conn_failed_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Failed Connections", + "step": 2 + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(increase(prometheus_evaluator_iterations_missed_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Missed Iterations", + "step": 2 + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(increase(prometheus_evaluator_iterations_skipped_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Skipped Iterations", + "step": 2 + }, + "version": "v0" + }, + "refId": "C" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(increase(prometheus_rule_evaluation_failures_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Evaluation", + "step": 2 + }, + "version": "v0" + }, + "refId": "D" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(increase(prometheus_sd_azure_refresh_failures_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Azure Refresh", + "step": 2 + }, + "version": "v0" + }, + "refId": "E" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(increase(prometheus_sd_consul_rpc_failures_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Consul RPC", + "step": 2 + }, + "version": "v0" + }, + "refId": "F" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(increase(prometheus_sd_dns_lookup_failures_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "DNS Lookup", + "step": 2 + }, + "version": "v0" + }, + "refId": "G" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(increase(prometheus_sd_ec2_refresh_failures_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "EC2 Refresh", + "step": 2 + }, + "version": "v0" + }, + "refId": "H" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(increase(prometheus_sd_gce_refresh_failures_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "GCE Refresh", + "step": 2 + }, + "version": "v0" + }, + "refId": "I" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(increase(prometheus_sd_marathon_refresh_failures_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Marathon Refresh", + "step": 2 + }, + "version": "v0" + }, + "refId": "J" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(increase(prometheus_sd_openstack_refresh_failures_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Openstack Refresh", + "step": 2 + }, + "version": "v0" + }, + "refId": "K" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(increase(prometheus_sd_triton_refresh_failures_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Triton Refresh", + "step": 2 + }, + "version": "v0" + }, + "refId": "L" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(increase(prometheus_target_scrapes_exceeded_sample_limit_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Sample Limit", + "step": 2 + }, + "version": "v0" + }, + "refId": "M" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(increase(prometheus_target_scrapes_sample_duplicate_timestamp_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Duplicate Timestamp", + "step": 2 + }, + "version": "v0" + }, + "refId": "N" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(increase(prometheus_target_scrapes_sample_out_of_bounds_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Timestamp Out of Bounds", + "step": 2 + }, + "version": "v0" + }, + "refId": "O" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(increase(prometheus_target_scrapes_sample_out_of_order_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Sample Out of Order", + "step": 2 + }, + "version": "v0" + }, + "refId": "P" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(increase(prometheus_treecache_zookeeper_failures_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Zookeeper", + "step": 2 + }, + "version": "v0" + }, + "refId": "Q" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(increase(prometheus_tsdb_compactions_failed_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "TSDB Compactions", + "step": 2 + }, + "version": "v0" + }, + "refId": "R" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(increase(prometheus_tsdb_head_series_not_found{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Series Not Found", + "step": 2 + }, + "version": "v0" + }, + "refId": "S" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(increase(prometheus_tsdb_reloads_failures_total{instance=~\"$instance\"}[5m])) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Reload", + "step": 2 + }, + "version": "v0" + }, + "refId": "T" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "All non-zero failures and errors", + "id": 33, + "links": [], + "title": "Failures and Errors", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Errors", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + } + }, + "version": "13.0.1" + } + } + }, + "panel-4": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "rate(prometheus_tsdb_head_samples_appended_total{job=~\"$job\",instance=~\"$instance\"}[1m])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "step": 2 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Rate of total number of appended samples", + "id": 4, + "links": [], + "title": "Appended Samples per Second", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.0.1" + } + } + }, + "panel-5": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "prometheus_tsdb_head_chunks{job=~\"$job\",instance=~\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "step": 4 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 5, + "links": [], + "title": "Storage Memory Chunks", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.0.1" + } + } + }, + "panel-6": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(prometheus_target_scrape_pool_sync_total{job=~\"$job\",instance=~\"$instance\"}) by (scrape_job)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{scrape_job}}", + "step": 4 + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Total number of syncs that were executed on a scrape pool.", + "id": 6, + "links": [], + "title": "Scrape Sync Total", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.0.1" + } + } + }, + "panel-7": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(rate(prometheus_target_interval_length_seconds_count{job=~\"$job\",instance=~\"$instance\"}[5m])) by (interval)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{interval}}", + "step": 20 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 7, + "links": [], + "title": "Target Scrapes / 5m", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": {} + }, + "version": "" + } + } + }, + "panel-9": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "prometheus_target_interval_length_seconds{instance=~\"$instance\", job=~\"$job\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{quantile}} {{interval}}", + "step": 20 + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 9, + "links": [], + "title": "Scrape Duration", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "options": {} + }, + "version": "" + } + } + } + }, + "layout": { + "kind": "RowsLayout", + "spec": { + "rows": [ + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-2" + }, + "height": 7, + "width": 6, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-25" + }, + "height": 7, + "width": 6, + "x": 6, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-12" + }, + "height": 7, + "width": 6, + "x": 12, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-14" + }, + "height": 7, + "width": 6, + "x": 18, + "y": 0 + } + } + ] + } + }, + "title": "at a glance" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-16" + }, + "height": 7, + "width": 4, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-18" + }, + "height": 7, + "width": 4, + "x": 4, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-19" + }, + "height": 7, + "width": 4, + "x": 8, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-13" + }, + "height": 7, + "width": 4, + "x": 12, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-20" + }, + "height": 7, + "width": 8, + "x": 16, + "y": 0 + } + } + ] + } + }, + "title": "quick numbers" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-33" + }, + "height": 7, + "width": 24, + "x": 0, + "y": 0 + } + } + ] + } + }, + "title": "errors" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-1" + }, + "height": 7, + "width": 12, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-5" + }, + "height": 7, + "width": 12, + "x": 12, + "y": 0 + } + } + ] + } + }, + "title": "up" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-3" + }, + "height": 7, + "width": 12, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-32" + }, + "height": 7, + "width": 12, + "x": 12, + "y": 0 + } + } + ] + } + }, + "title": "series" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-4" + }, + "height": 7, + "width": 24, + "x": 0, + "y": 0 + } + } + ] + } + }, + "title": "appended samples" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-6" + }, + "height": 7, + "width": 12, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-21" + }, + "height": 7, + "width": 12, + "x": 12, + "y": 0 + } + } + ] + } + }, + "title": "sync" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-29" + }, + "height": 7, + "width": 12, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-30" + }, + "height": 7, + "width": 12, + "x": 12, + "y": 0 + } + } + ] + } + }, + "title": "scrapes" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-10" + }, + "height": 7, + "width": 12, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-11" + }, + "height": 7, + "width": 12, + "x": 12, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-15" + }, + "height": 7, + "width": 12, + "x": 0, + "y": 7 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-31" + }, + "height": 7, + "width": 12, + "x": 12, + "y": 7 + } + } + ] + } + }, + "title": "durations" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-22" + }, + "height": 7, + "width": 24, + "x": 0, + "y": 0 + } + } + ] + } + }, + "title": "notifications" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-23" + }, + "height": 7, + "width": 12, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-24" + }, + "height": 7, + "width": 12, + "x": 12, + "y": 0 + } + } + ] + } + }, + "title": "config" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-28" + }, + "height": 7, + "width": 24, + "x": 0, + "y": 0 + } + } + ] + } + }, + "title": "garbage collection" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": true, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-26" + }, + "height": 7, + "width": 12, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-9" + }, + "height": 7, + "width": 6, + "x": 12, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-7" + }, + "height": 7, + "width": 6, + "x": 18, + "y": 0 + } + } + ] + } + }, + "title": "Broken, ignore" + } + } + ] + } + }, + "links": [], + "liveNow": false, + "preload": false, + "tags": [], + "timeSettings": { + "autoRefresh": "30s", + "autoRefreshIntervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "fiscalYearStartMonth": 0, + "from": "now-30m", + "hideTimepicker": false, + "timezone": "browser", + "to": "now" + }, + "title": "Prometheus 2.0 Overview", + "variables": [ + { + "kind": "QueryVariable", + "spec": { + "allowCustomValue": true, + "current": { + "text": "", + "value": "" + }, + "hide": "dontHide", + "includeAll": true, + "multi": true, + "name": "job", + "options": [], + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "__legacyStringValue": "query_result(prometheus_tsdb_head_samples_appended_total)" + }, + "version": "v0" + }, + "refresh": "onTimeRangeChanged", + "regex": "/.*job=\"([^\"]+)/", + "regexApplyTo": "value", + "skipUrlSync": false, + "sort": "alphabeticalAsc" + } + }, + { + "kind": "QueryVariable", + "spec": { + "allowCustomValue": true, + "current": { + "text": "", + "value": "" + }, + "hide": "dontHide", + "includeAll": true, + "multi": true, + "name": "instance", + "options": [], + "query": { + "datasource": { + "name": "prometheus" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "__legacyStringValue": "query_result(up{job=~\"$job\"})" + }, + "version": "v0" + }, + "refresh": "onTimeRangeChanged", + "regex": "/.*instance=\"([^\"]+).*/", + "regexApplyTo": "value", + "skipUrlSync": false, + "sort": "disabled" + } + }, + { + "kind": "CustomVariable", + "spec": { + "allowCustomValue": true, + "current": { + "text": "1h", + "value": "1h" + }, + "hide": "dontHide", + "includeAll": false, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "3h", + "value": "3h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "2d", + "value": "2d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + }, + { + "selected": false, + "text": "90d", + "value": "90d" + }, + { + "selected": false, + "text": "180d", + "value": "180d" + } + ], + "query": "1h, 3h, 6h, 12h, 1d, 2d, 7d, 30d, 90d, 180d", + "skipUrlSync": false, + "valuesFormat": "csv" + } + } + ] + } +} diff --git a/playbooks/monitoring/grafana/dashboards/backups.json.j2 b/playbooks/monitoring/grafana/dashboards/backups.json.j2 new file mode 100644 index 0000000..ead1709 --- /dev/null +++ b/playbooks/monitoring/grafana/dashboards/backups.json.j2 @@ -0,0 +1,215 @@ +{ + "id": null, + "uid": "beepi-backups", + "title": "Backups", + "tags": ["beepi", "backup"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "refresh": "1m", + "time": {"from": "now-7d", "to": "now"}, + "panels": [ + { + "type": "table", + "title": "Backup jobs", + "id": 1, + "gridPos": {"h": 7, "w": 24, "x": 0, "y": 0}, + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "targets": [ + { + "refId": "age", + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "expr": "time() - max by (backup_job) (backup_last_success_timestamp_seconds{backup_job!=\"\"})", + "instant": true, + "format": "table" + }, + { + "refId": "status", + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "expr": "max by (backup_job) (backup_last_status{backup_job!=\"\"})", + "instant": true, + "format": "table" + }, + { + "refId": "duration", + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "expr": "max by (backup_job) (backup_last_duration_seconds{backup_job!=\"\"})", + "instant": true, + "format": "table" + }, + { + "refId": "size", + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "expr": "max by (backup_job) (backup_last_archive_bytes{backup_job!=\"\"})", + "instant": true, + "format": "table" + } + ], + "transformations": [ + {"id": "joinByField", "options": {"byField": "backup_job", "mode": "outer"}}, + {"id": "organize", "options": { + "excludeByName": {"Time": true, "Time 1": true, "Time 2": true, "Time 3": true, "Time 4": true}, + "renameByName": { + "backup_job": "Job", + "Value #age": "Age", + "Value #status": "Status", + "Value #duration": "Duration", + "Value #size": "Archive" + } + }} + ], + "fieldConfig": { + "defaults": {"custom": {"align": "auto", "cellOptions": {"type": "auto"}}}, + "overrides": [ + {"matcher": {"id": "byName", "options": "Age"}, "properties": [ + {"id": "unit", "value": "s"}, + {"id": "custom.cellOptions", "value": {"type": "color-background"}}, + {"id": "thresholds", "value": {"mode": "absolute", "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 90000}, + {"color": "red", "value": 172800} + ]}} + ]}, + {"matcher": {"id": "byName", "options": "Duration"}, "properties": [{"id": "unit", "value": "s"}]}, + {"matcher": {"id": "byName", "options": "Archive"}, "properties": [{"id": "unit", "value": "bytes"}]}, + {"matcher": {"id": "byName", "options": "Status"}, "properties": [ + {"id": "custom.cellOptions", "value": {"type": "color-background"}}, + {"id": "mappings", "value": [{"type": "value", "options": { + "0": {"text": "FAILED", "color": "red", "index": 0}, + "1": {"text": "ok", "color": "green", "index": 1} + }}]}, + {"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}} + ]} + ] + }, + "options": {"showHeader": true} + }, + { + "type": "timeseries", + "title": "Archive size", + "id": 2, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 7}, + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "targets": [ + { + "refId": "A", + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "expr": "backup_archive_newest_bytes", + "legendFormat": "newest {{ '{{' }}kind{{ '}}' }}" + }, + { + "refId": "B", + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "expr": "backup_archive_total_bytes", + "legendFormat": "total {{ '{{' }}kind{{ '}}' }}" + } + ], + "fieldConfig": { + "defaults": {"unit": "bytes", "custom": {"lineWidth": 2, "fillOpacity": 8}}, + "overrides": [] + }, + "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi"}} + }, + { + "type": "timeseries", + "title": "Backup duration", + "id": 3, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 7}, + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "targets": [ + { + "refId": "A", + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "expr": "backup_last_duration_seconds{backup_job!=\"\"}", + "legendFormat": "{{ '{{' }}backup_job{{ '}}' }}" + } + ], + "fieldConfig": { + "defaults": {"unit": "s", "custom": {"lineWidth": 2, "fillOpacity": 8}}, + "overrides": [] + }, + "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi"}} + }, + { + "type": "stat", + "title": "Retained archives", + "id": 4, + "gridPos": {"h": 5, "w": 6, "x": 0, "y": 15}, + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "targets": [ + { + "refId": "A", + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "expr": "backup_archive_count", + "legendFormat": "{{ '{{' }}kind{{ '}}' }}", + "instant": true + } + ], + "fieldConfig": {"defaults": {"unit": "short", "color": {"mode": "thresholds"}, "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 2}]}}, "overrides": []}, + "options": {"colorMode": "background", "graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}} + }, + { + "type": "stat", + "title": "RAID mounted", + "id": 5, + "gridPos": {"h": 5, "w": 6, "x": 6, "y": 15}, + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "targets": [ + { + "refId": "A", + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "expr": "max(backup_raid_mounted) or vector(0)", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "mappings": [{"type": "value", "options": {"0": {"text": "NOT MOUNTED", "color": "red", "index": 0}, "1": {"text": "mounted", "color": "green", "index": 1}}}], + "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]} + }, + "overrides": [] + }, + "options": {"colorMode": "background", "graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}} + }, + { + "type": "timeseries", + "title": "RAID free space", + "id": 6, + "gridPos": {"h": 5, "w": 12, "x": 12, "y": 15}, + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "targets": [ + { + "refId": "A", + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "expr": "node_filesystem_avail_bytes{mountpoint=\"{{ backup_raid_mount }}\"}", + "legendFormat": "free" + } + ], + "fieldConfig": {"defaults": {"unit": "bytes", "custom": {"lineWidth": 2, "fillOpacity": 8}}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "single"}} + }, + { + "type": "logs", + "title": "Backup log", + "id": 7, + "gridPos": {"h": 12, "w": 24, "x": 0, "y": 20}, + "datasource": {"type": "loki", "uid": "{{ grafana_loki_uid }}"}, + "targets": [ + { + "refId": "A", + "datasource": {"type": "loki", "uid": "{{ grafana_loki_uid }}"}, + "expr": "{job=\"backup\"}", + "queryType": "range" + } + ], + "options": { + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true, + "enableLogDetails": true, + "dedupStrategy": "none", + "prettifyLogMessage": false + } + } + ] +} diff --git a/playbooks/monitoring/grafana/dashboards/infrastructure-map.json.j2 b/playbooks/monitoring/grafana/dashboards/infrastructure-map.json.j2 new file mode 100644 index 0000000..7ab3cd5 --- /dev/null +++ b/playbooks/monitoring/grafana/dashboards/infrastructure-map.json.j2 @@ -0,0 +1,364 @@ +{ + "id": null, + "uid": "beepi-map", + "title": "Infrastructure Map", + "tags": ["beepi", "overview"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": {"from": "now-6h", "to": "now"}, + "panels": [ + { + "type": "stat", + "title": "Services down", + "id": 1, + "gridPos": {"h": 4, "w": 4, "x": 0, "y": 0}, + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "targets": [ + { + "refId": "A", + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "expr": "count(min by (service) (probe_success{vantage=\"local\"}) == 0 and on (service) (max by (service) (service_enabled) == 1)) or vector(0)", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "red", "value": 1} + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false} + } + }, + { + "type": "stat", + "title": "Last daily backup", + "id": 2, + "gridPos": {"h": 4, "w": 4, "x": 4, "y": 0}, + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "targets": [ + { + "refId": "A", + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "expr": "time() - max(backup_last_success_timestamp_seconds{backup_job=\"daily\"})", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 90000}, + {"color": "red", "value": 172800} + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false} + } + }, + { + "type": "stat", + "title": "Last offsite sync", + "id": 3, + "gridPos": {"h": 4, "w": 4, "x": 8, "y": 0}, + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "targets": [ + { + "refId": "A", + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "expr": "time() - max(backup_last_success_timestamp_seconds{backup_job=\"offsite\"})", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 90000}, + {"color": "red", "value": 172800} + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false} + } + }, + { + "type": "stat", + "title": "Tunnel handshake", + "id": 4, + "gridPos": {"h": 4, "w": 4, "x": 12, "y": 0}, + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "targets": [ + { + "refId": "A", + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "expr": "max(wireguard_peer_handshake_age_seconds{peer=\"beepi\"})", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 180}, + {"color": "red", "value": 300} + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false} + } + }, + { + "type": "stat", + "title": "Ingress", + "id": 5, + "gridPos": {"h": 4, "w": 4, "x": 16, "y": 0}, + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "targets": [ + { + "refId": "A", + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "expr": "min(up{job=\"node\",host=\"ingress\"}) or vector(0)", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "mappings": [ + {"type": "value", "options": {"0": {"text": "DOWN", "color": "red", "index": 0}, "1": {"text": "UP", "color": "green", "index": 1}}} + ], + "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]} + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false} + } + }, + { + "type": "stat", + "title": "RAID free", + "id": 6, + "gridPos": {"h": 4, "w": 4, "x": 20, "y": 0}, + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "targets": [ + { + "refId": "A", + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "expr": "min(node_filesystem_avail_bytes{mountpoint=\"{{ backup_raid_mount }}\"})", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "orange", "value": 100000000000}, + {"color": "green", "value": 200000000000} + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "textMode": "auto", + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false} + } + }, + { + "type": "nodeGraph", + "title": "Infrastructure", + "id": 10, + "gridPos": {"h": 18, "w": 18, "x": 0, "y": 4}, + "datasource": {"type": "yesoreyeram-infinity-datasource", "uid": "{{ grafana_infinity_uid }}"}, + "targets": [ + { + "refId": "A", + "datasource": {"type": "yesoreyeram-infinity-datasource", "uid": "{{ grafana_infinity_uid }}"}, + "type": "json", + "source": "url", + "parser": "backend", + "format": "table", + "url": "http://topology:8088/topology.json", + "url_options": {"method": "GET", "data": ""}, + "root_selector": "nodes", + "columns": [ + {"selector": "id", "text": "id", "type": "string"}, + {"selector": "title", "text": "title", "type": "string"}, + {"selector": "subtitle", "text": "subtitle", "type": "string"}, + {"selector": "mainstat", "text": "mainstat", "type": "string"}, + {"selector": "arc__ok", "text": "arc__ok", "type": "number"}, + {"selector": "arc__fail", "text": "arc__fail", "type": "number"}, + {"selector": "arc__disabled", "text": "arc__disabled", "type": "number"}, + {"selector": "arc__unknown", "text": "arc__unknown", "type": "number"}, + {"selector": "arc__used", "text": "arc__used", "type": "number"}, + {"selector": "arc__free", "text": "arc__free", "type": "number"}, + {"selector": "detail__status", "text": "detail__status", "type": "string"}, + {"selector": "detail__group", "text": "detail__group", "type": "string"}, + {"selector": "detail__used", "text": "detail__used", "type": "string"}, + {"selector": "dashboard", "text": "dashboard", "type": "string"} + ] + }, + { + "refId": "B", + "datasource": {"type": "yesoreyeram-infinity-datasource", "uid": "{{ grafana_infinity_uid }}"}, + "type": "json", + "source": "url", + "parser": "backend", + "format": "table", + "url": "http://topology:8088/topology.json", + "url_options": {"method": "GET", "data": ""}, + "root_selector": "edges", + "columns": [ + {"selector": "id", "text": "id", "type": "string"}, + {"selector": "source", "text": "source", "type": "string"}, + {"selector": "target", "text": "target", "type": "string"}, + {"selector": "mainstat", "text": "mainstat", "type": "string"}, + {"selector": "color", "text": "color", "type": "string"}, + {"selector": "thickness", "text": "thickness", "type": "number"} + ] + } + ], + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + {"targetField": "arc__ok", "destinationType": "number"}, + {"targetField": "arc__fail", "destinationType": "number"}, + {"targetField": "arc__disabled", "destinationType": "number"}, + {"targetField": "arc__unknown", "destinationType": "number"}, + {"targetField": "arc__used", "destinationType": "number"}, + {"targetField": "arc__free", "destinationType": "number"} + ] + } + } + ], + "options": { + "nodes": { + "mainStatUnit": "", + "secondaryStatUnit": "", + "arcs": [ + {"field": "arc__ok", "color": "green"}, + {"field": "arc__fail", "color": "red"}, + {"field": "arc__disabled", "color": "text"}, + {"field": "arc__unknown", "color": "orange"}, + {"field": "arc__used", "color": "semi-dark-orange"}, + {"field": "arc__free", "color": "semi-dark-green"} + ] + }, + "edges": {"mainStatUnit": ""}, + "zoomMode": "cooperative" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [ + { + "matcher": {"id": "byName", "options": "arc__ok"}, + "properties": [{"id": "color", "value": {"mode": "fixed", "fixedColor": "green"}}] + }, + { + "matcher": {"id": "byName", "options": "arc__fail"}, + "properties": [{"id": "color", "value": {"mode": "fixed", "fixedColor": "red"}}] + }, + { + "matcher": {"id": "byName", "options": "arc__disabled"}, + "properties": [{"id": "color", "value": {"mode": "fixed", "fixedColor": "text"}}] + }, + { + "matcher": {"id": "byName", "options": "arc__unknown"}, + "properties": [{"id": "color", "value": {"mode": "fixed", "fixedColor": "orange"}}] + }, + { + "matcher": {"id": "byName", "options": "arc__used"}, + "properties": [{"id": "color", "value": {"mode": "fixed", "fixedColor": "semi-dark-orange"}}] + }, + { + "matcher": {"id": "byName", "options": "arc__free"}, + "properties": [{"id": "color", "value": {"mode": "fixed", "fixedColor": "semi-dark-green"}}] + }, + { + "matcher": {"id": "byName", "options": "id"}, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "Open related dashboard", + "url": "${__data.fields.dashboard}", + "targetBlank": false + } + ] + } + ] + } + ] + } + }, + { + "type": "alertlist", + "title": "Firing alerts", + "id": 11, + "gridPos": {"h": 18, "w": 6, "x": 18, "y": 4}, + "options": { + "alertInstanceLabelFilter": "", + "alertName": "", + "dashboardAlerts": false, + "groupBy": [], + "groupMode": "default", + "maxItems": 30, + "sortOrder": 3, + "stateFilter": {"error": true, "firing": true, "noData": true, "normal": false, "pending": true}, + "viewMode": "list" + } + } + ] +} diff --git a/playbooks/monitoring/grafana/dashboards/ingress.json.j2 b/playbooks/monitoring/grafana/dashboards/ingress.json.j2 new file mode 100644 index 0000000..96565f4 --- /dev/null +++ b/playbooks/monitoring/grafana/dashboards/ingress.json.j2 @@ -0,0 +1,260 @@ +{ + "id": null, + "uid": "beepi-ingress", + "title": "Ingress", + "tags": ["beepi", "ingress"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": {"from": "now-24h", "to": "now"}, + "panels": [ + { + "type": "stat", + "title": "Ingress uptime", + "id": 1, + "gridPos": {"h": 4, "w": 5, "x": 0, "y": 0}, + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "targets": [ + { + "refId": "A", + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "expr": "time() - max(node_boot_time_seconds{host=\"ingress\"})", + "instant": true + } + ], + "fieldConfig": {"defaults": {"unit": "s", "color": {"mode": "fixed", "fixedColor": "blue"}}, "overrides": []}, + "options": {"colorMode": "value", "graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}} + }, + { + "type": "stat", + "title": "Tunnel RTT", + "id": 2, + "gridPos": {"h": 4, "w": 5, "x": 5, "y": 0}, + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "targets": [ + { + "refId": "A", + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "expr": "max(probe_duration_seconds{probe=\"tunnel-rtt\"})", + "instant": true + } + ], + "fieldConfig": { + "defaults": {"unit": "s", "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 0.1}, {"color": "red", "value": 0.3}]}}, + "overrides": [] + }, + "options": {"colorMode": "background", "graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}} + }, + { + "type": "stat", + "title": "Banned hosts (24h)", + "id": 3, + "gridPos": {"h": 4, "w": 5, "x": 10, "y": 0}, + "datasource": {"type": "loki", "uid": "{{ grafana_loki_uid }}"}, + "targets": [ + { + "refId": "A", + "datasource": {"type": "loki", "uid": "{{ grafana_loki_uid }}"}, + "expr": "sum(count_over_time({job=\"fail2ban\", host=\"ingress\"} |= \"Ban\" [24h]))", + "queryType": "instant" + } + ], + "fieldConfig": {"defaults": {"unit": "short", "color": {"mode": "fixed", "fixedColor": "orange"}}, "overrides": []}, + "options": {"colorMode": "value", "graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}} + }, + { + "type": "stat", + "title": "Blocked packets (24h)", + "id": 4, + "gridPos": {"h": 4, "w": 9, "x": 15, "y": 0}, + "datasource": {"type": "loki", "uid": "{{ grafana_loki_uid }}"}, + "targets": [ + { + "refId": "A", + "datasource": {"type": "loki", "uid": "{{ grafana_loki_uid }}"}, + "expr": "sum(count_over_time({job=\"ufw\", host=\"ingress\"} [24h]))", + "queryType": "instant" + } + ], + "fieldConfig": {"defaults": {"unit": "short", "color": {"mode": "fixed", "fixedColor": "purple"}}, "overrides": []}, + "options": {"colorMode": "value", "graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}} + }, + { + "type": "table", + "title": "WireGuard peers", + "id": 5, + "gridPos": {"h": 7, "w": 12, "x": 0, "y": 4}, + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "targets": [ + { + "refId": "age", + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "expr": "max by (peer) (wireguard_peer_handshake_age_seconds)", + "instant": true, + "format": "table" + }, + { + "refId": "rx", + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "expr": "max by (peer) (wireguard_peer_receive_bytes_total)", + "instant": true, + "format": "table" + }, + { + "refId": "tx", + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "expr": "max by (peer) (wireguard_peer_transmit_bytes_total)", + "instant": true, + "format": "table" + } + ], + "transformations": [ + {"id": "joinByField", "options": {"byField": "peer", "mode": "outer"}}, + {"id": "organize", "options": { + "excludeByName": {"Time": true, "Time 1": true, "Time 2": true, "Time 3": true}, + "renameByName": {"peer": "Peer", "Value #age": "Last handshake", "Value #rx": "Received", "Value #tx": "Sent"} + }} + ], + "fieldConfig": { + "defaults": {"custom": {"align": "auto"}}, + "overrides": [ + {"matcher": {"id": "byName", "options": "Last handshake"}, "properties": [ + {"id": "unit", "value": "s"}, + {"id": "custom.cellOptions", "value": {"type": "color-background"}}, + {"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 300}, {"color": "red", "value": 3600}]}} + ]}, + {"matcher": {"id": "byRegexp", "options": "Received|Sent"}, "properties": [{"id": "unit", "value": "bytes"}]} + ] + }, + "options": {"showHeader": true} + }, + { + "type": "state-timeline", + "title": "Tunnel path probes", + "id": 6, + "gridPos": {"h": 7, "w": 12, "x": 12, "y": 4}, + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "targets": [ + { + "refId": "A", + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "expr": "min by (probe) (probe_success{vantage=\"ingress\"})", + "legendFormat": "{{ '{{' }}probe{{ '}}' }}" + } + ], + "fieldConfig": { + "defaults": { + "custom": {"fillOpacity": 90, "lineWidth": 0}, + "mappings": [{"type": "value", "options": {"0": {"text": "down", "color": "red", "index": 0}, "1": {"text": "up", "color": "green", "index": 1}}}], + "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]} + }, + "overrides": [] + }, + "options": {"mergeValues": true, "showValue": "never", "legend": {"displayMode": "list", "placement": "bottom", "showLegend": false}} + }, + { + "type": "timeseries", + "title": "Ingress host", + "id": 7, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 11}, + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "targets": [ + { + "refId": "A", + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "expr": "100 - (avg(rate(node_cpu_seconds_total{host=\"ingress\",mode=\"idle\"}[5m])) * 100)", + "legendFormat": "cpu" + }, + { + "refId": "B", + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "expr": "(1 - (node_memory_MemAvailable_bytes{host=\"ingress\"} / node_memory_MemTotal_bytes{host=\"ingress\"})) * 100", + "legendFormat": "memory" + }, + { + "refId": "C", + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "expr": "(1 - (node_filesystem_avail_bytes{host=\"ingress\",mountpoint=\"/\"} / node_filesystem_size_bytes{host=\"ingress\",mountpoint=\"/\"})) * 100", + "legendFormat": "disk" + } + ], + "fieldConfig": {"defaults": {"unit": "percent", "max": 100, "min": 0, "custom": {"lineWidth": 2, "fillOpacity": 8}}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi"}} + }, + { + "type": "timeseries", + "title": "Ingress network", + "id": 8, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 11}, + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "targets": [ + { + "refId": "A", + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "expr": "rate(node_network_receive_bytes_total{host=\"ingress\",device!~\"lo\"}[5m]) * 8", + "legendFormat": "in {{ '{{' }}device{{ '}}' }}" + }, + { + "refId": "B", + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "expr": "rate(node_network_transmit_bytes_total{host=\"ingress\",device!~\"lo\"}[5m]) * 8", + "legendFormat": "out {{ '{{' }}device{{ '}}' }}" + } + ], + "fieldConfig": {"defaults": {"unit": "bps", "custom": {"lineWidth": 2, "fillOpacity": 8}}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi"}} + }, + { + "type": "geomap", + "title": "Ban origins", + "id": 9, + "gridPos": {"h": 11, "w": 12, "x": 0, "y": 19}, + "datasource": {"type": "loki", "uid": "{{ grafana_loki_uid }}"}, + "targets": [ + { + "refId": "A", + "datasource": {"type": "loki", "uid": "{{ grafana_loki_uid }}"}, + "expr": "sum by (geoip_location_latitude, geoip_location_longitude, geoip_country_name) (count_over_time({job=\"fail2ban\"} |= \"Ban\" [$__range]))", + "queryType": "instant" + } + ], + "options": { + "basemap": {"type": "default", "name": "Basemap"}, + "layers": [ + { + "type": "markers", + "name": "bans", + "location": {"mode": "coords", "latitude": "geoip_location_latitude", "longitude": "geoip_location_longitude"}, + "config": {"style": {"color": {"fixed": "red"}, "size": {"field": "Value", "min": 4, "max": 20}, "opacity": 0.6}} + } + ], + "view": {"id": "zero", "lat": 25, "lon": 10, "zoom": 1.6} + }, + "fieldConfig": {"defaults": {}, "overrides": []} + }, + { + "type": "logs", + "title": "Ingress journal and firewall", + "id": 10, + "gridPos": {"h": 11, "w": 12, "x": 12, "y": 19}, + "datasource": {"type": "loki", "uid": "{{ grafana_loki_uid }}"}, + "targets": [ + { + "refId": "A", + "datasource": {"type": "loki", "uid": "{{ grafana_loki_uid }}"}, + "expr": "{host=\"ingress\"}", + "queryType": "range" + } + ], + "options": { + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true, + "enableLogDetails": true, + "dedupStrategy": "none", + "prettifyLogMessage": false + } + } + ] +} diff --git a/playbooks/monitoring/grafana/dashboards/services.json.j2 b/playbooks/monitoring/grafana/dashboards/services.json.j2 new file mode 100644 index 0000000..89a4f04 --- /dev/null +++ b/playbooks/monitoring/grafana/dashboards/services.json.j2 @@ -0,0 +1,195 @@ +{ + "id": null, + "uid": "beepi-services", + "title": "Services", + "tags": ["beepi", "services"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": {"from": "now-24h", "to": "now"}, + "panels": [ + { + "type": "table", + "title": "Service health", + "id": 1, + "gridPos": {"h": 10, "w": 24, "x": 0, "y": 0}, + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "targets": [ + { + "refId": "enabled", + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "expr": "max by (service) (service_enabled)", + "instant": true, + "format": "table" + }, + { + "refId": "units", + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "expr": "max by (service) (service_units_active) / max by (service) (service_units_total)", + "instant": true, + "format": "table" + }, + { + "refId": "local", + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "expr": "min by (service) (probe_success{vantage=\"local\"})", + "instant": true, + "format": "table" + }, + { + "refId": "vhost", + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "expr": "min by (service) (probe_success{vantage=\"nginx\"})", + "instant": true, + "format": "table" + }, + { + "refId": "latency", + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "expr": "max by (service) (probe_duration_seconds{vantage=\"nginx\"})", + "instant": true, + "format": "table" + } + ], + "transformations": [ + {"id": "joinByField", "options": {"byField": "service", "mode": "outer"}}, + {"id": "organize", "options": { + "excludeByName": {"Time": true, "Time 1": true, "Time 2": true, "Time 3": true, "Time 4": true, "Time 5": true}, + "renameByName": { + "service": "Service", + "Value #enabled": "Expected", + "Value #units": "Units", + "Value #local": "Local probe", + "Value #vhost": "Public vhost", + "Value #latency": "Latency" + } + }} + ], + "fieldConfig": { + "defaults": {"custom": {"align": "auto", "cellOptions": {"type": "auto"}}}, + "overrides": [ + {"matcher": {"id": "byName", "options": "Expected"}, "properties": [ + {"id": "mappings", "value": [{"type": "value", "options": {"0": {"text": "disabled", "color": "text", "index": 0}, "1": {"text": "yes", "color": "green", "index": 1}}}]}, + {"id": "custom.cellOptions", "value": {"type": "color-text"}} + ]}, + {"matcher": {"id": "byName", "options": "Units"}, "properties": [ + {"id": "unit", "value": "percentunit"}, + {"id": "custom.cellOptions", "value": {"type": "color-background"}}, + {"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "orange", "value": 0.5}, {"color": "green", "value": 1}]}} + ]}, + {"matcher": {"id": "byName", "options": "Local probe"}, "properties": [ + {"id": "custom.cellOptions", "value": {"type": "color-background"}}, + {"id": "mappings", "value": [ + {"type": "value", "options": {"0": {"text": "DOWN", "color": "red", "index": 0}, "1": {"text": "up", "color": "green", "index": 1}}}, + {"type": "special", "options": {"match": "null", "result": {"text": "no check", "color": "#3a3f47", "index": 2}}} + ]}, + {"id": "noValue", "value": "no check"}, + {"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}} + ]}, + {"matcher": {"id": "byName", "options": "Public vhost"}, "properties": [ + {"id": "custom.cellOptions", "value": {"type": "color-background"}}, + {"id": "mappings", "value": [ + {"type": "value", "options": {"0": {"text": "DOWN", "color": "red", "index": 0}, "1": {"text": "up", "color": "green", "index": 1}}}, + {"type": "special", "options": {"match": "null", "result": {"text": "internal only", "color": "#3a3f47", "index": 2}}} + ]}, + {"id": "noValue", "value": "internal only"}, + {"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}} + ]}, + {"matcher": {"id": "byName", "options": "Latency"}, "properties": [{"id": "unit", "value": "s"}]} + ] + }, + "options": {"showHeader": true, "sortBy": [{"displayName": "Service", "desc": false}]} + }, + { + "type": "state-timeline", + "title": "Probe history", + "id": 2, + "gridPos": {"h": 9, "w": 24, "x": 0, "y": 10}, + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "targets": [ + { + "refId": "A", + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "expr": "min by (service) (probe_success{vantage=\"local\"})", + "legendFormat": "{{ '{{' }}service{{ '}}' }}" + } + ], + "fieldConfig": { + "defaults": { + "custom": {"fillOpacity": 90, "lineWidth": 0}, + "mappings": [{"type": "value", "options": {"0": {"text": "down", "color": "red", "index": 0}, "1": {"text": "up", "color": "green", "index": 1}}}], + "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]} + }, + "overrides": [] + }, + "options": {"mergeValues": true, "showValue": "never", "legend": {"displayMode": "list", "placement": "bottom", "showLegend": false}} + }, + { + "type": "table", + "title": "Units and containers", + "id": 3, + "gridPos": {"h": 10, "w": 12, "x": 0, "y": 19}, + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "targets": [ + { + "refId": "A", + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "expr": "service_unit_active", + "instant": true, + "format": "table" + } + ], + "transformations": [ + {"id": "organize", "options": { + "excludeByName": {"Time": true, "__name__": true, "instance": true, "job": true, "host": true}, + "renameByName": {"service": "Service", "unit": "Unit", "kind": "Kind", "Value": "Active"} + }} + ], + "fieldConfig": { + "defaults": {"custom": {"align": "auto"}}, + "overrides": [ + {"matcher": {"id": "byName", "options": "Active"}, "properties": [ + {"id": "custom.cellOptions", "value": {"type": "color-background"}}, + {"id": "mappings", "value": [{"type": "value", "options": {"0": {"text": "stopped", "color": "red", "index": 0}, "1": {"text": "running", "color": "green", "index": 1}}}]}, + {"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}} + ]} + ] + }, + "options": {"showHeader": true} + }, + { + "type": "table", + "title": "TLS certificates", + "id": 4, + "gridPos": {"h": 10, "w": 12, "x": 12, "y": 19}, + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "targets": [ + { + "refId": "A", + "datasource": {"type": "prometheus", "uid": "{{ grafana_prometheus_uid }}"}, + "expr": "min by (service, instance) (probe_ssl_earliest_cert_expiry - time())", + "instant": true, + "format": "table" + } + ], + "transformations": [ + {"id": "organize", "options": { + "excludeByName": {"Time": true}, + "renameByName": {"service": "Service", "instance": "Endpoint", "Value": "Expires in"} + }} + ], + "fieldConfig": { + "defaults": {"custom": {"align": "auto"}}, + "overrides": [ + {"matcher": {"id": "byName", "options": "Expires in"}, "properties": [ + {"id": "unit", "value": "s"}, + {"id": "custom.cellOptions", "value": {"type": "color-background"}}, + {"id": "thresholds", "value": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "orange", "value": 1209600}, {"color": "green", "value": 2592000}]}} + ]} + ] + }, + "options": {"showHeader": true} + } + ] +} diff --git a/playbooks/monitoring/grafana/dashboards/uptimerobot.json.j2 b/playbooks/monitoring/grafana/dashboards/uptimerobot.json.j2 new file mode 100644 index 0000000..c02111e --- /dev/null +++ b/playbooks/monitoring/grafana/dashboards/uptimerobot.json.j2 @@ -0,0 +1,149 @@ +{ + "id": null, + "uid": "beepi-uptimerobot", + "title": "External Checks", + "tags": ["beepi", "uptimerobot"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "refresh": "5m", + "time": {"from": "now-24h", "to": "now"}, + "panels": [ + { + "type": "stat", + "title": "Monitors down", + "id": 1, + "gridPos": {"h": 5, "w": 6, "x": 0, "y": 0}, + "datasource": {"type": "yesoreyeram-infinity-datasource", "uid": "{{ grafana_uptimerobot_uid }}"}, + "targets": [ + { + "refId": "A", + "datasource": {"type": "yesoreyeram-infinity-datasource", "uid": "{{ grafana_uptimerobot_uid }}"}, + "type": "json", + "source": "url", + "parser": "backend", + "format": "table", + "url": "https://api.uptimerobot.com/v2/getMonitors?format=json&custom_uptime_ratios=1-7-30", + "url_options": {"method": "POST", "data": ""}, + "root_selector": "monitors", + "columns": [ + {"selector": "friendly_name", "text": "Monitor", "type": "string"}, + {"selector": "status", "text": "Status", "type": "number"} + ], + "filterExpression": "status == 8 || status == 9" + } + ], + "transformations": [{"id": "reduce", "options": {"reducers": ["count"]}}], + "fieldConfig": { + "defaults": { + "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "red", "value": 1}]}, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "textMode": "value", + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "/^Status$/", "values": false} + } + }, + { + "type": "table", + "title": "UptimeRobot monitors", + "id": 2, + "gridPos": {"h": 12, "w": 18, "x": 6, "y": 0}, + "datasource": {"type": "yesoreyeram-infinity-datasource", "uid": "{{ grafana_uptimerobot_uid }}"}, + "targets": [ + { + "refId": "A", + "datasource": {"type": "yesoreyeram-infinity-datasource", "uid": "{{ grafana_uptimerobot_uid }}"}, + "type": "json", + "source": "url", + "parser": "backend", + "format": "table", + "url": "https://api.uptimerobot.com/v2/getMonitors?format=json&custom_uptime_ratios=1-7-30", + "url_options": {"method": "POST", "data": ""}, + "root_selector": "monitors", + "columns": [ + {"selector": "friendly_name", "text": "Monitor", "type": "string"}, + {"selector": "status", "text": "Status", "type": "number"}, + {"selector": "custom_uptime_ratio", "text": "Uptime 1d-7d-30d", "type": "string"}, + {"selector": "interval", "text": "Interval", "type": "number"}, + {"selector": "url", "text": "URL", "type": "string"} + ] + } + ], + "fieldConfig": { + "defaults": {"custom": {"align": "auto", "cellOptions": {"type": "auto"}}}, + "overrides": [ + { + "matcher": {"id": "byName", "options": "Status"}, + "properties": [ + {"id": "custom.cellOptions", "value": {"type": "color-background"}}, + {"id": "mappings", "value": [{"type": "value", "options": { + "0": {"text": "paused", "color": "text", "index": 0}, + "1": {"text": "not checked", "color": "blue", "index": 1}, + "2": {"text": "UP", "color": "green", "index": 2}, + "8": {"text": "SEEMS DOWN", "color": "orange", "index": 3}, + "9": {"text": "DOWN", "color": "red", "index": 4} + }}]}, + {"id": "custom.width", "value": 130} + ] + }, + { + "matcher": {"id": "byName", "options": "Interval"}, + "properties": [{"id": "unit", "value": "s"}, {"id": "custom.width", "value": 100}] + }, + { + "matcher": {"id": "byName", "options": "Uptime 1d-7d-30d"}, + "properties": [{"id": "custom.width", "value": 190}] + } + ] + }, + "options": {"showHeader": true, "sortBy": [{"displayName": "Status", "desc": true}]} + }, + { + "type": "stat", + "title": "Monitors configured", + "id": 3, + "gridPos": {"h": 5, "w": 6, "x": 0, "y": 5}, + "datasource": {"type": "yesoreyeram-infinity-datasource", "uid": "{{ grafana_uptimerobot_uid }}"}, + "targets": [ + { + "refId": "A", + "datasource": {"type": "yesoreyeram-infinity-datasource", "uid": "{{ grafana_uptimerobot_uid }}"}, + "type": "json", + "source": "url", + "parser": "backend", + "format": "table", + "url": "https://api.uptimerobot.com/v2/getMonitors?format=json", + "url_options": {"method": "POST", "data": ""}, + "root_selector": "monitors", + "columns": [{"selector": "status", "text": "Status", "type": "number"}] + } + ], + "transformations": [{"id": "reduce", "options": {"reducers": ["count"]}}], + "fieldConfig": { + "defaults": {"color": {"mode": "fixed", "fixedColor": "blue"}, "unit": "short"}, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "textMode": "value", + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "/^Status$/", "values": false} + } + }, + { + "type": "text", + "title": "About", + "id": 4, + "gridPos": {"h": 4, "w": 24, "x": 0, "y": 12}, + "options": { + "mode": "markdown", + "content": "External reachability is checked by UptimeRobot from outside the network, which is the one thing this stack cannot do for itself: everything else runs on beepi, so it cannot report that beepi is down.\n\nThe free plan allows 10 API requests per minute. This dashboard refreshes every 5 minutes." + } + } + ] +} diff --git a/playbooks/monitoring/grafana/provisioning/alerting/contact-points.yml.j2 b/playbooks/monitoring/grafana/provisioning/alerting/contact-points.yml.j2 new file mode 100644 index 0000000..d7aa5ec --- /dev/null +++ b/playbooks/monitoring/grafana/provisioning/alerting/contact-points.yml.j2 @@ -0,0 +1,34 @@ +apiVersion: 1 + +contactPoints: + - orgId: 1 + name: discord + receivers: + - uid: discord-beepi + type: discord + settings: + url: {{ discord_webhook }} + title: '{{ '{{' }} template "default.title" . {{ '}}' }}' + message: | + {{ '{{' }} range .Alerts {{ '}}' }}**{{ '{{' }} .Labels.alertname {{ '}}' }}** {{ '{{' }} if .Labels.service {{ '}}' }}({{ '{{' }} .Labels.service {{ '}}' }}){{ '{{' }} end {{ '}}' }} + {{ '{{' }} .Annotations.summary {{ '}}' }} + {{ '{{' }} end {{ '}}' }} + use_discord_username: true + disableResolveMessage: false + +policies: + - orgId: 1 + receiver: discord + group_by: + - alertname + - service + - job + group_wait: 30s + group_interval: 5m + repeat_interval: 12h + routes: + - receiver: discord + object_matchers: + - ['severity', '=', 'critical'] + group_wait: 10s + repeat_interval: 4h diff --git a/playbooks/monitoring/grafana/provisioning/alerting/rules.yml.j2 b/playbooks/monitoring/grafana/provisioning/alerting/rules.yml.j2 new file mode 100644 index 0000000..1ea1120 --- /dev/null +++ b/playbooks/monitoring/grafana/provisioning/alerting/rules.yml.j2 @@ -0,0 +1,57 @@ +apiVersion: 1 + +groups: +{% for group in alert_groups %} + - orgId: 1 + name: {{ group.name }} + folder: {{ grafana_folder }} + interval: {{ group.interval }} + rules: +{% for rule in group.rules %} + - uid: {{ rule.uid }} + title: {{ rule.title }} + condition: C + for: {{ rule['for'] }} + noDataState: {{ rule.no_data | default('NoData') }} + execErrState: Alerting + labels: + severity: {{ rule.severity }} + annotations: + summary: {{ rule.summary }} + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: {{ grafana_loki_uid if rule.source | default('prometheus') == 'loki' else grafana_prometheus_uid }} + model: + refId: A + expr: '{{ rule.expr }}' + instant: true +{% if rule.source | default('prometheus') == 'loki' %} + queryType: instant +{% else %} + range: false +{% endif %} + - refId: B + datasourceUid: __expr__ + model: + refId: B + type: reduce + expression: A + reducer: last + settings: + mode: dropNN + - refId: C + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: B + conditions: + - evaluator: + type: {{ rule.op }} + params: + - {{ rule.threshold }} +{% endfor %} +{% endfor %} diff --git a/playbooks/monitoring/grafana/provisioning/dashboards/dashboards.yml.j2 b/playbooks/monitoring/grafana/provisioning/dashboards/dashboards.yml.j2 new file mode 100644 index 0000000..37b2043 --- /dev/null +++ b/playbooks/monitoring/grafana/provisioning/dashboards/dashboards.yml.j2 @@ -0,0 +1,26 @@ +apiVersion: 1 + +providers: + - name: beepi + orgId: 1 + folder: {{ grafana_folder }} + folderUid: {{ grafana_folder_uid }} + type: file + disableDeletion: false + updateIntervalSeconds: 30 + allowUiUpdates: false + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false + + - name: general + orgId: 1 + folder: {{ grafana_general_folder }} + folderUid: {{ grafana_general_folder_uid }} + type: file + disableDeletion: false + updateIntervalSeconds: 30 + allowUiUpdates: false + options: + path: /var/lib/grafana/dashboards-general + foldersFromFilesStructure: false diff --git a/playbooks/monitoring/grafana/provisioning/datasources/datasources.yml.j2 b/playbooks/monitoring/grafana/provisioning/datasources/datasources.yml.j2 new file mode 100644 index 0000000..d098fb0 --- /dev/null +++ b/playbooks/monitoring/grafana/provisioning/datasources/datasources.yml.j2 @@ -0,0 +1,49 @@ +apiVersion: 1 + +datasources: + - name: prometheus + uid: {{ grafana_prometheus_uid }} + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false + jsonData: + timeInterval: 15s + httpMethod: POST + + - name: loki + uid: {{ grafana_loki_uid }} + type: loki + access: proxy + url: http://loki:3100 + editable: false + jsonData: + maxLines: 2000 + + - name: topology + uid: {{ grafana_infinity_uid }} + type: yesoreyeram-infinity-datasource + access: proxy + editable: false + jsonData: + allowedHosts: + - http://topology:8088 + global_queries: [] +{% if uptimerobot_enabled %} + + - name: uptimerobot + uid: {{ grafana_uptimerobot_uid }} + type: yesoreyeram-infinity-datasource + access: proxy + editable: false + jsonData: + auth_method: apiKey + apiKeyKey: api_key + apiKeyType: query + allowedHosts: + - https://api.uptimerobot.com + global_queries: [] + secureJsonData: + apiKeyValue: {{ uptimerobot_api_key }} +{% endif %} diff --git a/playbooks/monitoring/local-config.yml.j2 b/playbooks/monitoring/local-config.yml.j2 index ab4b60f..1fff7bd 100644 --- a/playbooks/monitoring/local-config.yml.j2 +++ b/playbooks/monitoring/local-config.yml.j2 @@ -34,9 +34,6 @@ schema_config: prefix: index_ period: 24h -ruler: - alertmanager_url: http://localhost:9093 - ingester: wal: enabled: true diff --git a/playbooks/monitoring/monitoring.yml b/playbooks/monitoring/monitoring.yml index cadc6df..d4001d3 100644 --- a/playbooks/monitoring/monitoring.yml +++ b/playbooks/monitoring/monitoring.yml @@ -1,6 +1,8 @@ - name: Prometheus hosts: beepi become: true + vars_files: + - vars/alert_rules.yml vars: grafana_secret: !vault | $ANSIBLE_VAULT;1.1;AES256 @@ -25,19 +27,21 @@ 36376463313964333834373435653564346333396538306337333762613737643363636364353139 3032383564333638613566633363666430356230326231633636 tasks: - - name: Install podman - ansible.builtin.apt: - name: podman - state: present + - name: Determine whether alert delivery is configured + ansible.builtin.set_fact: + alerting_enabled: "{{ discord_webhook is defined and 'REPLACE_ME' not in discord_webhook }}" - - name: Install podman-compose - ansible.builtin.apt: - name: podman-compose - state: present + - name: Determine whether uptimerobot is configured + ansible.builtin.set_fact: + uptimerobot_enabled: "{{ uptimerobot_api_key is defined and 'REPLACE_ME' not in uptimerobot_api_key }}" - - name: Install prometheus-node-exporter + - name: Install packages ansible.builtin.apt: - name: prometheus-node-exporter + name: + - podman + - podman-compose + - prometheus-node-exporter + - wireguard-tools state: present - name: Create monitoring config directory @@ -46,6 +50,154 @@ state: directory mode: '0755' + - name: Create grafana provisioning directories + ansible.builtin.file: + path: "/opt/beeserver/monitoring/grafana/{{ item }}" + state: directory + mode: '0755' + loop: + - provisioning/datasources + - provisioning/dashboards + - provisioning/alerting + - dashboards + - dashboards-general + + - name: Create topology directory + ansible.builtin.file: + path: /opt/beeserver/monitoring/topology + state: directory + mode: '0755' + + - name: Create textfile directory + ansible.builtin.file: + path: "{{ node_exporter_textfile_dir }}" + state: directory + mode: '0755' + + - name: Configure node exporter + ansible.builtin.copy: + dest: /etc/default/prometheus-node-exporter + content: | + ARGS="--collector.textfile.directory={{ node_exporter_textfile_dir }} --collector.systemd --collector.processes" + mode: '0644' + notify: Restart node exporter + + - name: Copy textfile collectors + ansible.builtin.template: + src: "textfile/{{ item }}.sh.j2" + dest: "/usr/local/bin/{{ item }}.sh" + mode: '0755' + loop: + - service_metrics + - wg_metrics + + - name: Copy uptimerobot collector + ansible.builtin.template: + src: textfile/uptimerobot_metrics.sh.j2 + dest: /usr/local/bin/uptimerobot_metrics.sh + mode: '0700' + when: uptimerobot_enabled + + - name: Install uptimerobot timer + ansible.builtin.copy: + dest: "/etc/systemd/system/beepi-uptimerobot-metrics.{{ item.kind }}" + content: "{{ item.content }}" + mode: '0644' + loop: + - kind: service + content: | + [Unit] + Description=Collect uptimerobot metrics + + [Service] + Type=oneshot + ExecStart=/usr/local/bin/uptimerobot_metrics.sh + - kind: timer + content: | + [Unit] + Description=Collect uptimerobot metrics + + [Timer] + OnBootSec=3min + OnUnitActiveSec=5min + AccuracySec=30s + + [Install] + WantedBy=timers.target + when: uptimerobot_enabled + notify: Reload systemd + + - name: Install collector timers + ansible.builtin.copy: + dest: "/etc/systemd/system/{{ item.name }}.{{ item.kind }}" + content: "{{ item.content }}" + mode: '0644' + loop: + - name: beepi-service-metrics + kind: service + content: | + [Unit] + Description=Collect service unit metrics + + [Service] + Type=oneshot + ExecStart=/usr/local/bin/service_metrics.sh + - name: beepi-service-metrics + kind: timer + content: | + [Unit] + Description=Collect service unit metrics + + [Timer] + OnBootSec=1min + OnUnitActiveSec=30s + AccuracySec=5s + + [Install] + WantedBy=timers.target + - name: beepi-wg-metrics + kind: service + content: | + [Unit] + Description=Collect wireguard peer metrics + + [Service] + Type=oneshot + ExecStart=/usr/local/bin/wg_metrics.sh + - name: beepi-wg-metrics + kind: timer + content: | + [Unit] + Description=Collect wireguard peer metrics + + [Timer] + OnBootSec=1min + OnUnitActiveSec=30s + AccuracySec=5s + + [Install] + WantedBy=timers.target + notify: Reload systemd + + - name: Flush handlers + ansible.builtin.meta: flush_handlers + + - name: Enable collector timers + ansible.builtin.systemd_service: + name: "{{ item }}.timer" + state: started + enabled: true + loop: + - beepi-service-metrics + - beepi-wg-metrics + + - name: Enable uptimerobot timer + ansible.builtin.systemd_service: + name: beepi-uptimerobot-metrics.timer + state: started + enabled: true + when: uptimerobot_enabled + - name: Compose down changed_when: true failed_when: false @@ -60,11 +212,17 @@ mode: '0600' - name: Copy prometheus config - ansible.builtin.copy: - src: prometheus_config.yml + ansible.builtin.template: + src: prometheus.yml.j2 dest: /opt/beeserver/monitoring/prometheus.yml mode: '0644' + - name: Copy blackbox config + ansible.builtin.template: + src: blackbox.yml.j2 + dest: /opt/beeserver/monitoring/blackbox.yml + mode: '0644' + - name: Copy loki config ansible.builtin.template: src: local-config.yml.j2 @@ -77,17 +235,104 @@ dest: /opt/beeserver/monitoring/config.alloy mode: '0644' + - name: Copy topology service + ansible.builtin.copy: + src: topology/topology.py + dest: /opt/beeserver/monitoring/topology/topology.py + mode: '0644' + + - name: Copy topology spec + ansible.builtin.template: + src: topology/topology-spec.json.j2 + dest: /opt/beeserver/monitoring/topology/topology-spec.json + mode: '0644' + + - name: Copy grafana datasources + ansible.builtin.template: + src: grafana/provisioning/datasources/datasources.yml.j2 + dest: /opt/beeserver/monitoring/grafana/provisioning/datasources/datasources.yml + owner: '472' + group: '472' + mode: '0600' + + - name: Copy grafana dashboard provider + ansible.builtin.template: + src: grafana/provisioning/dashboards/dashboards.yml.j2 + dest: /opt/beeserver/monitoring/grafana/provisioning/dashboards/dashboards.yml + mode: '0644' + + - name: Copy hand made dashboards + ansible.builtin.copy: + src: grafana/dashboards-general/ + dest: /opt/beeserver/monitoring/grafana/dashboards-general/ + mode: '0644' + + - name: Copy grafana contact points + ansible.builtin.template: + src: grafana/provisioning/alerting/contact-points.yml.j2 + dest: /opt/beeserver/monitoring/grafana/provisioning/alerting/contact-points.yml + owner: '472' + group: '472' + mode: '0600' + when: alerting_enabled + + - name: Copy grafana alert rules + ansible.builtin.template: + src: grafana/provisioning/alerting/rules.yml.j2 + dest: /opt/beeserver/monitoring/grafana/provisioning/alerting/rules.yml + mode: '0644' + + - name: Copy grafana dashboards + ansible.builtin.template: + src: "grafana/dashboards/{{ item }}.json.j2" + dest: "/opt/beeserver/monitoring/grafana/dashboards/{{ item }}.json" + mode: '0644' + loop: + - infrastructure-map + - backups + - services + - ingress + + - name: Copy uptimerobot dashboard + ansible.builtin.template: + src: grafana/dashboards/uptimerobot.json.j2 + dest: /opt/beeserver/monitoring/grafana/dashboards/uptimerobot.json + mode: '0644' + when: uptimerobot_enabled + - name: Enable podman.socket ansible.builtin.service: name: podman.socket state: started enabled: true + - name: Check the geoip database + ansible.builtin.stat: + path: /opt/beeserver/monitoring/database/GeoLite2-City.mmdb + register: geoip_db + + - name: Check when geoip was last downloaded + ansible.builtin.stat: + path: /opt/beeserver/monitoring/.geoip_downloaded + register: geoip_stamp + + - name: Decide whether the geoip database needs refreshing + ansible.builtin.set_fact: + geoip_stale: "{{ not geoip_db.stat.exists or not geoip_stamp.stat.exists or (ansible_facts.date_time.epoch | int - geoip_stamp.stat.mtime | int) > geoip_max_age_seconds | int }}" + + - name: Report the geoip database age + ansible.builtin.debug: + msg: >- + geoip + {{ 'not present, downloading' if not (geoip_db.stat.exists and geoip_stamp.stat.exists) + else 'downloaded ' + (((ansible_facts.date_time.epoch | int - geoip_stamp.stat.mtime | int) / 3600) | round(1) | string) + 'h ago, ' + ('refreshing' if geoip_stale else 'reusing') }} + - name: Delete database directory ansible.builtin.file: path: /opt/beeserver/monitoring/database state: absent mode: '0755' + when: geoip_stale - name: Create database directory ansible.builtin.file: @@ -95,6 +340,12 @@ state: directory mode: '0755' + - name: Remove stale geoip archives + ansible.builtin.shell: + cmd: rm -f /opt/beeserver/monitoring/GeoLite2-City_*.tar.gz + changed_when: true + when: geoip_stale + - name: Download geoip database ansible.builtin.get_url: url: "https://download.maxmind.com/geoip/databases/GeoLite2-City/download?suffix=tar.gz" @@ -102,6 +353,7 @@ username: "{{ geoip_username }}" password: "{{ geoip_license }}" mode: '0644' + when: geoip_stale - name: Get geoip tar file ansible.builtin.find: @@ -111,6 +363,7 @@ use_regex: true limit: 1 register: found + when: geoip_stale - name: Extract geopip database ansible.builtin.unarchive: @@ -118,9 +371,50 @@ src: "{{ found.files[0].path }}" dest: /opt/beeserver/monitoring/database extra_opts: ['--strip-components=1', '--show-stored-names'] + when: geoip_stale and found.files | length > 0 + + - name: Stamp the geoip download time + ansible.builtin.file: + path: /opt/beeserver/monitoring/.geoip_downloaded + state: touch + mode: '0644' + when: geoip_stale and found.files | length > 0 - name: Compose up changed_when: true ansible.builtin.command: cmd: podman-compose up -d chdir: /opt/beeserver/monitoring + + - name: Run collectors once + changed_when: true + ansible.builtin.command: + cmd: "/usr/local/bin/{{ item }}.sh" + loop: + - service_metrics + - wg_metrics + + - name: Run uptimerobot collector once + changed_when: true + ansible.builtin.command: + cmd: /usr/local/bin/uptimerobot_metrics.sh + when: uptimerobot_enabled + + - name: Debug + ansible.builtin.debug: + msg: + - "Grafana: https://grafana.secretbee.buzz/" + - "Topology: curl -s localhost:8088/topology.json | head" + - "Prometheus targets: curl -s localhost:9090/api/v1/targets | jq '.data.activeTargets[] | {job: .labels.job, health}'" + tags: + - always + + handlers: + - name: Reload systemd + ansible.builtin.systemd_service: + daemon_reload: true + + - name: Restart node exporter + ansible.builtin.service: + name: prometheus-node-exporter + state: restarted diff --git a/playbooks/monitoring/prometheus.yml.j2 b/playbooks/monitoring/prometheus.yml.j2 new file mode 100644 index 0000000..30375f7 --- /dev/null +++ b/playbooks/monitoring/prometheus.yml.j2 @@ -0,0 +1,154 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + external_labels: + monitor: beepi + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ['localhost:9090'] + + - job_name: node + static_configs: + - targets: ['host.containers.internal:9100'] + labels: + host: beepi + - targets: ['{{ wireguard_ingress_ip }}:9100'] + labels: + host: ingress + - targets: ['192.168.178.51:9100'] + labels: + host: beeserver + + - job_name: podman + static_configs: + - targets: ['podman-exporter:9882'] + labels: + host: beepi + + - job_name: alloy + static_configs: + - targets: ['alloy:12345'] + labels: + host: beepi + - targets: ['{{ wireguard_ingress_ip }}:12345'] + labels: + host: ingress + + - job_name: loki + static_configs: + - targets: ['loki:3100'] + + - job_name: grafana + static_configs: + - targets: ['grafana:4000'] + + - job_name: blackbox + static_configs: + - targets: ['host.containers.internal:9115'] + + - job_name: probe-app + metrics_path: /probe + params: + module: [http_2xx_any] + static_configs: +{% for service in monitored_services %} +{% if service.probe is defined %} + - targets: ['{{ service.probe }}'] + labels: + service: {{ service.name }} + group: {{ service.group }} + vantage: local +{% endif %} +{% endfor %} + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: host.containers.internal:9115 + + - job_name: probe-vhost + metrics_path: /probe + params: + module: [https_vhost] + static_configs: +{% for service in monitored_services %} +{% if service.vhost is defined %} + - targets: ['https://{{ service.vhost }}'] + labels: + service: {{ service.name }} + group: {{ service.group }} + vantage: nginx +{% endif %} +{% endfor %} + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: host.containers.internal:9115 + + - job_name: probe-tcp + metrics_path: /probe + params: + module: [tcp_connect] + static_configs: +{% for service in monitored_services %} +{% for probe in service.tcp | default([]) %} + - targets: ['{{ probe.target }}'] + labels: + service: {{ service.name }} + group: {{ service.group }} + probe: {{ probe.name }} + vantage: local +{% endfor %} +{% endfor %} + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: host.containers.internal:9115 + + - job_name: probe-tunnel + scrape_interval: 30s + metrics_path: /probe + params: + module: [tcp_connect] + static_configs: +{% for probe in ingress_probe_targets %} + - targets: ['{{ probe.target }}'] + labels: + probe: {{ probe.name }} + vantage: ingress +{% endfor %} + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: '{{ wireguard_ingress_ip }}:9115' + + - job_name: probe-tunnel-icmp + scrape_interval: 30s + metrics_path: /probe + params: + module: [icmp] + static_configs: + - targets: ['{{ wireguard_beepi_ip }}'] + labels: + probe: tunnel-rtt + vantage: ingress + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: '{{ wireguard_ingress_ip }}:9115' diff --git a/playbooks/monitoring/prometheus_config.yml b/playbooks/monitoring/prometheus_config.yml deleted file mode 100644 index 3907eba..0000000 --- a/playbooks/monitoring/prometheus_config.yml +++ /dev/null @@ -1,16 +0,0 @@ -global: - scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. - evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. - -scrape_configs: - - job_name: 'prometheus' - static_configs: - - targets: ['localhost:9090'] - - job_name: node - static_configs: - - targets: - - 'host.containers.internal:9100' - - '192.168.178.51:9100' - - job_name: podman - static_configs: - - targets: ['podman-exporter:9882'] \ No newline at end of file diff --git a/playbooks/monitoring/textfile/service_metrics.sh.j2 b/playbooks/monitoring/textfile/service_metrics.sh.j2 new file mode 100644 index 0000000..0f673d0 --- /dev/null +++ b/playbooks/monitoring/textfile/service_metrics.sh.j2 @@ -0,0 +1,105 @@ +#!/bin/bash + +set -uo pipefail + +textfile_dir={{ node_exporter_textfile_dir }} +file=${textfile_dir}/services.prom +tmp=${file}.$$ + +mkdir -p ${textfile_dir} + +declare -A active_count +declare -A unit_count +units="" + +record() { + local service=$1 + local unit=$2 + local kind=$3 + local state=$4 + + units+="service_unit_active{service=\"${service}\",unit=\"${unit}\",kind=\"${kind}\"} ${state}"$'\n' + unit_count[${service}]=$(( ${unit_count[${service}]:-0} + 1 )) + active_count[${service}]=$(( ${active_count[${service}]:-0} + state )) +} + +check_user_unit() { + local service=$1 + local owner=$2 + local unit=$3 + local state=0 + + if systemctl --user --machine ${owner}@.host is-active --quiet ${unit} 2>/dev/null + then + state=1 + fi + + record ${service} ${unit} rootless ${state} +} + +check_system_unit() { + local service=$1 + local unit=$2 + local state=0 + + if systemctl is-active --quiet ${unit} 2>/dev/null + then + state=1 + fi + + record ${service} ${unit} system ${state} +} + +check_container() { + local service=$1 + local container=$2 + local state=0 + + if [ "$(podman container inspect -f '{% raw %}{{.State.Running}}{% endraw %}' ${container} 2>/dev/null)" = "true" ] + then + state=1 + fi + + record ${service} ${container} container ${state} +} + +{% for service in monitored_services %} +{% for unit in service.units | default([]) %} +check_user_unit {{ service.name }} {{ service.owner }} {{ unit }} +{% endfor %} +{% for unit in service.system_units | default([]) %} +check_system_unit {{ service.name }} {{ unit }} +{% endfor %} +{% for container in service.containers | default([]) %} +check_container {{ service.name }} {{ container }} +{% endfor %} +{% endfor %} + +{ + echo "# HELP service_unit_active Whether a unit or container backing a service is running." + echo "# TYPE service_unit_active gauge" + printf '%s' "${units}" + + echo "# HELP service_enabled Whether a service is expected to be running." + echo "# TYPE service_enabled gauge" +{% for service in monitored_services %} + echo "service_enabled{service=\"{{ service.name }}\",group=\"{{ service.group }}\"} {{ 1 if service.enabled else 0 }}" +{% endfor %} + + echo "# HELP service_units_total Number of units or containers backing a service." + echo "# TYPE service_units_total gauge" + for service in "${!unit_count[@]}" + do + echo "service_units_total{service=\"${service}\"} ${unit_count[${service}]}" + done + + echo "# HELP service_units_active Number of running units or containers backing a service." + echo "# TYPE service_units_active gauge" + for service in "${!active_count[@]}" + do + echo "service_units_active{service=\"${service}\"} ${active_count[${service}]}" + done +} > "${tmp}" + +chmod 0644 "${tmp}" +mv "${tmp}" "${file}" diff --git a/playbooks/monitoring/textfile/uptimerobot_metrics.sh.j2 b/playbooks/monitoring/textfile/uptimerobot_metrics.sh.j2 new file mode 100644 index 0000000..7049e9b --- /dev/null +++ b/playbooks/monitoring/textfile/uptimerobot_metrics.sh.j2 @@ -0,0 +1,71 @@ +#!/bin/bash + +set -uo pipefail + +textfile_dir={{ node_exporter_textfile_dir }} +file=${textfile_dir}/uptimerobot.prom +tmp=${file}.$$ + +mkdir -p ${textfile_dir} + +response=$(curl -sS -m 25 -X POST \ + "https://api.uptimerobot.com/v2/getMonitors?api_key={{ uptimerobot_api_key }}&format=json&custom_uptime_ratios=1-7-30" \ + 2>/dev/null) + +{% raw %} +printf '%s' "${response}" | python3 -c ' +import json +import sys + +print("# HELP uptimerobot_api_ok Whether the uptimerobot api answered successfully.") +print("# TYPE uptimerobot_api_ok gauge") +print("# HELP uptimerobot_monitor_up Whether an external monitor reports the target as up.") +print("# TYPE uptimerobot_monitor_up gauge") +print("# HELP uptimerobot_monitor_enabled Whether an external monitor is active rather than paused.") +print("# TYPE uptimerobot_monitor_enabled gauge") +print("# HELP uptimerobot_monitor_status Raw uptimerobot status, 0 paused 1 unchecked 2 up 8 seems down 9 down.") +print("# TYPE uptimerobot_monitor_status gauge") +print("# HELP uptimerobot_uptime_ratio Uptime percentage over the given window.") +print("# TYPE uptimerobot_uptime_ratio gauge") + +try: + payload = json.load(sys.stdin) +except ValueError: + print("uptimerobot_api_ok 0") + sys.exit(0) + +if payload.get("stat") != "ok": + print("uptimerobot_api_ok 0") + sys.exit(0) + +print("uptimerobot_api_ok 1") + +def escape(value): + return value.replace("\\", "\\\\").replace("\"", "\\\"") + +for monitor in payload.get("monitors", []): + name = escape(str(monitor.get("friendly_name", "unknown"))) + status = int(monitor.get("status", 1)) + labels = "monitor=\"%s\"" % name + + print("uptimerobot_monitor_status{%s} %d" % (labels, status)) + print("uptimerobot_monitor_enabled{%s} %d" % (labels, 0 if status == 0 else 1)) + print("uptimerobot_monitor_up{%s} %d" % (labels, 1 if status == 2 else 0)) + + ratios = str(monitor.get("custom_uptime_ratio", "")).split("-") + for window, ratio in zip(("1d", "7d", "30d"), ratios): + try: + print("uptimerobot_uptime_ratio{%s,window=\"%s\"} %s" % (labels, window, float(ratio))) + except ValueError: + pass +' > "${tmp}" +{% endraw %} + +if [ ! -s "${tmp}" ] +then + rm -f "${tmp}" + exit 1 +fi + +chmod 0644 "${tmp}" +mv "${tmp}" "${file}" diff --git a/playbooks/monitoring/textfile/wg_metrics.sh.j2 b/playbooks/monitoring/textfile/wg_metrics.sh.j2 new file mode 100644 index 0000000..df6803b --- /dev/null +++ b/playbooks/monitoring/textfile/wg_metrics.sh.j2 @@ -0,0 +1,60 @@ +#!/bin/bash + +set -uo pipefail + +textfile_dir={{ node_exporter_textfile_dir }} +file=${textfile_dir}/wireguard.prom +tmp=${file}.$$ +now=$(date +%s) + +mkdir -p ${textfile_dir} + +peer_name() { + case "$1" in +{% for key, name in wireguard_peer_names.items() %} + "{{ key }}") echo "{{ name }}" ;; +{% endfor %} + *) echo "unknown" ;; + esac +} + +{ + echo "# HELP wireguard_peer_last_handshake_seconds Unix time of the last handshake with a peer." + echo "# TYPE wireguard_peer_last_handshake_seconds gauge" + echo "# HELP wireguard_peer_handshake_age_seconds Seconds since the last handshake with a peer." + echo "# TYPE wireguard_peer_handshake_age_seconds gauge" + echo "# HELP wireguard_peer_up Whether a peer handshaked within the last five minutes." + echo "# TYPE wireguard_peer_up gauge" + echo "# HELP wireguard_peer_receive_bytes_total Bytes received from a peer." + echo "# TYPE wireguard_peer_receive_bytes_total counter" + echo "# HELP wireguard_peer_transmit_bytes_total Bytes sent to a peer." + echo "# TYPE wireguard_peer_transmit_bytes_total counter" + + wg show all dump 2>/dev/null | awk 'NF >= 8' | while read -r interface key psk endpoint allowed handshake rx tx keepalive + do + name=$(peer_name "${key}") + labels="interface=\"${interface}\",peer=\"${name}\"" + + age=$((now - handshake)) + up=0 + + if [ "${handshake}" -gt 0 ] && [ "${age}" -lt 300 ] + then + up=1 + fi + + if [ "${handshake}" -eq 0 ] + then + age=-1 + fi + + echo "wireguard_peer_last_handshake_seconds{${labels}} ${handshake}" + echo "wireguard_peer_handshake_age_seconds{${labels}} ${age}" + echo "wireguard_peer_up{${labels}} ${up}" + echo "wireguard_peer_receive_bytes_total{${labels}} ${rx}" + echo "wireguard_peer_transmit_bytes_total{${labels}} ${tx}" + done +} > "${tmp}" + +chmod 0644 "${tmp}" +mv "${tmp}" "${file}" diff --git a/playbooks/monitoring/topology/topology-spec.json.j2 b/playbooks/monitoring/topology/topology-spec.json.j2 new file mode 100644 index 0000000..8c522cf --- /dev/null +++ b/playbooks/monitoring/topology/topology-spec.json.j2 @@ -0,0 +1,130 @@ +{ + "nodes": [ + { + "id": "internet", + "title": "Internet", + "subtitle": "public", + "group": "edge" + }, + { + "id": "ingress", + "title": "Ingress", + "subtitle": "animeistrash", + "group": "edge", + "up": "up{job=\"node\",host=\"ingress\"}", + "stat": "100 - (avg(rate(node_cpu_seconds_total{job=\"node\",host=\"ingress\",mode=\"idle\"}[5m])) * 100)", + "stat_unit": "pct" + }, + { + "id": "tunnel", + "title": "WireGuard", + "subtitle": "{{ wireguard_ingress_ip }} to {{ wireguard_beepi_ip }}", + "group": "edge", + "up": "max(wireguard_peer_up{peer=\"beepi\"})", + "stat": "max(wireguard_peer_handshake_age_seconds{peer=\"beepi\"})", + "stat_unit": "h" + }, + { + "id": "beepi", + "title": "beepi", + "subtitle": "raspberry pi", + "group": "host", + "up": "up{job=\"node\",host=\"beepi\"}", + "stat": "100 - (avg(rate(node_cpu_seconds_total{job=\"node\",host=\"beepi\",mode=\"idle\"}[5m])) * 100)", + "stat_unit": "pct" + }, +{% for service in monitored_services %} + { + "id": "{{ service.name }}", + "title": "{{ service.title }}", + "subtitle": "{{ service.vhost | default(service.group) }}", + "group": "{{ service.group }}", + "enabled": "max(service_enabled{service=\"{{ service.name }}\"})", +{% if service.probe is defined or service.tcp is defined %} + "up": "min(probe_success{service=\"{{ service.name }}\",vantage=\"local\"})", + "stat": "max(probe_duration_seconds{service=\"{{ service.name }}\",vantage=\"local\"})", + "stat_unit": "s" +{% elif service.vhost is defined %} + "up": "min(probe_success{service=\"{{ service.name }}\",vantage=\"nginx\"})", + "stat": "max(probe_duration_seconds{service=\"{{ service.name }}\",vantage=\"nginx\"})", + "stat_unit": "s" +{% else %} + "up": "min(service_units_active{service=\"{{ service.name }}\"} / service_units_total{service=\"{{ service.name }}\"})" +{% endif %} + }, +{% endfor %} + { + "id": "raid", + "title": "RAID", + "subtitle": "{{ backup_raid_mount }}", + "group": "storage", + "up": "max(backup_raid_mounted)", + "stat": "max(node_filesystem_avail_bytes{mountpoint=\"{{ backup_raid_mount }}\"})", + "stat_unit": "bytes", + "fill": "1 - (max(node_filesystem_avail_bytes{mountpoint=\"{{ backup_raid_mount }}\"}) / max(node_filesystem_size_bytes{mountpoint=\"{{ backup_raid_mount }}\"}))" + }, +{% for job in backup_jobs %} +{% if job.name != 'offsite' %} + { + "id": "backup-{{ job.name }}", + "title": "{{ job.title }}", + "subtitle": "backup job", + "group": "backup", + "up": "(max(backup_last_status{backup_job=\"{{ job.name }}\"}) == 1) and ((time() - max(backup_last_success_timestamp_seconds{backup_job=\"{{ job.name }}\"})) < {{ job.max_age_hours * 3600 }})", + "stat": "time() - max(backup_last_success_timestamp_seconds{backup_job=\"{{ job.name }}\"})", + "stat_unit": "h" + }, +{% endif %} +{% endfor %} + { + "id": "offsite", + "title": "Offsite sync", + "subtitle": "daily 07:00", + "group": "storage", + "up": "(time() - max(backup_last_success_timestamp_seconds{backup_job=\"offsite\"})) < {{ 26 * 3600 }}", + "stat": "time() - max(backup_last_success_timestamp_seconds{backup_job=\"offsite\"})", + "stat_unit": "h" + }, + { + "id": "hetzner", + "title": "Hetzner box", + "subtitle": "{{ hetzner_host | default('storage box') }}", + "group": "storage", + "up": "(max(hetzner_reachable) == 1) and (max(hetzner_remote_files{path=\"beepi\"}) > 0)", + "stat": "max(hetzner_remote_bytes{path=\"beepi\"})", + "stat_unit": "bytes", + "fill": "sum(hetzner_remote_bytes) / {{ hetzner_box_bytes }}" + }, + { + "id": "uptimerobot", + "title": "External checks", + "subtitle": "uptimerobot", + "group": "edge", + "dashboard": "/d/beepi-uptimerobot", + "enabled": "max(uptimerobot_api_ok)", + "up": "min(uptimerobot_monitor_up and on (monitor) (uptimerobot_monitor_enabled == 1))", + "stat": "count(uptimerobot_monitor_enabled == 1)" + } + ], + "edges": [ + {"source": "uptimerobot", "target": "internet", "label": "probes"}, + {"source": "internet", "target": "ingress", "label": "public"}, + {"source": "ingress", "target": "tunnel", "label": "dnat"}, + {"source": "tunnel", "target": "beepi", "label": "wg0"}, +{% for service in monitored_services %} +{% if service.vhost is defined and service.name != 'nginx' %} + {"source": "nginx", "target": "{{ service.name }}", "label": "proxy"}, +{% endif %} +{% if service.vhost is not defined %} + {"source": "beepi", "target": "{{ service.name }}", "label": "{{ service.group }}"}, +{% endif %} +{% endfor %} + {"source": "beepi", "target": "nginx", "label": "tls"}, + {"source": "beepi", "target": "backup-daily", "label": "03:00"}, + {"source": "backup-daily", "target": "backup-weekly", "label": "sun 05:00"}, + {"source": "backup-daily", "target": "backup-restoretest", "label": "sun 06:00"}, + {"source": "backup-daily", "target": "raid", "label": "archive"}, + {"source": "raid", "target": "offsite", "label": "rsync"}, + {"source": "offsite", "target": "hetzner", "label": "ssh 23"} + ] +} diff --git a/playbooks/monitoring/topology/topology.py b/playbooks/monitoring/topology/topology.py new file mode 100644 index 0000000..3b992ef --- /dev/null +++ b/playbooks/monitoring/topology/topology.py @@ -0,0 +1,218 @@ +import json +import os +import threading +import time +import urllib.error +import urllib.parse +import urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +PROMETHEUS_URL = os.environ.get("PROMETHEUS_URL", "http://prometheus:9090") +LISTEN_PORT = int(os.environ.get("LISTEN_PORT", "8088")) +SPEC_PATH = os.environ.get("SPEC_PATH", "/app/topology-spec.json") +CACHE_SECONDS = int(os.environ.get("CACHE_SECONDS", "10")) + +STATUS_OK = "ok" +STATUS_FAIL = "fail" +STATUS_DISABLED = "disabled" +STATUS_UNKNOWN = "unknown" + +ARCS = { + STATUS_OK: "arc__ok", + STATUS_FAIL: "arc__fail", + STATUS_DISABLED: "arc__disabled", + STATUS_UNKNOWN: "arc__unknown", +} + +ARC_FIELDS = list(ARCS.values()) + ["arc__used", "arc__free"] + +COLORS = { + STATUS_OK: "green", + STATUS_FAIL: "red", + STATUS_DISABLED: "#6e7079", + STATUS_UNKNOWN: "orange", +} + +DASHBOARDS = { + "apps": "/d/beepi-services", + "platform": "/d/beepi-services", + "backup": "/d/beepi-backups", + "storage": "/d/beepi-backups", + "edge": "/d/beepi-ingress", + "host": "/d/beepi-ingress", +} + +_lock = threading.Lock() +_cache = {"at": 0.0, "body": None} + + +def query(expr): + url = PROMETHEUS_URL + "/api/v1/query?" + urllib.parse.urlencode({"query": expr}) + + try: + with urllib.request.urlopen(url, timeout=8) as response: + payload = json.load(response) + except (urllib.error.URLError, OSError, ValueError): + return None + + if payload.get("status") != "success": + return None + + result = payload.get("data", {}).get("result", []) + + if not result: + return None + + try: + return float(result[0]["value"][1]) + except (KeyError, IndexError, TypeError, ValueError): + return None + + +def format_stat(value, unit): + if value is None: + return "" + + if unit == "s": + return "%.0f ms" % (value * 1000) if value < 1 else "%.1f s" % value + + if unit == "h": + return "%.1f h" % (value / 3600.0) + + if unit == "d": + return "%.1f d" % (value / 86400.0) + + if unit == "bytes": + size = float(value) + for suffix in ("B", "KiB", "MiB", "GiB", "TiB"): + if size < 1024 or suffix == "TiB": + return "%.1f %s" % (size, suffix) + size /= 1024.0 + + if unit == "pct": + return "%.0f%%" % value + + return "%.0f" % value + + +def evaluate(node): + enabled = node.get("enabled") + + if enabled: + value = query(enabled) + + if value is not None and value < 1: + return STATUS_DISABLED + + up = node.get("up") + + if not up: + return STATUS_OK + + value = query(up) + + if value is None: + return STATUS_UNKNOWN + + return STATUS_OK if value >= 1 else STATUS_FAIL + + +def build(): + with open(SPEC_PATH) as handle: + spec = json.load(handle) + + nodes = [] + statuses = {} + + for node in spec["nodes"]: + status = evaluate(node) + statuses[node["id"]] = status + + entry = { + "id": node["id"], + "title": node["title"], + "subtitle": node.get("subtitle", ""), + "mainstat": format_stat(query(node["stat"]), node.get("stat_unit", "")) if node.get("stat") else "", + "detail__group": node.get("group", ""), + "detail__status": status, + "detail__used": "", + "dashboard": node.get("dashboard") or DASHBOARDS.get(node.get("group", ""), "/d/beepi-map"), + } + + for name in ARC_FIELDS: + entry[name] = 0.0 + + used = query(node["fill"]) if node.get("fill") and status == STATUS_OK else None + + if used is None: + entry[ARCS[status]] = 1.0 + else: + used = min(max(used, 0.0), 1.0) + entry["arc__used"] = used + entry["arc__free"] = 1.0 - used + entry["detail__used"] = "%.0f%% used" % (used * 100) + + nodes.append(entry) + + edges = [] + + for index, edge in enumerate(spec["edges"]): + source = statuses.get(edge["source"], STATUS_UNKNOWN) + target = statuses.get(edge["target"], STATUS_UNKNOWN) + degraded = STATUS_FAIL in (source, target) + idle = STATUS_DISABLED in (source, target) + + if degraded: + color = COLORS[STATUS_FAIL] + elif idle: + color = COLORS[STATUS_DISABLED] + else: + color = COLORS[STATUS_OK] + + edges.append({ + "id": str(index), + "source": edge["source"], + "target": edge["target"], + "mainstat": edge.get("label", ""), + "color": color, + "thickness": 3 if degraded else 1, + }) + + return {"nodes": nodes, "edges": edges} + + +def cached(): + with _lock: + now = time.time() + + if _cache["body"] is None or now - _cache["at"] > CACHE_SECONDS: + _cache["body"] = json.dumps(build()).encode() + _cache["at"] = now + + return _cache["body"] + + +class Handler(BaseHTTPRequestHandler): + def do_GET(self): + if self.path.rstrip("/") not in ("", "/topology.json", "/topology"): + self.send_error(404) + return + + try: + body = cached() + except Exception: + self.send_error(500) + return + + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, fmt, *args): + pass + + +if __name__ == "__main__": + ThreadingHTTPServer(("0.0.0.0", LISTEN_PORT), Handler).serve_forever() diff --git a/playbooks/monitoring/vars/alert_rules.yml b/playbooks/monitoring/vars/alert_rules.yml new file mode 100644 index 0000000..2036e7c --- /dev/null +++ b/playbooks/monitoring/vars/alert_rules.yml @@ -0,0 +1,242 @@ +alert_groups: + - name: backup + interval: 5m + rules: + - uid: backup-daily-stale + no_data: Alerting + title: Daily backup stale + expr: '(time() - max(backup_last_success_timestamp_seconds{backup_job="daily"})) / 3600' + op: gt + threshold: 26 + for: 15m + severity: critical + summary: No successful daily backup for more than 26 hours. + + - uid: backup-weekly-stale + no_data: Alerting + title: Weekly backup stale + expr: '(time() - max(backup_last_success_timestamp_seconds{backup_job="weekly"})) / 3600' + op: gt + threshold: 192 + for: 30m + severity: critical + summary: No successful weekly backup for more than 8 days. + + - uid: backup-restoretest-stale + no_data: Alerting + title: Restore test stale + expr: '(time() - max(backup_last_success_timestamp_seconds{backup_job="restoretest"})) / 3600' + op: gt + threshold: 192 + for: 30m + severity: warning + summary: The vaultwarden restore test has not passed in more than 8 days, backups are unverified. + + - uid: backup-offsite-stale + no_data: Alerting + title: Offsite sync stale + expr: '(time() - max(backup_last_success_timestamp_seconds{backup_job="offsite"})) / 3600' + op: gt + threshold: 26 + for: 15m + severity: critical + summary: Backups have not reached the Hetzner storage box in more than 26 hours. + + - uid: backup-job-failed + no_data: Alerting + title: Backup job failed + expr: 'min by (backup_job) (backup_last_status)' + op: lt + threshold: 1 + for: 5m + severity: critical + summary: The last run of this backup job exited with a failure. + + - uid: backup-archive-shrunk + title: Backup archive shrunk + expr: 'backup_archive_newest_bytes{kind="daily"} / avg_over_time(backup_archive_newest_bytes{kind="daily"}[7d])' + op: lt + threshold: 0.5 + for: 1h + severity: warning + summary: The newest daily archive is less than half the size of the weekly average, data may be missing. + + - uid: backup-raid-unmounted + no_data: Alerting + title: Backup RAID unmounted + expr: 'min(backup_raid_mounted)' + op: lt + threshold: 1 + for: 5m + severity: critical + summary: The backup RAID is not mounted, backups cannot be written. + + - uid: backup-raid-space + title: Backup RAID low on space + expr: 'min(node_filesystem_avail_bytes{mountpoint="/home/beeshare/raid"} / node_filesystem_size_bytes{mountpoint="/home/beeshare/raid"}) * 100' + op: lt + threshold: 15 + for: 30m + severity: warning + summary: Less than 15 percent free space left on the backup RAID. + + - uid: offsite-empty + no_data: Alerting + title: Offsite storage empty + expr: 'max(hetzner_remote_files{path="beepi"})' + op: lt + threshold: 1 + for: 30m + severity: critical + summary: The Hetzner storage box holds no completed backup files, offsite recovery is not possible. + + - uid: offsite-unreachable + no_data: Alerting + title: Offsite storage unreachable + expr: 'max(hetzner_reachable)' + op: lt + threshold: 1 + for: 2h + severity: warning + summary: The Hetzner storage box could not be listed, check external reachability on the box. + + - uid: offsite-content-stale + title: Offsite content stale + expr: 'min(hetzner_remote_newest_age_seconds{path="beepi"}) / 3600' + op: gt + threshold: 48 + for: 1h + severity: critical + summary: The newest backup on the Hetzner storage box is more than 48 hours old. + + - uid: backup-log-errors + title: Backup log errors + no_data: OK + source: loki + expr: 'sum(count_over_time({job="backup"} |~ "(?i)(FAILED|verify failed|refusing to sync)" [15m]))' + op: gt + threshold: 0 + for: 0s + severity: critical + summary: The backup log reported a failure in the last 15 minutes. + + - name: services + interval: 1m + rules: + - uid: service-probe-down + title: Service probe failing + expr: 'min by (service) (probe_success{vantage="local"}) and on (service) (max by (service) (service_enabled) == 1)' + op: lt + threshold: 1 + for: 5m + severity: critical + summary: A service that should be running is not answering its health probe. + + - uid: service-unit-down + title: Service unit not running + expr: 'min by (service) (service_units_active / service_units_total) and on (service) (max by (service) (service_enabled) == 1)' + op: lt + threshold: 1 + for: 5m + severity: critical + summary: A unit or container backing an enabled service is not running. + + - uid: service-vhost-down + title: Public vhost failing + expr: 'min by (service) (probe_success{vantage="nginx"}) and on (service) (max by (service) (service_enabled) == 1)' + op: lt + threshold: 1 + for: 5m + severity: critical + summary: An enabled service is not reachable through nginx and TLS. + + - uid: external-check-down + title: External check failing + expr: 'min by (monitor) (uptimerobot_monitor_up) and on (monitor) (max by (monitor) (uptimerobot_monitor_enabled) == 1)' + op: lt + threshold: 1 + for: 5m + severity: critical + summary: UptimeRobot reports this endpoint as unreachable from outside the network. + + - uid: service-cert-expiry + title: TLS certificate expiring + expr: 'min by (instance) (probe_ssl_earliest_cert_expiry - time()) / 86400' + op: lt + threshold: 14 + for: 1h + severity: warning + summary: A TLS certificate expires in less than 14 days. + + - uid: scrape-target-down + title: Scrape target down + expr: 'min by (job, instance) (up{host!~"{{ alert_excluded_hosts }}"})' + op: lt + threshold: 1 + for: 10m + severity: warning + summary: Prometheus cannot scrape this target. + + - name: infra + interval: 1m + rules: + - uid: ingress-down + title: Ingress host down + expr: 'min(up{job="node",host="ingress"})' + op: lt + threshold: 1 + for: 5m + severity: critical + summary: The ingress VPS is not answering scrapes, inbound traffic is likely broken. + + - uid: tunnel-peer-down + title: WireGuard tunnel down + expr: 'min by (peer) (wireguard_peer_up{peer=~"beepi|ingress"})' + op: lt + threshold: 1 + for: 5m + severity: critical + summary: The site to site WireGuard peer has not handshaked in the last five minutes. + + - uid: tunnel-probe-down + title: Tunnel path failing + expr: 'min by (probe) (probe_success{vantage="ingress"})' + op: lt + threshold: 1 + for: 5m + severity: critical + summary: The ingress host cannot reach this port on beepi through the tunnel. + + - uid: host-disk-space + title: Host low on disk + expr: >- + min by (host, device) + (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|ramfs",host!~"{{ alert_excluded_hosts }}"} + / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|ramfs",host!~"{{ alert_excluded_hosts }}"}) * 100 + op: lt + threshold: 10 + for: 30m + severity: warning + summary: Less than 10 percent free space left on this filesystem. + + - uid: host-memory + title: Host low on memory + expr: >- + min by (host) (node_memory_MemAvailable_bytes{host!~"{{ alert_excluded_hosts }}"} + / node_memory_MemTotal_bytes{host!~"{{ alert_excluded_hosts }}"}) * 100 + op: lt + threshold: 10 + for: 15m + severity: warning + summary: Less than 10 percent available memory on this host. + + - uid: host-systemd-failed + title: Failed systemd units + expr: >- + sum by (host) (node_systemd_unit_state{state="failed", + host!~"{{ alert_excluded_hosts }}", name!~"{{ alert_ignored_units }}"}) + op: gt + threshold: 0 + for: 15m + severity: warning + summary: A systemd unit outside the known-broken list is in a failed state on this host. diff --git a/playbooks/nginx/sites-enabled/lounge.secretbee.buzz b/playbooks/nginx/sites-enabled/lounge.secretbee.buzz deleted file mode 100644 index 5a116b9..0000000 --- a/playbooks/nginx/sites-enabled/lounge.secretbee.buzz +++ /dev/null @@ -1,36 +0,0 @@ -server { - listen 80; - server_name lounge.secretbee.buzz; - - location / { - return 301 https://$host$request_uri; - } -} - -server { - listen 443 ssl; - server_name lounge.secretbee.buzz; - - limit_req zone=mylimit burst=20; - - add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; - - ssl_certificate /etc/letsencrypt/live/lounge.secretbee.buzz/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/lounge.secretbee.buzz/privkey.pem; - include /etc/nginx/ssl.conf; - - location / { - proxy_pass http://127.0.0.1:9000/; - proxy_http_version 1.1; - proxy_set_header Connection "upgrade"; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header X-Forwarded-For $remote_addr; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_read_timeout 1d; - } - - location /folder/ { - proxy_pass http://127.0.0.1:9000/uploads/; - proxy_set_header X-Forwarded-For $remote_addr; - } -} diff --git a/playbooks/ufw-ingress/ufw-ingress.yml b/playbooks/ufw-ingress/ufw-ingress.yml index edf2007..ca5f6ca 100644 --- a/playbooks/ufw-ingress/ufw-ingress.yml +++ b/playbooks/ufw-ingress/ufw-ingress.yml @@ -41,6 +41,18 @@ - "80" - "443" + - name: UFW - Allow monitoring scrapes from beepi + community.general.ufw: + rule: allow + interface_in: wg0 + proto: tcp + from_ip: "{{ wireguard_beepi_ip }}" + to_port: '{{ item }}' + loop: + - "9100" # node exporter + - "9115" # blackbox exporter + - "12345" # alloy + - name: UFW - Allow wireguard outgoing community.general.ufw: route: true diff --git a/playbooks/ufw/ufw.yml b/playbooks/ufw/ufw.yml index cbba976..84921b7 100644 --- a/playbooks/ufw/ufw.yml +++ b/playbooks/ufw/ufw.yml @@ -111,6 +111,13 @@ interface_out: eth0 route: true + - name: UFW - Allow podman to wireguard + community.general.ufw: + rule: allow + interface_in: podman+ + interface_out: wg0 + route: true + - name: UFW - Allow node exporter scraping community.general.ufw: rule: allow @@ -125,6 +132,31 @@ proto: "tcp" to_port: 9882 + - name: UFW - Allow blackbox exporter scraping + community.general.ufw: + rule: allow + interface_in: podman+ + proto: "tcp" + to_port: 9115 + + - name: UFW - Allow loki push from ingress + community.general.ufw: + rule: allow + interface_in: wg0 + proto: tcp + from_ip: "{{ wireguard_ingress_ip }}" + to_port: "3100" + + - name: UFW - Allow loki push routing from ingress + community.general.ufw: + rule: allow + interface_in: wg0 + interface_out: podman+ + route: true + proto: tcp + from_ip: "{{ wireguard_ingress_ip }}" + to_port: "3100" + - name: UFW - Enable and deny by default community.general.ufw: state: enabled diff --git a/vaultShow.py b/vaultShow.py new file mode 100755 index 0000000..cba351f --- /dev/null +++ b/vaultShow.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 + +import argparse +import os +import re +import subprocess +import sys +import textwrap +from pathlib import Path + +ROOT = Path(__file__).resolve().parent +KEY = re.compile(r"^(?P[ \t]*)(?:-\s+)?(?P[\w.-]+):\s*!vault\s*\|-?\s*$") +HEADER = "$ANSIBLE_VAULT" + + +def password_file(): + override = os.environ.get("ANSIBLE_VAULT_PASSWORD_FILE") + path = Path(override) if override else ROOT / ".vault-password" + if not path.is_file(): + sys.exit(f"vault password file not found: {path}") + return str(path) + + +def decrypt(blob, pw): + result = subprocess.run( + ["ansible-vault", "decrypt", "--vault-password-file", pw, "--output", "-"], + input=blob, + capture_output=True, + text=True, + ) + if result.returncode != 0: + return None, result.stderr.strip() + return result.stdout, None + + +def extract(text): + lines = text.splitlines() + found = [] + i = 0 + while i < len(lines): + match = KEY.match(lines[i]) + if not match: + i += 1 + continue + indent = len(match.group("indent").expandtabs(8)) + body = [] + i += 1 + while i < len(lines): + line = lines[i] + if not line.strip(): + body.append("") + i += 1 + continue + width = len(line.expandtabs(8)) - len(line.expandtabs(8).lstrip()) + if width <= indent: + break + body.append(line) + i += 1 + blob = textwrap.dedent("\n".join(body)).strip() + if blob.startswith(HEADER): + found.append((match.group("name"), blob)) + return found + + +def show(name, value): + value = value.rstrip("\n") + if "\n" in value: + print(f"{name}:") + print(textwrap.indent(value, " ")) + else: + print(f"{name}: {value}") + + +def main(): + parser = argparse.ArgumentParser( + description="Decrypt inline !vault variables in a playbook or vars file." + ) + parser.add_argument("file", help="path to the file, or - to read a blob from stdin") + parser.add_argument("names", nargs="*", help="only show these variable names") + args = parser.parse_args() + + pw = password_file() + + if args.file == "-": + blob = textwrap.dedent(sys.stdin.read()).strip() + if not blob.startswith(HEADER): + sys.exit("stdin does not contain a vault blob") + value, error = decrypt(blob, pw) + if error: + sys.exit(error) + print(value.rstrip("\n")) + return + + path = Path(args.file) + text = path.read_text() + + if text.lstrip().startswith(HEADER): + value, error = decrypt(text.strip(), pw) + if error: + sys.exit(error) + sys.stdout.write(value) + return + + variables = extract(text) + if args.names: + variables = [(n, b) for n, b in variables if n in args.names] + if not variables: + sys.exit(f"no inline vault variables found in {path}") + + failures = 0 + for name, blob in variables: + value, error = decrypt(blob, pw) + if error: + print(f"{name}: ", file=sys.stderr) + failures += 1 + continue + show(name, value) + + sys.exit(1 if failures else 0) + + +if __name__ == "__main__": + main()