lots of stuff
This commit is contained in:
@@ -1,6 +1,8 @@
|
||||
- name: Prometheus
|
||||
hosts: beepi
|
||||
become: true
|
||||
vars_files:
|
||||
- vars/alert_rules.yml
|
||||
vars:
|
||||
grafana_secret: !vault |
|
||||
$ANSIBLE_VAULT;1.1;AES256
|
||||
@@ -25,19 +27,21 @@
|
||||
36376463313964333834373435653564346333396538306337333762613737643363636364353139
|
||||
3032383564333638613566633363666430356230326231633636
|
||||
tasks:
|
||||
- name: Install podman
|
||||
ansible.builtin.apt:
|
||||
name: podman
|
||||
state: present
|
||||
- name: Determine whether alert delivery is configured
|
||||
ansible.builtin.set_fact:
|
||||
alerting_enabled: "{{ discord_webhook is defined and 'REPLACE_ME' not in discord_webhook }}"
|
||||
|
||||
- name: Install podman-compose
|
||||
ansible.builtin.apt:
|
||||
name: podman-compose
|
||||
state: present
|
||||
- name: Determine whether uptimerobot is configured
|
||||
ansible.builtin.set_fact:
|
||||
uptimerobot_enabled: "{{ uptimerobot_api_key is defined and 'REPLACE_ME' not in uptimerobot_api_key }}"
|
||||
|
||||
- name: Install prometheus-node-exporter
|
||||
- name: Install packages
|
||||
ansible.builtin.apt:
|
||||
name: prometheus-node-exporter
|
||||
name:
|
||||
- podman
|
||||
- podman-compose
|
||||
- prometheus-node-exporter
|
||||
- wireguard-tools
|
||||
state: present
|
||||
|
||||
- name: Create monitoring config directory
|
||||
@@ -46,6 +50,154 @@
|
||||
state: directory
|
||||
mode: '0755'
|
||||
|
||||
- name: Create grafana provisioning directories
|
||||
ansible.builtin.file:
|
||||
path: "/opt/beeserver/monitoring/grafana/{{ item }}"
|
||||
state: directory
|
||||
mode: '0755'
|
||||
loop:
|
||||
- provisioning/datasources
|
||||
- provisioning/dashboards
|
||||
- provisioning/alerting
|
||||
- dashboards
|
||||
- dashboards-general
|
||||
|
||||
- name: Create topology directory
|
||||
ansible.builtin.file:
|
||||
path: /opt/beeserver/monitoring/topology
|
||||
state: directory
|
||||
mode: '0755'
|
||||
|
||||
- name: Create textfile directory
|
||||
ansible.builtin.file:
|
||||
path: "{{ node_exporter_textfile_dir }}"
|
||||
state: directory
|
||||
mode: '0755'
|
||||
|
||||
- name: Configure node exporter
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/default/prometheus-node-exporter
|
||||
content: |
|
||||
ARGS="--collector.textfile.directory={{ node_exporter_textfile_dir }} --collector.systemd --collector.processes"
|
||||
mode: '0644'
|
||||
notify: Restart node exporter
|
||||
|
||||
- name: Copy textfile collectors
|
||||
ansible.builtin.template:
|
||||
src: "textfile/{{ item }}.sh.j2"
|
||||
dest: "/usr/local/bin/{{ item }}.sh"
|
||||
mode: '0755'
|
||||
loop:
|
||||
- service_metrics
|
||||
- wg_metrics
|
||||
|
||||
- name: Copy uptimerobot collector
|
||||
ansible.builtin.template:
|
||||
src: textfile/uptimerobot_metrics.sh.j2
|
||||
dest: /usr/local/bin/uptimerobot_metrics.sh
|
||||
mode: '0700'
|
||||
when: uptimerobot_enabled
|
||||
|
||||
- name: Install uptimerobot timer
|
||||
ansible.builtin.copy:
|
||||
dest: "/etc/systemd/system/beepi-uptimerobot-metrics.{{ item.kind }}"
|
||||
content: "{{ item.content }}"
|
||||
mode: '0644'
|
||||
loop:
|
||||
- kind: service
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Collect uptimerobot metrics
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/bin/uptimerobot_metrics.sh
|
||||
- kind: timer
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Collect uptimerobot metrics
|
||||
|
||||
[Timer]
|
||||
OnBootSec=3min
|
||||
OnUnitActiveSec=5min
|
||||
AccuracySec=30s
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
when: uptimerobot_enabled
|
||||
notify: Reload systemd
|
||||
|
||||
- name: Install collector timers
|
||||
ansible.builtin.copy:
|
||||
dest: "/etc/systemd/system/{{ item.name }}.{{ item.kind }}"
|
||||
content: "{{ item.content }}"
|
||||
mode: '0644'
|
||||
loop:
|
||||
- name: beepi-service-metrics
|
||||
kind: service
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Collect service unit metrics
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/bin/service_metrics.sh
|
||||
- name: beepi-service-metrics
|
||||
kind: timer
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Collect service unit metrics
|
||||
|
||||
[Timer]
|
||||
OnBootSec=1min
|
||||
OnUnitActiveSec=30s
|
||||
AccuracySec=5s
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
- name: beepi-wg-metrics
|
||||
kind: service
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Collect wireguard peer metrics
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/bin/wg_metrics.sh
|
||||
- name: beepi-wg-metrics
|
||||
kind: timer
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Collect wireguard peer metrics
|
||||
|
||||
[Timer]
|
||||
OnBootSec=1min
|
||||
OnUnitActiveSec=30s
|
||||
AccuracySec=5s
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
notify: Reload systemd
|
||||
|
||||
- name: Flush handlers
|
||||
ansible.builtin.meta: flush_handlers
|
||||
|
||||
- name: Enable collector timers
|
||||
ansible.builtin.systemd_service:
|
||||
name: "{{ item }}.timer"
|
||||
state: started
|
||||
enabled: true
|
||||
loop:
|
||||
- beepi-service-metrics
|
||||
- beepi-wg-metrics
|
||||
|
||||
- name: Enable uptimerobot timer
|
||||
ansible.builtin.systemd_service:
|
||||
name: beepi-uptimerobot-metrics.timer
|
||||
state: started
|
||||
enabled: true
|
||||
when: uptimerobot_enabled
|
||||
|
||||
- name: Compose down
|
||||
changed_when: true
|
||||
failed_when: false
|
||||
@@ -60,11 +212,17 @@
|
||||
mode: '0600'
|
||||
|
||||
- name: Copy prometheus config
|
||||
ansible.builtin.copy:
|
||||
src: prometheus_config.yml
|
||||
ansible.builtin.template:
|
||||
src: prometheus.yml.j2
|
||||
dest: /opt/beeserver/monitoring/prometheus.yml
|
||||
mode: '0644'
|
||||
|
||||
- name: Copy blackbox config
|
||||
ansible.builtin.template:
|
||||
src: blackbox.yml.j2
|
||||
dest: /opt/beeserver/monitoring/blackbox.yml
|
||||
mode: '0644'
|
||||
|
||||
- name: Copy loki config
|
||||
ansible.builtin.template:
|
||||
src: local-config.yml.j2
|
||||
@@ -77,17 +235,104 @@
|
||||
dest: /opt/beeserver/monitoring/config.alloy
|
||||
mode: '0644'
|
||||
|
||||
- name: Copy topology service
|
||||
ansible.builtin.copy:
|
||||
src: topology/topology.py
|
||||
dest: /opt/beeserver/monitoring/topology/topology.py
|
||||
mode: '0644'
|
||||
|
||||
- name: Copy topology spec
|
||||
ansible.builtin.template:
|
||||
src: topology/topology-spec.json.j2
|
||||
dest: /opt/beeserver/monitoring/topology/topology-spec.json
|
||||
mode: '0644'
|
||||
|
||||
- name: Copy grafana datasources
|
||||
ansible.builtin.template:
|
||||
src: grafana/provisioning/datasources/datasources.yml.j2
|
||||
dest: /opt/beeserver/monitoring/grafana/provisioning/datasources/datasources.yml
|
||||
owner: '472'
|
||||
group: '472'
|
||||
mode: '0600'
|
||||
|
||||
- name: Copy grafana dashboard provider
|
||||
ansible.builtin.template:
|
||||
src: grafana/provisioning/dashboards/dashboards.yml.j2
|
||||
dest: /opt/beeserver/monitoring/grafana/provisioning/dashboards/dashboards.yml
|
||||
mode: '0644'
|
||||
|
||||
- name: Copy hand made dashboards
|
||||
ansible.builtin.copy:
|
||||
src: grafana/dashboards-general/
|
||||
dest: /opt/beeserver/monitoring/grafana/dashboards-general/
|
||||
mode: '0644'
|
||||
|
||||
- name: Copy grafana contact points
|
||||
ansible.builtin.template:
|
||||
src: grafana/provisioning/alerting/contact-points.yml.j2
|
||||
dest: /opt/beeserver/monitoring/grafana/provisioning/alerting/contact-points.yml
|
||||
owner: '472'
|
||||
group: '472'
|
||||
mode: '0600'
|
||||
when: alerting_enabled
|
||||
|
||||
- name: Copy grafana alert rules
|
||||
ansible.builtin.template:
|
||||
src: grafana/provisioning/alerting/rules.yml.j2
|
||||
dest: /opt/beeserver/monitoring/grafana/provisioning/alerting/rules.yml
|
||||
mode: '0644'
|
||||
|
||||
- name: Copy grafana dashboards
|
||||
ansible.builtin.template:
|
||||
src: "grafana/dashboards/{{ item }}.json.j2"
|
||||
dest: "/opt/beeserver/monitoring/grafana/dashboards/{{ item }}.json"
|
||||
mode: '0644'
|
||||
loop:
|
||||
- infrastructure-map
|
||||
- backups
|
||||
- services
|
||||
- ingress
|
||||
|
||||
- name: Copy uptimerobot dashboard
|
||||
ansible.builtin.template:
|
||||
src: grafana/dashboards/uptimerobot.json.j2
|
||||
dest: /opt/beeserver/monitoring/grafana/dashboards/uptimerobot.json
|
||||
mode: '0644'
|
||||
when: uptimerobot_enabled
|
||||
|
||||
- name: Enable podman.socket
|
||||
ansible.builtin.service:
|
||||
name: podman.socket
|
||||
state: started
|
||||
enabled: true
|
||||
|
||||
- name: Check the geoip database
|
||||
ansible.builtin.stat:
|
||||
path: /opt/beeserver/monitoring/database/GeoLite2-City.mmdb
|
||||
register: geoip_db
|
||||
|
||||
- name: Check when geoip was last downloaded
|
||||
ansible.builtin.stat:
|
||||
path: /opt/beeserver/monitoring/.geoip_downloaded
|
||||
register: geoip_stamp
|
||||
|
||||
- name: Decide whether the geoip database needs refreshing
|
||||
ansible.builtin.set_fact:
|
||||
geoip_stale: "{{ not geoip_db.stat.exists or not geoip_stamp.stat.exists or (ansible_facts.date_time.epoch | int - geoip_stamp.stat.mtime | int) > geoip_max_age_seconds | int }}"
|
||||
|
||||
- name: Report the geoip database age
|
||||
ansible.builtin.debug:
|
||||
msg: >-
|
||||
geoip
|
||||
{{ 'not present, downloading' if not (geoip_db.stat.exists and geoip_stamp.stat.exists)
|
||||
else 'downloaded ' + (((ansible_facts.date_time.epoch | int - geoip_stamp.stat.mtime | int) / 3600) | round(1) | string) + 'h ago, ' + ('refreshing' if geoip_stale else 'reusing') }}
|
||||
|
||||
- name: Delete database directory
|
||||
ansible.builtin.file:
|
||||
path: /opt/beeserver/monitoring/database
|
||||
state: absent
|
||||
mode: '0755'
|
||||
when: geoip_stale
|
||||
|
||||
- name: Create database directory
|
||||
ansible.builtin.file:
|
||||
@@ -95,6 +340,12 @@
|
||||
state: directory
|
||||
mode: '0755'
|
||||
|
||||
- name: Remove stale geoip archives
|
||||
ansible.builtin.shell:
|
||||
cmd: rm -f /opt/beeserver/monitoring/GeoLite2-City_*.tar.gz
|
||||
changed_when: true
|
||||
when: geoip_stale
|
||||
|
||||
- name: Download geoip database
|
||||
ansible.builtin.get_url:
|
||||
url: "https://download.maxmind.com/geoip/databases/GeoLite2-City/download?suffix=tar.gz"
|
||||
@@ -102,6 +353,7 @@
|
||||
username: "{{ geoip_username }}"
|
||||
password: "{{ geoip_license }}"
|
||||
mode: '0644'
|
||||
when: geoip_stale
|
||||
|
||||
- name: Get geoip tar file
|
||||
ansible.builtin.find:
|
||||
@@ -111,6 +363,7 @@
|
||||
use_regex: true
|
||||
limit: 1
|
||||
register: found
|
||||
when: geoip_stale
|
||||
|
||||
- name: Extract geopip database
|
||||
ansible.builtin.unarchive:
|
||||
@@ -118,9 +371,50 @@
|
||||
src: "{{ found.files[0].path }}"
|
||||
dest: /opt/beeserver/monitoring/database
|
||||
extra_opts: ['--strip-components=1', '--show-stored-names']
|
||||
when: geoip_stale and found.files | length > 0
|
||||
|
||||
- name: Stamp the geoip download time
|
||||
ansible.builtin.file:
|
||||
path: /opt/beeserver/monitoring/.geoip_downloaded
|
||||
state: touch
|
||||
mode: '0644'
|
||||
when: geoip_stale and found.files | length > 0
|
||||
|
||||
- name: Compose up
|
||||
changed_when: true
|
||||
ansible.builtin.command:
|
||||
cmd: podman-compose up -d
|
||||
chdir: /opt/beeserver/monitoring
|
||||
|
||||
- name: Run collectors once
|
||||
changed_when: true
|
||||
ansible.builtin.command:
|
||||
cmd: "/usr/local/bin/{{ item }}.sh"
|
||||
loop:
|
||||
- service_metrics
|
||||
- wg_metrics
|
||||
|
||||
- name: Run uptimerobot collector once
|
||||
changed_when: true
|
||||
ansible.builtin.command:
|
||||
cmd: /usr/local/bin/uptimerobot_metrics.sh
|
||||
when: uptimerobot_enabled
|
||||
|
||||
- name: Debug
|
||||
ansible.builtin.debug:
|
||||
msg:
|
||||
- "Grafana: https://grafana.secretbee.buzz/"
|
||||
- "Topology: curl -s localhost:8088/topology.json | head"
|
||||
- "Prometheus targets: curl -s localhost:9090/api/v1/targets | jq '.data.activeTargets[] | {job: .labels.job, health}'"
|
||||
tags:
|
||||
- always
|
||||
|
||||
handlers:
|
||||
- name: Reload systemd
|
||||
ansible.builtin.systemd_service:
|
||||
daemon_reload: true
|
||||
|
||||
- name: Restart node exporter
|
||||
ansible.builtin.service:
|
||||
name: prometheus-node-exporter
|
||||
state: restarted
|
||||
|
||||
Reference in New Issue
Block a user