421 lines
13 KiB
YAML
421 lines
13 KiB
YAML
- name: Prometheus
|
|
hosts: beepi
|
|
become: true
|
|
vars_files:
|
|
- vars/alert_rules.yml
|
|
vars:
|
|
grafana_secret: !vault |
|
|
$ANSIBLE_VAULT;1.1;AES256
|
|
38343463346232356634353236356534666166626665663237306130623336373062363563386537
|
|
6665356165346134643235383164383266373363316262310a613332386564636430643561373463
|
|
36346437353963333839656163623933616662326132373036636166616538343966356361636164
|
|
3035656163616631330a663363343363643065356535356633623638376430326539363536666638
|
|
65333930613135333364383462363233386662386531323935353932373465363234
|
|
geoip_username: !vault |
|
|
$ANSIBLE_VAULT;1.1;AES256
|
|
30656365643463396265633130356339386433633331613130633134336633643637653761623962
|
|
3633336466326564616432373031303530376166353436610a323033353164313436613363613836
|
|
30303931323234306162343361643162313238313234383837323530343936343134326365666166
|
|
3438643238626266320a333732363461613834613833303436363534393031323333613534333534
|
|
3438
|
|
geoip_license: !vault |
|
|
$ANSIBLE_VAULT;1.1;AES256
|
|
32336437653435396231663666643631373638616533653538333934356339666661313439343566
|
|
6462303439666233316162653430363239393031633362630a323533393233633164613031646633
|
|
62366362383162306563383235373262326262323337303333313163653838613436316337633063
|
|
3630356262666338320a333137333063303534386437343762616665383034336365663732343566
|
|
36376463313964333834373435653564346333396538306337333762613737643363636364353139
|
|
3032383564333638613566633363666430356230326231633636
|
|
tasks:
|
|
- name: Determine whether alert delivery is configured
|
|
ansible.builtin.set_fact:
|
|
alerting_enabled: "{{ discord_webhook is defined and 'REPLACE_ME' not in discord_webhook }}"
|
|
|
|
- name: Determine whether uptimerobot is configured
|
|
ansible.builtin.set_fact:
|
|
uptimerobot_enabled: "{{ uptimerobot_api_key is defined and 'REPLACE_ME' not in uptimerobot_api_key }}"
|
|
|
|
- name: Install packages
|
|
ansible.builtin.apt:
|
|
name:
|
|
- podman
|
|
- podman-compose
|
|
- prometheus-node-exporter
|
|
- wireguard-tools
|
|
state: present
|
|
|
|
- name: Create monitoring config directory
|
|
ansible.builtin.file:
|
|
path: /opt/beeserver/monitoring
|
|
state: directory
|
|
mode: '0755'
|
|
|
|
- name: Create grafana provisioning directories
|
|
ansible.builtin.file:
|
|
path: "/opt/beeserver/monitoring/grafana/{{ item }}"
|
|
state: directory
|
|
mode: '0755'
|
|
loop:
|
|
- provisioning/datasources
|
|
- provisioning/dashboards
|
|
- provisioning/alerting
|
|
- dashboards
|
|
- dashboards-general
|
|
|
|
- name: Create topology directory
|
|
ansible.builtin.file:
|
|
path: /opt/beeserver/monitoring/topology
|
|
state: directory
|
|
mode: '0755'
|
|
|
|
- name: Create textfile directory
|
|
ansible.builtin.file:
|
|
path: "{{ node_exporter_textfile_dir }}"
|
|
state: directory
|
|
mode: '0755'
|
|
|
|
- name: Configure node exporter
|
|
ansible.builtin.copy:
|
|
dest: /etc/default/prometheus-node-exporter
|
|
content: |
|
|
ARGS="--collector.textfile.directory={{ node_exporter_textfile_dir }} --collector.systemd --collector.processes"
|
|
mode: '0644'
|
|
notify: Restart node exporter
|
|
|
|
- name: Copy textfile collectors
|
|
ansible.builtin.template:
|
|
src: "textfile/{{ item }}.sh.j2"
|
|
dest: "/usr/local/bin/{{ item }}.sh"
|
|
mode: '0755'
|
|
loop:
|
|
- service_metrics
|
|
- wg_metrics
|
|
|
|
- name: Copy uptimerobot collector
|
|
ansible.builtin.template:
|
|
src: textfile/uptimerobot_metrics.sh.j2
|
|
dest: /usr/local/bin/uptimerobot_metrics.sh
|
|
mode: '0700'
|
|
when: uptimerobot_enabled
|
|
|
|
- name: Install uptimerobot timer
|
|
ansible.builtin.copy:
|
|
dest: "/etc/systemd/system/beepi-uptimerobot-metrics.{{ item.kind }}"
|
|
content: "{{ item.content }}"
|
|
mode: '0644'
|
|
loop:
|
|
- kind: service
|
|
content: |
|
|
[Unit]
|
|
Description=Collect uptimerobot metrics
|
|
|
|
[Service]
|
|
Type=oneshot
|
|
ExecStart=/usr/local/bin/uptimerobot_metrics.sh
|
|
- kind: timer
|
|
content: |
|
|
[Unit]
|
|
Description=Collect uptimerobot metrics
|
|
|
|
[Timer]
|
|
OnBootSec=3min
|
|
OnUnitActiveSec=5min
|
|
AccuracySec=30s
|
|
|
|
[Install]
|
|
WantedBy=timers.target
|
|
when: uptimerobot_enabled
|
|
notify: Reload systemd
|
|
|
|
- name: Install collector timers
|
|
ansible.builtin.copy:
|
|
dest: "/etc/systemd/system/{{ item.name }}.{{ item.kind }}"
|
|
content: "{{ item.content }}"
|
|
mode: '0644'
|
|
loop:
|
|
- name: beepi-service-metrics
|
|
kind: service
|
|
content: |
|
|
[Unit]
|
|
Description=Collect service unit metrics
|
|
|
|
[Service]
|
|
Type=oneshot
|
|
ExecStart=/usr/local/bin/service_metrics.sh
|
|
- name: beepi-service-metrics
|
|
kind: timer
|
|
content: |
|
|
[Unit]
|
|
Description=Collect service unit metrics
|
|
|
|
[Timer]
|
|
OnBootSec=1min
|
|
OnUnitActiveSec=30s
|
|
AccuracySec=5s
|
|
|
|
[Install]
|
|
WantedBy=timers.target
|
|
- name: beepi-wg-metrics
|
|
kind: service
|
|
content: |
|
|
[Unit]
|
|
Description=Collect wireguard peer metrics
|
|
|
|
[Service]
|
|
Type=oneshot
|
|
ExecStart=/usr/local/bin/wg_metrics.sh
|
|
- name: beepi-wg-metrics
|
|
kind: timer
|
|
content: |
|
|
[Unit]
|
|
Description=Collect wireguard peer metrics
|
|
|
|
[Timer]
|
|
OnBootSec=1min
|
|
OnUnitActiveSec=30s
|
|
AccuracySec=5s
|
|
|
|
[Install]
|
|
WantedBy=timers.target
|
|
notify: Reload systemd
|
|
|
|
- name: Flush handlers
|
|
ansible.builtin.meta: flush_handlers
|
|
|
|
- name: Enable collector timers
|
|
ansible.builtin.systemd_service:
|
|
name: "{{ item }}.timer"
|
|
state: started
|
|
enabled: true
|
|
loop:
|
|
- beepi-service-metrics
|
|
- beepi-wg-metrics
|
|
|
|
- name: Enable uptimerobot timer
|
|
ansible.builtin.systemd_service:
|
|
name: beepi-uptimerobot-metrics.timer
|
|
state: started
|
|
enabled: true
|
|
when: uptimerobot_enabled
|
|
|
|
- name: Compose down
|
|
changed_when: true
|
|
failed_when: false
|
|
ansible.builtin.command:
|
|
cmd: podman-compose down
|
|
chdir: /opt/beeserver/monitoring
|
|
|
|
- name: Copy compose
|
|
ansible.builtin.template:
|
|
src: docker-compose.yml.j2
|
|
dest: /opt/beeserver/monitoring/docker-compose.yml
|
|
mode: '0600'
|
|
|
|
- name: Copy prometheus config
|
|
ansible.builtin.template:
|
|
src: prometheus.yml.j2
|
|
dest: /opt/beeserver/monitoring/prometheus.yml
|
|
mode: '0644'
|
|
|
|
- name: Copy blackbox config
|
|
ansible.builtin.template:
|
|
src: blackbox.yml.j2
|
|
dest: /opt/beeserver/monitoring/blackbox.yml
|
|
mode: '0644'
|
|
|
|
- name: Copy loki config
|
|
ansible.builtin.template:
|
|
src: local-config.yml.j2
|
|
dest: /opt/beeserver/monitoring/local-config.yml
|
|
mode: '0644'
|
|
|
|
- name: Copy alloy config
|
|
ansible.builtin.template:
|
|
src: config.alloy.j2
|
|
dest: /opt/beeserver/monitoring/config.alloy
|
|
mode: '0644'
|
|
|
|
- name: Copy topology service
|
|
ansible.builtin.copy:
|
|
src: topology/topology.py
|
|
dest: /opt/beeserver/monitoring/topology/topology.py
|
|
mode: '0644'
|
|
|
|
- name: Copy topology spec
|
|
ansible.builtin.template:
|
|
src: topology/topology-spec.json.j2
|
|
dest: /opt/beeserver/monitoring/topology/topology-spec.json
|
|
mode: '0644'
|
|
|
|
- name: Copy grafana datasources
|
|
ansible.builtin.template:
|
|
src: grafana/provisioning/datasources/datasources.yml.j2
|
|
dest: /opt/beeserver/monitoring/grafana/provisioning/datasources/datasources.yml
|
|
owner: '472'
|
|
group: '472'
|
|
mode: '0600'
|
|
|
|
- name: Copy grafana dashboard provider
|
|
ansible.builtin.template:
|
|
src: grafana/provisioning/dashboards/dashboards.yml.j2
|
|
dest: /opt/beeserver/monitoring/grafana/provisioning/dashboards/dashboards.yml
|
|
mode: '0644'
|
|
|
|
- name: Copy hand made dashboards
|
|
ansible.builtin.copy:
|
|
src: grafana/dashboards-general/
|
|
dest: /opt/beeserver/monitoring/grafana/dashboards-general/
|
|
mode: '0644'
|
|
|
|
- name: Copy grafana contact points
|
|
ansible.builtin.template:
|
|
src: grafana/provisioning/alerting/contact-points.yml.j2
|
|
dest: /opt/beeserver/monitoring/grafana/provisioning/alerting/contact-points.yml
|
|
owner: '472'
|
|
group: '472'
|
|
mode: '0600'
|
|
when: alerting_enabled
|
|
|
|
- name: Copy grafana alert rules
|
|
ansible.builtin.template:
|
|
src: grafana/provisioning/alerting/rules.yml.j2
|
|
dest: /opt/beeserver/monitoring/grafana/provisioning/alerting/rules.yml
|
|
mode: '0644'
|
|
|
|
- name: Copy grafana dashboards
|
|
ansible.builtin.template:
|
|
src: "grafana/dashboards/{{ item }}.json.j2"
|
|
dest: "/opt/beeserver/monitoring/grafana/dashboards/{{ item }}.json"
|
|
mode: '0644'
|
|
loop:
|
|
- infrastructure-map
|
|
- backups
|
|
- services
|
|
- ingress
|
|
|
|
- name: Copy uptimerobot dashboard
|
|
ansible.builtin.template:
|
|
src: grafana/dashboards/uptimerobot.json.j2
|
|
dest: /opt/beeserver/monitoring/grafana/dashboards/uptimerobot.json
|
|
mode: '0644'
|
|
when: uptimerobot_enabled
|
|
|
|
- name: Enable podman.socket
|
|
ansible.builtin.service:
|
|
name: podman.socket
|
|
state: started
|
|
enabled: true
|
|
|
|
- name: Check the geoip database
|
|
ansible.builtin.stat:
|
|
path: /opt/beeserver/monitoring/database/GeoLite2-City.mmdb
|
|
register: geoip_db
|
|
|
|
- name: Check when geoip was last downloaded
|
|
ansible.builtin.stat:
|
|
path: /opt/beeserver/monitoring/.geoip_downloaded
|
|
register: geoip_stamp
|
|
|
|
- name: Decide whether the geoip database needs refreshing
|
|
ansible.builtin.set_fact:
|
|
geoip_stale: "{{ not geoip_db.stat.exists or not geoip_stamp.stat.exists or (ansible_facts.date_time.epoch | int - geoip_stamp.stat.mtime | int) > geoip_max_age_seconds | int }}"
|
|
|
|
- name: Report the geoip database age
|
|
ansible.builtin.debug:
|
|
msg: >-
|
|
geoip
|
|
{{ 'not present, downloading' if not (geoip_db.stat.exists and geoip_stamp.stat.exists)
|
|
else 'downloaded ' + (((ansible_facts.date_time.epoch | int - geoip_stamp.stat.mtime | int) / 3600) | round(1) | string) + 'h ago, ' + ('refreshing' if geoip_stale else 'reusing') }}
|
|
|
|
- name: Delete database directory
|
|
ansible.builtin.file:
|
|
path: /opt/beeserver/monitoring/database
|
|
state: absent
|
|
mode: '0755'
|
|
when: geoip_stale
|
|
|
|
- name: Create database directory
|
|
ansible.builtin.file:
|
|
path: /opt/beeserver/monitoring/database
|
|
state: directory
|
|
mode: '0755'
|
|
|
|
- name: Remove stale geoip archives
|
|
ansible.builtin.shell:
|
|
cmd: rm -f /opt/beeserver/monitoring/GeoLite2-City_*.tar.gz
|
|
changed_when: true
|
|
when: geoip_stale
|
|
|
|
- name: Download geoip database
|
|
ansible.builtin.get_url:
|
|
url: "https://download.maxmind.com/geoip/databases/GeoLite2-City/download?suffix=tar.gz"
|
|
dest: /opt/beeserver/monitoring
|
|
username: "{{ geoip_username }}"
|
|
password: "{{ geoip_license }}"
|
|
mode: '0644'
|
|
when: geoip_stale
|
|
|
|
- name: Get geoip tar file
|
|
ansible.builtin.find:
|
|
paths: /opt/beeserver/monitoring
|
|
file_type: file
|
|
patterns: '^GeoLite.*.tar.gz$'
|
|
use_regex: true
|
|
limit: 1
|
|
register: found
|
|
when: geoip_stale
|
|
|
|
- name: Extract geopip database
|
|
ansible.builtin.unarchive:
|
|
remote_src: true
|
|
src: "{{ found.files[0].path }}"
|
|
dest: /opt/beeserver/monitoring/database
|
|
extra_opts: ['--strip-components=1', '--show-stored-names']
|
|
when: geoip_stale and found.files | length > 0
|
|
|
|
- name: Stamp the geoip download time
|
|
ansible.builtin.file:
|
|
path: /opt/beeserver/monitoring/.geoip_downloaded
|
|
state: touch
|
|
mode: '0644'
|
|
when: geoip_stale and found.files | length > 0
|
|
|
|
- name: Compose up
|
|
changed_when: true
|
|
ansible.builtin.command:
|
|
cmd: podman-compose up -d
|
|
chdir: /opt/beeserver/monitoring
|
|
|
|
- name: Run collectors once
|
|
changed_when: true
|
|
ansible.builtin.command:
|
|
cmd: "/usr/local/bin/{{ item }}.sh"
|
|
loop:
|
|
- service_metrics
|
|
- wg_metrics
|
|
|
|
- name: Run uptimerobot collector once
|
|
changed_when: true
|
|
ansible.builtin.command:
|
|
cmd: /usr/local/bin/uptimerobot_metrics.sh
|
|
when: uptimerobot_enabled
|
|
|
|
- name: Debug
|
|
ansible.builtin.debug:
|
|
msg:
|
|
- "Grafana: https://grafana.secretbee.buzz/"
|
|
- "Topology: curl -s localhost:8088/topology.json | head"
|
|
- "Prometheus targets: curl -s localhost:9090/api/v1/targets | jq '.data.activeTargets[] | {job: .labels.job, health}'"
|
|
tags:
|
|
- always
|
|
|
|
handlers:
|
|
- name: Reload systemd
|
|
ansible.builtin.systemd_service:
|
|
daemon_reload: true
|
|
|
|
- name: Restart node exporter
|
|
ansible.builtin.service:
|
|
name: prometheus-node-exporter
|
|
state: restarted
|