298 lines
9.3 KiB
YAML
298 lines
9.3 KiB
YAML
- name: Monitoring-ingress
|
|
hosts: animeistrash
|
|
become: true
|
|
vars:
|
|
geoip_username: !vault |
|
|
$ANSIBLE_VAULT;1.1;AES256
|
|
30656365643463396265633130356339386433633331613130633134336633643637653761623962
|
|
3633336466326564616432373031303530376166353436610a323033353164313436613363613836
|
|
30303931323234306162343361643162313238313234383837323530343936343134326365666166
|
|
3438643238626266320a333732363461613834613833303436363534393031323333613534333534
|
|
3438
|
|
geoip_license: !vault |
|
|
$ANSIBLE_VAULT;1.1;AES256
|
|
32336437653435396231663666643631373638616533653538333934356339666661313439343566
|
|
6462303439666233316162653430363239393031633362630a323533393233633164613031646633
|
|
62366362383162306563383235373262326262323337303333313163653838613436316337633063
|
|
3630356262666338320a333137333063303534386437343762616665383034336365663732343566
|
|
36376463313964333834373435653564346333396538306337333762613737643363636364353139
|
|
3032383564333638613566633363666430356230326231633636
|
|
tasks:
|
|
- name: Install exporters
|
|
ansible.builtin.apt:
|
|
name:
|
|
- prometheus-node-exporter
|
|
- prometheus-blackbox-exporter
|
|
- wireguard-tools
|
|
state: present
|
|
update_cache: true
|
|
|
|
- name: Create textfile directory
|
|
ansible.builtin.file:
|
|
path: "{{ node_exporter_textfile_dir }}"
|
|
state: directory
|
|
mode: '0755'
|
|
|
|
- name: Configure node exporter
|
|
ansible.builtin.copy:
|
|
dest: /etc/default/prometheus-node-exporter
|
|
content: |
|
|
ARGS="--web.listen-address={{ wireguard_ingress_ip }}:9100 --collector.textfile.directory={{ node_exporter_textfile_dir }} --collector.systemd"
|
|
mode: '0644'
|
|
notify: Restart node exporter
|
|
|
|
- name: Configure blackbox exporter
|
|
ansible.builtin.copy:
|
|
dest: /etc/default/prometheus-blackbox-exporter
|
|
content: |
|
|
ARGS="--config.file=/etc/prometheus/blackbox.yml --web.listen-address={{ wireguard_ingress_ip }}:9115"
|
|
mode: '0644'
|
|
notify: Restart blackbox exporter
|
|
|
|
- name: Copy blackbox config
|
|
ansible.builtin.copy:
|
|
src: ../monitoring/blackbox.yml
|
|
dest: /etc/prometheus/blackbox.yml
|
|
mode: '0644'
|
|
notify: Restart blackbox exporter
|
|
|
|
- name: Look up the prometheus group
|
|
ansible.builtin.getent:
|
|
database: group
|
|
key: prometheus
|
|
|
|
- name: Allow the prometheus group to send icmp
|
|
ansible.builtin.copy:
|
|
dest: /etc/sysctl.d/99-blackbox-icmp.conf
|
|
content: |
|
|
net.ipv4.ping_group_range = {{ prometheus_gid }} {{ prometheus_gid }}
|
|
mode: '0644'
|
|
vars:
|
|
prometheus_gid: "{{ ansible_facts.getent_group.prometheus[1] }}"
|
|
notify: Apply sysctl
|
|
|
|
- name: Create blackbox override directory
|
|
ansible.builtin.file:
|
|
path: /etc/systemd/system/prometheus-blackbox-exporter.service.d
|
|
state: directory
|
|
mode: '0755'
|
|
|
|
- name: Allow blackbox to send icmp
|
|
ansible.builtin.copy:
|
|
dest: /etc/systemd/system/prometheus-blackbox-exporter.service.d/capabilities.conf
|
|
content: |
|
|
[Service]
|
|
AmbientCapabilities=CAP_NET_RAW
|
|
CapabilityBoundingSet=CAP_NET_RAW
|
|
mode: '0644'
|
|
notify:
|
|
- Reload systemd
|
|
- Restart blackbox exporter
|
|
|
|
- name: Copy wireguard collector
|
|
ansible.builtin.template:
|
|
src: ../monitoring/textfile/wg_metrics.sh.j2
|
|
dest: /usr/local/bin/wg_metrics.sh
|
|
mode: '0755'
|
|
|
|
- name: Install collector timer
|
|
ansible.builtin.copy:
|
|
dest: "/etc/systemd/system/ingress-wg-metrics.{{ item.kind }}"
|
|
content: "{{ item.content }}"
|
|
mode: '0644'
|
|
loop:
|
|
- kind: service
|
|
content: |
|
|
[Unit]
|
|
Description=Collect wireguard peer metrics
|
|
|
|
[Service]
|
|
Type=oneshot
|
|
ExecStart=/usr/local/bin/wg_metrics.sh
|
|
- kind: timer
|
|
content: |
|
|
[Unit]
|
|
Description=Collect wireguard peer metrics
|
|
|
|
[Timer]
|
|
OnBootSec=1min
|
|
OnUnitActiveSec=30s
|
|
AccuracySec=5s
|
|
|
|
[Install]
|
|
WantedBy=timers.target
|
|
notify: Reload systemd
|
|
|
|
- name: Flush handlers
|
|
ansible.builtin.meta: flush_handlers
|
|
|
|
- name: Enable collector timer
|
|
ansible.builtin.systemd_service:
|
|
name: ingress-wg-metrics.timer
|
|
state: started
|
|
enabled: true
|
|
|
|
- name: Create alloy directories
|
|
ansible.builtin.file:
|
|
path: "{{ item }}"
|
|
state: directory
|
|
mode: '0755'
|
|
loop:
|
|
- /etc/alloy
|
|
- /opt/database
|
|
|
|
- name: Create apt keyring directory
|
|
ansible.builtin.file:
|
|
path: /etc/apt/keyrings
|
|
state: directory
|
|
mode: '0755'
|
|
|
|
- name: Add grafana apt key
|
|
ansible.builtin.get_url:
|
|
url: https://apt.grafana.com/gpg.key
|
|
dest: /etc/apt/keyrings/grafana.asc
|
|
mode: '0644'
|
|
|
|
- name: Add grafana apt repository
|
|
ansible.builtin.apt_repository:
|
|
repo: "deb [signed-by=/etc/apt/keyrings/grafana.asc] https://apt.grafana.com stable main"
|
|
filename: grafana
|
|
state: present
|
|
|
|
- name: Install alloy
|
|
ansible.builtin.apt:
|
|
name: alloy
|
|
state: present
|
|
update_cache: true
|
|
|
|
- name: Create alloy data directory
|
|
ansible.builtin.file:
|
|
path: /var/lib/alloy/data
|
|
state: directory
|
|
owner: alloy
|
|
group: alloy
|
|
mode: '0755'
|
|
notify: Restart alloy
|
|
|
|
- name: Check the geoip database
|
|
ansible.builtin.stat:
|
|
path: /opt/database/GeoLite2-City.mmdb
|
|
register: geoip_db
|
|
|
|
- name: Check when geoip was last downloaded
|
|
ansible.builtin.stat:
|
|
path: /opt/.geoip_downloaded
|
|
register: geoip_stamp
|
|
|
|
- name: Decide whether the geoip database needs refreshing
|
|
ansible.builtin.set_fact:
|
|
geoip_stale: "{{ not geoip_db.stat.exists or not geoip_stamp.stat.exists or (ansible_facts.date_time.epoch | int - geoip_stamp.stat.mtime | int) > geoip_max_age_seconds | int }}"
|
|
|
|
- name: Report the geoip database age
|
|
ansible.builtin.debug:
|
|
msg: >-
|
|
geoip
|
|
{{ 'not present, downloading' if not (geoip_db.stat.exists and geoip_stamp.stat.exists)
|
|
else 'downloaded ' + (((ansible_facts.date_time.epoch | int - geoip_stamp.stat.mtime | int) / 3600) | round(1) | string) + 'h ago, ' + ('refreshing' if geoip_stale else 'reusing') }}
|
|
|
|
- name: Download geoip database
|
|
ansible.builtin.get_url:
|
|
url: "https://download.maxmind.com/geoip/databases/GeoLite2-City/download?suffix=tar.gz"
|
|
dest: /opt/geoip.tar.gz
|
|
username: "{{ geoip_username }}"
|
|
password: "{{ geoip_license }}"
|
|
mode: '0644'
|
|
force: true
|
|
when: geoip_stale
|
|
|
|
- name: Extract geoip database
|
|
ansible.builtin.unarchive:
|
|
remote_src: true
|
|
src: /opt/geoip.tar.gz
|
|
dest: /opt/database
|
|
extra_opts: ['--strip-components=1', '--show-stored-names']
|
|
when: geoip_stale
|
|
notify: Restart alloy
|
|
|
|
- name: Stamp the geoip download time
|
|
ansible.builtin.file:
|
|
path: /opt/.geoip_downloaded
|
|
state: touch
|
|
mode: '0644'
|
|
when: geoip_stale
|
|
|
|
- name: Copy alloy config
|
|
ansible.builtin.template:
|
|
src: config.alloy.j2
|
|
dest: /etc/alloy/config.alloy
|
|
mode: '0644'
|
|
notify: Restart alloy
|
|
|
|
- name: Configure alloy
|
|
ansible.builtin.copy:
|
|
dest: /etc/default/alloy
|
|
content: |
|
|
CONFIG_FILE="/etc/alloy/config.alloy"
|
|
CUSTOM_ARGS="--server.http.listen-addr={{ wireguard_ingress_ip }}:12345 --stability.level=experimental"
|
|
RESTART_ON_UPGRADE=true
|
|
mode: '0644'
|
|
notify: Restart alloy
|
|
|
|
- name: Allow alloy to read logs
|
|
ansible.builtin.user:
|
|
name: alloy
|
|
groups:
|
|
- adm
|
|
- systemd-journal
|
|
append: true
|
|
notify: Restart alloy
|
|
|
|
- name: Enable services
|
|
ansible.builtin.service:
|
|
name: "{{ item }}"
|
|
state: started
|
|
enabled: true
|
|
loop:
|
|
- prometheus-node-exporter
|
|
- prometheus-blackbox-exporter
|
|
- alloy
|
|
|
|
- name: Run collector once
|
|
changed_when: true
|
|
ansible.builtin.command:
|
|
cmd: /usr/local/bin/wg_metrics.sh
|
|
|
|
- name: Debug
|
|
ansible.builtin.debug:
|
|
msg:
|
|
- "Node metrics: curl -s {{ wireguard_ingress_ip }}:9100/metrics | head"
|
|
- "Blackbox probe: curl -s '{{ wireguard_ingress_ip }}:9115/probe?target={{ wireguard_beepi_ip }}:443&module=tcp_connect'"
|
|
- "Alloy logs: journalctl -u alloy -n 50"
|
|
tags:
|
|
- always
|
|
|
|
handlers:
|
|
- name: Apply sysctl
|
|
changed_when: true
|
|
ansible.builtin.command:
|
|
cmd: sysctl --system
|
|
|
|
- name: Reload systemd
|
|
ansible.builtin.systemd_service:
|
|
daemon_reload: true
|
|
|
|
- name: Restart node exporter
|
|
ansible.builtin.service:
|
|
name: prometheus-node-exporter
|
|
state: restarted
|
|
|
|
- name: Restart blackbox exporter
|
|
ansible.builtin.service:
|
|
name: prometheus-blackbox-exporter
|
|
state: restarted
|
|
|
|
- name: Restart alloy
|
|
ansible.builtin.service:
|
|
name: alloy
|
|
state: restarted
|