lots of stuff
This commit is contained in:
@@ -0,0 +1,297 @@
|
||||
- name: Monitoring-ingress
|
||||
hosts: animeistrash
|
||||
become: true
|
||||
vars:
|
||||
geoip_username: !vault |
|
||||
$ANSIBLE_VAULT;1.1;AES256
|
||||
30656365643463396265633130356339386433633331613130633134336633643637653761623962
|
||||
3633336466326564616432373031303530376166353436610a323033353164313436613363613836
|
||||
30303931323234306162343361643162313238313234383837323530343936343134326365666166
|
||||
3438643238626266320a333732363461613834613833303436363534393031323333613534333534
|
||||
3438
|
||||
geoip_license: !vault |
|
||||
$ANSIBLE_VAULT;1.1;AES256
|
||||
32336437653435396231663666643631373638616533653538333934356339666661313439343566
|
||||
6462303439666233316162653430363239393031633362630a323533393233633164613031646633
|
||||
62366362383162306563383235373262326262323337303333313163653838613436316337633063
|
||||
3630356262666338320a333137333063303534386437343762616665383034336365663732343566
|
||||
36376463313964333834373435653564346333396538306337333762613737643363636364353139
|
||||
3032383564333638613566633363666430356230326231633636
|
||||
tasks:
|
||||
- name: Install exporters
|
||||
ansible.builtin.apt:
|
||||
name:
|
||||
- prometheus-node-exporter
|
||||
- prometheus-blackbox-exporter
|
||||
- wireguard-tools
|
||||
state: present
|
||||
update_cache: true
|
||||
|
||||
- name: Create textfile directory
|
||||
ansible.builtin.file:
|
||||
path: "{{ node_exporter_textfile_dir }}"
|
||||
state: directory
|
||||
mode: '0755'
|
||||
|
||||
- name: Configure node exporter
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/default/prometheus-node-exporter
|
||||
content: |
|
||||
ARGS="--web.listen-address={{ wireguard_ingress_ip }}:9100 --collector.textfile.directory={{ node_exporter_textfile_dir }} --collector.systemd"
|
||||
mode: '0644'
|
||||
notify: Restart node exporter
|
||||
|
||||
- name: Configure blackbox exporter
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/default/prometheus-blackbox-exporter
|
||||
content: |
|
||||
ARGS="--config.file=/etc/prometheus/blackbox.yml --web.listen-address={{ wireguard_ingress_ip }}:9115"
|
||||
mode: '0644'
|
||||
notify: Restart blackbox exporter
|
||||
|
||||
- name: Copy blackbox config
|
||||
ansible.builtin.copy:
|
||||
src: ../monitoring/blackbox.yml
|
||||
dest: /etc/prometheus/blackbox.yml
|
||||
mode: '0644'
|
||||
notify: Restart blackbox exporter
|
||||
|
||||
- name: Look up the prometheus group
|
||||
ansible.builtin.getent:
|
||||
database: group
|
||||
key: prometheus
|
||||
|
||||
- name: Allow the prometheus group to send icmp
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/sysctl.d/99-blackbox-icmp.conf
|
||||
content: |
|
||||
net.ipv4.ping_group_range = {{ prometheus_gid }} {{ prometheus_gid }}
|
||||
mode: '0644'
|
||||
vars:
|
||||
prometheus_gid: "{{ ansible_facts.getent_group.prometheus[1] }}"
|
||||
notify: Apply sysctl
|
||||
|
||||
- name: Create blackbox override directory
|
||||
ansible.builtin.file:
|
||||
path: /etc/systemd/system/prometheus-blackbox-exporter.service.d
|
||||
state: directory
|
||||
mode: '0755'
|
||||
|
||||
- name: Allow blackbox to send icmp
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/systemd/system/prometheus-blackbox-exporter.service.d/capabilities.conf
|
||||
content: |
|
||||
[Service]
|
||||
AmbientCapabilities=CAP_NET_RAW
|
||||
CapabilityBoundingSet=CAP_NET_RAW
|
||||
mode: '0644'
|
||||
notify:
|
||||
- Reload systemd
|
||||
- Restart blackbox exporter
|
||||
|
||||
- name: Copy wireguard collector
|
||||
ansible.builtin.template:
|
||||
src: ../monitoring/textfile/wg_metrics.sh.j2
|
||||
dest: /usr/local/bin/wg_metrics.sh
|
||||
mode: '0755'
|
||||
|
||||
- name: Install collector timer
|
||||
ansible.builtin.copy:
|
||||
dest: "/etc/systemd/system/ingress-wg-metrics.{{ item.kind }}"
|
||||
content: "{{ item.content }}"
|
||||
mode: '0644'
|
||||
loop:
|
||||
- kind: service
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Collect wireguard peer metrics
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/bin/wg_metrics.sh
|
||||
- kind: timer
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Collect wireguard peer metrics
|
||||
|
||||
[Timer]
|
||||
OnBootSec=1min
|
||||
OnUnitActiveSec=30s
|
||||
AccuracySec=5s
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
notify: Reload systemd
|
||||
|
||||
- name: Flush handlers
|
||||
ansible.builtin.meta: flush_handlers
|
||||
|
||||
- name: Enable collector timer
|
||||
ansible.builtin.systemd_service:
|
||||
name: ingress-wg-metrics.timer
|
||||
state: started
|
||||
enabled: true
|
||||
|
||||
- name: Create alloy directories
|
||||
ansible.builtin.file:
|
||||
path: "{{ item }}"
|
||||
state: directory
|
||||
mode: '0755'
|
||||
loop:
|
||||
- /etc/alloy
|
||||
- /opt/database
|
||||
|
||||
- name: Create apt keyring directory
|
||||
ansible.builtin.file:
|
||||
path: /etc/apt/keyrings
|
||||
state: directory
|
||||
mode: '0755'
|
||||
|
||||
- name: Add grafana apt key
|
||||
ansible.builtin.get_url:
|
||||
url: https://apt.grafana.com/gpg.key
|
||||
dest: /etc/apt/keyrings/grafana.asc
|
||||
mode: '0644'
|
||||
|
||||
- name: Add grafana apt repository
|
||||
ansible.builtin.apt_repository:
|
||||
repo: "deb [signed-by=/etc/apt/keyrings/grafana.asc] https://apt.grafana.com stable main"
|
||||
filename: grafana
|
||||
state: present
|
||||
|
||||
- name: Install alloy
|
||||
ansible.builtin.apt:
|
||||
name: alloy
|
||||
state: present
|
||||
update_cache: true
|
||||
|
||||
- name: Create alloy data directory
|
||||
ansible.builtin.file:
|
||||
path: /var/lib/alloy/data
|
||||
state: directory
|
||||
owner: alloy
|
||||
group: alloy
|
||||
mode: '0755'
|
||||
notify: Restart alloy
|
||||
|
||||
- name: Check the geoip database
|
||||
ansible.builtin.stat:
|
||||
path: /opt/database/GeoLite2-City.mmdb
|
||||
register: geoip_db
|
||||
|
||||
- name: Check when geoip was last downloaded
|
||||
ansible.builtin.stat:
|
||||
path: /opt/.geoip_downloaded
|
||||
register: geoip_stamp
|
||||
|
||||
- name: Decide whether the geoip database needs refreshing
|
||||
ansible.builtin.set_fact:
|
||||
geoip_stale: "{{ not geoip_db.stat.exists or not geoip_stamp.stat.exists or (ansible_facts.date_time.epoch | int - geoip_stamp.stat.mtime | int) > geoip_max_age_seconds | int }}"
|
||||
|
||||
- name: Report the geoip database age
|
||||
ansible.builtin.debug:
|
||||
msg: >-
|
||||
geoip
|
||||
{{ 'not present, downloading' if not (geoip_db.stat.exists and geoip_stamp.stat.exists)
|
||||
else 'downloaded ' + (((ansible_facts.date_time.epoch | int - geoip_stamp.stat.mtime | int) / 3600) | round(1) | string) + 'h ago, ' + ('refreshing' if geoip_stale else 'reusing') }}
|
||||
|
||||
- name: Download geoip database
|
||||
ansible.builtin.get_url:
|
||||
url: "https://download.maxmind.com/geoip/databases/GeoLite2-City/download?suffix=tar.gz"
|
||||
dest: /opt/geoip.tar.gz
|
||||
username: "{{ geoip_username }}"
|
||||
password: "{{ geoip_license }}"
|
||||
mode: '0644'
|
||||
force: true
|
||||
when: geoip_stale
|
||||
|
||||
- name: Extract geoip database
|
||||
ansible.builtin.unarchive:
|
||||
remote_src: true
|
||||
src: /opt/geoip.tar.gz
|
||||
dest: /opt/database
|
||||
extra_opts: ['--strip-components=1', '--show-stored-names']
|
||||
when: geoip_stale
|
||||
notify: Restart alloy
|
||||
|
||||
- name: Stamp the geoip download time
|
||||
ansible.builtin.file:
|
||||
path: /opt/.geoip_downloaded
|
||||
state: touch
|
||||
mode: '0644'
|
||||
when: geoip_stale
|
||||
|
||||
- name: Copy alloy config
|
||||
ansible.builtin.template:
|
||||
src: config.alloy.j2
|
||||
dest: /etc/alloy/config.alloy
|
||||
mode: '0644'
|
||||
notify: Restart alloy
|
||||
|
||||
- name: Configure alloy
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/default/alloy
|
||||
content: |
|
||||
CONFIG_FILE="/etc/alloy/config.alloy"
|
||||
CUSTOM_ARGS="--server.http.listen-addr={{ wireguard_ingress_ip }}:12345 --stability.level=experimental"
|
||||
RESTART_ON_UPGRADE=true
|
||||
mode: '0644'
|
||||
notify: Restart alloy
|
||||
|
||||
- name: Allow alloy to read logs
|
||||
ansible.builtin.user:
|
||||
name: alloy
|
||||
groups:
|
||||
- adm
|
||||
- systemd-journal
|
||||
append: true
|
||||
notify: Restart alloy
|
||||
|
||||
- name: Enable services
|
||||
ansible.builtin.service:
|
||||
name: "{{ item }}"
|
||||
state: started
|
||||
enabled: true
|
||||
loop:
|
||||
- prometheus-node-exporter
|
||||
- prometheus-blackbox-exporter
|
||||
- alloy
|
||||
|
||||
- name: Run collector once
|
||||
changed_when: true
|
||||
ansible.builtin.command:
|
||||
cmd: /usr/local/bin/wg_metrics.sh
|
||||
|
||||
- name: Debug
|
||||
ansible.builtin.debug:
|
||||
msg:
|
||||
- "Node metrics: curl -s {{ wireguard_ingress_ip }}:9100/metrics | head"
|
||||
- "Blackbox probe: curl -s '{{ wireguard_ingress_ip }}:9115/probe?target={{ wireguard_beepi_ip }}:443&module=tcp_connect'"
|
||||
- "Alloy logs: journalctl -u alloy -n 50"
|
||||
tags:
|
||||
- always
|
||||
|
||||
handlers:
|
||||
- name: Apply sysctl
|
||||
changed_when: true
|
||||
ansible.builtin.command:
|
||||
cmd: sysctl --system
|
||||
|
||||
- name: Reload systemd
|
||||
ansible.builtin.systemd_service:
|
||||
daemon_reload: true
|
||||
|
||||
- name: Restart node exporter
|
||||
ansible.builtin.service:
|
||||
name: prometheus-node-exporter
|
||||
state: restarted
|
||||
|
||||
- name: Restart blackbox exporter
|
||||
ansible.builtin.service:
|
||||
name: prometheus-blackbox-exporter
|
||||
state: restarted
|
||||
|
||||
- name: Restart alloy
|
||||
ansible.builtin.service:
|
||||
name: alloy
|
||||
state: restarted
|
||||
Reference in New Issue
Block a user