- name: Monitoring-ingress
  hosts: animeistrash
  become: true
  vars:
    geoip_username: !vault |
              $ANSIBLE_VAULT;1.1;AES256
              30656365643463396265633130356339386433633331613130633134336633643637653761623962
              3633336466326564616432373031303530376166353436610a323033353164313436613363613836
              30303931323234306162343361643162313238313234383837323530343936343134326365666166
              3438643238626266320a333732363461613834613833303436363534393031323333613534333534
              3438
    geoip_license: !vault |
              $ANSIBLE_VAULT;1.1;AES256
              32336437653435396231663666643631373638616533653538333934356339666661313439343566
              6462303439666233316162653430363239393031633362630a323533393233633164613031646633
              62366362383162306563383235373262326262323337303333313163653838613436316337633063
              3630356262666338320a333137333063303534386437343762616665383034336365663732343566
              36376463313964333834373435653564346333396538306337333762613737643363636364353139
              3032383564333638613566633363666430356230326231633636
  tasks:
    - name: Install exporters
      ansible.builtin.apt:
        name:
          - prometheus-node-exporter
          - prometheus-blackbox-exporter
          - wireguard-tools
        state: present
        update_cache: true

    - name: Create textfile directory
      ansible.builtin.file:
        path: "{{ node_exporter_textfile_dir }}"
        state: directory
        mode: '0755'

    - name: Configure node exporter
      ansible.builtin.copy:
        dest: /etc/default/prometheus-node-exporter
        content: |
          ARGS="--web.listen-address={{ wireguard_ingress_ip }}:9100 --collector.textfile.directory={{ node_exporter_textfile_dir }} --collector.systemd"
        mode: '0644'
      notify: Restart node exporter

    - name: Configure blackbox exporter
      ansible.builtin.copy:
        dest: /etc/default/prometheus-blackbox-exporter
        content: |
          ARGS="--config.file=/etc/prometheus/blackbox.yml --web.listen-address={{ wireguard_ingress_ip }}:9115"
        mode: '0644'
      notify: Restart blackbox exporter

    - name: Copy blackbox config
      ansible.builtin.copy:
        src: ../monitoring/blackbox.yml
        dest: /etc/prometheus/blackbox.yml
        mode: '0644'
      notify: Restart blackbox exporter

    - name: Look up the prometheus group
      ansible.builtin.getent:
        database: group
        key: prometheus

    - name: Allow the prometheus group to send icmp
      ansible.builtin.copy:
        dest: /etc/sysctl.d/99-blackbox-icmp.conf
        content: |
          net.ipv4.ping_group_range = {{ prometheus_gid }} {{ prometheus_gid }}
        mode: '0644'
      vars:
        prometheus_gid: "{{ ansible_facts.getent_group.prometheus[1] }}"
      notify: Apply sysctl

    - name: Create blackbox override directory
      ansible.builtin.file:
        path: /etc/systemd/system/prometheus-blackbox-exporter.service.d
        state: directory
        mode: '0755'

    - name: Allow blackbox to send icmp
      ansible.builtin.copy:
        dest: /etc/systemd/system/prometheus-blackbox-exporter.service.d/capabilities.conf
        content: |
          [Service]
          AmbientCapabilities=CAP_NET_RAW
          CapabilityBoundingSet=CAP_NET_RAW
        mode: '0644'
      notify:
        - Reload systemd
        - Restart blackbox exporter

    - name: Copy wireguard collector
      ansible.builtin.template:
        src: ../monitoring/textfile/wg_metrics.sh.j2
        dest: /usr/local/bin/wg_metrics.sh
        mode: '0755'

    - name: Install collector timer
      ansible.builtin.copy:
        dest: "/etc/systemd/system/ingress-wg-metrics.{{ item.kind }}"
        content: "{{ item.content }}"
        mode: '0644'
      loop:
        - kind: service
          content: |
            [Unit]
            Description=Collect wireguard peer metrics

            [Service]
            Type=oneshot
            ExecStart=/usr/local/bin/wg_metrics.sh
        - kind: timer
          content: |
            [Unit]
            Description=Collect wireguard peer metrics

            [Timer]
            OnBootSec=1min
            OnUnitActiveSec=30s
            AccuracySec=5s

            [Install]
            WantedBy=timers.target
      notify: Reload systemd

    - name: Flush handlers
      ansible.builtin.meta: flush_handlers

    - name: Enable collector timer
      ansible.builtin.systemd_service:
        name: ingress-wg-metrics.timer
        state: started
        enabled: true

    - name: Create alloy directories
      ansible.builtin.file:
        path: "{{ item }}"
        state: directory
        mode: '0755'
      loop:
        - /etc/alloy
        - /opt/database

    - name: Create apt keyring directory
      ansible.builtin.file:
        path: /etc/apt/keyrings
        state: directory
        mode: '0755'

    - name: Add grafana apt key
      ansible.builtin.get_url:
        url: https://apt.grafana.com/gpg.key
        dest: /etc/apt/keyrings/grafana.asc
        mode: '0644'

    - name: Add grafana apt repository
      ansible.builtin.apt_repository:
        repo: "deb [signed-by=/etc/apt/keyrings/grafana.asc] https://apt.grafana.com stable main"
        filename: grafana
        state: present

    - name: Install alloy
      ansible.builtin.apt:
        name: alloy
        state: present
        update_cache: true

    - name: Create alloy data directory
      ansible.builtin.file:
        path: /var/lib/alloy/data
        state: directory
        owner: alloy
        group: alloy
        mode: '0755'
      notify: Restart alloy

    - name: Check the geoip database
      ansible.builtin.stat:
        path: /opt/database/GeoLite2-City.mmdb
      register: geoip_db

    - name: Check when geoip was last downloaded
      ansible.builtin.stat:
        path: /opt/.geoip_downloaded
      register: geoip_stamp

    - name: Decide whether the geoip database needs refreshing
      ansible.builtin.set_fact:
        geoip_stale: "{{ not geoip_db.stat.exists or not geoip_stamp.stat.exists or (ansible_facts.date_time.epoch | int - geoip_stamp.stat.mtime | int) > geoip_max_age_seconds | int }}"

    - name: Report the geoip database age
      ansible.builtin.debug:
        msg: >-
          geoip
          {{ 'not present, downloading' if not (geoip_db.stat.exists and geoip_stamp.stat.exists)
             else 'downloaded ' + (((ansible_facts.date_time.epoch | int - geoip_stamp.stat.mtime | int) / 3600) | round(1) | string) + 'h ago, ' + ('refreshing' if geoip_stale else 'reusing') }}

    - name: Download geoip database
      ansible.builtin.get_url:
        url: "https://download.maxmind.com/geoip/databases/GeoLite2-City/download?suffix=tar.gz"
        dest: /opt/geoip.tar.gz
        username: "{{ geoip_username }}"
        password: "{{ geoip_license }}"
        mode: '0644'
        force: true
      when: geoip_stale

    - name: Extract geoip database
      ansible.builtin.unarchive:
        remote_src: true
        src: /opt/geoip.tar.gz
        dest: /opt/database
        extra_opts: ['--strip-components=1', '--show-stored-names']
      when: geoip_stale
      notify: Restart alloy

    - name: Stamp the geoip download time
      ansible.builtin.file:
        path: /opt/.geoip_downloaded
        state: touch
        mode: '0644'
      when: geoip_stale

    - name: Copy alloy config
      ansible.builtin.template:
        src: config.alloy.j2
        dest: /etc/alloy/config.alloy
        mode: '0644'
      notify: Restart alloy

    - name: Configure alloy
      ansible.builtin.copy:
        dest: /etc/default/alloy
        content: |
          CONFIG_FILE="/etc/alloy/config.alloy"
          CUSTOM_ARGS="--server.http.listen-addr={{ wireguard_ingress_ip }}:12345 --stability.level=experimental"
          RESTART_ON_UPGRADE=true
        mode: '0644'
      notify: Restart alloy

    - name: Allow alloy to read logs
      ansible.builtin.user:
        name: alloy
        groups:
          - adm
          - systemd-journal
        append: true
      notify: Restart alloy

    - name: Enable services
      ansible.builtin.service:
        name: "{{ item }}"
        state: started
        enabled: true
      loop:
        - prometheus-node-exporter
        - prometheus-blackbox-exporter
        - alloy

    - name: Run collector once
      changed_when: true
      ansible.builtin.command:
        cmd: /usr/local/bin/wg_metrics.sh

    - name: Debug
      ansible.builtin.debug:
        msg:
          - "Node metrics: curl -s {{ wireguard_ingress_ip }}:9100/metrics | head"
          - "Blackbox probe: curl -s '{{ wireguard_ingress_ip }}:9115/probe?target={{ wireguard_beepi_ip }}:443&module=tcp_connect'"
          - "Alloy logs: journalctl -u alloy -n 50"
      tags:
        - always

  handlers:
    - name: Apply sysctl
      changed_when: true
      ansible.builtin.command:
        cmd: sysctl --system

    - name: Reload systemd
      ansible.builtin.systemd_service:
        daemon_reload: true

    - name: Restart node exporter
      ansible.builtin.service:
        name: prometheus-node-exporter
        state: restarted

    - name: Restart blackbox exporter
      ansible.builtin.service:
        name: prometheus-blackbox-exporter
        state: restarted

    - name: Restart alloy
      ansible.builtin.service:
        name: alloy
        state: restarted