- name: Prometheus
  hosts: beepi
  become: true
  vars_files:
    - vars/alert_rules.yml
  vars:
    grafana_secret: !vault |
          $ANSIBLE_VAULT;1.1;AES256
          38343463346232356634353236356534666166626665663237306130623336373062363563386537
          6665356165346134643235383164383266373363316262310a613332386564636430643561373463
          36346437353963333839656163623933616662326132373036636166616538343966356361636164
          3035656163616631330a663363343363643065356535356633623638376430326539363536666638
          65333930613135333364383462363233386662386531323935353932373465363234
    geoip_username: !vault |
              $ANSIBLE_VAULT;1.1;AES256
              30656365643463396265633130356339386433633331613130633134336633643637653761623962
              3633336466326564616432373031303530376166353436610a323033353164313436613363613836
              30303931323234306162343361643162313238313234383837323530343936343134326365666166
              3438643238626266320a333732363461613834613833303436363534393031323333613534333534
              3438
    geoip_license: !vault |
              $ANSIBLE_VAULT;1.1;AES256
              32336437653435396231663666643631373638616533653538333934356339666661313439343566
              6462303439666233316162653430363239393031633362630a323533393233633164613031646633
              62366362383162306563383235373262326262323337303333313163653838613436316337633063
              3630356262666338320a333137333063303534386437343762616665383034336365663732343566
              36376463313964333834373435653564346333396538306337333762613737643363636364353139
              3032383564333638613566633363666430356230326231633636
  tasks:
    - name: Determine whether alert delivery is configured
      ansible.builtin.set_fact:
        alerting_enabled: "{{ discord_webhook is defined and 'REPLACE_ME' not in discord_webhook }}"

    - name: Determine whether uptimerobot is configured
      ansible.builtin.set_fact:
        uptimerobot_enabled: "{{ uptimerobot_api_key is defined and 'REPLACE_ME' not in uptimerobot_api_key }}"

    - name: Install packages
      ansible.builtin.apt:
        name:
          - podman
          - podman-compose
          - prometheus-node-exporter
          - wireguard-tools
        state: present

    - name: Create monitoring config directory
      ansible.builtin.file:
        path: /opt/beeserver/monitoring
        state: directory
        mode: '0755'

    - name: Create grafana provisioning directories
      ansible.builtin.file:
        path: "/opt/beeserver/monitoring/grafana/{{ item }}"
        state: directory
        mode: '0755'
      loop:
        - provisioning/datasources
        - provisioning/dashboards
        - provisioning/alerting
        - dashboards
        - dashboards-general

    - name: Create topology directory
      ansible.builtin.file:
        path: /opt/beeserver/monitoring/topology
        state: directory
        mode: '0755'

    - name: Create textfile directory
      ansible.builtin.file:
        path: "{{ node_exporter_textfile_dir }}"
        state: directory
        mode: '0755'

    - name: Configure node exporter
      ansible.builtin.copy:
        dest: /etc/default/prometheus-node-exporter
        content: |
          ARGS="--collector.textfile.directory={{ node_exporter_textfile_dir }} --collector.systemd --collector.processes"
        mode: '0644'
      notify: Restart node exporter

    - name: Copy textfile collectors
      ansible.builtin.template:
        src: "textfile/{{ item }}.sh.j2"
        dest: "/usr/local/bin/{{ item }}.sh"
        mode: '0755'
      loop:
        - service_metrics
        - wg_metrics

    - name: Copy uptimerobot collector
      ansible.builtin.template:
        src: textfile/uptimerobot_metrics.sh.j2
        dest: /usr/local/bin/uptimerobot_metrics.sh
        mode: '0700'
      when: uptimerobot_enabled

    - name: Install uptimerobot timer
      ansible.builtin.copy:
        dest: "/etc/systemd/system/beepi-uptimerobot-metrics.{{ item.kind }}"
        content: "{{ item.content }}"
        mode: '0644'
      loop:
        - kind: service
          content: |
            [Unit]
            Description=Collect uptimerobot metrics

            [Service]
            Type=oneshot
            ExecStart=/usr/local/bin/uptimerobot_metrics.sh
        - kind: timer
          content: |
            [Unit]
            Description=Collect uptimerobot metrics

            [Timer]
            OnBootSec=3min
            OnUnitActiveSec=5min
            AccuracySec=30s

            [Install]
            WantedBy=timers.target
      when: uptimerobot_enabled
      notify: Reload systemd

    - name: Install collector timers
      ansible.builtin.copy:
        dest: "/etc/systemd/system/{{ item.name }}.{{ item.kind }}"
        content: "{{ item.content }}"
        mode: '0644'
      loop:
        - name: beepi-service-metrics
          kind: service
          content: |
            [Unit]
            Description=Collect service unit metrics

            [Service]
            Type=oneshot
            ExecStart=/usr/local/bin/service_metrics.sh
        - name: beepi-service-metrics
          kind: timer
          content: |
            [Unit]
            Description=Collect service unit metrics

            [Timer]
            OnBootSec=1min
            OnUnitActiveSec=30s
            AccuracySec=5s

            [Install]
            WantedBy=timers.target
        - name: beepi-wg-metrics
          kind: service
          content: |
            [Unit]
            Description=Collect wireguard peer metrics

            [Service]
            Type=oneshot
            ExecStart=/usr/local/bin/wg_metrics.sh
        - name: beepi-wg-metrics
          kind: timer
          content: |
            [Unit]
            Description=Collect wireguard peer metrics

            [Timer]
            OnBootSec=1min
            OnUnitActiveSec=30s
            AccuracySec=5s

            [Install]
            WantedBy=timers.target
      notify: Reload systemd

    - name: Flush handlers
      ansible.builtin.meta: flush_handlers

    - name: Enable collector timers
      ansible.builtin.systemd_service:
        name: "{{ item }}.timer"
        state: started
        enabled: true
      loop:
        - beepi-service-metrics
        - beepi-wg-metrics

    - name: Enable uptimerobot timer
      ansible.builtin.systemd_service:
        name: beepi-uptimerobot-metrics.timer
        state: started
        enabled: true
      when: uptimerobot_enabled

    - name: Compose down
      changed_when: true
      failed_when: false
      ansible.builtin.command:
        cmd: podman-compose down
        chdir: /opt/beeserver/monitoring

    - name: Copy compose
      ansible.builtin.template:
        src: docker-compose.yml.j2
        dest: /opt/beeserver/monitoring/docker-compose.yml
        mode: '0600'

    - name: Copy prometheus config
      ansible.builtin.template:
        src: prometheus.yml.j2
        dest: /opt/beeserver/monitoring/prometheus.yml
        mode: '0644'

    - name: Copy blackbox config
      ansible.builtin.template:
        src: blackbox.yml.j2
        dest: /opt/beeserver/monitoring/blackbox.yml
        mode: '0644'

    - name: Copy loki config
      ansible.builtin.template:
        src: local-config.yml.j2
        dest: /opt/beeserver/monitoring/local-config.yml
        mode: '0644'

    - name: Copy alloy config
      ansible.builtin.template:
        src: config.alloy.j2
        dest: /opt/beeserver/monitoring/config.alloy
        mode: '0644'

    - name: Copy topology service
      ansible.builtin.copy:
        src: topology/topology.py
        dest: /opt/beeserver/monitoring/topology/topology.py
        mode: '0644'

    - name: Copy topology spec
      ansible.builtin.template:
        src: topology/topology-spec.json.j2
        dest: /opt/beeserver/monitoring/topology/topology-spec.json
        mode: '0644'

    - name: Copy grafana datasources
      ansible.builtin.template:
        src: grafana/provisioning/datasources/datasources.yml.j2
        dest: /opt/beeserver/monitoring/grafana/provisioning/datasources/datasources.yml
        owner: '472'
        group: '472'
        mode: '0600'

    - name: Copy grafana dashboard provider
      ansible.builtin.template:
        src: grafana/provisioning/dashboards/dashboards.yml.j2
        dest: /opt/beeserver/monitoring/grafana/provisioning/dashboards/dashboards.yml
        mode: '0644'

    - name: Copy hand made dashboards
      ansible.builtin.copy:
        src: grafana/dashboards-general/
        dest: /opt/beeserver/monitoring/grafana/dashboards-general/
        mode: '0644'

    - name: Copy grafana contact points
      ansible.builtin.template:
        src: grafana/provisioning/alerting/contact-points.yml.j2
        dest: /opt/beeserver/monitoring/grafana/provisioning/alerting/contact-points.yml
        owner: '472'
        group: '472'
        mode: '0600'
      when: alerting_enabled

    - name: Copy grafana alert rules
      ansible.builtin.template:
        src: grafana/provisioning/alerting/rules.yml.j2
        dest: /opt/beeserver/monitoring/grafana/provisioning/alerting/rules.yml
        mode: '0644'

    - name: Copy grafana dashboards
      ansible.builtin.template:
        src: "grafana/dashboards/{{ item }}.json.j2"
        dest: "/opt/beeserver/monitoring/grafana/dashboards/{{ item }}.json"
        mode: '0644'
      loop:
        - infrastructure-map
        - backups
        - services
        - ingress

    - name: Copy uptimerobot dashboard
      ansible.builtin.template:
        src: grafana/dashboards/uptimerobot.json.j2
        dest: /opt/beeserver/monitoring/grafana/dashboards/uptimerobot.json
        mode: '0644'
      when: uptimerobot_enabled

    - name: Enable podman.socket
      ansible.builtin.service:
        name: podman.socket
        state: started
        enabled: true

    - name: Check the geoip database
      ansible.builtin.stat:
        path: /opt/beeserver/monitoring/database/GeoLite2-City.mmdb
      register: geoip_db

    - name: Check when geoip was last downloaded
      ansible.builtin.stat:
        path: /opt/beeserver/monitoring/.geoip_downloaded
      register: geoip_stamp

    - name: Decide whether the geoip database needs refreshing
      ansible.builtin.set_fact:
        geoip_stale: "{{ not geoip_db.stat.exists or not geoip_stamp.stat.exists or (ansible_facts.date_time.epoch | int - geoip_stamp.stat.mtime | int) > geoip_max_age_seconds | int }}"

    - name: Report the geoip database age
      ansible.builtin.debug:
        msg: >-
          geoip
          {{ 'not present, downloading' if not (geoip_db.stat.exists and geoip_stamp.stat.exists)
             else 'downloaded ' + (((ansible_facts.date_time.epoch | int - geoip_stamp.stat.mtime | int) / 3600) | round(1) | string) + 'h ago, ' + ('refreshing' if geoip_stale else 'reusing') }}

    - name: Delete database directory
      ansible.builtin.file:
        path: /opt/beeserver/monitoring/database
        state: absent
        mode: '0755'
      when: geoip_stale

    - name: Create database directory
      ansible.builtin.file:
        path: /opt/beeserver/monitoring/database
        state: directory
        mode: '0755'

    - name: Remove stale geoip archives
      ansible.builtin.shell:
        cmd: rm -f /opt/beeserver/monitoring/GeoLite2-City_*.tar.gz
      changed_when: true
      when: geoip_stale

    - name: Download geoip database
      ansible.builtin.get_url:
        url: "https://download.maxmind.com/geoip/databases/GeoLite2-City/download?suffix=tar.gz"
        dest: /opt/beeserver/monitoring
        username: "{{ geoip_username }}"
        password: "{{ geoip_license }}"
        mode: '0644'
      when: geoip_stale

    - name: Get geoip tar file
      ansible.builtin.find:
        paths: /opt/beeserver/monitoring
        file_type: file
        patterns: '^GeoLite.*.tar.gz$'
        use_regex: true
        limit: 1
      register: found
      when: geoip_stale

    - name: Extract geopip database
      ansible.builtin.unarchive:
        remote_src: true
        src: "{{ found.files[0].path }}"
        dest: /opt/beeserver/monitoring/database
        extra_opts: ['--strip-components=1', '--show-stored-names']
      when: geoip_stale and found.files | length > 0

    - name: Stamp the geoip download time
      ansible.builtin.file:
        path: /opt/beeserver/monitoring/.geoip_downloaded
        state: touch
        mode: '0644'
      when: geoip_stale and found.files | length > 0

    - name: Compose up
      changed_when: true
      ansible.builtin.command:
        cmd: podman-compose up -d
        chdir: /opt/beeserver/monitoring

    - name: Run collectors once
      changed_when: true
      ansible.builtin.command:
        cmd: "/usr/local/bin/{{ item }}.sh"
      loop:
        - service_metrics
        - wg_metrics

    - name: Run uptimerobot collector once
      changed_when: true
      ansible.builtin.command:
        cmd: /usr/local/bin/uptimerobot_metrics.sh
      when: uptimerobot_enabled

    - name: Debug
      ansible.builtin.debug:
        msg:
          - "Grafana: https://grafana.secretbee.buzz/"
          - "Topology: curl -s localhost:8088/topology.json | head"
          - "Prometheus targets: curl -s localhost:9090/api/v1/targets | jq '.data.activeTargets[] | {job: .labels.job, health}'"
      tags:
        - always

  handlers:
    - name: Reload systemd
      ansible.builtin.systemd_service:
        daemon_reload: true

    - name: Restart node exporter
      ansible.builtin.service:
        name: prometheus-node-exporter
        state: restarted