logging stack and firewall

playbooks/monitoring/config.alloy.j2

@@ -0,0 +1,22 @@
+loki.source.journal "system" {
+  forward_to = [loki.write.default.receiver]
+  labels = {
+    job = "systemd-journal",
+    host = "beepi",
+  }
+}
+
+local.file_match "varlog" {
+  path_targets = [{"__path__" = "/var/log/*.log", job = "varlog", host = "beepi"}]
+}
+
+loki.source.file "varlog" {
+  targets    = local.file_match.varlog.targets
+  forward_to = [loki.write.default.receiver]
+}
+
+loki.write "default" {
+  endpoint {
+    url = "http://loki:3100/loki/api/v1/push"
+  }
+}

playbooks/monitoring/docker-compose.yml.j2

@@ -30,4 +30,44 @@ services:
       - GF_SERVER_HTTP_PORT=4000
       - GF_SERVER_PROTOCOL=HTTP
     networks:
-      - monitoring
+      - monitoring
+
+  loki:
+    image: docker.io/grafana/loki
+    container_name: loki
+    ports:
+      - '127.0.0.1:3100:3100'
+      - '127.0.0.1:9096:9096'
+    volumes:
+      - loki-data:/loki
+      - /opt/monitoring/local-config.yml:/etc/loki/local-config.yaml
+    command: -config.file=/etc/loki/local-config.yaml
+    restart: unless-stopped
+    networks:
+      - monitoring
+
+  alloy:
+    image: docker.io/grafana/alloy
+    container_name: alloy
+    ports:
+      - '127.0.0.1:12345:12345'
+    volumes:
+      - /opt/monitoring/config.alloy:/etc/alloy/config.alloy
+      - /var/log:/var/log:ro
+      - /run/log/journal:/run/log/journal:ro
+      - /var/log/journal:/var/log/journal:ro
+      - /etc/machine-id:/etc/machine-id:ro
+      - alloy-data:/var/lib/alloy/data
+    command:
+      - run
+      - /etc/alloy/config.alloy
+      - --storage.path=/var/lib/alloy/data
+      - --server.http.listen-addr=0.0.0.0:12345
+      - --stability.level=experimental
+    networks:
+      - monitoring
+
+volumes:
+  grafana-storage:
+  loki-data:
+  alloy-data:

playbooks/monitoring/local-config.yml.j2

@@ -0,0 +1,77 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+  log_level: info
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /loki
+  storage:
+    filesystem:
+      chunks_directory: /loki/chunks
+      rules_directory: /loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+query_range:
+  results_cache:
+    cache:
+      embedded_cache:
+        enabled: true
+        max_size_mb: 100
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+ruler:
+  alertmanager_url: http://localhost:9093
+
+ingester:
+  wal:
+    enabled: true
+    dir: /loki/wal
+  lifecycler:
+    ring:
+      replication_factor: 1
+
+limits_config:
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+  ingestion_rate_mb: 16
+  ingestion_burst_size_mb: 24
+  max_streams_per_user: 10000
+  max_line_size: 256kb
+
+chunk_store_config:
+  chunk_cache_config:
+    embedded_cache:
+      enabled: true
+      max_size_mb: 100
+
+storage_config:
+  tsdb_shipper:
+    active_index_directory: /loki/tsdb-index
+    cache_location: /loki/tsdb-cache
+
+compactor:
+  working_directory: /loki/compactor
+  compaction_interval: 10m
+  retention_enabled: true
+  retention_delete_delay: 2h
+  retention_delete_worker_count: 150
+  delete_request_store: filesystem
+
+table_manager:
+  retention_deletes_enabled: true
+  retention_period: 720h

playbooks/monitoring/monitoring.yml

@@ -25,18 +25,30 @@
         name: prometheus-node-exporter
         state: present
 
-    - name: Create prometheus config directory
+    - name: Create monitoring config directory
       ansible.builtin.file:
         path: /opt/monitoring
         state: directory
         mode: '0755'
 
-    - name: Copy config
+    - name: Copy prometheus config
       ansible.builtin.copy:
         src: prometheus_config.yml
         dest: /opt/monitoring/prometheus.yml
         mode: '0644'
 
+    - name: Copy loki config
+      ansible.builtin.template:
+        src: local-config.yml.j2
+        dest: /opt/monitoring/local-config.yml
+        mode: '0644'
+
+    - name: Copy alloy config
+      ansible.builtin.template:
+        src: config.alloy.j2
+        dest: /opt/monitoring/config.alloy
+        mode: '0644'
+
     - name: Copy compose
       ansible.builtin.template:
         src: docker-compose.yml.j2