lots of stuff

playbooks/monitoring/config.alloy.j2

@@ -10,8 +10,17 @@ local.file_match "fail2ban" {
   path_targets = [{"__path__" = "/var/log/fail2ban.log", job = "fail2ban", host = "beepi"}]
 }
 
+local.file_match "backup" {
+  path_targets = [{"__path__" = "/var/log/beepi-backup.log", job = "backup", host = "beepi"}]
+}
+
 local.file_match "varlog" {
-  path_targets = [{"__path__" = "/var/log/*.log", job = "varlog", "__path_exclude__" = "/var/log/fail2ban.log", host = "beepi"}]
+  path_targets = [{
+    "__path__" = "/var/log/*.log",
+    job = "varlog",
+    "__path_exclude__" = "/var/log/{fail2ban,beepi-backup}.log",
+    host = "beepi",
+  }]
 }
 
 loki.source.file "varlog" {
@@ -19,6 +28,34 @@ loki.source.file "varlog" {
   forward_to = [loki.write.default.receiver]
 }
 
+loki.source.file "backup" {
+  targets    = local.file_match.backup.targets
+  forward_to = [loki.process.backup.receiver]
+}
+
+loki.process "backup" {
+    forward_to = [loki.write.default.receiver]
+
+    stage.regex {
+        expression = `^\[(?P<ts>[^\]]+)\]\s(?P<backup_job>\w+):\s(?P<message>.*)$`
+    }
+
+    stage.labels {
+        values = {
+            backup_job = "",
+        }
+    }
+
+    stage.match {
+        selector = `{job="backup"} |~ "(?i)(FAILED|verify failed|refusing to sync|error)"`
+        stage.static_labels {
+            values = {
+                level = "error",
+            }
+        }
+    }
+}
+
 loki.source.file "fail2ban_source" {
   targets    = local.file_match.fail2ban.targets
   forward_to = [loki.process.geoip.receiver]
@@ -58,4 +95,4 @@ loki.write "default" {
   endpoint {
     url = "http://loki:3100/loki/api/v1/push"
   }
-}
+}