stuff

playbooks/backup/backup.yml

@@ -12,7 +12,9 @@
   tasks:
     - name: Install rsync
       ansible.builtin.apt:
-        name: rsync
+        name:
+          - rsync
+          - sqlite3
         state: present
 
     - name: Create backup directory
@@ -33,6 +35,12 @@
         dest: /opt/backup/backupWeekly.sh
         mode: '0700'
 
+    - name: Copy restore test
+      ansible.builtin.copy:
+        src: restoreTest.sh
+        dest: /opt/backup/restoreTest.sh
+        mode: '0700'
+
     - name: Copy passphrase
       ansible.builtin.template:
         src: passphrase.txt.j2
@@ -61,3 +69,12 @@
         hour: '5'
         weekday: '0'
         job: /opt/backup/backupWeekly.sh >> /var/log/beepi-backup.log 2>&1
+
+    - name: Schedule weekly restore test
+      ansible.builtin.cron:
+        name: beepi restore test
+        user: root
+        minute: '0'
+        hour: '6'
+        weekday: '0'
+        job: /opt/backup/restoreTest.sh >> /var/log/beepi-backup.log 2>&1

playbooks/backup/backupDaily.sh

@@ -28,6 +28,7 @@ restore() {
 
     systemctl --user --machine gitea@.host start gitea-db.service gitea-server.service || true
     systemctl --user --machine immich@.host start immich-db.service immich-redis.service immich-server.service || true
+    systemctl --user --machine vaultwarden@.host start vaultwarden.service || true
     rm -f /home/immich/systemd-immich-*.tar
 }
 
@@ -69,4 +70,17 @@ systemctl --user --machine immich@.host start immich-db.service immich-redis.ser
 mkdir -p /opt/backup/beeserver/immich/
 mv /home/immich/systemd-immich-*.tar /opt/backup/beeserver/immich/
 
-tar -czf - /opt/backup/beeserver | gpg --batch --passphrase-file /opt/backup/passphrase.txt -c -o /home/beeshare/raid/serverBackup/beeserver_daily_${now}.tar.gz.gpg
+systemctl --user --machine vaultwarden@.host stop vaultwarden.service
+rsync -aH /home/vaultwarden/data /opt/backup/beeserver/vaultwarden
+systemctl --user --machine vaultwarden@.host start vaultwarden.service
+
+archive=/home/beeshare/raid/serverBackup/beeserver_daily_${now}.tar.gz.gpg
+
+tar -czf - /opt/backup/beeserver | gpg --batch --passphrase-file /opt/backup/passphrase.txt -c -o ${archive}
+
+if ! gpg --batch --passphrase-file /opt/backup/passphrase.txt -d ${archive} | tar -tzf - >/dev/null
+then
+    echo "[${now}] verify failed, discarding ${archive}"
+    rm -f ${archive}
+    exit 1
+fi

playbooks/backup/backupWeekly.sh

@@ -4,12 +4,18 @@ set -euo pipefail
 
 now=$(date +"%Y%m%d%H%M")
 
-tar -czf - /opt/backup/beeserver | gpg --batch --passphrase-file /opt/backup/passphrase.txt -c -o /home/beeshare/raid/serverBackup/beeserver_weekly_${now}.tar.gz.gpg
+archive=/home/beeshare/raid/serverBackup/beeserver_weekly_${now}.tar.gz.gpg
 
-if [ -s /home/beeshare/raid/serverBackup/beeserver_weekly_${now}.tar.gz.gpg ]
+tar -czf - /opt/backup/beeserver | gpg --batch --passphrase-file /opt/backup/passphrase.txt -c -o ${archive}
+
+if ! gpg --batch --passphrase-file /opt/backup/passphrase.txt -d ${archive} | tar -tzf - >/dev/null
 then
-    rm -rf /opt/backup/beeserver
+    echo "[${now}] verify failed, discarding ${archive} and keeping staging"
+    rm -f ${archive}
+    exit 1
 fi
 
+rm -rf /opt/backup/beeserver
+
 find /home/beeshare/raid/serverBackup -maxdepth 1 -name 'beeserver_weekly_*.tar.gz.gpg' -mtime +30 -delete
 find /home/beeshare/raid/serverBackup -maxdepth 1 -name 'beeserver_daily_*.tar.gz.gpg' -mtime +7 -delete

playbooks/backup/restoreTest.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+set -euo pipefail
+
+now=$(date +"%Y%m%d%H%M")
+backup_dir=/home/beeshare/raid/serverBackup
+image=docker.io/vaultwarden/server:1.37.1
+container=vaultwarden-restoretest
+port=18222
+work=$(mktemp -d)
+
+cleanup() {
+    podman rm -f ${container} >/dev/null 2>&1 || true
+    rm -rf "${work}"
+}
+
+trap cleanup EXIT
+
+latest=$(ls -1t ${backup_dir}/beeserver_*.tar.gz.gpg | head -1)
+echo "[${now}] restore test using ${latest}"
+
+gpg --batch --passphrase-file /opt/backup/passphrase.txt -d "${latest}" \
+    | tar -xzf - -C "${work}" opt/backup/beeserver/vaultwarden
+
+data="${work}/opt/backup/beeserver/vaultwarden/data"
+
+[[ -s "${data}/db.sqlite3" ]]
+[[ -s "${data}/rsa_key.pem" ]]
+echo "  db and rsa_key present"
+
+integrity=$(sqlite3 "${data}/db.sqlite3" "PRAGMA integrity_check;")
+[[ "${integrity}" == "ok" ]]
+echo "  integrity_check ok"
+
+users=$(sqlite3 "${data}/db.sqlite3" "select count(*) from users;")
+[[ "${users}" -ge 1 ]]
+echo "  users ${users}"
+
+email=$(sqlite3 "${data}/db.sqlite3" "select email from users limit 1;")
+
+podman run -d --rm --name ${container} -p 127.0.0.1:${port}:80 -v "${data}":/data ${image} >/dev/null
+
+for _ in $(seq 60)
+do
+    if curl -sf http://127.0.0.1:${port}/api/config >/dev/null 2>&1
+    then
+        break
+    fi
+    sleep 1
+done
+
+config=$(curl -sf http://127.0.0.1:${port}/api/config)
+[[ "${config}" == *'"version"'* ]]
+echo "  api/config ok"
+
+prelogin=$(curl -sf -X POST http://127.0.0.1:${port}/identity/accounts/prelogin \
+    -H 'Content-Type: application/json' \
+    -d "{\"email\":\"${email}\"}")
+[[ "${prelogin}" == *[Kk]df* ]]
+echo "  prelogin ok for ${email}"
+
+echo "[${now}] restore test PASSED"