bee/beepi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

lots of stuff

Browse Source
This commit is contained in:
bee
2026-08-16 14:48:04 +02:00
parent c63db540e8
commit 49fa352476
45 changed files with 32520 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files
playbooks/backup/backup.yml
+65
View File
@@ -41,12 +41,72 @@
dest: /opt/backup/restoreTest.sh
mode: '0700'
- name: Copy metrics library
ansible.builtin.template:
src: metrics.sh.j2
dest: /opt/backup/metrics.sh
mode: '0700'
- name: Copy inventory collector
ansible.builtin.template:
src: backupInventory.sh.j2
dest: /opt/backup/backupInventory.sh
mode: '0700'
- name: Copy passphrase
ansible.builtin.template:
src: passphrase.txt.j2
dest: /opt/backup/passphrase.txt
mode: '0600'
- name: Create textfile directory
ansible.builtin.file:
path: "{{ node_exporter_textfile_dir }}"
state: directory
mode: '0755'
- name: Install inventory timer
ansible.builtin.copy:
dest: "/etc/systemd/system/beepi-backup-inventory.{{ item.kind }}"
content: "{{ item.content }}"
mode: '0644'
loop:
- kind: service
content: |
[Unit]
Description=Collect backup inventory metrics
[Service]
Type=oneshot
ExecStart=/opt/backup/backupInventory.sh
- kind: timer
content: |
[Unit]
Description=Collect backup inventory metrics
[Timer]
OnBootSec=2min
OnUnitActiveSec=15min
AccuracySec=1min
[Install]
WantedBy=timers.target
notify: Reload systemd
- name: Flush handlers
ansible.builtin.meta: flush_handlers
- name: Enable inventory timer
ansible.builtin.systemd_service:
name: beepi-backup-inventory.timer
state: started
enabled: true
- name: Run inventory collector once
changed_when: true
ansible.builtin.command:
cmd: /opt/backup/backupInventory.sh
- name: Set path
ansible.builtin.cron:
name: PATH
@@ -78,3 +138,8 @@
hour: '6'
weekday: '0'
job: /opt/backup/restoreTest.sh >> /var/log/beepi-backup.log 2>&1
handlers:
- name: Reload systemd
ansible.builtin.systemd_service:
daemon_reload: true
playbooks/backup/backupDaily.sh
+24 -2
View File
@@ -2,6 +2,8 @@
set -euo pipefail
source /opt/backup/metrics.sh
now=$(date +"%Y%m%d%H%M")
declare -a arr=("mail" "matrix" "monitoring")
@@ -12,6 +14,8 @@ declare -a immich=("systemd-immich-pgdata" "systemd-immich-upload")
gitea_uid=$(id -u gitea)
immich_uid=$(id -u immich)
metric_init daily
restore() {
for i in "${arr[@]}"; do
cd /opt/beeserver/${i}
@@ -32,10 +36,21 @@ restore() {
rm -f /home/immich/systemd-immich-*.tar
}
trap restore EXIT
finish() {
local raw=$?
local status
status=$(metric_status ${raw})
restore
metric_finish ${status}
exit ${status}
}
trap finish EXIT
for i in "${arr[@]}"
do
log "staging ${i}"
cd /opt/beeserver/${i}
podman-compose down -t 60
if [[ "$i" == "monitoring" ]]
@@ -57,10 +72,12 @@ do
podman-compose up -d
done
log "staging gitea"
systemctl --user --machine gitea@.host stop gitea-server.service gitea-db.service
rsync -aH /home/gitea/data /opt/backup/beeserver/gitea
systemctl --user --machine gitea@.host start gitea-db.service gitea-server.service
log "staging immich"
systemctl --user --machine immich@.host stop immich-server.service immich-db.service immich-redis.service
for v in "${immich[@]}"
do
@@ -70,17 +87,22 @@ systemctl --user --machine immich@.host start immich-db.service immich-redis.ser
mkdir -p /opt/backup/beeserver/immich/
mv /home/immich/systemd-immich-*.tar /opt/backup/beeserver/immich/
log "staging vaultwarden"
systemctl --user --machine vaultwarden@.host stop vaultwarden.service
rsync -aH /home/vaultwarden/data /opt/backup/beeserver/vaultwarden
systemctl --user --machine vaultwarden@.host start vaultwarden.service
archive=/home/beeshare/raid/serverBackup/beeserver_daily_${now}.tar.gz.gpg
log "encrypting ${archive}"
tar -czf - /opt/backup/beeserver | gpg --batch --passphrase-file /opt/backup/passphrase.txt -c -o ${archive}
log "verifying ${archive}"
if ! gpg --batch --passphrase-file /opt/backup/passphrase.txt -d ${archive} | tar -tzf - >/dev/null
then
echo "[${now}] verify failed, discarding ${archive}"
log "verify failed, discarding ${archive}"
rm -f ${archive}
exit 1
fi
metric_archive ${archive}
playbooks/backup/backupInventory.sh.j2
+65
View File
@@ -0,0 +1,65 @@
#!/bin/bash
set -uo pipefail
textfile_dir={{ node_exporter_textfile_dir }}
archive_dir={{ backup_archive_dir }}
raid_mount={{ backup_raid_mount }}
file=${textfile_dir}/backup_inventory.prom
tmp=${file}.$$
now=$(date +%s)
mkdir -p ${textfile_dir}
mounted=0
if mountpoint -q ${raid_mount}
then
mounted=1
fi
emit_kind() {
local kind=$1
local newest
local count=0
local age=0
local bytes=0
local total=0
if [ ${mounted} -eq 1 ]
then
count=$(find ${archive_dir} -maxdepth 1 -name "beeserver_${kind}_*.tar.gz.gpg" -type f 2>/dev/null | wc -l)
newest=$(find ${archive_dir} -maxdepth 1 -name "beeserver_${kind}_*.tar.gz.gpg" -type f -printf '%T@ %p\n' 2>/dev/null | sort -rn | head -1 | cut -d' ' -f2-)
total=$(find ${archive_dir} -maxdepth 1 -name "beeserver_${kind}_*.tar.gz.gpg" -type f -printf '%s\n' 2>/dev/null | awk '{ s += $1 } END { print s + 0 }')
if [ -n "${newest:-}" ]
then
bytes=$(stat -c %s "${newest}")
age=$((now - $(stat -c %Y "${newest}")))
fi
fi
echo "backup_archive_count{kind=\"${kind}\"} ${count}"
echo "backup_archive_newest_age_seconds{kind=\"${kind}\"} ${age}"
echo "backup_archive_newest_bytes{kind=\"${kind}\"} ${bytes}"
echo "backup_archive_total_bytes{kind=\"${kind}\"} ${total}"
}
{
echo "# HELP backup_raid_mounted Whether the backup raid is mounted."
echo "# TYPE backup_raid_mounted gauge"
echo "backup_raid_mounted ${mounted}"
echo "# HELP backup_archive_count Number of retained archives."
echo "# TYPE backup_archive_count gauge"
echo "# HELP backup_archive_newest_age_seconds Age of the newest archive."
echo "# TYPE backup_archive_newest_age_seconds gauge"
echo "# HELP backup_archive_newest_bytes Size of the newest archive."
echo "# TYPE backup_archive_newest_bytes gauge"
echo "# HELP backup_archive_total_bytes Size of all retained archives."
echo "# TYPE backup_archive_total_bytes gauge"
emit_kind daily
emit_kind weekly
} > "${tmp}"
chmod 0644 "${tmp}"
mv "${tmp}" "${file}"
playbooks/backup/backupWeekly.sh
+12 -2
View File
@@ -2,20 +2,30 @@
set -euo pipefail
source /opt/backup/metrics.sh
now=$(date +"%Y%m%d%H%M")
archive=/home/beeshare/raid/serverBackup/beeserver_weekly_${now}.tar.gz.gpg
metric_init weekly
trap metric_guard EXIT
log "encrypting ${archive}"
tar -czf - /opt/backup/beeserver | gpg --batch --passphrase-file /opt/backup/passphrase.txt -c -o ${archive}
log "verifying ${archive}"
if ! gpg --batch --passphrase-file /opt/backup/passphrase.txt -d ${archive} | tar -tzf - >/dev/null
then
echo "[${now}] verify failed, discarding ${archive} and keeping staging"
log "verify failed, discarding ${archive} and keeping staging"
rm -f ${archive}
exit 1
fi
metric_archive ${archive}
rm -rf /opt/backup/beeserver
log "pruning expired archives"
find /home/beeshare/raid/serverBackup -maxdepth 1 -name 'beeserver_weekly_*.tar.gz.gpg' -mtime +30 -delete
find /home/beeshare/raid/serverBackup -maxdepth 1 -name 'beeserver_daily_*.tar.gz.gpg' -mtime +7 -delete
find /home/beeshare/raid/serverBackup -maxdepth 1 -name 'beeserver_daily_*.tar.gz.gpg' -mtime +7 -delete
playbooks/backup/metrics.sh.j2
+104
View File
@@ -0,0 +1,104 @@
#!/bin/bash
textfile_dir={{ node_exporter_textfile_dir }}
metric_job=""
metric_started=0
metric_bytes=0
metric_signalled=""
metric_on_signal() {
metric_signalled=$1
exit $((128 + $1))
}
metric_install_traps() {
trap 'metric_on_signal 1' HUP
trap 'metric_on_signal 2' INT
trap 'metric_on_signal 15' TERM
}
metric_status() {
if [ -n "${metric_signalled}" ]
then
echo $((128 + metric_signalled))
else
echo "$1"
fi
}
log() {
echo "[$(date --rfc-3339=seconds)] ${metric_job}: $*"
}
metric_previous_success() {
local file=${textfile_dir}/backup_${metric_job}.prom
if [ -r "${file}" ]
then
grep "^backup_last_success_timestamp_seconds{backup_job=\"${metric_job}\"} " "${file}" | awk '{ print $2 }'
fi
}
metric_init() {
metric_job=$1
metric_started=$(date +%s)
mkdir -p ${textfile_dir}
metric_install_traps
log "start"
}
metric_archive() {
metric_bytes=$(stat -c %s "$1" 2>/dev/null || echo 0)
}
metric_finish() {
local status=$1
local now
local success
local previous
local file=${textfile_dir}/backup_${metric_job}.prom
local tmp=${file}.$$
now=$(date +%s)
previous=$(metric_previous_success)
if [ "${status}" -eq 0 ]
then
success=${now}
log "done in $((now - metric_started))s"
else
success=${previous:-0}
log "FAILED with status ${status} after $((now - metric_started))s"
fi
cat > "${tmp}" <<EOF
# HELP backup_last_run_timestamp_seconds Unix time of the last run of this backup job.
# TYPE backup_last_run_timestamp_seconds gauge
backup_last_run_timestamp_seconds{backup_job="${metric_job}"} ${now}
# HELP backup_last_success_timestamp_seconds Unix time of the last successful run of this backup job.
# TYPE backup_last_success_timestamp_seconds gauge
backup_last_success_timestamp_seconds{backup_job="${metric_job}"} ${success}
# HELP backup_last_duration_seconds Duration of the last run of this backup job.
# TYPE backup_last_duration_seconds gauge
backup_last_duration_seconds{backup_job="${metric_job}"} $((now - metric_started))
# HELP backup_last_status Exit status of the last run of this backup job, 1 is success.
# TYPE backup_last_status gauge
backup_last_status{backup_job="${metric_job}"} $([ "${status}" -eq 0 ] && echo 1 || echo 0)
# HELP backup_last_archive_bytes Size of the archive produced by the last run of this backup job.
# TYPE backup_last_archive_bytes gauge
backup_last_archive_bytes{backup_job="${metric_job}"} ${metric_bytes}
EOF
chmod 0644 "${tmp}"
mv "${tmp}" "${file}"
}
metric_guard() {
local raw=$?
local status
status=$(metric_status ${raw})
metric_finish ${status}
exit ${status}
}
playbooks/backup/restoreTest.sh
+19 -7
View File
@@ -2,6 +2,8 @@
set -euo pipefail
source /opt/backup/metrics.sh
now=$(date +"%Y%m%d%H%M")
backup_dir=/home/beeshare/raid/serverBackup
image=docker.io/vaultwarden/server:1.37.1
@@ -9,15 +11,23 @@ container=vaultwarden-restoretest
port=18222
work=$(mktemp -d)
metric_init restoretest
cleanup() {
local raw=$?
local status
status=$(metric_status ${raw})
podman rm -f ${container} >/dev/null 2>&1 || true
rm -rf "${work}"
metric_finish ${status}
exit ${status}
}
trap cleanup EXIT
latest=$(ls -1t ${backup_dir}/beeserver_*.tar.gz.gpg | head -1)
echo "[${now}] restore test using ${latest}"
log "restore test using ${latest}"
gpg --batch --passphrase-file /opt/backup/passphrase.txt -d "${latest}" \
| tar -xzf - -C "${work}" opt/backup/beeserver/vaultwarden
@@ -26,15 +36,15 @@ data="${work}/opt/backup/beeserver/vaultwarden/data"
[[ -s "${data}/db.sqlite3" ]]
[[ -s "${data}/rsa_key.pem" ]]
echo " db and rsa_key present"
log "db and rsa_key present"
integrity=$(sqlite3 "${data}/db.sqlite3" "PRAGMA integrity_check;")
[[ "${integrity}" == "ok" ]]
echo " integrity_check ok"
log "integrity_check ok"
users=$(sqlite3 "${data}/db.sqlite3" "select count(*) from users;")
[[ "${users}" -ge 1 ]]
echo " users ${users}"
log "users ${users}"
email=$(sqlite3 "${data}/db.sqlite3" "select email from users limit 1;")
@@ -51,12 +61,14 @@ done
config=$(curl -sf http://127.0.0.1:${port}/api/config)
[[ "${config}" == *'"version"'* ]]
echo " api/config ok"
log "api/config ok"
prelogin=$(curl -sf -X POST http://127.0.0.1:${port}/identity/accounts/prelogin \
-H 'Content-Type: application/json' \
-d "{\"email\":\"${email}\"}")
[[ "${prelogin}" == *[Kk]df* ]]
echo " prelogin ok for ${email}"
log "prelogin ok for ${email}"
echo "[${now}] restore test PASSED"
metric_archive "${latest}"
log "restore test PASSED"