lots of stuff
This commit is contained in:
@@ -6,7 +6,6 @@
|
||||
user_home: /home/hetzner
|
||||
|
||||
hetzner_raid_mount: /home/beeshare/raid
|
||||
hetzner_port: 23
|
||||
|
||||
hetzner_paths:
|
||||
- src: /home/beeshare/raid/serverBackup
|
||||
@@ -14,10 +13,6 @@
|
||||
- src: /home/beeshare/raid/foundry/backup
|
||||
dest: ./foundry
|
||||
|
||||
# Fill in after creating the storage box sub-account, then re-run
|
||||
hetzner_host: "u651287.your-storagebox.de"
|
||||
hetzner_user: "u651287"
|
||||
|
||||
tasks:
|
||||
- name: Install rsync
|
||||
ansible.builtin.apt:
|
||||
@@ -88,6 +83,58 @@
|
||||
job: /opt/backup/hetznerSync.sh >> /var/log/beepi-backup.log 2>&1
|
||||
when: hetzner_host | length > 0
|
||||
|
||||
- name: Copy metrics collector
|
||||
ansible.builtin.template:
|
||||
src: hetznerMetrics.sh.j2
|
||||
dest: /opt/backup/hetznerMetrics.sh
|
||||
mode: '0700'
|
||||
when: hetzner_host | length > 0
|
||||
|
||||
- name: Install metrics timer
|
||||
ansible.builtin.copy:
|
||||
dest: "/etc/systemd/system/beepi-hetzner-metrics.{{ item.kind }}"
|
||||
content: "{{ item.content }}"
|
||||
mode: '0644'
|
||||
loop:
|
||||
- kind: service
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Collect hetzner storage box metrics
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/opt/backup/hetznerMetrics.sh
|
||||
- kind: timer
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Collect hetzner storage box metrics
|
||||
|
||||
[Timer]
|
||||
OnBootSec=5min
|
||||
OnUnitActiveSec=1h
|
||||
AccuracySec=5min
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
when: hetzner_host | length > 0
|
||||
notify: Reload systemd
|
||||
|
||||
- name: Flush handlers
|
||||
ansible.builtin.meta: flush_handlers
|
||||
|
||||
- name: Enable metrics timer
|
||||
ansible.builtin.systemd_service:
|
||||
name: beepi-hetzner-metrics.timer
|
||||
state: started
|
||||
enabled: true
|
||||
when: hetzner_host | length > 0
|
||||
|
||||
- name: Run metrics collector once
|
||||
changed_when: true
|
||||
ansible.builtin.command:
|
||||
cmd: /opt/backup/hetznerMetrics.sh
|
||||
when: hetzner_host | length > 0
|
||||
|
||||
- name: Debug
|
||||
ansible.builtin.debug:
|
||||
msg:
|
||||
@@ -95,3 +142,8 @@
|
||||
- "Read logs: sudo tail -50 /var/log/beepi-backup.log"
|
||||
tags:
|
||||
- always
|
||||
|
||||
handlers:
|
||||
- name: Reload systemd
|
||||
ansible.builtin.systemd_service:
|
||||
daemon_reload: true
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
textfile_dir={{ node_exporter_textfile_dir }}
|
||||
file=${textfile_dir}/hetzner.prom
|
||||
tmp=${file}.$$
|
||||
now=$(date +%s)
|
||||
|
||||
ssh_opts="-p {{ hetzner_port }} -i {{ user_home }}/.ssh/id_ed25519 -o UserKnownHostsFile={{ user_home }}/.ssh/known_hosts -o StrictHostKeyChecking=yes -o BatchMode=yes -o ConnectTimeout=15"
|
||||
|
||||
mkdir -p ${textfile_dir}
|
||||
|
||||
reachable=0
|
||||
body=""
|
||||
|
||||
emit_path() {
|
||||
local label=$1
|
||||
local remote=$2
|
||||
local listing
|
||||
local files
|
||||
local bytes
|
||||
local pending
|
||||
local newest
|
||||
local age=-1
|
||||
|
||||
if ! listing=$(sudo -u {{ user }} rsync --list-only -e "ssh ${ssh_opts}" {{ hetzner_user }}@{{ hetzner_host }}:${remote}/ 2>/dev/null)
|
||||
then
|
||||
body+="hetzner_remote_listing_ok{path=\"${label}\"} 0"$'\n'
|
||||
return
|
||||
fi
|
||||
|
||||
reachable=1
|
||||
|
||||
files=$(echo "${listing}" | awk '$1 ~ /^-/ && substr($5,1,1) != "." { c++ } END { print c + 0 }')
|
||||
bytes=$(echo "${listing}" | awk '$1 ~ /^-/ && substr($5,1,1) != "." { gsub(",", "", $2); s += $2 } END { printf "%.0f", s + 0 }')
|
||||
pending=$(echo "${listing}" | awk '$1 ~ /^-/ && substr($5,1,1) == "." { gsub(",", "", $2); s += $2 } END { printf "%.0f", s + 0 }')
|
||||
newest=$(echo "${listing}" | awk '$1 ~ /^-/ && substr($5,1,1) != "." { print $3" "$4 }' | sort -r | head -1)
|
||||
|
||||
if [ -n "${newest}" ]
|
||||
then
|
||||
age=$((now - $(date -d "$(echo ${newest} | tr '/' '-')" +%s)))
|
||||
fi
|
||||
|
||||
body+="hetzner_remote_listing_ok{path=\"${label}\"} 1"$'\n'
|
||||
body+="hetzner_remote_files{path=\"${label}\"} ${files}"$'\n'
|
||||
body+="hetzner_remote_bytes{path=\"${label}\"} ${bytes}"$'\n'
|
||||
body+="hetzner_remote_pending_bytes{path=\"${label}\"} ${pending}"$'\n'
|
||||
body+="hetzner_remote_newest_age_seconds{path=\"${label}\"} ${age}"$'\n'
|
||||
}
|
||||
|
||||
{% for path in hetzner_paths %}
|
||||
emit_path "{{ path.dest | replace('./', '') }}" "{{ path.dest }}"
|
||||
{% endfor %}
|
||||
|
||||
{
|
||||
echo "# HELP hetzner_reachable Whether the storage box answered at least one listing."
|
||||
echo "# TYPE hetzner_reachable gauge"
|
||||
echo "hetzner_reachable ${reachable}"
|
||||
echo "# HELP hetzner_last_check_timestamp_seconds Unix time of the last storage box check."
|
||||
echo "# TYPE hetzner_last_check_timestamp_seconds gauge"
|
||||
echo "hetzner_last_check_timestamp_seconds ${now}"
|
||||
echo "# HELP hetzner_remote_listing_ok Whether this remote path could be listed."
|
||||
echo "# TYPE hetzner_remote_listing_ok gauge"
|
||||
echo "# HELP hetzner_remote_files Number of completed files stored at this remote path."
|
||||
echo "# TYPE hetzner_remote_files gauge"
|
||||
echo "# HELP hetzner_remote_bytes Bytes of completed files stored at this remote path."
|
||||
echo "# TYPE hetzner_remote_bytes gauge"
|
||||
echo "# HELP hetzner_remote_pending_bytes Bytes of in flight partial transfers at this remote path."
|
||||
echo "# TYPE hetzner_remote_pending_bytes gauge"
|
||||
echo "# HELP hetzner_remote_newest_age_seconds Age of the newest completed file at this remote path."
|
||||
echo "# TYPE hetzner_remote_newest_age_seconds gauge"
|
||||
printf '%s' "${body}"
|
||||
} > "${tmp}"
|
||||
|
||||
chmod 0644 "${tmp}"
|
||||
mv "${tmp}" "${file}"
|
||||
@@ -2,11 +2,22 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
now=$(date +"%Y%m%d%H%M")
|
||||
exec 9>/var/lock/hetznerSync.lock
|
||||
|
||||
if ! flock -n 9
|
||||
then
|
||||
echo "[$(date --rfc-3339=seconds)] offsite: another sync is still running, skipping this run"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
source /opt/backup/metrics.sh
|
||||
|
||||
metric_init offsite
|
||||
trap metric_guard EXIT
|
||||
|
||||
if ! mountpoint -q {{ hetzner_raid_mount }}
|
||||
then
|
||||
echo "[${now}] {{ hetzner_raid_mount }} not mounted, refusing to sync"
|
||||
log "{{ hetzner_raid_mount }} not mounted, refusing to sync"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -16,19 +27,17 @@ sync_path() {
|
||||
|
||||
if [ -z "$(ls -A ${src} 2>/dev/null)" ]
|
||||
then
|
||||
echo "[${now}] ${src} empty, refusing to sync"
|
||||
log "${src} empty, refusing to sync"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[${now}] syncing ${src} to ${dest}"
|
||||
log "syncing ${src} to ${dest}"
|
||||
sudo -u {{ user }} rsync -a --delete --partial \
|
||||
-e "ssh -p {{ hetzner_port }} -i {{ user_home }}/.ssh/id_ed25519 -o UserKnownHostsFile={{ user_home }}/.ssh/known_hosts -o StrictHostKeyChecking=yes" \
|
||||
${src}/ \
|
||||
{{ hetzner_user }}@{{ hetzner_host }}:${dest}/
|
||||
}
|
||||
|
||||
echo "[${now}] offsite sync start"
|
||||
{% for path in hetzner_paths %}
|
||||
sync_path {{ path.src }} {{ path.dest }}
|
||||
{% endfor %}
|
||||
echo "[${now}] offsite sync done"
|
||||
|
||||
Reference in New Issue
Block a user