summaryrefslogtreecommitdiff
path: root/bin
diff options
context:
space:
mode:
Diffstat (limited to 'bin')
-rw-r--r--bin/README.md3
-rw-r--r--bin/systemd/vpsbackup.service8
-rw-r--r--bin/systemd/vpsbackup.timer10
-rwxr-xr-xbin/vpsbackup402
4 files changed, 423 insertions, 0 deletions
diff --git a/bin/README.md b/bin/README.md
index 6c32c13..52593e9 100644
--- a/bin/README.md
+++ b/bin/README.md
@@ -9,10 +9,13 @@ and the config looks for it on `PATH` before falling back to this directory.
| `docsync` | Backups | restic, ssh, jq, notify-send |
| `imgcollage` | Media thumbnails > Collage | Python 3, Pillow |
| `system-theme` | Appearance > Theme | the COSMIC desktop |
+| `vpsbackup` | (standalone) | restic, rsync, ssh |
Each script's header or usage message (`docsync help`) says how to set it up.
`systemd/` holds docsync's user units: copy them to `~/.config/systemd/user/`,
put `docsync` at `~/.local/bin/docsync`, and enable the three timers.
+`vpsbackup.timer` and `.service` likewise run `~/.local/bin/vpsbackup`, which
+pulls the VPS here daily; `vpsbackup help` covers its setup and restores.
`photocollage/` is vendored unchanged from
[PhotoCollage](https://github.com/adrienverge/PhotoCollage) by Adrien Vergé,
diff --git a/bin/systemd/vpsbackup.service b/bin/systemd/vpsbackup.service
new file mode 100644
index 0000000..f2bb392
--- /dev/null
+++ b/bin/systemd/vpsbackup.service
@@ -0,0 +1,8 @@
+[Unit]
+Description=vpsbackup: pull the VPS and snapshot it
+
+[Service]
+Type=oneshot
+ExecStart=%h/.local/bin/vpsbackup run
+Nice=10
+IOSchedulingClass=idle
diff --git a/bin/systemd/vpsbackup.timer b/bin/systemd/vpsbackup.timer
new file mode 100644
index 0000000..d35268b
--- /dev/null
+++ b/bin/systemd/vpsbackup.timer
@@ -0,0 +1,10 @@
+[Unit]
+Description=vpsbackup: hourly (backs up once a day)
+
+[Timer]
+OnCalendar=hourly
+RandomizedDelaySec=5min
+Persistent=true
+
+[Install]
+WantedBy=timers.target
diff --git a/bin/vpsbackup b/bin/vpsbackup
new file mode 100755
index 0000000..ccede77
--- /dev/null
+++ b/bin/vpsbackup
@@ -0,0 +1,402 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+CONFIG_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/vpsbackup"
+STATE_DIR="${XDG_STATE_HOME:-$HOME/.local/state}/vpsbackup"
+
+# Pulls the whole VPS (less what a reinstall brings back) to this machine and
+# keeps its history in a local restic repository. The VPS never gets write
+# access here: this machine logs in with a key whose forced command
+# (vpsbackup-serve, below) allows a read-only rsync of / and three small
+# commands, so a compromised VPS can't touch its own backups.
+#
+# $CONFIG_DIR/config sets SSH_HOST, a Host in ~/.ssh/config that logs in as
+# root on the VPS with that key (IdentitiesOnly, BatchMode-safe), and
+# ADMIN_HOST, an ordinary root login used only by install-server. It may also
+# set MAIL_TO (the VPS mailbox problems are mailed to, default root),
+# BACKUP_DIR (default ~/Backups/vps), SNAPSHOT_HOST, and EXTRA_FILTER, rsync
+# filter rules added to FILTER below, e.g. ('- /srv/scratch/').
+SSH_HOST=
+ADMIN_HOST=
+MAIL_TO=root
+BACKUP_DIR="$HOME/Backups/vps"
+SNAPSHOT_HOST=vps
+EXTRA_FILTER=()
+# shellcheck source=/dev/null
+[[ -f "$CONFIG_DIR/config" ]] && . "$CONFIG_DIR/config"
+MIRROR="$BACKUP_DIR/mirror"
+export RESTIC_REPOSITORY="$BACKUP_DIR/restic"
+export RESTIC_PASSWORD_FILE="$CONFIG_DIR/restic-password"
+
+TAG=vps
+
+# The hourly timer runs 'vpsbackup run', which does nothing until a day has
+# passed since the last snapshot: one backup a day, retried every hour while
+# the laptop is off the network or the VPS is unreachable.
+RUN_EVERY=$((20 * 3600))
+FORGET_EVERY=$((7 * 86400))
+CHECK_EVERY=$((30 * 86400))
+CHECK_SUBSET=10%
+KEEP=(--keep-daily 30 --keep-weekly 52 --keep-monthly unlimited)
+
+# A failed run mails its log to MAIL_TO (through the VPS's own postfix), at
+# most once per window. Silence is covered on the VPS side: vpsbackup-watch
+# mails daily once no snapshot has been reported for WATCH_DAYS days.
+ERROR_MAIL_EVERY=$((12 * 3600))
+WATCH_DAYS=3
+
+# What isn't worth pulling: kernel filesystems, what the packages bring back,
+# caches, and PostgreSQL's live data files, which aren't consistent mid-write
+# (vpsbackup-serve's prepare dumps them to /var/backups/vpsbackup instead).
+# Exclude the repository docsync writes to on the VPS (if it does) through
+# EXTRA_FILTER, or this machine's own backups come back here.
+FILTER=(
+ '- /proc/' '- /sys/' '- /dev/' '- /run/' '- /tmp/' '- /var/tmp/'
+ '- /mnt/' '- /media/' '- /lost+found/' '- /swapfile' '- /boot/'
+ '+ /usr/' '+ /usr/local/' '- /usr/*'
+ '- /var/cache/' '- /var/lib/apt/lists/' '- /var/lib/clamav/'
+ '- /var/lib/postgresql/' '- /var/log/journal/'
+ '- /root/.cache/' '- /home/*/.cache/'
+ "${EXTRA_FILTER[@]}"
+)
+
+usage() {
+ cat <<EOF
+Usage: vpsbackup COMMAND
+
+ run back up the VPS if a day has passed since the last snapshot
+ (the hourly timer runs this); 'run --now' backs up regardless
+ status when the last backup landed, and the latest snapshots
+ restic ARGS any restic command against the repository, e.g.
+ vpsbackup restic snapshots
+ vpsbackup restic mount ~/mnt/vps
+ init create the restic password and repository (once)
+ install-server
+ install or update the VPS side through ADMIN_HOST: the
+ forced command, the backup key and the daily watch timer
+
+Files: the rsync mirror is $MIRROR, the repository
+$RESTIC_REPOSITORY. The mirror keeps owners, groups and special files in
+user.rsync.* xattrs (rsync --fake-super); restic keeps xattrs, so a restore
+carries them. To put files back on the VPS:
+ vpsbackup restic restore latest:$MIRROR --target DIR --include /etc/nginx
+ rsync -aH --numeric-ids --fake-super DIR/etc/nginx/ root@VPS:/etc/nginx/
+The PostgreSQL dump is var/backups/vpsbackup/postgres.sql (psql -f it as
+postgres), and the package lists sit beside it.
+
+Setup, all in $CONFIG_DIR:
+ config SSH_HOST=<Host using the backup key>, ADMIN_HOST=<root login>,
+ optionally MAIL_TO, BACKUP_DIR, SNAPSHOT_HOST
+ restic-password made by 'vpsbackup init', mode 600
+ key the backup key's .pub is read from SSH_HOST's IdentityFile
+Then: vpsbackup init; vpsbackup install-server; enable vpsbackup.timer.
+EOF
+}
+
+die() { echo "vpsbackup: $*" >&2; exit 1; }
+
+now() { date +%s; }
+
+put_state() { # NAME LINE...
+ if (( $# > 1 )); then printf '%s\n' "${@:2}"; fi > "$STATE_DIR/$1.tmp"
+ mv "$STATE_DIR/$1.tmp" "$STATE_DIR/$1"
+}
+
+get_state() { # NAME DEFAULT
+ if [[ -s "$STATE_DIR/$1" ]]; then head -n1 "$STATE_DIR/$1"; else printf '%s\n' "$2"; fi
+}
+
+since() { # NAME: seconds since the epoch stored in state NAME (0 if never)
+ echo $(( $(now) - $(get_state "$1" 0) ))
+}
+
+ago() { # EPOCH
+ local s=$(( $(now) - $1 ))
+ if (( s < 90 )); then echo "just now"
+ elif (( s < 5400 )); then echo "$(( s / 60 )) min ago"
+ elif (( s < 172800 )); then echo "$(( s / 3600 )) h ago"
+ else echo "$(( s / 86400 )) days ago"
+ fi
+}
+
+# Keepalives notice a dropped home connection in two minutes rather than
+# waiting out rsync's --timeout.
+SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=30 -o ServerAliveInterval=30 -o ServerAliveCountMax=4)
+
+remote() { # COMMAND [< stdin]
+ ssh "${SSH_OPTS[@]}" "$SSH_HOST" "$@"
+}
+
+reachable() {
+ remote ping </dev/null >/dev/null 2>&1
+}
+
+# ---------------------------------------------------------------- the VPS side
+
+# The forced command for the backup key. rrsync -ro / refuses anything but a
+# read (no --remove-source-files, no writes); the rest is an allow list.
+serve_script() {
+ cat <<'EOF'
+#!/bin/bash
+# vpsbackup-serve: forced command for the home machine's backup key, put here
+# by `vpsbackup install-server` (see that script on the home machine).
+set -euo pipefail
+DUMPS=/var/backups/vpsbackup
+STATE=/var/lib/vpsbackup
+MAIL_TO=$(cat "$STATE/mail-to")
+
+mail_me() { # SUBJECT, body on stdin
+ { printf 'To: %s\nSubject: %s\nAuto-Submitted: auto-generated\n\n' "$MAIL_TO" "$1"
+ cat; } | /usr/sbin/sendmail -i -- "$MAIL_TO"
+}
+
+case "${SSH_ORIGINAL_COMMAND:-}" in
+ ping) ;;
+ prepare)
+ umask 077
+ mkdir -p "$DUMPS"
+ cd /
+ runuser -u postgres -- pg_dumpall --clean --if-exists > "$DUMPS/postgres.sql.tmp"
+ mv "$DUMPS/postgres.sql.tmp" "$DUMPS/postgres.sql"
+ dpkg --get-selections > "$DUMPS/packages.selections"
+ apt-mark showmanual > "$DUMPS/packages.manual" ;;
+ report-ok)
+ date +%s > "$STATE/last-ok.tmp"
+ mv "$STATE/last-ok.tmp" "$STATE/last-ok" ;;
+ report-error)
+ head -c 65536 | tr -cd '\11\12\15\40-\176' |
+ mail_me "VPS backup failed" ;;
+ "rsync --server --sender "*)
+ exec /usr/bin/rrsync -ro / ;;
+ *)
+ echo "vpsbackup-serve: not allowed: ${SSH_ORIGINAL_COMMAND:-(shell)}" >&2
+ exit 1 ;;
+esac
+EOF
+}
+
+watch_script() {
+ cat <<EOF
+#!/bin/bash
+# vpsbackup-watch: mails once a day while no backup has reached the home
+# machine for $WATCH_DAYS days. Put here by \`vpsbackup install-server\`.
+set -euo pipefail
+STATE=/var/lib/vpsbackup
+EOF
+ cat <<'EOF'
+MAIL_TO=$(cat "$STATE/mail-to")
+last=$(cat "$STATE/last-ok" 2>/dev/null || echo 0)
+days=$(( ($(date +%s) - last) / 86400 ))
+EOF
+ echo "(( days >= $WATCH_DAYS )) || exit 0"
+ cat <<'EOF'
+{ printf 'To: %s\nSubject: No VPS backup for %s days\nAuto-Submitted: auto-generated\n\n' \
+ "$MAIL_TO" "$days"
+ echo "No backup of $(hostname -f) has reached the home machine since"
+ echo "$(date -d "@$last" '+%F %R %Z')."
+ echo
+ echo "The home machine retries hourly whenever it's on. If it has been on,"
+ echo "look at 'vpsbackup status' and 'journalctl --user -u vpsbackup' there."
+} | /usr/sbin/sendmail -i -- "$MAIL_TO"
+EOF
+}
+
+cmd_install_server() {
+ [[ -n "$ADMIN_HOST" ]] || die "set ADMIN_HOST in $CONFIG_DIR/config"
+ local keyfile pub
+ keyfile=$(ssh -G "$SSH_HOST" | awk '$1 == "identityfile" { print $2; exit }')
+ keyfile="${keyfile/#\~/$HOME}"
+ [[ -f "$keyfile.pub" ]] || die "no public key at $keyfile.pub (SSH_HOST's IdentityFile)"
+ pub=$(awk '{ print $1, $2 }' "$keyfile.pub")
+
+ # Global, not local: the EXIT trap runs after this function has returned.
+ tmp=$(mktemp -d)
+ trap 'rm -rf "$tmp"' EXIT
+ serve_script > "$tmp/vpsbackup-serve"
+ watch_script > "$tmp/vpsbackup-watch"
+ cat > "$tmp/vpsbackup-watch.service" <<'EOF'
+[Unit]
+Description=vpsbackup: mail if no backup has reached home lately
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/vpsbackup-watch
+EOF
+ cat > "$tmp/vpsbackup-watch.timer" <<'EOF'
+[Unit]
+Description=vpsbackup: daily check that backups reach home
+
+[Timer]
+OnCalendar=*-*-* 17:00:00
+Persistent=true
+
+[Install]
+WantedBy=timers.target
+EOF
+ printf 'restrict,command="/usr/local/sbin/vpsbackup-serve" %s vpsbackup\n' "$pub" \
+ > "$tmp/authorized_key"
+
+ printf '%s\n' "$MAIL_TO" > "$tmp/mail-to"
+ cat > "$tmp/install" <<'EOF'
+set -euo pipefail
+src=$(dirname "$0")
+install -m 755 "$src/vpsbackup-serve" "$src/vpsbackup-watch" /usr/local/sbin/
+install -m 644 "$src/vpsbackup-watch.service" "$src/vpsbackup-watch.timer" /etc/systemd/system/
+install -d -m 700 /var/lib/vpsbackup
+install -m 600 "$src/mail-to" /var/lib/vpsbackup/mail-to
+# A fresh install starts the watch clock now rather than at the epoch.
+[ -s /var/lib/vpsbackup/last-ok ] || date +%s > /var/lib/vpsbackup/last-ok
+keys=/root/.ssh/authorized_keys
+cp -p "$keys" "$keys.bak-vpsbackup-$(date +%Y%m%d-%H%M%S)"
+grep -v " vpsbackup$" "$keys" > "$keys.new" || true
+cat "$src/authorized_key" >> "$keys.new"
+chmod 600 "$keys.new"
+mv "$keys.new" "$keys"
+systemctl daemon-reload
+systemctl enable --now vpsbackup-watch.timer
+echo "installed on $(hostname -f)"
+EOF
+ tar -C "$tmp" -cf - . |
+ ssh "$ADMIN_HOST" 'd=$(mktemp -d) && tar -C "$d" -xf - && bash "$d/install"; rc=$?; rm -rf "$d"; exit $rc'
+}
+
+# ---------------------------------------------------------------- commands
+
+cmd_init() {
+ mkdir -p "$CONFIG_DIR" "$BACKUP_DIR"
+ chmod 700 "$BACKUP_DIR"
+ if [[ ! -f "$RESTIC_PASSWORD_FILE" ]]; then
+ (umask 077; head -c 32 /dev/urandom | base64 > "$RESTIC_PASSWORD_FILE")
+ echo "vpsbackup: new password in $RESTIC_PASSWORD_FILE; keep a copy somewhere safe"
+ fi
+ restic cat config >/dev/null 2>&1 || restic init
+}
+
+# A failed run: record it, and mail the log unless one went out recently or
+# the VPS can't be reached to send it (vpsbackup-watch covers that case).
+fail() { # MESSAGE
+ put_state last-result "$(now) error $1"
+ echo "vpsbackup: $1" >&2
+ if [[ -n "${INVOCATION_ID:-}" ]] && (( $(since mailed) >= ERROR_MAIL_EVERY )) && reachable; then
+ { echo "vpsbackup on $(uname -n) failed: $1"; echo; tail -n 200 "$tmp/log"; } |
+ remote report-error && put_state mailed "$(now)"
+ fi
+ exit 1
+}
+
+# Runs a step with its stderr in the journal and in the log a failure mails.
+step() { # COMMAND...
+ "$@" 2> >(tee -a "$tmp/log" >&2)
+}
+
+cmd_run() {
+ mkdir -p "$STATE_DIR"
+ exec 9>"$STATE_DIR/lock"
+ flock -n 9 || { echo "vpsbackup: a backup is already running" >&2; exit 0; }
+ [[ -f "$RESTIC_PASSWORD_FILE" ]] || die "no password file; run 'vpsbackup init'"
+
+ if [[ "${1:-}" != --now ]] && (( $(since last-success) < RUN_EVERY )); then
+ exit 0
+ fi
+
+ # Global, not local: the EXIT trap runs after this function has returned.
+ tmp=$(mktemp -d)
+ trap 'rm -rf "$tmp"' EXIT
+ : > "$tmp/log"
+
+ if ! reachable; then
+ put_state last-result "$(now) offline can't reach $SSH_HOST"
+ echo "vpsbackup: can't reach $SSH_HOST, trying again next hour" >&2
+ exit 0
+ fi
+
+ step remote prepare </dev/null || fail "couldn't dump PostgreSQL on the VPS"
+
+ mkdir -p "$MIRROR"
+ chmod 700 "$BACKUP_DIR"
+ local -a filter=()
+ local f
+ for f in "${FILTER[@]}"; do filter+=(--filter "$f"); done
+ local rc=0
+ # --partial keeps a half-pulled file, so the next run continues it; restic
+ # only snapshots the mirror after a complete pull.
+ step rsync -aH --numeric-ids --fake-super --delete --delete-excluded --partial \
+ --stats --timeout=600 -e "ssh ${SSH_OPTS[*]}" \
+ "${filter[@]}" "$SSH_HOST:/" "$MIRROR/" >"$tmp/stats" || rc=$?
+ grep -E '^(Number of (regular )?files transferred|Total transferred file size|Total bytes received):' \
+ "$tmp/stats" || true
+ case $rc in
+ # 24 is "files vanished during the transfer", normal on a live system.
+ 0|24) ;;
+ # The connection dropping (home network or IP changing mid-pull) is
+ # retried next hour without mail; if it keeps happening, no snapshot
+ # is reported and vpsbackup-watch mails after WATCH_DAYS.
+ 12|30|35|255)
+ put_state last-result "$(now) interrupted rsync lost the connection ($rc)"
+ echo "vpsbackup: lost the connection mid-pull; the next run continues it" >&2
+ exit 0 ;;
+ *) fail "rsync exited with $rc" ;;
+ esac
+
+ unlock_stale
+ rc=0
+ step restic backup "$MIRROR" --host "$SNAPSHOT_HOST" --tag "$TAG" \
+ --skip-if-unchanged --retry-lock 1h || rc=$?
+ case $rc in
+ 0) ;;
+ 3) fail "restic couldn't read some files in the mirror" ;;
+ *) fail "restic backup exited with $rc" ;;
+ esac
+ put_state last-success "$(now)"
+ put_state last-result "$(now) ok"
+ remote report-ok </dev/null || echo "vpsbackup: couldn't report success to the VPS" >&2
+
+ if (( $(since last-forget) >= FORGET_EVERY )); then
+ step restic forget --host "$SNAPSHOT_HOST" --tag "$TAG" "${KEEP[@]}" --prune \
+ --retry-lock 1h || fail "restic forget/prune failed"
+ put_state last-forget "$(now)"
+ fi
+ if (( $(since last-check) >= CHECK_EVERY )); then
+ step restic check --read-data-subset "$CHECK_SUBSET" --retry-lock 1h ||
+ fail "restic check found a problem with the repository"
+ put_state last-check "$(now)"
+ fi
+}
+
+# A run that died mid-operation leaves its lock behind; without --remove-all,
+# unlock removes only stale ones.
+unlock_stale() {
+ restic unlock 2>/dev/null || true
+}
+
+cmd_status() {
+ local last result
+ last=$(get_state last-success "")
+ result=$(get_state last-result "")
+ if [[ -n "$last" ]]; then
+ echo "Last backup: $(date -d "@$last" '+%F %R') ($(ago "$last"))"
+ else
+ echo "Last backup: never"
+ fi
+ if [[ -n "$result" ]]; then
+ read -r when status msg <<<"$result"
+ echo "Last run: $(date -d "@$when" '+%F %R') $status${msg:+: $msg}"
+ fi
+ echo
+ restic snapshots --tag "$TAG" --latest 5 --compact
+}
+
+case "${1:-}" in
+ run|install-server)
+ [[ -n "$SSH_HOST" ]] || die "set SSH_HOST in $CONFIG_DIR/config" ;;
+esac
+
+case "${1:-}" in
+ run) shift; cmd_run "$@" ;;
+ status) cmd_status ;;
+ init) cmd_init ;;
+ install-server) cmd_install_server ;;
+ restic) shift; exec restic "$@" ;;
+ -h|--help|help|"") usage ;;
+ *) usage >&2; exit 2 ;;
+esac
generated by cgit v1.2.3 (git 2.46.0) at 2026-11-02 13:20:22 +0000