bin: add vpsbackup, a daily pull backup of the VPS

vpsbackup pulls the VPS over a read-only rsync key into a local mirror
(--fake-super keeps owners, modes and symlinks in xattrs) and snapshots it
into a local restic repository, hourly until a day has passed. Failures are
mailed through the VPS's own postfix, and a watch timer it installs there
mails when no backup has arrived for three days. A dropped connection is
retried next hour rather than reported.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

4 files changed, 423 insertions, 0 deletions

diff --git a/bin/README.md b/bin/README.md
index 6c32c13..52593e9 100644
--- a/bin/README.md
+++ b/bin/README.md
@@ -9,10 +9,13 @@ and the config looks for it on `PATH` before falling back to this directory.
 | `docsync`        | Backups                    | restic, ssh, jq, notify-send            |
 | `imgcollage`     | Media thumbnails > Collage | Python 3, Pillow                        |
 | `system-theme`   | Appearance > Theme         | the COSMIC desktop                      |
+| `vpsbackup`      | (standalone)               | restic, rsync, ssh                      |
 
 Each script's header or usage message (`docsync help`) says how to set it up.
 `systemd/` holds docsync's user units: copy them to `~/.config/systemd/user/`,
 put `docsync` at `~/.local/bin/docsync`, and enable the three timers.
+`vpsbackup.timer` and `.service` likewise run `~/.local/bin/vpsbackup`, which
+pulls the VPS here daily; `vpsbackup help` covers its setup and restores.
 
 `photocollage/` is vendored unchanged from
 [PhotoCollage](https://github.com/adrienverge/PhotoCollage) by Adrien Vergé,
diff --git a/bin/systemd/vpsbackup.service b/bin/systemd/vpsbackup.service
new file mode 100644
index 0000000..f2bb392
--- /dev/null
+++ b/bin/systemd/vpsbackup.service
@@ -0,0 +1,8 @@
+[Unit]
+Description=vpsbackup: pull the VPS and snapshot it
+
+[Service]
+Type=oneshot
+ExecStart=%h/.local/bin/vpsbackup run
+Nice=10
+IOSchedulingClass=idle
diff --git a/bin/systemd/vpsbackup.timer b/bin/systemd/vpsbackup.timer
new file mode 100644
index 0000000..d35268b
--- /dev/null
+++ b/bin/systemd/vpsbackup.timer
@@ -0,0 +1,10 @@
+[Unit]
+Description=vpsbackup: hourly (backs up once a day)
+
+[Timer]
+OnCalendar=hourly
+RandomizedDelaySec=5min
+Persistent=true
+
+[Install]
+WantedBy=timers.target
diff --git a/bin/vpsbackup b/bin/vpsbackup
new file mode 100755
index 0000000..ccede77
--- /dev/null
+++ b/bin/vpsbackup
@@ -0,0 +1,402 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+CONFIG_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/vpsbackup"
+STATE_DIR="${XDG_STATE_HOME:-$HOME/.local/state}/vpsbackup"
+
+# Pulls the whole VPS (less what a reinstall brings back) to this machine and
+# keeps its history in a local restic repository. The VPS never gets write
+# access here: this machine logs in with a key whose forced command
+# (vpsbackup-serve, below) allows a read-only rsync of / and three small
+# commands, so a compromised VPS can't touch its own backups.
+#
+# $CONFIG_DIR/config sets SSH_HOST, a Host in ~/.ssh/config that logs in as
+# root on the VPS with that key (IdentitiesOnly, BatchMode-safe), and
+# ADMIN_HOST, an ordinary root login used only by install-server. It may also
+# set MAIL_TO (the VPS mailbox problems are mailed to, default root),
+# BACKUP_DIR (default ~/Backups/vps), SNAPSHOT_HOST, and EXTRA_FILTER, rsync
+# filter rules added to FILTER below, e.g. ('- /srv/scratch/').
+SSH_HOST=
+ADMIN_HOST=
+MAIL_TO=root
+BACKUP_DIR="$HOME/Backups/vps"
+SNAPSHOT_HOST=vps
+EXTRA_FILTER=()
+# shellcheck source=/dev/null
+[[ -f "$CONFIG_DIR/config" ]] && . "$CONFIG_DIR/config"
+MIRROR="$BACKUP_DIR/mirror"
+export RESTIC_REPOSITORY="$BACKUP_DIR/restic"
+export RESTIC_PASSWORD_FILE="$CONFIG_DIR/restic-password"
+
+TAG=vps
+
+# The hourly timer runs 'vpsbackup run', which does nothing until a day has
+# passed since the last snapshot: one backup a day, retried every hour while
+# the laptop is off the network or the VPS is unreachable.
+RUN_EVERY=$((20 * 3600))
+FORGET_EVERY=$((7 * 86400))
+CHECK_EVERY=$((30 * 86400))
+CHECK_SUBSET=10%
+KEEP=(--keep-daily 30 --keep-weekly 52 --keep-monthly unlimited)
+
+# A failed run mails its log to MAIL_TO (through the VPS's own postfix), at
+# most once per window. Silence is covered on the VPS side: vpsbackup-watch
+# mails daily once no snapshot has been reported for WATCH_DAYS days.
+ERROR_MAIL_EVERY=$((12 * 3600))
+WATCH_DAYS=3
+
+# What isn't worth pulling: kernel filesystems, what the packages bring back,
+# caches, and PostgreSQL's live data files, which aren't consistent mid-write
+# (vpsbackup-serve's prepare dumps them to /var/backups/vpsbackup instead).
+# Exclude the repository docsync writes to on the VPS (if it does) through
+# EXTRA_FILTER, or this machine's own backups come back here.
+FILTER=(
+    '- /proc/' '- /sys/' '- /dev/' '- /run/' '- /tmp/' '- /var/tmp/'
+    '- /mnt/' '- /media/' '- /lost+found/' '- /swapfile' '- /boot/'
+    '+ /usr/' '+ /usr/local/' '- /usr/*'
+    '- /var/cache/' '- /var/lib/apt/lists/' '- /var/lib/clamav/'
+    '- /var/lib/postgresql/' '- /var/log/journal/'
+    '- /root/.cache/' '- /home/*/.cache/'
+    "${EXTRA_FILTER[@]}"
+)
+
+usage() {
+    cat <<EOF
+Usage: vpsbackup COMMAND
+
+  run           back up the VPS if a day has passed since the last snapshot
+                (the hourly timer runs this); 'run --now' backs up regardless
+  status        when the last backup landed, and the latest snapshots
+  restic ARGS   any restic command against the repository, e.g.
+                  vpsbackup restic snapshots
+                  vpsbackup restic mount ~/mnt/vps
+  init          create the restic password and repository (once)
+  install-server
+                install or update the VPS side through ADMIN_HOST: the
+                forced command, the backup key and the daily watch timer
+
+Files: the rsync mirror is $MIRROR, the repository
+$RESTIC_REPOSITORY. The mirror keeps owners, groups and special files in
+user.rsync.* xattrs (rsync --fake-super); restic keeps xattrs, so a restore
+carries them. To put files back on the VPS:
+  vpsbackup restic restore latest:$MIRROR --target DIR --include /etc/nginx
+  rsync -aH --numeric-ids --fake-super DIR/etc/nginx/ root@VPS:/etc/nginx/
+The PostgreSQL dump is var/backups/vpsbackup/postgres.sql (psql -f it as
+postgres), and the package lists sit beside it.
+
+Setup, all in $CONFIG_DIR:
+  config            SSH_HOST=<Host using the backup key>, ADMIN_HOST=<root login>,
+                    optionally MAIL_TO, BACKUP_DIR, SNAPSHOT_HOST
+  restic-password   made by 'vpsbackup init', mode 600
+  key               the backup key's .pub is read from SSH_HOST's IdentityFile
+Then: vpsbackup init; vpsbackup install-server; enable vpsbackup.timer.
+EOF
+}
+
+die() { echo "vpsbackup: $*" >&2; exit 1; }
+
+now() { date +%s; }
+
+put_state() { # NAME LINE...
+    if (( $# > 1 )); then printf '%s\n' "${@:2}"; fi > "$STATE_DIR/$1.tmp"
+    mv "$STATE_DIR/$1.tmp" "$STATE_DIR/$1"
+}
+
+get_state() { # NAME DEFAULT
+    if [[ -s "$STATE_DIR/$1" ]]; then head -n1 "$STATE_DIR/$1"; else printf '%s\n' "$2"; fi
+}
+
+since() { # NAME: seconds since the epoch stored in state NAME (0 if never)
+    echo $(( $(now) - $(get_state "$1" 0) ))
+}
+
+ago() { # EPOCH
+    local s=$(( $(now) - $1 ))
+    if (( s < 90 )); then echo "just now"
+    elif (( s < 5400 )); then echo "$(( s / 60 )) min ago"
+    elif (( s < 172800 )); then echo "$(( s / 3600 )) h ago"
+    else echo "$(( s / 86400 )) days ago"
+    fi
+}
+
+# Keepalives notice a dropped home connection in two minutes rather than
+# waiting out rsync's --timeout.
+SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=30 -o ServerAliveInterval=30 -o ServerAliveCountMax=4)
+
+remote() { # COMMAND [< stdin]
+    ssh "${SSH_OPTS[@]}" "$SSH_HOST" "$@"
+}
+
+reachable() {
+    remote ping </dev/null >/dev/null 2>&1
+}
+
+# ---------------------------------------------------------------- the VPS side
+
+# The forced command for the backup key. rrsync -ro / refuses anything but a
+# read (no --remove-source-files, no writes); the rest is an allow list.
+serve_script() {
+    cat <<'EOF'
+#!/bin/bash
+# vpsbackup-serve: forced command for the home machine's backup key, put here
+# by `vpsbackup install-server` (see that script on the home machine).
+set -euo pipefail
+DUMPS=/var/backups/vpsbackup
+STATE=/var/lib/vpsbackup
+MAIL_TO=$(cat "$STATE/mail-to")
+
+mail_me() { # SUBJECT, body on stdin
+    { printf 'To: %s\nSubject: %s\nAuto-Submitted: auto-generated\n\n' "$MAIL_TO" "$1"
+      cat; } | /usr/sbin/sendmail -i -- "$MAIL_TO"
+}
+
+case "${SSH_ORIGINAL_COMMAND:-}" in
+    ping) ;;
+    prepare)
+        umask 077
+        mkdir -p "$DUMPS"
+        cd /
+        runuser -u postgres -- pg_dumpall --clean --if-exists > "$DUMPS/postgres.sql.tmp"
+        mv "$DUMPS/postgres.sql.tmp" "$DUMPS/postgres.sql"
+        dpkg --get-selections > "$DUMPS/packages.selections"
+        apt-mark showmanual > "$DUMPS/packages.manual" ;;
+    report-ok)
+        date +%s > "$STATE/last-ok.tmp"
+        mv "$STATE/last-ok.tmp" "$STATE/last-ok" ;;
+    report-error)
+        head -c 65536 | tr -cd '\11\12\15\40-\176' |
+            mail_me "VPS backup failed" ;;
+    "rsync --server --sender "*)
+        exec /usr/bin/rrsync -ro / ;;
+    *)
+        echo "vpsbackup-serve: not allowed: ${SSH_ORIGINAL_COMMAND:-(shell)}" >&2
+        exit 1 ;;
+esac
+EOF
+}
+
+watch_script() {
+    cat <<EOF
+#!/bin/bash
+# vpsbackup-watch: mails once a day while no backup has reached the home
+# machine for $WATCH_DAYS days. Put here by \`vpsbackup install-server\`.
+set -euo pipefail
+STATE=/var/lib/vpsbackup
+EOF
+    cat <<'EOF'
+MAIL_TO=$(cat "$STATE/mail-to")
+last=$(cat "$STATE/last-ok" 2>/dev/null || echo 0)
+days=$(( ($(date +%s) - last) / 86400 ))
+EOF
+    echo "(( days >= $WATCH_DAYS )) || exit 0"
+    cat <<'EOF'
+{ printf 'To: %s\nSubject: No VPS backup for %s days\nAuto-Submitted: auto-generated\n\n' \
+      "$MAIL_TO" "$days"
+  echo "No backup of $(hostname -f) has reached the home machine since"
+  echo "$(date -d "@$last" '+%F %R %Z')."
+  echo
+  echo "The home machine retries hourly whenever it's on. If it has been on,"
+  echo "look at 'vpsbackup status' and 'journalctl --user -u vpsbackup' there."
+} | /usr/sbin/sendmail -i -- "$MAIL_TO"
+EOF
+}
+
+cmd_install_server() {
+    [[ -n "$ADMIN_HOST" ]] || die "set ADMIN_HOST in $CONFIG_DIR/config"
+    local keyfile pub
+    keyfile=$(ssh -G "$SSH_HOST" | awk '$1 == "identityfile" { print $2; exit }')
+    keyfile="${keyfile/#\~/$HOME}"
+    [[ -f "$keyfile.pub" ]] || die "no public key at $keyfile.pub (SSH_HOST's IdentityFile)"
+    pub=$(awk '{ print $1, $2 }' "$keyfile.pub")
+
+    # Global, not local: the EXIT trap runs after this function has returned.
+    tmp=$(mktemp -d)
+    trap 'rm -rf "$tmp"' EXIT
+    serve_script > "$tmp/vpsbackup-serve"
+    watch_script > "$tmp/vpsbackup-watch"
+    cat > "$tmp/vpsbackup-watch.service" <<'EOF'
+[Unit]
+Description=vpsbackup: mail if no backup has reached home lately
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/vpsbackup-watch
+EOF
+    cat > "$tmp/vpsbackup-watch.timer" <<'EOF'
+[Unit]
+Description=vpsbackup: daily check that backups reach home
+
+[Timer]
+OnCalendar=*-*-* 17:00:00
+Persistent=true
+
+[Install]
+WantedBy=timers.target
+EOF
+    printf 'restrict,command="/usr/local/sbin/vpsbackup-serve" %s vpsbackup\n' "$pub" \
+        > "$tmp/authorized_key"
+
+    printf '%s\n' "$MAIL_TO" > "$tmp/mail-to"
+    cat > "$tmp/install" <<'EOF'
+set -euo pipefail
+src=$(dirname "$0")
+install -m 755 "$src/vpsbackup-serve" "$src/vpsbackup-watch" /usr/local/sbin/
+install -m 644 "$src/vpsbackup-watch.service" "$src/vpsbackup-watch.timer" /etc/systemd/system/
+install -d -m 700 /var/lib/vpsbackup
+install -m 600 "$src/mail-to" /var/lib/vpsbackup/mail-to
+# A fresh install starts the watch clock now rather than at the epoch.
+[ -s /var/lib/vpsbackup/last-ok ] || date +%s > /var/lib/vpsbackup/last-ok
+keys=/root/.ssh/authorized_keys
+cp -p "$keys" "$keys.bak-vpsbackup-$(date +%Y%m%d-%H%M%S)"
+grep -v " vpsbackup$" "$keys" > "$keys.new" || true
+cat "$src/authorized_key" >> "$keys.new"
+chmod 600 "$keys.new"
+mv "$keys.new" "$keys"
+systemctl daemon-reload
+systemctl enable --now vpsbackup-watch.timer
+echo "installed on $(hostname -f)"
+EOF
+    tar -C "$tmp" -cf - . |
+        ssh "$ADMIN_HOST" 'd=$(mktemp -d) && tar -C "$d" -xf - && bash "$d/install"; rc=$?; rm -rf "$d"; exit $rc'
+}
+
+# ---------------------------------------------------------------- commands
+
+cmd_init() {
+    mkdir -p "$CONFIG_DIR" "$BACKUP_DIR"
+    chmod 700 "$BACKUP_DIR"
+    if [[ ! -f "$RESTIC_PASSWORD_FILE" ]]; then
+        (umask 077; head -c 32 /dev/urandom | base64 > "$RESTIC_PASSWORD_FILE")
+        echo "vpsbackup: new password in $RESTIC_PASSWORD_FILE; keep a copy somewhere safe"
+    fi
+    restic cat config >/dev/null 2>&1 || restic init
+}
+
+# A failed run: record it, and mail the log unless one went out recently or
+# the VPS can't be reached to send it (vpsbackup-watch covers that case).
+fail() { # MESSAGE
+    put_state last-result "$(now) error $1"
+    echo "vpsbackup: $1" >&2
+    if [[ -n "${INVOCATION_ID:-}" ]] && (( $(since mailed) >= ERROR_MAIL_EVERY )) && reachable; then
+        { echo "vpsbackup on $(uname -n) failed: $1"; echo; tail -n 200 "$tmp/log"; } |
+            remote report-error && put_state mailed "$(now)"
+    fi
+    exit 1
+}
+
+# Runs a step with its stderr in the journal and in the log a failure mails.
+step() { # COMMAND...
+    "$@" 2> >(tee -a "$tmp/log" >&2)
+}
+
+cmd_run() {
+    mkdir -p "$STATE_DIR"
+    exec 9>"$STATE_DIR/lock"
+    flock -n 9 || { echo "vpsbackup: a backup is already running" >&2; exit 0; }
+    [[ -f "$RESTIC_PASSWORD_FILE" ]] || die "no password file; run 'vpsbackup init'"
+
+    if [[ "${1:-}" != --now ]] && (( $(since last-success) < RUN_EVERY )); then
+        exit 0
+    fi
+
+    # Global, not local: the EXIT trap runs after this function has returned.
+    tmp=$(mktemp -d)
+    trap 'rm -rf "$tmp"' EXIT
+    : > "$tmp/log"
+
+    if ! reachable; then
+        put_state last-result "$(now) offline can't reach $SSH_HOST"
+        echo "vpsbackup: can't reach $SSH_HOST, trying again next hour" >&2
+        exit 0
+    fi
+
+    step remote prepare </dev/null || fail "couldn't dump PostgreSQL on the VPS"
+
+    mkdir -p "$MIRROR"
+    chmod 700 "$BACKUP_DIR"
+    local -a filter=()
+    local f
+    for f in "${FILTER[@]}"; do filter+=(--filter "$f"); done
+    local rc=0
+    # --partial keeps a half-pulled file, so the next run continues it; restic
+    # only snapshots the mirror after a complete pull.
+    step rsync -aH --numeric-ids --fake-super --delete --delete-excluded --partial \
+        --stats --timeout=600 -e "ssh ${SSH_OPTS[*]}" \
+        "${filter[@]}" "$SSH_HOST:/" "$MIRROR/" >"$tmp/stats" || rc=$?
+    grep -E '^(Number of (regular )?files transferred|Total transferred file size|Total bytes received):' \
+        "$tmp/stats" || true
+    case $rc in
+        # 24 is "files vanished during the transfer", normal on a live system.
+        0|24) ;;
+        # The connection dropping (home network or IP changing mid-pull) is
+        # retried next hour without mail; if it keeps happening, no snapshot
+        # is reported and vpsbackup-watch mails after WATCH_DAYS.
+        12|30|35|255)
+            put_state last-result "$(now) interrupted rsync lost the connection ($rc)"
+            echo "vpsbackup: lost the connection mid-pull; the next run continues it" >&2
+            exit 0 ;;
+        *) fail "rsync exited with $rc" ;;
+    esac
+
+    unlock_stale
+    rc=0
+    step restic backup "$MIRROR" --host "$SNAPSHOT_HOST" --tag "$TAG" \
+        --skip-if-unchanged --retry-lock 1h || rc=$?
+    case $rc in
+        0) ;;
+        3) fail "restic couldn't read some files in the mirror" ;;
+        *) fail "restic backup exited with $rc" ;;
+    esac
+    put_state last-success "$(now)"
+    put_state last-result "$(now) ok"
+    remote report-ok </dev/null || echo "vpsbackup: couldn't report success to the VPS" >&2
+
+    if (( $(since last-forget) >= FORGET_EVERY )); then
+        step restic forget --host "$SNAPSHOT_HOST" --tag "$TAG" "${KEEP[@]}" --prune \
+            --retry-lock 1h || fail "restic forget/prune failed"
+        put_state last-forget "$(now)"
+    fi
+    if (( $(since last-check) >= CHECK_EVERY )); then
+        step restic check --read-data-subset "$CHECK_SUBSET" --retry-lock 1h ||
+            fail "restic check found a problem with the repository"
+        put_state last-check "$(now)"
+    fi
+}
+
+# A run that died mid-operation leaves its lock behind; without --remove-all,
+# unlock removes only stale ones.
+unlock_stale() {
+    restic unlock 2>/dev/null || true
+}
+
+cmd_status() {
+    local last result
+    last=$(get_state last-success "")
+    result=$(get_state last-result "")
+    if [[ -n "$last" ]]; then
+        echo "Last backup: $(date -d "@$last" '+%F %R') ($(ago "$last"))"
+    else
+        echo "Last backup: never"
+    fi
+    if [[ -n "$result" ]]; then
+        read -r when status msg <<<"$result"
+        echo "Last run:    $(date -d "@$when" '+%F %R') $status${msg:+: $msg}"
+    fi
+    echo
+    restic snapshots --tag "$TAG" --latest 5 --compact
+}
+
+case "${1:-}" in
+    run|install-server)
+        [[ -n "$SSH_HOST" ]] || die "set SSH_HOST in $CONFIG_DIR/config" ;;
+esac
+
+case "${1:-}" in
+    run) shift; cmd_run "$@" ;;
+    status) cmd_status ;;
+    init) cmd_init ;;
+    install-server) cmd_install_server ;;
+    restic) shift; exec restic "$@" ;;
+    -h|--help|help|"") usage ;;
+    *) usage >&2; exit 2 ;;
+esac