path: root/bin/vpsbackup

#!/usr/bin/env bash
set -euo pipefail

CONFIG_DIR="${XDG_CONFIG_HOME:-$HOME/.config}/vpsbackup"
STATE_DIR="${XDG_STATE_HOME:-$HOME/.local/state}/vpsbackup"

# Pulls the whole VPS (less what a reinstall brings back) to this machine and
# keeps its history in a local restic repository. The VPS never gets write
# access here: this machine logs in with a key whose forced command
# (vpsbackup-serve, below) allows a read-only rsync of / and three small
# commands, so a compromised VPS can't touch its own backups.
#
# $CONFIG_DIR/config sets SSH_HOST, a Host in ~/.ssh/config that logs in as
# root on the VPS with that key (IdentitiesOnly, BatchMode-safe), and
# ADMIN_HOST, an ordinary root login used only by install-server. It may also
# set MAIL_TO (the VPS mailbox problems are mailed to, default root),
# BACKUP_DIR (default ~/Backups/vps), SNAPSHOT_HOST, and EXTRA_FILTER, rsync
# filter rules added to FILTER below, e.g. ('- /srv/scratch/').
SSH_HOST=
ADMIN_HOST=
MAIL_TO=root
BACKUP_DIR="$HOME/Backups/vps"
SNAPSHOT_HOST=vps
EXTRA_FILTER=()
# shellcheck source=/dev/null
[[ -f "$CONFIG_DIR/config" ]] && . "$CONFIG_DIR/config"
MIRROR="$BACKUP_DIR/mirror"
export RESTIC_REPOSITORY="$BACKUP_DIR/restic"
export RESTIC_PASSWORD_FILE="$CONFIG_DIR/restic-password"

TAG=vps

# The hourly timer runs 'vpsbackup run', which does nothing until a day has
# passed since the last snapshot: one backup a day, retried every hour while
# the laptop is off the network or the VPS is unreachable.
RUN_EVERY=$((20 * 3600))
FORGET_EVERY=$((7 * 86400))
CHECK_EVERY=$((30 * 86400))
CHECK_SUBSET=10%
KEEP=(--keep-daily 30 --keep-weekly 52 --keep-monthly unlimited)

# A failed run mails its log to MAIL_TO (through the VPS's own postfix), at
# most once per window. Silence is covered on the VPS side: vpsbackup-watch
# mails daily once no snapshot has been reported for WATCH_DAYS days.
ERROR_MAIL_EVERY=$((12 * 3600))
WATCH_DAYS=3

# What isn't worth pulling: kernel filesystems, what the packages bring back,
# caches, and PostgreSQL's live data files, which aren't consistent mid-write
# (vpsbackup-serve's prepare dumps them to /var/backups/vpsbackup instead).
# Exclude the repository docsync writes to on the VPS (if it does) through
# EXTRA_FILTER, or this machine's own backups come back here.
FILTER=(
    '- /proc/' '- /sys/' '- /dev/' '- /run/' '- /tmp/' '- /var/tmp/'
    '- /mnt/' '- /media/' '- /lost+found/' '- /swapfile' '- /boot/'
    '+ /usr/' '+ /usr/local/' '- /usr/*'
    '- /var/cache/' '- /var/lib/apt/lists/' '- /var/lib/clamav/'
    '- /var/lib/postgresql/' '- /var/log/journal/'
    '- /root/.cache/' '- /home/*/.cache/'
    "${EXTRA_FILTER[@]}"
)

usage() {
    cat <<EOF
Usage: vpsbackup COMMAND

  run           back up the VPS if a day has passed since the last snapshot
                (the hourly timer runs this); 'run --now' backs up regardless
  status        when the last backup landed, and the latest snapshots
  restic ARGS   any restic command against the repository, e.g.
                  vpsbackup restic snapshots
                  vpsbackup restic mount ~/mnt/vps
  init          create the restic password and repository (once)
  install-server
                install or update the VPS side through ADMIN_HOST: the
                forced command, the backup key and the daily watch timer

Files: the rsync mirror is $MIRROR, the repository
$RESTIC_REPOSITORY. The mirror keeps owners, groups and special files in
user.rsync.* xattrs (rsync --fake-super); restic keeps xattrs, so a restore
carries them. To put files back on the VPS:
  vpsbackup restic restore latest:$MIRROR --target DIR --include /etc/nginx
  rsync -aH --numeric-ids --fake-super DIR/etc/nginx/ root@VPS:/etc/nginx/
The PostgreSQL dump is var/backups/vpsbackup/postgres.sql (psql -f it as
postgres), and the package lists sit beside it.

Setup, all in $CONFIG_DIR:
  config            SSH_HOST=<Host using the backup key>, ADMIN_HOST=<root login>,
                    optionally MAIL_TO, BACKUP_DIR, SNAPSHOT_HOST
  restic-password   made by 'vpsbackup init', mode 600
  key               the backup key's .pub is read from SSH_HOST's IdentityFile
Then: vpsbackup init; vpsbackup install-server; enable vpsbackup.timer.
EOF
}

die() { echo "vpsbackup: $*" >&2; exit 1; }

now() { date +%s; }

put_state() { # NAME LINE...
    if (( $# > 1 )); then printf '%s\n' "${@:2}"; fi > "$STATE_DIR/$1.tmp"
    mv "$STATE_DIR/$1.tmp" "$STATE_DIR/$1"
}

get_state() { # NAME DEFAULT
    if [[ -s "$STATE_DIR/$1" ]]; then head -n1 "$STATE_DIR/$1"; else printf '%s\n' "$2"; fi
}

since() { # NAME: seconds since the epoch stored in state NAME (0 if never)
    echo $(( $(now) - $(get_state "$1" 0) ))
}

ago() { # EPOCH
    local s=$(( $(now) - $1 ))
    if (( s < 90 )); then echo "just now"
    elif (( s < 5400 )); then echo "$(( s / 60 )) min ago"
    elif (( s < 172800 )); then echo "$(( s / 3600 )) h ago"
    else echo "$(( s / 86400 )) days ago"
    fi
}

# Keepalives notice a dropped home connection in two minutes rather than
# waiting out rsync's --timeout.
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=30 -o ServerAliveInterval=30 -o ServerAliveCountMax=4)

remote() { # COMMAND [< stdin]
    ssh "${SSH_OPTS[@]}" "$SSH_HOST" "$@"
}

reachable() {
    remote ping </dev/null >/dev/null 2>&1
}

# ---------------------------------------------------------------- the VPS side

# The forced command for the backup key. rrsync -ro / refuses anything but a
# read (no --remove-source-files, no writes); the rest is an allow list.
serve_script() {
    cat <<'EOF'
#!/bin/bash
# vpsbackup-serve: forced command for the home machine's backup key, put here
# by `vpsbackup install-server` (see that script on the home machine).
set -euo pipefail
DUMPS=/var/backups/vpsbackup
STATE=/var/lib/vpsbackup
MAIL_TO=$(cat "$STATE/mail-to")

mail_me() { # SUBJECT, body on stdin
    { printf 'To: %s\nSubject: %s\nAuto-Submitted: auto-generated\n\n' "$MAIL_TO" "$1"
      cat; } | /usr/sbin/sendmail -i -- "$MAIL_TO"
}

case "${SSH_ORIGINAL_COMMAND:-}" in
    ping) ;;
    prepare)
        umask 077
        mkdir -p "$DUMPS"
        cd /
        runuser -u postgres -- pg_dumpall --clean --if-exists > "$DUMPS/postgres.sql.tmp"
        mv "$DUMPS/postgres.sql.tmp" "$DUMPS/postgres.sql"
        dpkg --get-selections > "$DUMPS/packages.selections"
        apt-mark showmanual > "$DUMPS/packages.manual" ;;
    report-ok)
        date +%s > "$STATE/last-ok.tmp"
        mv "$STATE/last-ok.tmp" "$STATE/last-ok" ;;
    report-error)
        head -c 65536 | tr -cd '\11\12\15\40-\176' |
            mail_me "VPS backup failed" ;;
    "rsync --server --sender "*)
        exec /usr/bin/rrsync -ro / ;;
    *)
        echo "vpsbackup-serve: not allowed: ${SSH_ORIGINAL_COMMAND:-(shell)}" >&2
        exit 1 ;;
esac
EOF
}

watch_script() {
    cat <<EOF
#!/bin/bash
# vpsbackup-watch: mails once a day while no backup has reached the home
# machine for $WATCH_DAYS days. Put here by \`vpsbackup install-server\`.
set -euo pipefail
STATE=/var/lib/vpsbackup
EOF
    cat <<'EOF'
MAIL_TO=$(cat "$STATE/mail-to")
last=$(cat "$STATE/last-ok" 2>/dev/null || echo 0)
days=$(( ($(date +%s) - last) / 86400 ))
EOF
    echo "(( days >= $WATCH_DAYS )) || exit 0"
    cat <<'EOF'
{ printf 'To: %s\nSubject: No VPS backup for %s days\nAuto-Submitted: auto-generated\n\n' \
      "$MAIL_TO" "$days"
  echo "No backup of $(hostname -f) has reached the home machine since"
  echo "$(date -d "@$last" '+%F %R %Z')."
  echo
  echo "The home machine retries hourly whenever it's on. If it has been on,"
  echo "look at 'vpsbackup status' and 'journalctl --user -u vpsbackup' there."
} | /usr/sbin/sendmail -i -- "$MAIL_TO"
EOF
}

cmd_install_server() {
    [[ -n "$ADMIN_HOST" ]] || die "set ADMIN_HOST in $CONFIG_DIR/config"
    local keyfile pub
    keyfile=$(ssh -G "$SSH_HOST" | awk '$1 == "identityfile" { print $2; exit }')
    keyfile="${keyfile/#\~/$HOME}"
    [[ -f "$keyfile.pub" ]] || die "no public key at $keyfile.pub (SSH_HOST's IdentityFile)"
    pub=$(awk '{ print $1, $2 }' "$keyfile.pub")

    # Global, not local: the EXIT trap runs after this function has returned.
    tmp=$(mktemp -d)
    trap 'rm -rf "$tmp"' EXIT
    serve_script > "$tmp/vpsbackup-serve"
    watch_script > "$tmp/vpsbackup-watch"
    cat > "$tmp/vpsbackup-watch.service" <<'EOF'
[Unit]
Description=vpsbackup: mail if no backup has reached home lately

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/vpsbackup-watch
EOF
    cat > "$tmp/vpsbackup-watch.timer" <<'EOF'
[Unit]
Description=vpsbackup: daily check that backups reach home

[Timer]
OnCalendar=*-*-* 17:00:00
Persistent=true

[Install]
WantedBy=timers.target
EOF
    printf 'restrict,command="/usr/local/sbin/vpsbackup-serve" %s vpsbackup\n' "$pub" \
        > "$tmp/authorized_key"

    printf '%s\n' "$MAIL_TO" > "$tmp/mail-to"
    cat > "$tmp/install" <<'EOF'
set -euo pipefail
src=$(dirname "$0")
install -m 755 "$src/vpsbackup-serve" "$src/vpsbackup-watch" /usr/local/sbin/
install -m 644 "$src/vpsbackup-watch.service" "$src/vpsbackup-watch.timer" /etc/systemd/system/
install -d -m 700 /var/lib/vpsbackup
install -m 600 "$src/mail-to" /var/lib/vpsbackup/mail-to
# A fresh install starts the watch clock now rather than at the epoch.
[ -s /var/lib/vpsbackup/last-ok ] || date +%s > /var/lib/vpsbackup/last-ok
keys=/root/.ssh/authorized_keys
cp -p "$keys" "$keys.bak-vpsbackup-$(date +%Y%m%d-%H%M%S)"
grep -v " vpsbackup$" "$keys" > "$keys.new" || true
cat "$src/authorized_key" >> "$keys.new"
chmod 600 "$keys.new"
mv "$keys.new" "$keys"
systemctl daemon-reload
systemctl enable --now vpsbackup-watch.timer
echo "installed on $(hostname -f)"
EOF
    tar -C "$tmp" -cf - . |
        ssh "$ADMIN_HOST" 'd=$(mktemp -d) && tar -C "$d" -xf - && bash "$d/install"; rc=$?; rm -rf "$d"; exit $rc'
}

# ---------------------------------------------------------------- commands

cmd_init() {
    mkdir -p "$CONFIG_DIR" "$BACKUP_DIR"
    chmod 700 "$BACKUP_DIR"
    if [[ ! -f "$RESTIC_PASSWORD_FILE" ]]; then
        (umask 077; head -c 32 /dev/urandom | base64 > "$RESTIC_PASSWORD_FILE")
        echo "vpsbackup: new password in $RESTIC_PASSWORD_FILE; keep a copy somewhere safe"
    fi
    restic cat config >/dev/null 2>&1 || restic init
}

# A failed run: record it, and mail the log unless one went out recently or
# the VPS can't be reached to send it (vpsbackup-watch covers that case).
fail() { # MESSAGE
    put_state last-result "$(now) error $1"
    echo "vpsbackup: $1" >&2
    if [[ -n "${INVOCATION_ID:-}" ]] && (( $(since mailed) >= ERROR_MAIL_EVERY )) && reachable; then
        { echo "vpsbackup on $(uname -n) failed: $1"; echo; tail -n 200 "$tmp/log"; } |
            remote report-error && put_state mailed "$(now)"
    fi
    exit 1
}

# Runs a step with its stderr in the journal and in the log a failure mails.
step() { # COMMAND...
    "$@" 2> >(tee -a "$tmp/log" >&2)
}

cmd_run() {
    mkdir -p "$STATE_DIR"
    exec 9>"$STATE_DIR/lock"
    flock -n 9 || { echo "vpsbackup: a backup is already running" >&2; exit 0; }
    [[ -f "$RESTIC_PASSWORD_FILE" ]] || die "no password file; run 'vpsbackup init'"

    if [[ "${1:-}" != --now ]] && (( $(since last-success) < RUN_EVERY )); then
        exit 0
    fi

    # Global, not local: the EXIT trap runs after this function has returned.
    tmp=$(mktemp -d)
    trap 'rm -rf "$tmp"' EXIT
    : > "$tmp/log"

    if ! reachable; then
        put_state last-result "$(now) offline can't reach $SSH_HOST"
        echo "vpsbackup: can't reach $SSH_HOST, trying again next hour" >&2
        exit 0
    fi

    step remote prepare </dev/null || fail "couldn't dump PostgreSQL on the VPS"

    mkdir -p "$MIRROR"
    chmod 700 "$BACKUP_DIR"
    local -a filter=()
    local f
    for f in "${FILTER[@]}"; do filter+=(--filter "$f"); done
    local rc=0
    # --partial keeps a half-pulled file, so the next run continues it; restic
    # only snapshots the mirror after a complete pull.
    step rsync -aH --numeric-ids --fake-super --delete --delete-excluded --partial \
        --stats --timeout=600 -e "ssh ${SSH_OPTS[*]}" \
        "${filter[@]}" "$SSH_HOST:/" "$MIRROR/" >"$tmp/stats" || rc=$?
    grep -E '^(Number of (regular )?files transferred|Total transferred file size|Total bytes received):' \
        "$tmp/stats" || true
    case $rc in
        # 24 is "files vanished during the transfer", normal on a live system.
        0|24) ;;
        # The connection dropping (home network or IP changing mid-pull) is
        # retried next hour without mail; if it keeps happening, no snapshot
        # is reported and vpsbackup-watch mails after WATCH_DAYS.
        12|30|35|255)
            put_state last-result "$(now) interrupted rsync lost the connection ($rc)"
            echo "vpsbackup: lost the connection mid-pull; the next run continues it" >&2
            exit 0 ;;
        *) fail "rsync exited with $rc" ;;
    esac

    unlock_stale
    rc=0
    step restic backup "$MIRROR" --host "$SNAPSHOT_HOST" --tag "$TAG" \
        --skip-if-unchanged --retry-lock 1h || rc=$?
    case $rc in
        0) ;;
        3) fail "restic couldn't read some files in the mirror" ;;
        *) fail "restic backup exited with $rc" ;;
    esac
    put_state last-success "$(now)"
    put_state last-result "$(now) ok"
    remote report-ok </dev/null || echo "vpsbackup: couldn't report success to the VPS" >&2

    if (( $(since last-forget) >= FORGET_EVERY )); then
        step restic forget --host "$SNAPSHOT_HOST" --tag "$TAG" "${KEEP[@]}" --prune \
            --retry-lock 1h || fail "restic forget/prune failed"
        put_state last-forget "$(now)"
    fi
    if (( $(since last-check) >= CHECK_EVERY )); then
        step restic check --read-data-subset "$CHECK_SUBSET" --retry-lock 1h ||
            fail "restic check found a problem with the repository"
        put_state last-check "$(now)"
    fi
}

# A run that died mid-operation leaves its lock behind; without --remove-all,
# unlock removes only stale ones.
unlock_stale() {
    restic unlock 2>/dev/null || true
}

cmd_status() {
    local last result
    last=$(get_state last-success "")
    result=$(get_state last-result "")
    if [[ -n "$last" ]]; then
        echo "Last backup: $(date -d "@$last" '+%F %R') ($(ago "$last"))"
    else
        echo "Last backup: never"
    fi
    if [[ -n "$result" ]]; then
        read -r when status msg <<<"$result"
        echo "Last run:    $(date -d "@$when" '+%F %R') $status${msg:+: $msg}"
    fi
    echo
    restic snapshots --tag "$TAG" --latest 5 --compact
}

case "${1:-}" in
    run|install-server)
        [[ -n "$SSH_HOST" ]] || die "set SSH_HOST in $CONFIG_DIR/config" ;;
esac

case "${1:-}" in
    run) shift; cmd_run "$@" ;;
    status) cmd_status ;;
    init) cmd_init ;;
    install-server) cmd_install_server ;;
    restic) shift; exec restic "$@" ;;
    -h|--help|help|"") usage ;;
    *) usage >&2; exit 2 ;;
esac