From a34eb98b5e7433fd4d631dbed90d13a8b04ee5aa Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 22 Oct 2026 03:27:42 +0000 Subject: [PATCH] Backups: survive a day-long first upload over a home line The first real run died 90 seconds into a stall: ServerAliveInterval 30 x CountMax 3, five parallel restic connections saturating a home NAS, and restic cannot reconnect once its ssh exits, so the run was lost. - ssh waits five minutes of real silence (15s x 20), TCPKeepAlive on - two SFTP connections by default (BACKUP_SFTP_CONNECTIONS), and an optional upload cap for restic and the mirror (BACKUP_LIMIT_UPLOAD_KBPS) - one run at a time: an hourly tick during a long run skips itself instead of failing on the repository lock - restic unlock first, so a crashed run's stale lock clears itself README: start the first backup inside the scheduled container, and what to do when a large first upload dies (rerun; restic reuses what landed). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XWoa1a1upTria7ntq56x3B --- .env.example | 6 +++++ docker-compose.yml | 3 +++ ops/backup/README.md | 47 ++++++++++++++++++++++++++++++++++++---- ops/backup/backup.sh | 43 ++++++++++++++++++++++++++++++++---- ops/backup/entrypoint.sh | 9 ++++++-- 5 files changed, 98 insertions(+), 10 deletions(-) diff --git a/.env.example b/.env.example index 3391858..11f42b9 100644 --- a/.env.example +++ b/.env.example @@ -155,6 +155,12 @@ BACKUP_SFTP_REMOTE_PATH=/GAM/restic # Optional plain-file mirror beside the restic repository: SFTP chroot path, e.g. # /GAM/mirror. Empty = off. See ops/backup/README.md step 13. BACKUP_MIRROR_PATH= +# Parallel SFTP connections to the NAS. restic's own default of 5 can stall a home +# line badly enough for ssh to drop mid-upload. +BACKUP_SFTP_CONNECTIONS=2 +# Optional upload cap in KiB/s for restic and the mirror, e.g. 5000 (about 40 Mbit/s). +# Empty = uncapped. Useful while the first upload of a large library runs. +BACKUP_LIMIT_UPLOAD_KBPS= BACKUP_CRON_SCHEDULE=0 * * * * BACKUP_KEEP_HOURLY=48 BACKUP_KEEP_DAILY=14 diff --git a/docker-compose.yml b/docker-compose.yml index e303179..645691e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -87,6 +87,9 @@ services: - BACKUP_SFTP_REMOTE_PATH=${BACKUP_SFTP_REMOTE_PATH:-/GAM/restic} # Empty leaves the plain-file mirror off; restic runs either way. - BACKUP_MIRROR_PATH=${BACKUP_MIRROR_PATH:-} + # Parallel SFTP connections, and an optional upload cap in KiB/s (empty = none). + - BACKUP_SFTP_CONNECTIONS=${BACKUP_SFTP_CONNECTIONS:-2} + - BACKUP_LIMIT_UPLOAD_KBPS=${BACKUP_LIMIT_UPLOAD_KBPS:-} - BACKUP_CRON_SCHEDULE=${BACKUP_CRON_SCHEDULE:-0 * * * *} - BACKUP_KEEP_HOURLY=${BACKUP_KEEP_HOURLY:-48} - BACKUP_KEEP_DAILY=${BACKUP_KEEP_DAILY:-14} diff --git a/ops/backup/README.md b/ops/backup/README.md index 835b3ce..c4cba80 100644 --- a/ops/backup/README.md +++ b/ops/backup/README.md @@ -109,7 +109,7 @@ Never Notes' `/GeckoNotes/restic`. restic would happily put two apps' snapshots repository, and `--host gam` would keep their retention apart — but the restic password would then be shared, and a repository corrupted by either app would take both histories with it. -**e. Carry on from step 11**, then 13–14 for the mirror, with +**e. Carry on from step 11** — including "A large first upload" — then 13–14 for the mirror, with `BACKUP_MIRROR_PATH=/GeckoNotes/gam/mirror` (and that path wherever step 14 says `/GAM/mirror`). On the NAS the mirror is then `/volume1/GeckoNotes/gam/mirror` — the `GAM_MIRROR_DIR` the archive needs ([`ops/archive/README.md`](../archive/README.md), @@ -404,12 +404,38 @@ Edit `.env`, not `.env.example` — that stays a template. ```bash docker compose build backup docker compose run --rm backup restic init -docker compose run --rm backup /usr/local/bin/backup.sh docker compose up -d backup +docker compose exec -d backup sh -c '/usr/local/bin/backup.sh >>/proc/1/fd/1 2>>/proc/1/fd/2' +docker logs -f backup ``` -`restic init` is one-time and creates the repository on the NAS. The third line -forces an immediate backup instead of waiting for the first cron tick. +`restic init` is one-time and creates the repository on the NAS. The `exec` line +starts the first backup now instead of waiting for the first cron tick, inside the +scheduled container and detached. Ctrl-C on `docker logs -f` stops only the watching, +not the backup. + +Inside that container rather than a `docker compose run --rm` one, because the run +lock lives there: a scheduled tick that arrives while this is still going sees it and +skips itself (`previous run still going, skipping`) instead of failing on restic's +repository lock. + +### A large first upload + +The first run sends the whole library, which over a home line can take most of a day +(restic prints an ETA). Later runs send only what changed, in minutes. + +- **If it dies, run the same `exec` line again.** restic saves its index as it goes, so + what already reached the NAS is not sent twice; the rerun picks up roughly where the + last one stopped. The dead run's leftover repository lock is cleared automatically. +- **If the home connection is sluggish meanwhile**, cap the upload in `.env`, e.g. + `BACKUP_LIMIT_UPLOAD_KBPS=5000` (about 40 Mbit/s; KiB/s, applies to restic and the + mirror), then `docker compose up -d --force-recreate backup` and start it again. + Remove the cap once the first run is through. +- `BACKUP_SFTP_CONNECTIONS` (default 2) is how many uploads run in parallel. Raise it + only if the NAS and both lines have room to spare; more connections is what makes a + saturated home NAS stall. +- With the mirror on, the same run then copies the library a second time into + `mirror/` — expect the first run to take about twice restic's ETA. `docker compose run --rm` creates a throwaway container each time, so it always picks up current files. `docker compose restart` does **not** — bind mounts and @@ -546,6 +572,19 @@ firewall rule, non-default port and Auto Block stop being load-bearing. The client-side error is always the same unhelpful `Permission denied (publickey,password)`. Get the server's actual reason instead. +### `Timeout, server … not responding` partway through a run + +Authentication worked — the run got as far as uploading — and then the NAS went quiet +for longer than ssh would wait, usually under the long first upload. ssh now waits five +minutes (`ServerAliveInterval 15` × `ServerAliveCountMax 20` in `entrypoint.sh`) and +restic uses two connections, not five; if it still happens, set +`BACKUP_LIMIT_UPLOAD_KBPS` ("A large first upload", step 11) and rerun. Nothing is +lost: the rerun reuses what was already uploaded. + +If it happens on *small* hourly runs too, the link itself is dropping — check the NAS's +**Log Center** for the connection, and whether Gecko Notes' backup over the same link +fails at the same times. + ### Read sshd's reasoning directly Task Scheduler, root. Runs a debug sshd on a spare port that logs why it refused: diff --git a/ops/backup/backup.sh b/ops/backup/backup.sh index 65bffd5..56a3cce 100644 --- a/ops/backup/backup.sh +++ b/ops/backup/backup.sh @@ -7,9 +7,38 @@ set -eu trap 'echo "[backup] FAILED (exit $?)" >&2' ERR +mkdir -p /backup/tmp + +# One run at a time. A first upload of a large library takes most of a day, and the +# schedule is hourly: without this, each tick would start a second run that fails on +# restic's repository lock and reports FAILED, and the real one would be buried. The +# lock lives in the container's tmpfs, so it is held exactly as long as this process +# is — which is also why a manual run goes through `docker compose exec`, not `run`. +exec 9>/backup/tmp/backup.lock +if ! flock -n 9; then + echo "[backup] $(date -u +%Y-%m-%dT%H:%M:%SZ) previous run still going, skipping" + exit 0 +fi + echo "[backup] $(date -u +%Y-%m-%dT%H:%M:%SZ) starting" -mkdir -p /backup/tmp +# Fewer than restic's default of 5 parallel SFTP connections: a home NAS on a home +# line, saturated for hours, stalls long enough for ssh to give up. Numbers only, so +# leaving these unquoted below to split into flags is safe. +CONNECTIONS="${BACKUP_SFTP_CONNECTIONS:-2}" +RESTIC_LIMIT="" +RCLONE_LIMIT="" +if [ -n "${BACKUP_LIMIT_UPLOAD_KBPS:-}" ]; then + RESTIC_LIMIT="--limit-upload ${BACKUP_LIMIT_UPLOAD_KBPS}" + RCLONE_LIMIT="--bwlimit ${BACKUP_LIMIT_UPLOAD_KBPS}k" +fi + +# A run that died — the connection dropped, the container was recreated — leaves its +# lock in the repository, and every later run would refuse to start until a person +# cleared it. Without --remove-all this removes only locks restic judges stale, never +# one a live process holds (Notes' repository is a different one, and untouched). +restic unlock -o sftp.connections="${CONNECTIONS}" + rm -f /backup/tmp/gam.db echo "[backup] taking WAL-safe sqlite snapshot" @@ -21,14 +50,18 @@ echo "[backup] running restic backup" # hostname fragmenting the snapshot history into groups of one. restic backup /backup/tmp/gam.db /backup/src/media \ --host gam \ - --tag scheduled + --tag scheduled \ + -o sftp.connections="${CONNECTIONS}" \ + ${RESTIC_LIMIT} echo "[backup] pruning per retention policy" restic forget \ --keep-hourly "${BACKUP_KEEP_HOURLY:-48}" \ --keep-daily "${BACKUP_KEEP_DAILY:-14}" \ --keep-weekly "${BACKUP_KEEP_WEEKLY:-8}" \ - --prune + --prune \ + -o sftp.connections="${CONNECTIONS}" \ + ${RESTIC_LIMIT} if [ -n "${BACKUP_MIRROR_PATH:-}" ]; then MIRROR="nas:${BACKUP_MIRROR_PATH%/}" @@ -49,12 +82,14 @@ if [ -n "${BACKUP_MIRROR_PATH:-}" ]; then echo "[backup] mirroring media to ${MIRROR}/media" rclone sync /backup/src/media "${MIRROR}/media" \ --exclude '*.partial' \ + --transfers "${CONNECTIONS}" \ + ${RCLONE_LIMIT} \ -v # Uploaded under a temporary name and renamed into place by rclone, so an archive # reading the mirror never opens half a database. echo "[backup] mirroring database to ${MIRROR}/db/gam.db" - rclone copyto /backup/tmp/mirror.db "${MIRROR}/db/gam.db" + rclone copyto /backup/tmp/mirror.db "${MIRROR}/db/gam.db" ${RCLONE_LIMIT} # What the archive's operator checks to know which code version to run: an archive # older than the schema it is given cannot read it. diff --git a/ops/backup/entrypoint.sh b/ops/backup/entrypoint.sh index a4f0fde..f872599 100644 --- a/ops/backup/entrypoint.sh +++ b/ops/backup/entrypoint.sh @@ -5,6 +5,10 @@ # through to running the schedule in the foreground. set -eu +# ServerAlive: give up after five minutes of silence, not ninety seconds. A NAS busy +# writing an upload that runs for hours can go quiet for over a minute and still be +# there; at 30s x 3 the first real run died that way, and restic cannot reconnect once +# its ssh has exited — the whole run is lost, not one file. mkdir -p /root/.ssh cat > /root/.ssh/config <