diff --git a/.gitignore b/.gitignore index ee069cc..40da8c4 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,7 @@ frontend/node_modules/ *.db *.db-wal *.db-shm + +# Dedicated Litestream SFTP backup keypair — lives only on the API box. +deploy/*key* +deploy/litestream_key* diff --git a/api/README.md b/api/README.md index 6feb6d1..ae1e773 100644 --- a/api/README.md +++ b/api/README.md @@ -36,5 +36,48 @@ which serves ``https://nestquest.cubecraftlabs.com`` (Let's Encrypt, auto-renewed, HTTP redirected to HTTPS) and forwards to the API box on port 8080 — so on that box set ``NESTQUEST_BIND_ADDRESS`` to its LAN address (``10.60.1.14``). Traefik only allows -``/api/v1/panel`` from the Home Assistant host. Network separation, -backup and restore are later Feature 17 tasks. \ No newline at end of file +``/api/v1/panel`` from the Home Assistant host. Network separation +and the full restore runbook are later Feature 17 tasks. + +## Continuous backup (Litestream) + +Compose also runs a ``litestream`` sidecar +(``litestream/litestream:0.5.17``, config ``deploy/litestream.yml``). +It mounts the same ``nestquest-db`` volume and continuously replicates +``/var/lib/nestquest/nestquest.db`` over SFTP to the Synology NAS +``CubeCraft-SRVR1`` (``joshua@10.60.1.50``), directory +``/volume1/homes/Joshua/nestquest-backups`` (``/homes/Joshua/...`` in +the NAS's chrooted SFTP view) — so the backup is outside the Docker +host and its volume. A snapshot is taken daily and a week of +snapshots plus incremental changes is retained; the NAS host key is +pinned in the config. + +The SFTP login uses a dedicated keypair that lives only on the API +box and is git-ignored. Create it once, before ``up``: + + ssh-keygen -t ed25519 -N "" -C nestquest-litestream -f deploy/litestream_key + +then append ``restrict `` + the contents of ``deploy/litestream_key.pub`` +to ``joshua@10.60.1.50:~/.ssh/authorized_keys``. Compose mounts the +private key read-only at ``/etc/litestream/sftp_key``. The sidecar runs +as uid/gid 999 (the API's ``nestquest`` user), so give it the key: + + sudo chown 999:999 deploy/litestream_key && chmod 600 deploy/litestream_key + +The sidecar needs WRITE access to the database volume: Litestream adds +its own ``_litestream_seq``/``_litestream_lock`` tables to the database, +writes the ``-wal``/``-shm`` files and checkpoints the WAL, and fails +with "unable to open database file" on a read-only mount. That access +is contained rather than removed — the sidecar runs as the non-root +volume owner with a read-only root filesystem, ``no-new-privileges`` +and all Linux capabilities dropped. + +Check replication with +``docker compose -f deploy/compose.yaml logs litestream`` or +``docker compose -f deploy/compose.yaml exec litestream litestream ltx /var/lib/nestquest/nestquest.db``. +To restore (high level): stop the ``api`` service, run +``litestream restore -config /etc/litestream.yml -o +/var/lib/nestquest/nestquest.db`` from a litestream container with the +same mounts, verify the restored file, move it into place on the +volume (removing any stale ``-wal``/``-shm`` files), and start ``api`` +again. \ No newline at end of file diff --git a/deploy/compose.yaml b/deploy/compose.yaml index fd0cf3d..742ba23 100644 --- a/deploy/compose.yaml +++ b/deploy/compose.yaml @@ -42,5 +42,38 @@ services: retries: 5 start_period: 15s + # Continuous backup: streams the SQLite database on the shared named + # volume to the Synology NAS over SFTP (config: deploy/litestream.yml). + # The SFTP private key deploy/litestream_key is git-ignored and must be + # created on the API box before starting (see api/README.md). + # + # The database volume is mounted READ-WRITE on purpose: Litestream + # cannot run against a read-only mount ("unable to open database file + # (14)", even with its metadata moved elsewhere) because it writes its + # _litestream_seq/_litestream_lock tables into the database, opens the + # -wal/-shm files for writing, and checkpoints the WAL. The write + # access is contained instead: it runs as the API's own non-root user + # (uid/gid 999, owner of the volume) with a read-only root filesystem, + # no privilege escalation and no Linux capabilities. + litestream: + image: litestream/litestream:0.5.17 + restart: unless-stopped + command: ["replicate", "-config", "/etc/litestream.yml"] + user: "999:999" + read_only: true + tmpfs: + - /tmp + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + depends_on: + api: + condition: service_healthy + volumes: + - nestquest-db:/var/lib/nestquest + - ./litestream.yml:/etc/litestream.yml:ro + - ./litestream_key:/etc/litestream/sftp_key:ro + volumes: nestquest-db: \ No newline at end of file diff --git a/deploy/litestream.yml b/deploy/litestream.yml new file mode 100644 index 0000000..698f3b1 --- /dev/null +++ b/deploy/litestream.yml @@ -0,0 +1,32 @@ +# Litestream continuous backup of the NestQuest SQLite database (Feature +# 17). The ``litestream`` compose service mounts the same named volume +# as the API and streams every committed change to the Synology NAS +# over SFTP, so the backup lives OUTSIDE the container's volume. +# +# Nothing here is secret. The SFTP private key is a dedicated, +# git-ignored keypair (deploy/litestream_key) mounted read-only by +# compose at /etc/litestream/sftp_key; its public half is in +# joshua@10.60.1.50:~/.ssh/authorized_keys. ``host-key`` pins the +# NAS's host key so the connection is verified; it must be the ECDSA +# key, because that is the algorithm Litestream's SSH client negotiates +# with this NAS (pinning the ed25519 key fails with "host key mismatch"). +# +# The Synology SFTP server is chrooted to the shared folders, so the +# directory seen over SSH as /volume1/homes/Joshua/nestquest-backups is +# /homes/Joshua/nestquest-backups over SFTP. + +# Take a full snapshot daily and keep a week of snapshots plus the +# incremental changes between them (point-in-time restore window). +snapshot: + interval: 24h + retention: 168h + +dbs: + - path: /var/lib/nestquest/nestquest.db + replicas: + - type: sftp + host: 10.60.1.50 + user: joshua + path: /homes/Joshua/nestquest-backups + key-path: /etc/litestream/sftp_key + host-key: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNujay9nYEiXuRQf4wDRDU1gp+CDvpeMdhuhC2XypIihzKGEtB654kwrUGfA2kDWAoSPGlmojeS3OIRtHPkj6GY=