Task 47b27ccb: Continuous backup of the SQLite file (#215)

.gitignore

@@ -14,3 +14,7 @@ frontend/node_modules/
 *.db
 *.db-wal
 *.db-shm
+
+# Dedicated Litestream SFTP backup keypair — lives only on the API box.
+deploy/*key*
+deploy/litestream_key*

api/README.md

@@ -36,5 +36,48 @@ which serves ``https://nestquest.cubecraftlabs.com`` (Let's Encrypt,
 auto-renewed, HTTP redirected to HTTPS) and forwards to the API box
 on port 8080 — so on that box set ``NESTQUEST_BIND_ADDRESS`` to its
 LAN address (``10.60.1.14``). Traefik only allows
-``/api/v1/panel`` from the Home Assistant host. Network separation,
-backup and restore are later Feature 17 tasks.
+``/api/v1/panel`` from the Home Assistant host. Network separation
+and the full restore runbook are later Feature 17 tasks.
+
+## Continuous backup (Litestream)
+
+Compose also runs a ``litestream`` sidecar
+(``litestream/litestream:0.5.17``, config ``deploy/litestream.yml``).
+It mounts the same ``nestquest-db`` volume and continuously replicates
+``/var/lib/nestquest/nestquest.db`` over SFTP to the Synology NAS
+``CubeCraft-SRVR1`` (``joshua@10.60.1.50``), directory
+``/volume1/homes/Joshua/nestquest-backups`` (``/homes/Joshua/...`` in
+the NAS's chrooted SFTP view) — so the backup is outside the Docker
+host and its volume. A snapshot is taken daily and a week of
+snapshots plus incremental changes is retained; the NAS host key is
+pinned in the config.
+
+The SFTP login uses a dedicated keypair that lives only on the API
+box and is git-ignored. Create it once, before ``up``:
+
+    ssh-keygen -t ed25519 -N "" -C nestquest-litestream -f deploy/litestream_key
+
+then append ``restrict `` + the contents of ``deploy/litestream_key.pub``
+to ``joshua@10.60.1.50:~/.ssh/authorized_keys``. Compose mounts the
+private key read-only at ``/etc/litestream/sftp_key``. The sidecar runs
+as uid/gid 999 (the API's ``nestquest`` user), so give it the key:
+
+    sudo chown 999:999 deploy/litestream_key && chmod 600 deploy/litestream_key
+
+The sidecar needs WRITE access to the database volume: Litestream adds
+its own ``_litestream_seq``/``_litestream_lock`` tables to the database,
+writes the ``-wal``/``-shm`` files and checkpoints the WAL, and fails
+with "unable to open database file" on a read-only mount. That access
+is contained rather than removed — the sidecar runs as the non-root
+volume owner with a read-only root filesystem, ``no-new-privileges``
+and all Linux capabilities dropped.
+
+Check replication with
+``docker compose -f deploy/compose.yaml logs litestream`` or
+``docker compose -f deploy/compose.yaml exec litestream litestream ltx /var/lib/nestquest/nestquest.db``.
+To restore (high level): stop the ``api`` service, run
+``litestream restore -config /etc/litestream.yml -o <new path>
+/var/lib/nestquest/nestquest.db`` from a litestream container with the
+same mounts, verify the restored file, move it into place on the
+volume (removing any stale ``-wal``/``-shm`` files), and start ``api``
+again.

deploy/compose.yaml

@@ -42,5 +42,38 @@ services:
       retries: 5
       start_period: 15s
 
+  # Continuous backup: streams the SQLite database on the shared named
+  # volume to the Synology NAS over SFTP (config: deploy/litestream.yml).
+  # The SFTP private key deploy/litestream_key is git-ignored and must be
+  # created on the API box before starting (see api/README.md).
+  #
+  # The database volume is mounted READ-WRITE on purpose: Litestream
+  # cannot run against a read-only mount ("unable to open database file
+  # (14)", even with its metadata moved elsewhere) because it writes its
+  # _litestream_seq/_litestream_lock tables into the database, opens the
+  # -wal/-shm files for writing, and checkpoints the WAL.  The write
+  # access is contained instead: it runs as the API's own non-root user
+  # (uid/gid 999, owner of the volume) with a read-only root filesystem,
+  # no privilege escalation and no Linux capabilities.
+  litestream:
+    image: litestream/litestream:0.5.17
+    restart: unless-stopped
+    command: ["replicate", "-config", "/etc/litestream.yml"]
+    user: "999:999"
+    read_only: true
+    tmpfs:
+      - /tmp
+    security_opt:
+      - no-new-privileges:true
+    cap_drop:
+      - ALL
+    depends_on:
+      api:
+        condition: service_healthy
+    volumes:
+      - nestquest-db:/var/lib/nestquest
+      - ./litestream.yml:/etc/litestream.yml:ro
+      - ./litestream_key:/etc/litestream/sftp_key:ro
+
 volumes:
   nestquest-db:

deploy/litestream.yml

@@ -0,0 +1,32 @@
+# Litestream continuous backup of the NestQuest SQLite database (Feature
+# 17).  The ``litestream`` compose service mounts the same named volume
+# as the API and streams every committed change to the Synology NAS
+# over SFTP, so the backup lives OUTSIDE the container's volume.
+#
+# Nothing here is secret.  The SFTP private key is a dedicated,
+# git-ignored keypair (deploy/litestream_key) mounted read-only by
+# compose at /etc/litestream/sftp_key; its public half is in
+# joshua@10.60.1.50:~/.ssh/authorized_keys.  ``host-key`` pins the
+# NAS's host key so the connection is verified; it must be the ECDSA
+# key, because that is the algorithm Litestream's SSH client negotiates
+# with this NAS (pinning the ed25519 key fails with "host key mismatch").
+#
+# The Synology SFTP server is chrooted to the shared folders, so the
+# directory seen over SSH as /volume1/homes/Joshua/nestquest-backups is
+# /homes/Joshua/nestquest-backups over SFTP.
+
+# Take a full snapshot daily and keep a week of snapshots plus the
+# incremental changes between them (point-in-time restore window).
+snapshot:
+  interval: 24h
+  retention: 168h
+
+dbs:
+  - path: /var/lib/nestquest/nestquest.db
+    replicas:
+      - type: sftp
+        host: 10.60.1.50
+        user: joshua
+        path: /homes/Joshua/nestquest-backups
+        key-path: /etc/litestream/sftp_key
+        host-key: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNujay9nYEiXuRQf4wDRDU1gp+CDvpeMdhuhC2XypIihzKGEtB654kwrUGfA2kDWAoSPGlmojeS3OIRtHPkj6GY=