From 67e44a506ac38cf84e370ecaa54338103571b001 Mon Sep 17 00:00:00 2001 From: dev_agent Date: Mon, 5 Oct 2026 20:59:47 -0400 Subject: [PATCH] Feature 07: Deployment, identity and backups (#75) Co-authored-by: dev_agent --- .gitignore | 2 + deploy/.env.example | 80 ++ deploy/RUNBOOK.md | 886 ++++++++++++++++++ deploy/compose.yaml | 100 ++ deploy/known_hosts | 6 + deploy/litestream.yml | 31 + deploy/systemd/mealquest-firewall.service | 22 + deploy/systemd/mealquest-firewall.sh | 53 ++ deploy/systemd/mealquest-media-backup.service | 29 + deploy/systemd/mealquest-media-backup.sh | 44 + deploy/systemd/mealquest-media-backup.timer | 11 + deploy/systemd/mealquest.service | 22 + 12 files changed, 1286 insertions(+) create mode 100644 deploy/.env.example create mode 100644 deploy/RUNBOOK.md create mode 100644 deploy/compose.yaml create mode 100644 deploy/known_hosts create mode 100644 deploy/litestream.yml create mode 100644 deploy/systemd/mealquest-firewall.service create mode 100755 deploy/systemd/mealquest-firewall.sh create mode 100644 deploy/systemd/mealquest-media-backup.service create mode 100755 deploy/systemd/mealquest-media-backup.sh create mode 100644 deploy/systemd/mealquest-media-backup.timer create mode 100644 deploy/systemd/mealquest.service diff --git a/.gitignore b/.gitignore index 60709ee..7f9b8de 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ design/.DS_Store # Secrets and local environment .env* +!deploy/.env.example +deploy/.env deploy/secrets/ # Build artifacts and dependencies diff --git a/deploy/.env.example b/deploy/.env.example new file mode 100644 index 0000000..d0e969e --- /dev/null +++ b/deploy/.env.example @@ -0,0 +1,80 @@ +# MealQuest production environment. Copy to deploy/.env (never committed) +# and replace the placeholders. The service fails fast: a missing or invalid +# required variable stops it at boot with the variable named. +# +# Secrets are never set here. They live in deploy/secrets/ (never committed), +# one value per file, owned by the container user and readable only by it: +# +# mkdir -p deploy/secrets && chmod 700 deploy/secrets +# head -c 32 /dev/urandom | base64 > deploy/secrets/secret_key +# printf '%s\n' '' > deploy/secrets/oidc_client_secret +# sudo chown 10001:10001 deploy/secrets/* && sudo chmod 600 deploy/secrets/* +# +# deploy/compose.yaml mounts them at /run/secrets/ and points the +# *_FILE variables at them. + +# --- Compose ----------------------------------------------------------------- + +# Host address the service is published on, as
:8090. Production +# binds the LAN address 10.60.1.14 only; 127.0.0.1 suits a dev machine. +MEALQUEST_BIND_ADDRESS=127.0.0.1 + +# --- Service ----------------------------------------------------------------- + +# Public origin the household opens. Fixes the OIDC redirect URI +# (/auth/callback) and the Secure cookies. Must be https:// (plain +# http:// is accepted only for localhost, 127.0.0.1 and [::1]). +MEALQUEST_PUBLIC_URL=https://mealquest.cubecraftlabs.com + +# How long a graceful shutdown may take before in-flight requests are cut. +MEALQUEST_SHUTDOWN_TIMEOUT=10s + +# Optional. Comma-separated IPs/CIDRs of reverse proxies whose +# X-Forwarded-For/-Proto headers are honoured. Production lists Traefik; +# unset trusts no peer. +MEALQUEST_TRUSTED_PROXY=10.60.1.33 + +# IANA zone the household settings are created with on first boot. Required +# until those settings exist; ignored afterwards (change the zone in +# household settings instead). Confirm the household's zone before the first +# deploy. +MEALQUEST_DEFAULT_TZ=America/New_York + +# --- Authentik OIDC ---------------------------------------------------------- + +# Issuer of the Authentik application "mealquest". Discovered at boot, so it +# must be reachable from the container. Must be https://. +MEALQUEST_OIDC_ISSUER=https://auth.cubecraftlabs.com/application/o/mealquest/ +MEALQUEST_OIDC_CLIENT_ID=mealquest +# Comma-separated Authentik groups whose members may sign in. +MEALQUEST_OIDC_ALLOWED_GROUPS=mealquest-admin + +# Optional session lifetimes. A session ends after IDLE unused and always MAX +# after sign-in; MAX must be at least IDLE. Defaults: 720h and 4320h. +MEALQUEST_SESSION_IDLE=720h +MEALQUEST_SESSION_MAX=4320h + +# --- Set by deploy/compose.yaml; do not set here ----------------------------- +# +# MEALQUEST_HTTP_ADDR=:8080 +# MEALQUEST_DATA_DIR=/data +# MEALQUEST_SECRET_KEY_FILE=/run/secrets/secret_key +# Base64 of 32 random bytes; encrypts secrets at rest (AES-256-GCM). +# MEALQUEST_OIDC_CLIENT_SECRET_FILE=/run/secrets/oidc_client_secret +# The Authentik provider's client secret. + +# --- Planned; not read by the service yet ------------------------------------- +# +# Added to deploy/compose.yaml and this file by the features that use them. +# +# Web Push (Feature 19). Generate the key pair with cmd/vapidgen. +# MEALQUEST_VAPID_PUBLIC_KEY= +# MEALQUEST_VAPID_PRIVATE_KEY_FILE=/run/secrets/vapid_private_key +# MEALQUEST_VAPID_SUBJECT=mailto: +# +# Google Calendar OAuth web client (Feature 16). +# MEALQUEST_GOOGLE_CLIENT_ID=.apps.googleusercontent.com +# MEALQUEST_GOOGLE_CLIENT_SECRET_FILE=/run/secrets/google_client_secret +# +# SearXNG recipe search sidecar on the compose network (Feature 07). +# MEALQUEST_SEARXNG_URL=http://searxng:8080 diff --git a/deploy/RUNBOOK.md b/deploy/RUNBOOK.md new file mode 100644 index 0000000..c255c4e --- /dev/null +++ b/deploy/RUNBOOK.md @@ -0,0 +1,886 @@ +# MealQuest deployment runbook + +How MealQuest is deployed on office-srvr (10.60.1.14), how it is reached +through Authentik, Traefik and DNS, how it is updated and rolled back, how +its data is backed up and restored, and how its secrets are rotated. Values +are the ones in use on 2026-10-05. Secrets themselves are never written +here. + +Paths used below: + +| Name | Path | +| --- | --- | +| Checkout on the host | `/home/overseer/mealquest` | +| Data volume (`mealquest-data`) on the host | `/var/lib/docker/volumes/mealquest-data/_data` | +| Same volume inside the containers | `/data` | +| Database | `mealquest.db` (+ `mealquest.db-wal`, `mealquest.db-shm` while open) | +| Media | `media/` in the data volume | +| Backup target (NAS, over SSH) | `joshua@10.60.1.50:/volume1/homes/Joshua/mealquest-backups` | +| Same target over SFTP (Synology chroot) | `/homes/Joshua/mealquest-backups` | + +## Prerequisites + +On office-srvr (10.60.1.14), as recorded on 2026-10-05: + +| Item | Value | +| --- | --- | +| OS | Ubuntu 26.04.1 LTS | +| Docker Engine / Compose plugin | 29.8.2 / 5.6.0 | +| git, rsync, iptables | 2.53.0, 3.4.1, 1.8.11 (nf_tables) | +| Account | `overseer`, with sudo | +| Network | LAN reachability to Traefik (10.60.1.33), Authentik (`auth.cubecraftlabs.com`) and the NAS (10.60.1.50, SSH) | + +Elsewhere: + +- Admin access to Authentik at . +- SSH access to the Traefik host 10.60.1.33. +- Cloudflare access for the `cubecraftlabs.com` zone, and admin access to + both Pi-holes (10.60.1.101 `dns.ccl`, 10.60.1.85 `dns2.ccl`). +- A NAS account `joshua` on 10.60.1.50 with the backup directory + `/volume1/homes/Joshua/mealquest-backups` (see "Backups"). +- A Gitea token able to read `Maniacs_Mansion/MealQuest`, for the clone. + +## Stack and first deploy + +The compose stack is `deploy/compose.yaml`: the `mealquest` service (built +from the repository `Dockerfile`, runs as uid/gid 10001, listens on 8080 in +the container) and the `litestream` service. Data lives in the named volume +`mealquest-data`. + +### 1. Choose and check the port + +The container's 8080 is published on `10.60.1.14:8090`. Ports 8080 and 8081 +on this host belong to NestQuest, so 8090 was chosen. Before the first +deploy it was checked free with: + +```sh +ss -ltnp | grep 8090 # printed nothing +``` + +After the deploy the same host shows NestQuest on 8080/8081 and MealQuest +on 8090: + +``` +LISTEN 0 4096 10.60.1.14:8080 0.0.0.0:* +LISTEN 0 4096 10.60.1.14:8081 0.0.0.0:* +LISTEN 0 4096 10.60.1.14:8090 0.0.0.0:* +``` + +The published host port is fixed in `deploy/compose.yaml`, in +`deploy/systemd/mealquest-firewall.sh` and in the Traefik route; change all +three together. + +### 2. Clone + +The checkout lives at `/home/overseer/mealquest`, next to NestQuest's +`~/nestquest` (not `/opt/mealquest`). Clone over HTTPS with a Gitea token, +then reset the remote to the clean URL so the token is not stored in +`.git/config`: + +```sh +cd /home/overseer +git clone https://:@code.cubecraftlabs.com/Maniacs_Mansion/MealQuest.git mealquest +cd mealquest +git remote set-url origin https://code.cubecraftlabs.com/Maniacs_Mansion/MealQuest.git +git remote -v # must show no token +git checkout main +``` + +### 3. Environment file + +`deploy/.env` (git-ignored) is copied from `deploy/.env.example` and is +mode 0600, owned by `overseer`: + +```sh +cp deploy/.env.example deploy/.env +chmod 600 deploy/.env +``` + +Then edit `deploy/.env` to the values below. In particular +**`MEALQUEST_BIND_ADDRESS` must be changed**: `.env.example` ships +`127.0.0.1` (a dev machine), and left as is the service listens only on +the host's loopback, which Traefik cannot reach. Set it to the LAN address +and check: + +```sh +sed -i 's/^MEALQUEST_BIND_ADDRESS=.*/MEALQUEST_BIND_ADDRESS=10.60.1.14/' deploy/.env +grep '^MEALQUEST_BIND_ADDRESS=' deploy/.env # MEALQUEST_BIND_ADDRESS=10.60.1.14 +docker compose -f deploy/compose.yaml config | grep -A4 'ports:' +# host_ip: 10.60.1.14, target: 8080, published: "8090" +``` + +After step 5 the listener is `10.60.1.14:8090` (container port 8080): + +```sh +docker compose -f deploy/compose.yaml port mealquest 8080 # 10.60.1.14:8090 +sudo ss -ltnp 'sport = :8090' # 10.60.1.14:8090 only, no 0.0.0.0 or 127.0.0.1 +``` + +Values in use: + +| Variable | Value | +| --- | --- | +| `MEALQUEST_BIND_ADDRESS` | `10.60.1.14` | +| `MEALQUEST_PUBLIC_URL` | `https://mealquest.cubecraftlabs.com` | +| `MEALQUEST_SHUTDOWN_TIMEOUT` | `10s` | +| `MEALQUEST_TRUSTED_PROXY` | `10.60.1.33` | +| `MEALQUEST_DEFAULT_TZ` | `America/New_York` | +| `MEALQUEST_OIDC_ISSUER` | `https://auth.cubecraftlabs.com/application/o/mealquest/` | +| `MEALQUEST_OIDC_CLIENT_ID` | `i6zraqmWsF552b8nmte070LXz0SI5pRdArBxFxeq` (not a secret) | +| `MEALQUEST_OIDC_ALLOWED_GROUPS` | `mealquest-admin` | +| `MEALQUEST_SESSION_IDLE` / `MEALQUEST_SESSION_MAX` | `720h` / `4320h` | + +`MEALQUEST_HTTP_ADDR`, `MEALQUEST_DATA_DIR` and the `*_FILE` variables are +set by `deploy/compose.yaml` and must not appear in `deploy/.env`. + +### 4. Secrets + +`deploy/secrets/` (git-ignored) is mode 0700, owned by `overseer`. Each +file holds one value, is owned by `10001:10001` (host user `mealquest`) and +is mode 0600. Compose bind-mounts file secrets with their host owner and +mode, so any other owner or mode stops the container reading them and the +service refuses to start. + +| File | Content | Mounted at | +| --- | --- | --- | +| `secret_key` | base64 of 32 random bytes; encrypts stored secrets (AES-256-GCM) | `/run/secrets/secret_key` | +| `oidc_client_secret` | Client Secret of the Authentik provider `mealquest` | `/run/secrets/oidc_client_secret` | +| `litestream_key` | dedicated SSH private key for the NAS (see "Backups") | `/etc/litestream/sftp_key` | + +```sh +mkdir -p deploy/secrets && chmod 700 deploy/secrets +head -c 32 /dev/urandom | base64 > deploy/secrets/secret_key +printf '%s\n' '' > deploy/secrets/oidc_client_secret +# litestream_key: see "Backups" > "Database — Litestream" +sudo chown 10001:10001 deploy/secrets/* +sudo chmod 600 deploy/secrets/* +sudo ls -la deploy/secrets # three files, mealquest mealquest, -rw------- +``` + +The `mealquest` host user (uid/gid 10001) is created in "Media — nightly +rsync"; until it exists `ls` shows the numeric ids. + +### 5. Firewall, then the stack + +Install the firewall unit and the stack unit first (see "Host firewall" +and "Stack unit"), so port 8090 is never open to the LAN. Create the media +directory (see "Media — nightly rsync"), then build and start: + +```sh +docker compose -f deploy/compose.yaml build +sudo systemctl enable --now mealquest # starts mealquest-firewall first +docker compose -f deploy/compose.yaml ps # mealquest healthy, litestream up +curl -fsS http://10.60.1.14:8090/health +curl -s -o /dev/null -w "%{http_code}\n" https://mealquest.cubecraftlabs.com/health # 200 +``` + +### Host checkout state on 2026-10-05 + +The first deploy ran before Feature 07 reached `main`. The checkout is at +`main` `9cddbbb` and the Feature 07 files under `deploy/` were copied in +from the feature branch: `deploy/compose.yaml` and `deploy/.env.example` +are listed in `.git/info/exclude`, the rest show as untracked. Before the +first routine deploy of a `main` that contains `deploy/`, remove those +`.git/info/exclude` lines and the copied files (keep `deploy/.env` and +`deploy/secrets/`), or `git checkout` refuses to overwrite them: + +```sh +cd /home/overseer/mealquest +git status --porcelain --untracked-files=all deploy # the copied files +sed -i '/^deploy\/\(compose.yaml\|.env.example\)$/d' .git/info/exclude +git clean -n deploy # dry run: must not list .env or secrets/ (both git-ignored) +git clean -f deploy +``` + +Then follow "Routine deploy"; the checkout brings the tracked copies back. + +## Authentik + +Sign-in is OpenID Connect against Authentik at +. + +| Object | Setting | +| --- | --- | +| Provider | OAuth2/OpenID provider `mealquest`, client type **Confidential** | +| Redirect URI | `https://mealquest.cubecraftlabs.com/auth/callback` (strict) | +| Signing | RS256, with the instance's signing key | +| Scopes | MealQuest requests `openid profile email groups offline_access`; the provider's scope mappings include openid, profile, email and offline_access, and the `groups` claim is delivered by a property mapping (the default profile mapping carries `groups`) | +| Application | name MealQuest, slug `mealquest`, provider `mealquest` | +| Group | `mealquest-admin`, bound to the application; household members are added to it | +| Client ID | `i6zraqmWsF552b8nmte070LXz0SI5pRdArBxFxeq` (not a secret; in `deploy/.env` as `MEALQUEST_OIDC_CLIENT_ID`) | +| Client Secret | in `deploy/secrets/oidc_client_secret` only | + +MealQuest refuses a sign-in unless the user's `groups` claim (from the ID +token, or from userinfo when the token has none) contains a group in +`MEALQUEST_OIDC_ALLOWED_GROUPS`. + +Check discovery from the Docker host — the service discovers the issuer +at boot and will not start if it cannot: + +```sh +curl -s -o /dev/null -w "%{http_code}\n" \ + https://auth.cubecraftlabs.com/application/o/mealquest/.well-known/openid-configuration # 200 +``` + +On 2026-10-05 discovery returned 200 with issuer +`https://auth.cubecraftlabs.com/application/o/mealquest/` and +`id_token_signing_alg_values_supported: ["RS256"]`. + +## Reverse proxy (Traefik) + +MealQuest is published at by the +shared Traefik instance on 10.60.1.33 (Feature 07, task 0c1dc172). The +route is a dynamic-configuration file that lives on the Traefik host only, +at `/home/overseer/Code/traefik/config/dynamic/mealquest-ccl.yml`. Its exact +contents: + +```yaml +# MealQuest - https://mealquest.cubecraftlabs.com +# Feature 07 task 0c1dc172. Single Go binary (API + embedded PWA) on +# office-srvr 10.60.1.14:8090. HTTP -> HTTPS is handled globally by +# http-conf.yml, so only a websecure router is defined here. +# This file lives on the Traefik host only; the repo copy is recorded in +# deploy/RUNBOOK.md by task 876134bc. +http: + routers: + mealquest: + entryPoints: + - websecure + rule: "Host(`mealquest.cubecraftlabs.com`)" + priority: 100 + middlewares: + - mealquest-headers + service: mealquest + tls: + certResolver: myresolver + domains: + - main: mealquest.cubecraftlabs.com + + services: + mealquest: + loadBalancer: + servers: + - url: "http://10.60.1.14:8090" + + middlewares: + mealquest-headers: + headers: + stsSeconds: 31536000 + stsIncludeSubdomains: true + contentTypeNosniff: true + frameDeny: true +``` + +Traefik watches the dynamic directory, so saving the file loads the route; +the certificate comes from `certResolver: myresolver` (Let's Encrypt, +DNS-01 through Cloudflare). Check from the LAN: + +```sh +curl -s -o /dev/null -w "%{http_code}\n" https://mealquest.cubecraftlabs.com/health # 200 +``` + +## DNS + +| Where | Record | +| --- | --- | +| Cloudflare, zone `cubecraftlabs.com` | `CNAME mealquest.cubecraftlabs.com` → `cubecraftlabs.com`, **proxied = true** (orange cloud). Because it is proxied, public resolvers answer with Cloudflare's anycast addresses, A and AAAA (AAAA on 2026-10-05: `2606:4700:3032::6815:34ed`, `2606:4700:3035::ac43:cd75`), never the origin | +| Pi-hole primary 10.60.1.101 (`dns.ccl`), Local DNS records | `A mealquest.cubecraftlabs.com` → `10.60.1.33` (Traefik) | +| Pi-hole secondary 10.60.1.85 (`dns2.ccl`), Local DNS records | `A mealquest.cubecraftlabs.com` → `10.60.1.33` (Traefik) | + +The local record must be on **both** Pi-holes, or clients using the +secondary go out through Cloudflare. Check: + +```sh +for s in 10.60.1.101 10.60.1.85; do dig +short @$s mealquest.cubecraftlabs.com A; done +# 10.60.1.33 from each +``` + +**AAAA caveat.** The Pi-hole local record is A only, so both Pi-holes still +forward AAAA queries and return Cloudflare's IPv6 addresses +(`2606:4700:3032::6815:34ed`, `2606:4700:3035::ac43:cd75` on 2026-10-05). An +IPv6-capable LAN client may prefer those and reach MealQuest through +Cloudflare instead of directly through Traefik. It still works (Cloudflare +proxies to the public route), but it is not the LAN path. If that matters, +add a local AAAA override or block AAAA for this name on both Pi-holes. + +## Host firewall + +Docker publishes `10.60.1.14:8090` past the host's INPUT rules, so the +port is restricted in the `DOCKER-USER` chain instead: only Traefik +(10.60.1.33) and the host itself (10.60.1.14) may connect; every other +source is dropped. + +Docker DNATs the published port to the container address and port 8080 +**before** `DOCKER-USER`, so a plain `-p tcp --dport 8090` rule never +matches. The rules match the original destination `10.60.1.14:8090` with +conntrack (`--ctorigdst 10.60.1.14 --ctorigdstport 8090 --ctdir ORIGINAL`) +and are tagged with the comment `mealquest-firewall`. The port is never +open while the rules change: if the chain does not already start with +exactly the three rules, the script first inserts the DROP at position 1, +then deletes its other tagged rules and inserts the two RETURN rules above +the DROP. A re-run on a correct chain changes nothing. NestQuest's +8080/8081 rules below them are left untouched. + +| Repository file | Installed as | +| --- | --- | +| `deploy/systemd/mealquest-firewall.sh` | `/usr/local/sbin/mealquest-firewall` (root, 0755) | +| `deploy/systemd/mealquest-firewall.service` | `/etc/systemd/system/mealquest-firewall.service` (root, 0644) | + +The unit is `Type=oneshot`, `RemainAfterExit=yes`, `Requires=` and +`PartOf=docker.service`, and `WantedBy=multi-user.target docker.service`: +it re-applies the rules whenever Docker starts or restarts (Docker +recreates `DOCKER-USER` empty), and a failure fails this unit only. + +Install (done on 10.60.1.14 on 2026-10-05): + +```sh +cd /home/overseer/mealquest +sudo install -o root -g root -m 0755 deploy/systemd/mealquest-firewall.sh /usr/local/sbin/mealquest-firewall +sudo install -o root -g root -m 0644 deploy/systemd/mealquest-firewall.service /etc/systemd/system/ +sudo systemctl daemon-reload +sudo systemctl enable --now mealquest-firewall +``` + +Check: + +```sh +systemctl is-enabled mealquest-firewall; systemctl is-active mealquest-firewall # enabled, active +sudo iptables -S DOCKER-USER +``` + +Expected head of the chain (recorded 2026-10-05; NestQuest's rules follow): + +``` +-A DOCKER-USER -s 10.60.1.33/32 -m conntrack --ctorigdst 10.60.1.14 --ctorigdstport 8090 --ctdir ORIGINAL -m comment --comment mealquest-firewall -j RETURN +-A DOCKER-USER -s 10.60.1.14/32 -m conntrack --ctorigdst 10.60.1.14 --ctorigdstport 8090 --ctdir ORIGINAL -m comment --comment mealquest-firewall -j RETURN +-A DOCKER-USER -m conntrack --ctorigdst 10.60.1.14 --ctorigdstport 8090 --ctdir ORIGINAL -m comment --comment mealquest-firewall -j DROP +``` + +From any other LAN machine, `curl -m 5 http://10.60.1.14:8090/health` must +time out; from 10.60.1.33 and 10.60.1.14 it returns 200. + +## Stack unit + +`deploy/systemd/mealquest.service` owns the compose stack and is the only +way it starts. Both compose services use `restart: "no"`, so Docker never +starts them on its own; the unit has `Requires=` and +`After=mealquest-firewall.service`, so the stack starts only after the +firewall rules are applied, and not at all if that unit fails. It is +`PartOf=docker.service`, so restarting Docker (which recreates +`DOCKER-USER` empty) stops the stack, re-applies the firewall, then starts +the stack again. `ExecStart` is `docker compose … up -d` and `ExecStop` is +`docker compose … down`; neither builds, so build before starting. + +Containers no longer restart after a crash; `systemctl restart mealquest` +brings them back. + +| Repository file | Installed as | +| --- | --- | +| `deploy/systemd/mealquest.service` | `/etc/systemd/system/mealquest.service` (root, 0644) | + +Install (done on 10.60.1.14 on 2026-10-05): + +```sh +cd /home/overseer/mealquest +sudo install -o root -g root -m 0644 deploy/systemd/mealquest.service /etc/systemd/system/ +sudo systemctl daemon-reload +sudo systemctl enable --now mealquest +``` + +Check: + +```sh +systemctl is-enabled mealquest; systemctl is-active mealquest # enabled, active +systemctl show mealquest -p Requires -p After # both list mealquest-firewall.service +``` + +Start, stop and restart the whole stack with `sudo systemctl start|stop|restart +mealquest`, never with `docker compose up` or `down`. Per-service compose +commands (`stop mealquest`, `up -d --force-recreate litestream`) are fine +while the unit is active, because the firewall is already in place. + +## Routine deploy and rollback + +Production runs `main` (a release); only `overseer` merges `dev` → `main`. +Every deploy builds the image on the host from the checked-out commit. + +### Deploy + +```sh +cd /home/overseer/mealquest +git fetch origin +git rev-parse --short HEAD # the SHA being replaced: note it for rollback +git checkout --detach origin/main # or a release tag +docker compose -f deploy/compose.yaml build +sudo systemctl restart mealquest # applies the firewall, then down + up -d +docker compose -f deploy/compose.yaml ps # mealquest healthy, litestream up +curl -fsS http://10.60.1.14:8090/health +curl -s -o /dev/null -w "%{http_code}\n" https://mealquest.cubecraftlabs.com/health # 200 +git rev-parse HEAD # the deployed SHA +``` + +Record the deployed SHA, the previous SHA and the date in the Maestro task +for the release. Migrations run at start-up; the service refuses to start +if the database has an applied migration it does not know. + +If `deploy/.env.example` gained a variable, add it to `deploy/.env` before +the restart; the service fails fast at boot naming any missing required variable. + +### Rollback to a previous SHA + +```sh +cd /home/overseer/mealquest +git checkout --detach +docker compose -f deploy/compose.yaml build +sudo systemctl restart mealquest +docker compose -f deploy/compose.yaml ps +curl -fsS http://10.60.1.14:8090/health +``` + +A rollback across a release that added a migration fails at start-up +(the older binary refuses the newer schema). In that case restore the +database to a point before the upgrade — "Restore the database" with +`-timestamp` set just before the deploy — then start the older SHA. +Changes made since that point are lost; say so to the household. + +## Backups + +### Database — Litestream + +The `litestream` compose service streams every committed change to +`mealquest.db` to the NAS over SFTP (`deploy/litestream.yml`). It takes a +full snapshot daily (compaction level 9) and keeps a week of snapshots +plus the incremental LTX files between them. + +One-time setup (already done on 10.60.1.14): + +1. Generate the dedicated keypair, git-ignored, owned by the container + user: + + ```sh + cd /home/overseer/mealquest + ssh-keygen -t ed25519 -N '' -C mealquest-litestream -f deploy/secrets/litestream_key + # append deploy/secrets/litestream_key.pub to joshua@10.60.1.50:~/.ssh/authorized_keys + rm deploy/secrets/litestream_key.pub + sudo chown 10001:10001 deploy/secrets/litestream_key + sudo chmod 0600 deploy/secrets/litestream_key + ``` + +2. The NAS host key is pinned in `deploy/litestream.yml` (`host-key`). + +Check replication — list every LTX file at every level. Level 9 holds the +full snapshots; the default listing (`-level 0`) shows only L0 files: + +```sh +docker compose -f deploy/compose.yaml exec -T litestream \ + litestream ltx -config /etc/litestream.yml -level all /data/mealquest.db +``` + +### Media — nightly rsync + +`mealquest-media-backup.timer` runs `mealquest-media-backup.service` at +03:30 every night. The service runs `/usr/local/sbin/mealquest-media-backup` +(`deploy/systemd/mealquest-media-backup.sh`) as the unprivileged host user +`mealquest` (uid/gid 10001, the app's own user) with no capabilities. +systemd bind-mounts the data volume read-only at `/run/mealquest-data` and +passes the Litestream key in as the credential `nas_key`; uid 10001 cannot +otherwise reach `/var/lib/docker` or `deploy/secrets/`. The script refuses a +`media` path that is a symlink and fails (exit 1) if `media` is missing. + +One-time setup (already done on 10.60.1.14): + +```sh +cd /home/overseer/mealquest +# Host user for the backup; ssh needs a passwd entry for its uid. +sudo groupadd --system --gid 10001 mealquest +sudo useradd --system --uid 10001 --gid 10001 --no-create-home \ + --home-dir /nonexistent --shell /usr/sbin/nologin mealquest +# Pinned NAS host keys, root-owned and world-readable. +sudo install -d -o root -g root -m 0755 /etc/mealquest +sudo install -o root -g root -m 0644 deploy/known_hosts /etc/mealquest/known_hosts +# The media directory; the backup fails if it is ever missing. +sudo install -d -o 10001 -g 10001 -m 0755 /var/lib/docker/volumes/mealquest-data/_data/media +# Script and units. +sudo install -o root -g root -m 0755 deploy/systemd/mealquest-media-backup.sh /usr/local/sbin/mealquest-media-backup +sudo install -o root -g root -m 0644 deploy/systemd/mealquest-media-backup.service \ + deploy/systemd/mealquest-media-backup.timer /etc/systemd/system/ +sudo systemctl daemon-reload +sudo systemctl enable --now mealquest-media-backup.timer +``` + +Run it by hand and read the result: + +```sh +sudo systemctl start mealquest-media-backup.service +journalctl -u mealquest-media-backup --no-pager -n 30 +``` + +To restore media, follow "Restore photos (media)" — never copy back while +the timer or the service can still run. + +## Restore the database + +SQLite in WAL mode keeps recent commits in `mealquest.db-wal`, indexed by +`mealquest.db-shm`. **A restored database must never sit beside an old WAL +or shared-memory file**: on open, SQLite would replay the old WAL's pages +onto the restored database and corrupt it or mix in state from the +database it replaced. So all three files are moved aside together, with +Litestream's local state directory `.mealquest.db-litestream`, before the +restore — and the restore runs into an empty volume. + +Run from `/home/overseer/mealquest`: + +```sh +D=/var/lib/docker/volumes/mealquest-data/_data +A=/var/tmp/mealquest-restore-aside-$(date +%Y%m%d%H%M) + +# 1. Stop the whole stack (app and Litestream). Nothing may hold the +# database open; a clean close checkpoints the WAL into mealquest.db. +sudo systemctl stop mealquest + +# 2. Move the database, its -wal and -shm files and Litestream's state +# aside — all of them, every time. Keep the directory until the +# restored stack is verified. +sudo mkdir "$A" +for f in mealquest.db mealquest.db-wal mealquest.db-shm .mealquest.db-litestream; do + if sudo test -e "$D/$f"; then sudo mv "$D/$f" "$A/"; fi +done +sudo ls -la "$D" # must list only media/ + +# 3. Restore from the NAS. --no-deps keeps the app stopped so it cannot +# create a fresh, empty database first. +docker compose -f deploy/compose.yaml run --rm --no-deps litestream \ + restore -config /etc/litestream.yml -o /data/mealquest.db /data/mealquest.db +sudo ls -la "$D" # mealquest.db only: no -wal, no -shm + +# 4. Check the restored file before starting the app (see "Count check"). + +# 5. Start the stack and check health. +sudo systemctl start mealquest +docker compose -f deploy/compose.yaml ps +curl -s -o /dev/null -w "health %{http_code}\n" http://10.60.1.14:8090/health +``` + +To restore to a point in time, add `-timestamp 2026-10-05T23:00:00Z` (or +`-txid`) to the `restore` command. + +### Count check + +`/tmp/mqcount.py` prints row counts for key tables, the number of +application tables, a SHA-256 over every application row (first 16 hex +digits; Litestream's own `_litestream*` tables are excluded) and +`PRAGMA integrity_check`. Run it on a **copy** of the database, never on the +live file; copy `-wal`/`-shm` along with it when the app is running. + +```python +import sqlite3, sys, hashlib +c = sqlite3.connect(sys.argv[1]) +tabs = [r[0] for r in c.execute("select name from sqlite_master where type=\"table\" and name not like \"sqlite_%\" and name not like \"_litestream%\" order by name")] +for t in ("dinners", "week_plans", "schema_migrations"): + print(t, c.execute(f"select count(*) from {t}").fetchone()[0]) +h = hashlib.sha256() +for t in tabs: + for row in c.execute(f"select * from \"{t}\" order by rowid"): + h.update(repr(row).encode()) +print("tables", len(tabs), "content-sha256", h.hexdigest()[:16]) +print("integrity", c.execute("pragma integrity_check").fetchone()[0]) +``` + +```sh +rm -rf /tmp/mqcheck && mkdir /tmp/mqcheck +sudo sh -c "cp -a $D/mealquest.db* /tmp/mqcheck/" && sudo chown -R "$USER" /tmp/mqcheck +python3 /tmp/mqcount.py /tmp/mqcheck/mealquest.db +``` + +Before and after must match exactly. + +## Restore photos (media) + +Recipe and dinner photos live in `media/` in the data volume and are +backed up nightly to `joshua@10.60.1.50:/volume1/homes/Joshua/mealquest-backups/media/` +(see "Media — nightly rsync"). The backup service mirrors with +`rsync --delete`. **An `rsync --delete` run must never see an empty or +partial `media/`**: it would delete the NAS copy you are about to restore +from. Stopping the timer is not enough — it does not stop a run that is +already in flight, and the service can still be started by hand. So stop +**both** the timer and the service, confirm no rsync is running, and do +not start the service until the restore is verified. + +Run from `/home/overseer/mealquest`: + +```sh +M=/var/lib/docker/volumes/mealquest-data/_data/media +NAS=/volume1/homes/Joshua/mealquest-backups/media +SSH="ssh -i deploy/secrets/litestream_key -o IdentitiesOnly=yes -o UserKnownHostsFile=/etc/mealquest/known_hosts -o StrictHostKeyChecking=yes" + +# 1. Stop the nightly mirror: the timer AND any run in flight. +sudo systemctl stop mealquest-media-backup.timer mealquest-media-backup.service + +# 2. Pre-check — all three must hold before media/ is touched. If any +# fails, stop here and find out why. +systemctl is-active mealquest-media-backup.timer # inactive +systemctl is-active mealquest-media-backup.service # inactive (or failed) +pgrep -a -u mealquest rsync || echo "no rsync" # no rsync +sudo $SSH joshua@10.60.1.50 "find $NAS -type f | wc -l" +# ^ note this NAS file count; it must be non-zero and is the target below. + +# 3. Stop the app (no writes into media/ meanwhile). +docker compose -f deploy/compose.yaml stop mealquest + +# 4. Recreate the directory if it is gone, owned by the app user. +sudo install -d -o 10001 -g 10001 -m 0755 "$M" + +# 5. Copy back from the NAS as uid 10001's files. No --delete in this +# direction: the NAS copy is the source and is only read. +sudo rsync -a --stats --chown=10001:10001 -e "$SSH" "joshua@10.60.1.50:$NAS/" "$M/" + +# 6. Post-condition — before anything can mirror again: +sudo find "$M" -type f | wc -l # >= the NAS count from step 2 +sudo $SSH joshua@10.60.1.50 "find $NAS -type f | wc -l" # unchanged from step 2 + +# 7. Start the app, then the timer again. Only now may the service run. +docker compose -f deploy/compose.yaml up -d +curl -fsS http://10.60.1.14:8090/health +sudo systemctl start mealquest-media-backup.timer +systemctl is-active mealquest-media-backup.timer # active +``` + +The local count may exceed the NAS count only if `media/` still held +files the NAS never received (photos added since the last nightly run); +the next nightly run then copies them up. If the local count is lower, +the copy did not complete: leave the timer stopped and re-run step 5. + +When the database is restored too, restore it first ("Restore the +database"), then the media, so every photo the database refers to is +present. This media restore has not been run against production yet; the +backup direction was checked on 2026-10-05 (see "Media backup checks"). + +## Restore drill record — 2026-10-05 + +Drill on 10.60.1.14 against the live (pre-launch, empty-household) +database, Litestream 0.5.17. Commands as in "Restore the database"; +output trimmed only of compose progress lines. + +Replica before the drill (`ltx -level all`; level 9 is the full snapshot): + +``` +level min_txid max_txid size created +0 0000000000000003 0000000000000003 14110 2026-10-05T23:13:30Z +1 0000000000000001 0000000000000002 14110 2026-10-05T23:12:38Z +1 0000000000000003 0000000000000003 14110 2026-10-05T23:13:30Z +2 0000000000000001 0000000000000002 14110 2026-10-05T23:12:38Z +2 0000000000000003 0000000000000003 14110 2026-10-05T23:13:30Z +3 0000000000000001 0000000000000002 14110 2026-10-05T23:12:38Z +9 0000000000000001 0000000000000002 14110 2026-10-05T23:12:38Z +``` + +1. `docker compose -f deploy/compose.yaml down` — both containers stopped + and removed. The volume then held `mealquest.db`, + `.mealquest.db-litestream/` and `media/`; the clean close had + checkpointed and removed `-wal` and `-shm`. +2. Before (copy of the stopped database): + + ``` + dinners 0 + week_plans 0 + schema_migrations 7 + tables 26 content-sha256 836e230e16c58dc7 + integrity ok + ``` + +3. Moved aside to `/var/tmp/mealquest-restore-aside-20261005`: + `mealquest.db` and `.mealquest.db-litestream` (no `-wal`/`-shm` existed + after the clean stop; the loop moves them whenever they do). Volume + afterwards: `media/` only. +4. `docker compose -f deploy/compose.yaml run --rm --no-deps litestream restore -config /etc/litestream.yml -o /data/mealquest.db /data/mealquest.db` + — volume afterwards: `mealquest.db` (258048 bytes) and `media/`; no + `-wal`, no `-shm`. +5. After (copy of the restored file, before the app started): + + ``` + dinners 0 + week_plans 0 + schema_migrations 7 + tables 26 content-sha256 836e230e16c58dc7 + integrity ok + ``` + +6. `docker compose -f deploy/compose.yaml up -d` — `mealquest-mealquest-1` + healthy, `mealquest-litestream-1` up; `GET /health` → `200`. Counts on a + copy of the running database (`mealquest.db`, `-wal`, `-shm`) were + identical to the above. +7. Litestream resumed against the restored database: + + ``` + level=INFO msg="detected database behind replica" db=mealquest.db db_txid=0000000000000000 replica_txid=0000000000000003 + level=INFO msg="fetched latest L0 file from replica" db=mealquest.db min_txid=0000000000000003 max_txid=0000000000000003 + level=INFO msg="compaction complete" db=mealquest.db level=1 txid.min=0000000000000004 txid.max=0000000000000004 size=14110 + ``` + + Replica after the drill (`ltx -level all`): + + ``` + level min_txid max_txid size created + 0 0000000000000004 0000000000000004 14110 2026-10-05T23:27:21Z + 1 0000000000000001 0000000000000002 14110 2026-10-05T23:12:38Z + 1 0000000000000003 0000000000000003 14110 2026-10-05T23:13:30Z + 1 0000000000000004 0000000000000004 14110 2026-10-05T23:27:21Z + 2 0000000000000001 0000000000000002 14110 2026-10-05T23:12:38Z + 2 0000000000000003 0000000000000003 14110 2026-10-05T23:13:30Z + 3 0000000000000001 0000000000000002 14110 2026-10-05T23:12:38Z + 9 0000000000000001 0000000000000002 14110 2026-10-05T23:12:38Z + ``` + +The first drill earlier the same day (before `-level all` was recorded) +gave the same before/after result: checksum `836e230e16c58dc7`, 26 tables, +integrity ok. + +### Media backup checks — 2026-10-05 + +`sudo systemctl start mealquest-media-backup.service` on 10.60.1.14: + +| Case | Result | +| --- | --- | +| Normal run | `Result=success`; journal: `synced /run/mealquest-data/media to joshua@10.60.1.50:/volume1/homes/Joshua/mealquest-backups/media/ as uid 10001` | +| A test file added | `Number of regular files transferred: 1`; removing it then gave `Number of deleted files: 1 (reg: 1)` | +| `media` replaced by a symlink to `/etc` | `Result=exit-code`, status 1; `/run/mealquest-data/media is a symlink to /etc; refusing to back it up` | +| `media` removed | `Result=exit-code`, status 1; `media directory /run/mealquest-data/media is missing` | +| `sudo -u mealquest` writing into the volume via `/var/lib/docker/...` | `Permission denied` — uid 10001 cannot traverse `/var/lib/docker` (mode 0710 root) | + +## Secret rotation + +Rotate a secret when it may have leaked, when someone with access leaves, +or on the owner's schedule. Never paste a secret into a ticket, chat or +this file. + +### App key (`deploy/secrets/secret_key`) + +The app key encrypts every stored secret: the `_ct` columns +`ai_settings.api_key_ct` (LLM API key), `recipe_search_settings.api_key_ct` +(recipe-search API key), `integrations_google.refresh_token_ct` (Google +Calendar refresh token) and `chat_settings.webhook_url_ct` (Google Chat +webhook URL). **Rotating it re-encrypts nothing**: a value encrypted with +the old key can no longer be read. A `mealquest rotate-key` helper that +re-encrypts in place is planned but does not exist yet. + +**What is recoverable today.** The service currently exposes only +household settings; no shipped flow writes or re-enters any of these four +secrets. The re-entry flows are planned: the LLM key in Settings → AI +assistant (the AI settings feature), the recipe-search key (Feature 10), +Google Calendar reconnect (Feature 16) and the Google Chat webhook +(Feature 19). So today the rotation is safe **only if no `_ct` value is +stored**, which the pre-check below proves. If the pre-check finds a +stored value, do not rotate: it could not be re-entered afterwards. Keep +the current key and wait for the matching flow (or the `rotate-key` +helper). + +**The pre-check and the key replacement run with the whole stack +stopped**, from the pre-check until the new key is in place. A raw copy of +`mealquest.db` and `mealquest.db-wal` taken while the stack runs is **not** +a consistent snapshot: the two files are copied one after the other, and a +checkpoint (by the app or by Litestream) between the two copies can move +committed pages out of the WAL into `mealquest.db` after `mealquest.db` was +already copied, then reset the WAL before it is copied. The copy then +lacks those commits — possibly a stored `_ct` value — and the pre-check +reports a false 0. A running app could also store a `_ct` value after a +correct pre-check and before the key is replaced. Stopping both containers +removes every writer (Litestream also writes and checkpoints), and the +clean close checkpoints the WAL into `mealquest.db`, so the file alone is +complete. + +Run from `/home/overseer/mealquest`: + +```sh +D=/var/lib/docker/volumes/mealquest-data/_data + +# 1. Stop the whole stack (app and Litestream) and leave it stopped until +# step 4. Nothing may hold the database open. +sudo systemctl stop mealquest +sudo ls -la "$D" # mealquest.db and media/ only: no -wal, no -shm; if + # either exists, the close was not clean — stop here + +# 2. Pre-check, on a copy of the now-quiescent database. +rm -rf /tmp/mqcheck && mkdir /tmp/mqcheck +sudo cp -a "$D/mealquest.db" /tmp/mqcheck/ && sudo chown -R "$USER" /tmp/mqcheck +python3 - /tmp/mqcheck/mealquest.db <<'PY' +import sqlite3, sys +c = sqlite3.connect(sys.argv[1]) +for t, col in (("ai_settings", "api_key_ct"), ("recipe_search_settings", "api_key_ct"), + ("integrations_google", "refresh_token_ct"), ("chat_settings", "webhook_url_ct")): + print(t, col, c.execute(f"select count(*) from {t} where {col} is not null").fetchone()[0]) +PY +rm -rf /tmp/mqcheck +# every count must be 0; otherwise start the stack again on the current +# key (sudo systemctl start mealquest) and stop here + +# 3. Replace the key, still with the stack stopped. +sudo cp -a deploy/secrets/secret_key /var/tmp/secret_key.old # keep until verified, then shred +head -c 32 /dev/urandom | base64 | sudo tee deploy/secrets/secret_key >/dev/null +sudo chown 10001:10001 deploy/secrets/secret_key +sudo chmod 600 deploy/secrets/secret_key + +# 4. Start the stack on the new key and check health. +sudo systemctl start mealquest +docker compose -f deploy/compose.yaml ps +curl -fsS http://10.60.1.14:8090/health +``` + +Then sign in and open household settings to confirm the service runs on +the new key. Finally `sudo shred -u /var/tmp/secret_key.old`. Once the +planned flows ship, this section gains a step per secret to re-enter it +and confirm it works. + +### OIDC client secret (`deploy/secrets/oidc_client_secret`) + +In Authentik, open the provider `mealquest` and regenerate the Client +Secret. Write it to the file (same owner and mode as above) and recreate +the service: + +```sh +printf '%s\n' '' | sudo tee deploy/secrets/oidc_client_secret >/dev/null +sudo chown 10001:10001 deploy/secrets/oidc_client_secret +sudo chmod 600 deploy/secrets/oidc_client_secret +docker compose -f deploy/compose.yaml up -d --force-recreate mealquest +``` + +Sign in at to confirm; sign-ins +fail between regenerating the secret in Authentik and recreating the +service. + +### LLM API key + +Stored encrypted in the database, not in `deploy/`. In the app, open +**Settings → AI assistant**, choose **Replace**, paste the new key and +**Verify**, and check the verified state is shown. Then revoke the +old key in the provider's console. (The AI assistant settings page is +delivered by its own feature; until it ships there is no stored LLM key +to rotate.) + +### VAPID keys (Web Push) + +Planned with Web Push (Feature 19): the public key goes in +`deploy/.env` as `MEALQUEST_VAPID_PUBLIC_KEY`, the private key in +`deploy/secrets/vapid_private_key` (owner `10001:10001`, mode 0600), and +the pair is generated with `cmd/vapidgen`. **Changing the VAPID pair +invalidates every push subscription**: each device must turn +notifications on again (`/settings/notifications`) before it receives +pushes. Rotate only when the private key may have leaked, write both +halves, recreate the service with `--force-recreate mealquest`, and tell +the household to re-enable notifications on each device. + +### Litestream / NAS key (`deploy/secrets/litestream_key`) + +Generate a new keypair as in "Database — Litestream", add the new public +key to `joshua@10.60.1.50:~/.ssh/authorized_keys`, replace the file +(owner `10001:10001`, mode 0600), run +`docker compose -f deploy/compose.yaml up -d --force-recreate litestream` +and `sudo systemctl start mealquest-media-backup.service`, check both +(`litestream ltx` and the journal), then remove the old public key from +the NAS. + +## Context records + +The confirmed values in this runbook (port 8090, checkout path, Authentik +group `mealquest-admin`, DNS, firewall, backup target) are also recorded in +the CTXD project `mealquest`: `RUNBOOKS.MD` (deployment procedure) and the +open questions in `CONTEXT.MD`, updated for Feature 07 task 876134bc. diff --git a/deploy/compose.yaml b/deploy/compose.yaml new file mode 100644 index 0000000..c5bae84 --- /dev/null +++ b/deploy/compose.yaml @@ -0,0 +1,100 @@ +# Production stack for MealQuest on 10.60.1.14 (D-018). Build on the host, +# then start through systemd: +# +# docker compose -f deploy/compose.yaml build +# sudo systemctl restart mealquest +# +# The services use restart: "no" on purpose: Docker must not start them +# before the DOCKER-USER firewall is applied. deploy/systemd/mealquest.service +# owns start and stop and requires mealquest-firewall.service first. +# +# Configuration comes from deploy/.env (copy deploy/.env.example) and secrets +# from deploy/secrets/; neither is committed. Compose bind-mounts file secrets +# with their host owner and mode, so each file in deploy/secrets/ must be +# owned by the container user 10001:10001 with mode 0600. +name: mealquest + +services: + mealquest: + build: + context: .. + dockerfile: Dockerfile + user: "10001:10001" + restart: "no" + env_file: + - path: .env + required: true + # Fixed by this file's container shape (listen port, volume, secret + # mounts); deploy/.env must not set them. + environment: + MEALQUEST_HTTP_ADDR: ":8080" + MEALQUEST_DATA_DIR: /data + MEALQUEST_SECRET_KEY_FILE: /run/secrets/secret_key + MEALQUEST_OIDC_CLIENT_SECRET_FILE: /run/secrets/oidc_client_secret + secrets: + - secret_key + - oidc_client_secret + volumes: + - mealquest-data:/data + ports: + - "${MEALQUEST_BIND_ADDRESS:?set MEALQUEST_BIND_ADDRESS in deploy/.env}:8090:8080" + networks: + - mealquest + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/health"] + interval: 30s + timeout: 3s + start_period: 10s + retries: 3 + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + + # Litestream streams the database to the Synology NAS over SFTP + # (config: deploy/litestream.yml). The SFTP private key + # deploy/secrets/litestream_key is a dedicated keypair, owned by + # 10001:10001 with mode 0600 like the other secrets. + # + # The data volume is mounted read-write on purpose: Litestream writes its + # tracking tables into the database and checkpoints the WAL. It runs as + # the app's own user (owner of the volume) with a read-only root + # filesystem, no privilege escalation and no Linux capabilities. + # + # Restore: follow "Restore the database" in deploy/RUNBOOK.md (stop the + # stack, move the database with its -wal and -shm aside, restore, check). + litestream: + image: litestream/litestream:0.5.17 + user: "10001:10001" + restart: "no" + command: ["replicate", "-config", "/etc/litestream.yml"] + depends_on: + mealquest: + condition: service_healthy + volumes: + - mealquest-data:/data + - ./litestream.yml:/etc/litestream.yml:ro + - ./secrets/litestream_key:/etc/litestream/sftp_key:ro + networks: + - mealquest + read_only: true + tmpfs: + - /tmp + security_opt: + - no-new-privileges:true + cap_drop: + - ALL + +secrets: + secret_key: + file: ./secrets/secret_key + oidc_client_secret: + file: ./secrets/oidc_client_secret + +volumes: + mealquest-data: + name: mealquest-data + +networks: + mealquest: + name: mealquest diff --git a/deploy/known_hosts b/deploy/known_hosts new file mode 100644 index 0000000..1b17352 --- /dev/null +++ b/deploy/known_hosts @@ -0,0 +1,6 @@ +# Host keys of the Synology NAS (10.60.1.50), the backup target. Installed +# root-owned with mode 0644 as /etc/mealquest/known_hosts and pinned by the +# media backup (UserKnownHostsFile, StrictHostKeyChecking=yes). The ECDSA +# key is also pinned in deploy/litestream.yml (host-key). +10.60.1.50 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKHxIfgjavBABMRE5GW/EFY4C4gO6mo0sKbhkUW2NcAo +10.60.1.50 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNujay9nYEiXuRQf4wDRDU1gp+CDvpeMdhuhC2XypIihzKGEtB654kwrUGfA2kDWAoSPGlmojeS3OIRtHPkj6GY= diff --git a/deploy/litestream.yml b/deploy/litestream.yml new file mode 100644 index 0000000..a825cea --- /dev/null +++ b/deploy/litestream.yml @@ -0,0 +1,31 @@ +# Litestream continuous backup of the MealQuest SQLite database (Feature +# 07). The ``litestream`` compose service mounts the same named volume as +# the app and streams every committed change to the Synology NAS over +# SFTP, so the backup lives outside the Docker host. +# +# Nothing here is secret. The SFTP private key is a dedicated, git-ignored +# keypair (deploy/secrets/litestream_key) mounted read-only by compose at +# /etc/litestream/sftp_key; its public half is in +# joshua@10.60.1.50:~/.ssh/authorized_keys. ``host-key`` pins the NAS's +# ECDSA host key, the algorithm Litestream's SSH client negotiates with +# this NAS. +# +# The Synology SFTP server is chrooted to the shared folders, so the +# directory seen over SSH as /volume1/homes/Joshua/mealquest-backups is +# /homes/Joshua/mealquest-backups over SFTP. + +# Take a full snapshot daily and keep a week of snapshots plus the +# incremental changes between them (point-in-time restore window). +snapshot: + interval: 24h + retention: 168h + +dbs: + - path: /data/mealquest.db + replicas: + - type: sftp + host: 10.60.1.50 + user: joshua + path: /homes/Joshua/mealquest-backups + key-path: /etc/litestream/sftp_key + host-key: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNujay9nYEiXuRQf4wDRDU1gp+CDvpeMdhuhC2XypIihzKGEtB654kwrUGfA2kDWAoSPGlmojeS3OIRtHPkj6GY= diff --git a/deploy/systemd/mealquest-firewall.service b/deploy/systemd/mealquest-firewall.service new file mode 100644 index 0000000..c7c5f8e --- /dev/null +++ b/deploy/systemd/mealquest-firewall.service @@ -0,0 +1,22 @@ +# Applies the MealQuest DOCKER-USER rules (/usr/local/sbin/mealquest-firewall). +# +# PartOf= re-runs this unit whenever docker.service is stopped or restarted, +# and WantedBy=docker.service starts it whenever docker.service starts. Both +# are one-way: a failure here fails this unit visibly and never fails +# docker.service. +# +# Start MealQuest only after this unit has succeeded: +# systemctl start mealquest-firewall && docker compose up -d +[Unit] +Description=Restrict MealQuest port 8090 to Traefik (DOCKER-USER) +After=docker.service +Requires=docker.service +PartOf=docker.service + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/local/sbin/mealquest-firewall + +[Install] +WantedBy=multi-user.target docker.service diff --git a/deploy/systemd/mealquest-firewall.sh b/deploy/systemd/mealquest-firewall.sh new file mode 100755 index 0000000..788baa8 --- /dev/null +++ b/deploy/systemd/mealquest-firewall.sh @@ -0,0 +1,53 @@ +#!/bin/sh +# Restrict the MealQuest published host port (10.60.1.14:8090) to Traefik +# (10.60.1.33) and the Docker host itself (10.60.1.14); drop every other +# source. +# +# DOCKER-USER runs after Docker's DNAT, so the destination there is the +# container address and port, not the published ones. Match the original +# destination address and port with conntrack instead of -d/--dport. +# +# The rules must sit at the top of DOCKER-USER, ahead of any other project's +# rules (an earlier RETURN or DROP would decide MealQuest traffic first). +# The port is never left open while the rules are updated: +# +# 1. If the chain already starts with exactly the three rules below and +# holds no other mealquest-firewall rule, change nothing. +# 2. Otherwise insert the DROP at position 1 first, so the port is closed +# to every source (including Traefik, briefly) from here on. +# 3. Delete every other rule tagged mealquest-firewall, by rule number, +# highest first. +# 4. Insert the two RETURN rules above the DROP. +# +# Re-runs leave the same ordered rules at positions 1-3 and no duplicates. +# Installed as /usr/local/sbin/mealquest-firewall. +set -eu + +chain=DOCKER-USER +tag="-m comment --comment mealquest-firewall" +match="-m conntrack --ctorigdst 10.60.1.14 --ctorigdstport 8090 --ctdir ORIGINAL" + +# The three rules as iptables -S prints them, in their required order. +want="-A $chain -s 10.60.1.33/32 $match $tag -j RETURN +-A $chain -s 10.60.1.14/32 $match $tag -j RETURN +-A $chain $match $tag -j DROP" + +rules=$(iptables -w -S "$chain" | grep -e '^-A ' || true) +head=$(printf '%s\n' "$rules" | head -n 3) +tagged=$(printf '%s\n' "$rules" | grep -c -e '--comment mealquest-firewall ' || true) +if [ "$head" = "$want" ] && [ "$tagged" -eq 3 ]; then + exit 0 +fi + +iptables -w -I "$chain" 1 $match $tag -j DROP + +iptables -w -S "$chain" | grep -e '^-A ' | + awk '/--comment mealquest-firewall / { print NR }' | + sort -rn | while read -r num; do + if [ "$num" -ne 1 ]; then + iptables -w -D "$chain" "$num" + fi +done + +iptables -w -I "$chain" 1 -s 10.60.1.14/32 $match $tag -j RETURN +iptables -w -I "$chain" 1 -s 10.60.1.33/32 $match $tag -j RETURN diff --git a/deploy/systemd/mealquest-media-backup.service b/deploy/systemd/mealquest-media-backup.service new file mode 100644 index 0000000..9f5a61c --- /dev/null +++ b/deploy/systemd/mealquest-media-backup.service @@ -0,0 +1,29 @@ +# Runs the nightly MealQuest media backup (/usr/local/sbin/mealquest-media-backup). +# Triggered by mealquest-media-backup.timer; run by hand with +# systemctl start mealquest-media-backup +# and read the result with +# journalctl -u mealquest-media-backup +# +# The script runs as the app's user (10001, host user "mealquest") with no +# capabilities. systemd, as root, only bind-mounts the data volume's root +# (a root-owned path the app cannot replace) read-only at +# /run/mealquest-data and copies the NAS key into the credentials directory. +[Unit] +Description=Back up MealQuest media to the Synology NAS +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/mealquest-media-backup +User=mealquest +Group=mealquest +LoadCredential=nas_key:/home/overseer/mealquest/deploy/secrets/litestream_key +BindReadOnlyPaths=/var/lib/docker/volumes/mealquest-data/_data:/run/mealquest-data +ProtectSystem=strict +ProtectHome=yes +PrivateTmp=yes +PrivateDevices=yes +NoNewPrivileges=yes +CapabilityBoundingSet= +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX diff --git a/deploy/systemd/mealquest-media-backup.sh b/deploy/systemd/mealquest-media-backup.sh new file mode 100755 index 0000000..f4ee14d --- /dev/null +++ b/deploy/systemd/mealquest-media-backup.sh @@ -0,0 +1,44 @@ +#!/bin/sh +# Nightly copy of the MealQuest media directory (recipe and dinner photos) +# to the Synology NAS. The database itself is replicated continuously by +# Litestream; this covers the files beside it. +# +# Runs as the unprivileged app user (10001, host user "mealquest"), never +# root: mealquest-media-backup.service bind-mounts the data volume +# read-only at /run/mealquest-data and hands the NAS key in as a systemd +# credential. Anything the app could plant in the volume is therefore read +# with the app's own permissions only. +# +# The script changes into the media directory and checks that the physical +# path is the expected one, so a media path that is a symlink is refused. +# rsync then copies "./" — the directory the shell is already in — so +# swapping the path afterwards cannot redirect it. Symlinks inside the +# directory are copied as links (-a), never followed. +# +# The media directory is created at deploy time (deploy/RUNBOOK.md). If it +# is missing the job fails; it is never treated as "nothing to back up". +# +# rsync mirrors into .../mealquest-backups/media/ only: the target ends in +# a slash, so --delete removes files from that directory and nothing +# outside it. Re-runs transfer only what changed. Output goes to the +# journal via mealquest-media-backup.service. +set -eu + +src=/run/mealquest-data/media +dest=joshua@10.60.1.50:/volume1/homes/Joshua/mealquest-backups/media/ +key=$CREDENTIALS_DIRECTORY/nas_key +known_hosts=/etc/mealquest/known_hosts + +if ! cd "$src"; then + echo "mealquest-media-backup: media directory $src is missing" >&2 + exit 1 +fi +if [ "$(pwd -P)" != "$src" ]; then + echo "mealquest-media-backup: $src is a symlink to $(pwd -P); refusing to back it up" >&2 + exit 1 +fi + +rsync -a --delete --stats \ + -e "ssh -F none -i $key -o IdentitiesOnly=yes -o BatchMode=yes -o UserKnownHostsFile=$known_hosts -o GlobalKnownHostsFile=/dev/null -o StrictHostKeyChecking=yes" \ + ./ "$dest" +echo "mealquest-media-backup: synced $src to $dest as uid $(id -u)" diff --git a/deploy/systemd/mealquest-media-backup.timer b/deploy/systemd/mealquest-media-backup.timer new file mode 100644 index 0000000..62fe968 --- /dev/null +++ b/deploy/systemd/mealquest-media-backup.timer @@ -0,0 +1,11 @@ +# Nightly trigger for mealquest-media-backup.service. Persistent= runs a +# missed backup at the next boot. +[Unit] +Description=Nightly MealQuest media backup + +[Timer] +OnCalendar=*-*-* 03:30:00 +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/deploy/systemd/mealquest.service b/deploy/systemd/mealquest.service new file mode 100644 index 0000000..4db5d55 --- /dev/null +++ b/deploy/systemd/mealquest.service @@ -0,0 +1,22 @@ +# Owns the MealQuest compose stack (deploy/compose.yaml). The services use +# restart: "no", so Docker never starts them on its own; this unit is the +# only start path. +# +# Requires= and After=mealquest-firewall.service start the stack only after +# the DOCKER-USER rules are applied, and not at all if that unit fails. +# PartOf=docker.service propagates a Docker stop or restart here, so a +# Docker restart re-applies the firewall before the stack comes back. +[Unit] +Description=MealQuest compose stack +Requires=mealquest-firewall.service +After=mealquest-firewall.service docker.service +PartOf=docker.service + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/bin/docker compose -f /home/overseer/mealquest/deploy/compose.yaml up -d +ExecStop=/usr/bin/docker compose -f /home/overseer/mealquest/deploy/compose.yaml down + +[Install] +WantedBy=multi-user.target