Feature 24: Automated deploys (pull-based, runbook-faithful) (#210)

Co-authored-by: orchestrator-agent <20+orchestrator-agent@code.cubecraftcreations.com>

deploy/auto-deploy.sh

@@ -0,0 +1,251 @@
+#!/usr/bin/env bash
+# MealQuest automated deploy: deploy/RUNBOOK.md, run unattended.
+#
+#   mealquest-auto-deploy              deploy origin/main if it is new and CI passed
+#   mealquest-auto-deploy --ref <ref>  deploy <ref> now, bypassing the CI and no-op gates
+#
+# Installed as /usr/local/sbin/mealquest-auto-deploy and run as overseer by
+# mealquest-auto-deploy.service (deploy/systemd); install, semantics and the
+# manual override: deploy/RUNBOOK.md "Automated deploys". Run as root (sudo)
+# it re-executes itself as overseer.
+#
+# Without --ref: exit 0 with no output when origin/main is the last deployed
+# SHA in the deploy record or its CI is pending; exit 1, deploying nothing,
+# when its CI failed or reports no status, or when the last attempt to deploy
+# that SHA failed. Every deploy runs the runbook's blocks in order and stops
+# at the first failure; nothing before the restart touches production. A
+# verified deploy appends one line to the deploy record:
+#   <UTC date> previous=<SHA> deployed=<SHA> restore_point=<UTC date>
+# a failed one appends:
+#   <UTC date> FAILED target=<SHA> status=<exit status>
+set -euo pipefail
+
+CHECKOUT=/home/overseer/mealquest
+RECORD=/home/overseer/mealquest-auto-deploy.log
+LOCK=/home/overseer/mealquest-auto-deploy.lock
+STATUS_API=https://code.cubecraftlabs.com/api/v1/repos/Maniacs_Mansion/MealQuest/commits
+PUBLIC=https://mealquest.cubecraftlabs.com
+D=/var/lib/docker/volumes/mealquest-data/_data
+COMPOSE=(docker compose -f deploy/compose.yaml)
+
+fail() { echo "$*" >&2; exit 1; }
+
+if [ "$(id -u)" -eq 0 ]; then
+  exec sudo -u overseer -H -- "$0" "$@"
+fi
+[ "$(id -un)" = overseer ] || fail "run as overseer (or root), not $(id -un)"
+
+ref=
+case "$#" in
+  0) ;;
+  2) [ "$1" = --ref ] && [ -n "$2" ] || fail "usage: $0 [--ref <ref>]"; ref=$2 ;;
+  *) fail "usage: $0 [--ref <ref>]" ;;
+esac
+
+exec 9>> "$LOCK"
+flock -n 9 || fail "another mealquest-auto-deploy run holds $LOCK"
+
+cd "$CHECKOUT"
+git fetch --quiet --tags origin || fail "git fetch origin failed"
+
+if [ -n "$ref" ]; then
+  target=$(git rev-parse --verify --quiet "$ref^{commit}") || fail "$ref is not a commit in $CHECKOUT"
+else
+  # The full remote-tracking ref: a bare origin/main resolves a tag or local
+  # branch of that name first, which could deploy a commit not on main.
+  target=$(git rev-parse --verify --quiet "refs/remotes/origin/main^{commit}") \
+    || fail "refs/remotes/origin/main is not a commit in $CHECKOUT"
+  [ -f "$RECORD" ] || fail "no deploy record at $RECORD: seed it with one manual deploy (deploy/RUNBOOK.md \"Automated deploys\")"
+  last=$(grep -oE 'deployed=[0-9a-f]{40}' "$RECORD" | tail -n 1 | cut -d = -f 2) || true
+  [ -n "$last" ] || fail "$RECORD records no deploy: seed it with one manual deploy (deploy/RUNBOOK.md \"Automated deploys\")"
+  [ "$target" != "$last" ] || exit 0
+  if tail -n 1 "$RECORD" | grep -q " FAILED target=$target status="; then
+    fail "auto-deploy of $target already failed (see $RECORD and the journal): fix it, then deploy with --ref"
+  fi
+  state=$(curl -fsS --max-time 30 "$STATUS_API/$target/status" \
+    | python3 -c 'import json, sys; d = json.load(sys.stdin); print(d["state"] if d["total_count"] else "absent")') \
+    || fail "could not read the CI status of $target from $STATUS_API"
+  case "$state" in
+    success) ;;
+    pending) exit 0 ;;
+    *) fail "CI for origin/main $target is $state, not success: not deploying" ;;
+  esac
+fi
+
+echo "auto-deploy: deploying $target${ref:+ (--ref $ref)}"
+restarted=
+W=
+on_exit() {
+  local st=$?
+  [ -z "$W" ] || rm -rf "$W" || st=1
+  if [ "$st" -ne 0 ]; then
+    echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) FAILED target=$target status=$st" >> "$RECORD"
+    if [ -n "$restarted" ]; then
+      echo "auto-deploy FAILED (status $st) after the restart: production may be down or unverified; read journalctl -u mealquest, then fix or roll back (deploy/RUNBOOK.md)" >&2
+    else
+      echo "auto-deploy FAILED (status $st): production untouched" >&2
+    fi
+  fi
+  exit "$st"
+}
+trap on_exit EXIT
+
+# deploy/RUNBOOK.md "Deploy" > "1. Build": record the rollback SHA from the
+# RUNNING release, check out the target, reject a dirty checkout, build.
+bind=$(grep -E '^MEALQUEST_BIND_ADDRESS=' deploy/.env | cut -d = -f 2-) || fail "deploy/.env sets no MEALQUEST_BIND_ADDRESS"
+[ -n "$bind" ] || fail "deploy/.env sets no MEALQUEST_BIND_ADDRESS"
+running=$(curl -fsS "http://$bind:8090/health" | python3 -c 'import json, sys; print(json.load(sys.stdin)["version"])') \
+  || fail "http://$bind:8090/health did not return a version: no running release to roll back to"
+case "$running" in
+  dev|*-dirty) fail "the running release reports $running, which names no commit" ;;
+esac
+rollback=$(git rev-parse --verify --quiet "$running^{commit}") \
+  || fail "running version $running is not a commit in this repository"
+echo "rollback SHA: $rollback (running version $running)"
+git checkout --quiet --detach "$target"
+dirty=$(git status --porcelain --untracked-files=no -- . ':(exclude)internal/frontend/dist')
+[ -z "$dirty" ] || fail "the checkout has modified tracked files; clean it first:
+$dirty"
+v=$(git describe --tags --always)${dirty:+-dirty}   # the Makefile's VERSION
+make docker
+docker image inspect "mealquest:$v" > /dev/null 2>&1 || fail "no image mealquest:$v: the build failed"
+echo "built: mealquest:$v"
+
+# deploy/RUNBOOK.md "Prerequisites": deploy/.env sets every variable of the
+# target's .env.example, and every secret is mode 0600.
+missing=$(comm -23 <(grep -oE '^MEALQUEST_[A-Z_]+' deploy/.env.example | sort -u) \
+                   <(grep -oE '^MEALQUEST_[A-Z_]+' deploy/.env | sort -u))
+[ -z "$missing" ] || fail "deploy/.env is missing: $missing"
+loose=$(sudo find deploy/secrets -type f ! -perm 600)
+[ -z "$loose" ] || fail "secrets not mode 0600: $loose"
+
+# CTXD RUNBOOKS.MD "Routine deploy and rollback" > "Deploy" step 2: a verified
+# pre-deploy restore point. The scripts are CTXD "Count check". They and the
+# check files live in a private mktemp directory (mode 0700), not at shared
+# /tmp paths, and python runs with -I: the snapshot runs as root, and a
+# shared directory on its import path would let any local user run code as
+# root.
+W=$(mktemp -d)
+cat > "$W/mqcount.py" <<'EOF'
+import sqlite3, sys, hashlib
+c = sqlite3.connect(sys.argv[1])
+tabs = [r[0] for r in c.execute("select name from sqlite_master where type=\"table\" and name not like \"sqlite_%\" and name not like \"_litestream%\" order by name")]
+for t in ("dinners", "week_plans", "schema_migrations"):
+    print(t, c.execute(f"select count(*) from {t}").fetchone()[0])
+h = hashlib.sha256()
+for t in tabs:
+    for row in c.execute(f"select * from \"{t}\" order by rowid"):
+        h.update(repr(row).encode())
+print("tables", len(tabs), "content-sha256", h.hexdigest()[:16])
+print("integrity", c.execute("pragma integrity_check").fetchone()[0])
+EOF
+cat > "$W/mqsnap.py" <<'EOF'
+import os, sqlite3, sys
+src, dst = sys.argv[1], sys.argv[2]
+if os.path.exists(dst):
+    sys.exit(f"{dst} already exists")
+s = sqlite3.connect(f"file:{src}?mode=ro", uri=True)
+d = sqlite3.connect(dst)
+s.backup(d)
+s.close()
+result = d.execute("pragma integrity_check").fetchall()
+d.close()
+if result != [("ok",)]:
+    sys.exit(f"snapshot integrity_check failed: {result}")
+print("snapshot", dst, "integrity ok")
+EOF
+point=$(
+set -eu
+point=
+trap 'st=$?
+rm -rf "$W" || st=1
+sudo sh -c "rm -f $D/predeploy-check.db*" || st=1
+if [ "$st" -eq 0 ] && [ -n "$point" ]; then echo "pre-deploy restore point: $point" >&2; echo "$point"
+else [ "$st" -ne 0 ] || st=1; echo "verification FAILED (status $st): do not restart" >&2; fi
+exit "$st"' EXIT
+exec 3>&1 1>&2   # block output to the journal; only the restore point to $point
+mkdir "$W/mqcheck"
+# a. Replication running, no log errors (grep: 0 match, 1 none, 2+ error), newest LTX listed
+id=$("${COMPOSE[@]}" ps --status running --quiet litestream) || fail "docker compose ps failed"
+[ -n "$id" ] || fail "litestream is not running"
+"${COMPOSE[@]}" logs --since 24h litestream > "$W/mqcheck/litestream.log" || fail "docker compose logs failed"
+gs=0
+grep -i error "$W/mqcheck/litestream.log" > "$W/mqcheck/litestream.errors" || gs=$?
+case "$gs" in
+  0) cat "$W/mqcheck/litestream.errors"; fail "litestream logged errors in the last 24h";;
+  1) ;;
+  *) fail "the litestream log scan failed (status $gs)";;
+esac
+"${COMPOSE[@]}" exec -T litestream \
+  litestream ltx -config /etc/litestream.yml -level all /data/mealquest.db
+# b. Restore replica to a scratch file (live files untouched), move off the volume, snapshot live
+"${COMPOSE[@]}" run --rm --no-deps litestream \
+  restore -config /etc/litestream.yml -o /data/predeploy-check.db /data/mealquest.db
+sudo sh -c "mv $D/predeploy-check.db* $W/mqcheck/"
+sudo python3 -I "$W/mqsnap.py" "$D/mealquest.db" "$W/mqcheck/live.db"
+sudo chown -R "$(id -un)" "$W/mqcheck"
+sudo ls -ln "$D"
+# mealquest.db-wal and -shm owned by 10001, never 0 (CTXD "Count check")
+root=$(sudo find "$D" -maxdepth 1 \( -name mealquest.db-wal -o -name mealquest.db-shm \) -uid 0)
+[ -z "$root" ] || fail "owned by root, not 10001 (CTXD RUNBOOKS.MD \"Count check\"): $root"
+# c. Restored replica: integrity ok and identical counts/content-sha256 to the live snapshot
+python3 -I "$W/mqcount.py" "$W/mqcheck/predeploy-check.db" > "$W/mqcheck/restored.txt"
+python3 -I "$W/mqcount.py" "$W/mqcheck/live.db" > "$W/mqcheck/live.txt"
+cat "$W/mqcheck/restored.txt"
+grep -qx 'integrity ok' "$W/mqcheck/restored.txt" || fail "the restored replica failed integrity_check"
+diff "$W/mqcheck/restored.txt" "$W/mqcheck/live.txt" || fail "the restored replica differs from the live snapshot"
+# d. Restore point: after every check, before the restart
+point=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+exec 1>&3
+)
+
+# CTXD RUNBOOKS.MD "Routine deploy and rollback" > "Deploy" step 3:
+# boot-required configuration, secrets read the way the service reads them.
+[ -f deploy/.env ] || fail "missing deploy/.env"
+cid=$(grep -m 1 -E '^MEALQUEST_GOOGLE_CLIENT_ID=' deploy/.env | cut -d = -f 2- | tr -d '"' | tr -d "'") \
+  || fail "missing MEALQUEST_GOOGLE_CLIENT_ID in deploy/.env"
+[ -n "$cid" ] || fail "missing MEALQUEST_GOOGLE_CLIENT_ID in deploy/.env"
+[ -e deploy/secrets/google_client_secret ] || fail "missing deploy/secrets/google_client_secret"
+[ -s deploy/secrets/google_client_secret ] || fail "deploy/secrets/google_client_secret is empty"
+id=$("${COMPOSE[@]}" ps --status running --quiet mealquest) || fail "docker compose ps failed"
+[ -n "$id" ] || fail "the mealquest container is not running: start it (the check reads secrets the way the service does)"
+docker exec "$id" sh -c 'test -s /run/secrets/google_client_secret && test -r /run/secrets/google_client_secret' \
+  || fail "google_client_secret is not readable by the service user inside the container"
+echo "boot-required configuration present"
+
+# deploy/RUNBOOK.md "2. Ship and restart".
+"${COMPOSE[@]}" pull litestream searxng
+docker tag "mealquest:$v" mealquest-mealquest
+restarted=yes
+sudo systemctl restart mealquest
+for s in mealquest litestream searxng; do
+  id=$("${COMPOSE[@]}" ps --status running --quiet "$s")
+  [ -n "$id" ] || fail "$s is not running"
+done
+id=$("${COMPOSE[@]}" ps --quiet mealquest)
+health=$(docker inspect -f '{{.State.Health.Status}}' "$id")
+[ "$health" = healthy ] || fail "mealquest is $health, not healthy"
+echo "restarted on: mealquest:$v"
+
+# deploy/RUNBOOK.md "3. Verify the deployed version and a real screen".
+for base in "http://$bind:8090" "$PUBLIC"; do
+  got=$(curl -fsS "$base/health" | python3 -c 'import json, sys; print(json.load(sys.stdin)["version"])') \
+    || fail "$base/health did not return a version"
+  [ "$got" = "$v" ] || fail "$base/health reports version $got, the checkout is $v"
+  for path in / /plan; do
+    page=$(curl -fsS "$base$path") || fail "$base$path did not return 200"
+    grep -q '<div id="root"></div>' <<< "$page" || fail "$base$path is not the PWA shell"
+    js=$(grep -oE 'src="/assets/[^"]+\.js"' <<< "$page" | head -n 1 | cut -d '"' -f 2) || true
+    [ -n "$js" ] || fail "$base$path references no built /assets/ script"
+    type=$(curl -fsS -o /dev/null -w '%{content_type}' "$base$js") || fail "$base$js did not return 200"
+    case "$type" in
+      text/javascript*) ;;
+      *) fail "$base$js is $type, not JavaScript" ;;
+    esac
+  done
+  echo "$base: version $got, PWA shell and $js served"
+done
+
+deployed=$(git rev-parse HEAD)
+echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) previous=$rollback deployed=$deployed restore_point=$point" >> "$RECORD"
+echo "deployed and verified: $v ($deployed), previous $rollback, restore point $point"