Maniacs_Mansion/NestQuest
11
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add versioned schema migration runner (#17)

Browse Source
This commit is contained in:
Joshua King
2026-09-14 06:54:45 -04:00
parent e98642e236
commit c6d42e8277
2 changed files with 549 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
custom_components/nestquest/migrations.py
+161
View File
@@ -0,0 +1,161 @@
"""Versioned schema migration runner for the NestQuest SQLite database.
The runner owns one metadata table, ``nestquest_schema_version``, holding
a single row with the schema version the database file currently
reflects. Migrations are declared as an ordered list: entry ``n``
upgrades version ``n-1`` to version ``n``. Applying a migration means
running its DDL inside one transaction and stamping the new version in
that same transaction, so a database never persists a half-applied
migration: any failure rolls the statements and the version stamp back
together, leaving the file at its prior version.
Version 0 means "the file exists but carries no NestQuest schema yet"
— a fresh file, or a file this integration has never written. Starting
the runner against version 0 applies migration 1, which applies the
full v1 DDL from :mod:`.schema` and stamps version 1.
Each migration's statements and its version stamp run inside ONE
transaction, and the stamp is written as DELETE + INSERT rather than
UPDATE because a version-0 database has no version row to update yet.
An UPDATE form would silently match zero rows there and leave the
database migrated but unstamped — reporting version 0 forever, and
re-applying migration 1 on every startup. The DELETE+INSERT shape is
also what makes the stamp a replace, so the row never duplicates.
Idempotence across restarts: a database already at the latest version
has no pending migrations, so the runner opens a read-only check, finds
nothing to do, and leaves the file byte-untouched (no write transaction
is opened, so no WAL churn).
Migration list policy: entries are append-only. Once shipped, a
migration's SQL must never be edited — an old database applying it for
the first time and a fresh database would then diverge. Fix-forward
with a new migration instead. The version stamp lives in the same
SQLite file as the data it describes, so restoring a backup restores
its matching schema version with it.
Corrupt or missing files are not this module's concern: startup
handling of those cases is task d5bca74b, which decides whether to
open fresh or raise ConfigEntryNotReady. This runner assumes it is
handed an open, working connection.
"""
from __future__ import annotations
import logging
from collections.abc import Sequence
from .db import NestQuestDatabase
from .schema import SCHEMA_V1_STATEMENTS
LOGGER = logging.getLogger(__name__)
#: Metadata table holding the applied schema version. Prefixed with the
#: integration domain so a config directory shared with other tools can
#: never collide with a foreign ``schema_version`` table.
VERSION_TABLE = "nestquest_schema_version"
#: One-based index into :data:`MIGRATIONS`: entry ``n`` upgrades
#: version ``n-1`` to version ``n``.
MIGRATION_1_V1_DDL: list[str] = list(SCHEMA_V1_STATEMENTS)
#: The ordered migration list. Append-only: never edit an applied
#: entry, add the next one instead.
MIGRATIONS: Sequence[Sequence[str]] = [
MIGRATION_1_V1_DDL,
]
async def _create_version_table(database: NestQuestDatabase) -> None:
"""Create the version metadata table if it does not exist.
The table is created outside the per-migration transaction: it is
the runner's own bookkeeping, needed before any migration can stamp
a version, and creating it is itself idempotent.
"""
await database.execute(
f"""
CREATE TABLE IF NOT EXISTS {VERSION_TABLE} (
version INTEGER NOT NULL
)
"""
)
async def read_schema_version(database: NestQuestDatabase) -> int:
"""Return the schema version recorded in the database.
Returns 0 when the version table or its row is missing — the
"fresh file" case. A stored version higher than the runner knows
is surfaced as-is; the caller decides how to treat a future version.
Reading is deliberately passive: a missing table raises, it is not
swallowed into 0, because :func:`apply_migrations` always creates
the table before reading and a direct reader hitting "no such
table" is a caller bug worth surfacing, not a fresh file.
"""
row = await database.fetch_one(
f"SELECT version FROM {VERSION_TABLE} LIMIT 1"
)
if row is None:
return 0
version = row[0]
if not isinstance(version, int):
raise RuntimeError(
f"Schema version metadata is not an integer: {version!r}"
)
return version
async def apply_migrations(
database: NestQuestDatabase,
migrations: Sequence[Sequence[str]] = MIGRATIONS,
) -> int:
"""Bring the database schema up to the latest version.
Reads the current version, applies every pending migration in order
— each inside one transaction that also stamps the new version —
and returns the resulting version. Applying zero pending
migrations (database already current) is a no-op that logs at debug
level and opens no write transaction.
Raises whatever the failing migration raised, with the database
rolled back to its prior version and prior rows untouched.
"""
await _create_version_table(database)
current = await read_schema_version(database)
latest = len(migrations)
if current > latest:
raise RuntimeError(
f"Database schema version {current} is newer than this "
f"integration understands (latest known: {latest}); "
"downgrades are not supported"
)
if current == latest:
LOGGER.debug(
"NestQuest schema already at version %d; no migrations to apply",
current,
)
return current
for target in range(current + 1, latest + 1):
statements = migrations[target - 1]
async with database.transaction():
for sql in statements:
await database.execute(sql)
# DELETE + INSERT, not UPDATE: a fresh file has no version
# row yet, and UPDATE would silently match zero rows.
await database.execute(f"DELETE FROM {VERSION_TABLE}")
await database.execute(
f"INSERT INTO {VERSION_TABLE} (version) VALUES (?)",
(target,),
)
LOGGER.info(
"NestQuest schema migrated to version %d (%d statement%s)",
target,
len(statements),
"s" if len(statements) != 1 else "",
)
return latest
tests/test_migrations.py
+388
View File
@@ -0,0 +1,388 @@
"""Tests for migrations.py: the versioned schema migration runner."""
from __future__ import annotations
import asyncio
import logging
import sqlite3
from unittest.mock import AsyncMock
import pytest
from custom_components.nestquest.db import NestQuestDatabase
from custom_components.nestquest.migrations import (
MIGRATIONS,
VERSION_TABLE,
apply_migrations,
read_schema_version,
)
from custom_components.nestquest.schema import SCHEMA_V1_STATEMENTS
def _run(coro):
return asyncio.new_event_loop().run_until_complete(coro)
def _make_hass_mock():
from unittest.mock import AsyncMock, MagicMock
hass = MagicMock()
hass.async_add_executor_job = AsyncMock(side_effect=(lambda fn, *a: fn(*a)))
return hass
def _open_db(path) -> NestQuestDatabase:
database = NestQuestDatabase(_make_hass_mock())
_run(database.open(path))
return database
def _migrate(database, migrations=MIGRATIONS):
return _run(apply_migrations(database, migrations))
def _version(database) -> int:
return _run(read_schema_version(database))
def _tables(database) -> set[str]:
rows = _run(
database.fetch_all(
"SELECT name FROM sqlite_master WHERE type = 'table' "
"AND name NOT LIKE 'sqlite_%'"
)
)
return {row[0] for row in rows}
def _counts(database) -> dict[str, int]:
"""Row counts for every v1 data table plus the version table."""
tables = (
"children",
"admin_users",
"schedule_rules",
"task_definitions",
"presence_schedules",
"presence_overrides",
"task_instances",
"completion_events",
)
counts = {}
for table in tables:
row = _run(database.fetch_one(f"SELECT COUNT(*) FROM {table}"))
counts[table] = row[0]
return counts
# ---------------------------------------------------------------------------
# Fresh file: version 0 -> version 1
# ---------------------------------------------------------------------------
def test_fresh_file_migrates_to_version_1(tmp_path) -> None:
database = _open_db(tmp_path / "fresh.db")
try:
final = _migrate(database)
assert final == 1
assert _version(database) == 1
finally:
_run(database.close())
def test_fresh_file_migration_creates_all_v1_tables(tmp_path) -> None:
database = _open_db(tmp_path / "fresh-tables.db")
try:
_migrate(database)
expected = {
"children",
"admin_users",
"schedule_rules",
"task_definitions",
"presence_schedules",
"presence_overrides",
"task_instances",
"completion_events",
VERSION_TABLE,
}
assert _tables(database) == expected
finally:
_run(database.close())
def test_missing_version_table_raises_on_direct_read(tmp_path) -> None:
database = _open_db(tmp_path / "no-table.db")
try:
with pytest.raises(sqlite3.OperationalError, match="no such table"):
_run(read_schema_version(database))
row = _run(
database.fetch_one(
"SELECT name FROM sqlite_master WHERE type = 'table' "
f"AND name = '{VERSION_TABLE}'"
)
)
assert row is None, "read_schema_version must not create the table"
finally:
_run(database.close())
def test_empty_version_table_reads_as_version_0(tmp_path) -> None:
database = _open_db(tmp_path / "empty-table.db")
try:
_run(
database.execute(
f"CREATE TABLE {VERSION_TABLE} (version INTEGER NOT NULL)"
)
)
assert _version(database) == 0
finally:
_run(database.close())
def test_migration_1_is_the_full_v1_ddl() -> None:
assert MIGRATIONS[0] == SCHEMA_V1_STATEMENTS
assert len(MIGRATIONS) == 1
# ---------------------------------------------------------------------------
# Idempotence: second start is a no-op
# ---------------------------------------------------------------------------
def test_second_run_applies_nothing_and_keeps_rows(tmp_path) -> None:
database = _open_db(tmp_path / "idempotent.db")
try:
_migrate(database)
_run(
database.execute(
"INSERT INTO children (display_name, created_at) VALUES (?, ?)",
("Ada", "2026-09-13T00:00:00+00:00"),
)
)
_run(
database.execute(
"INSERT INTO admin_users (ha_user_id, added_at) VALUES (?, ?)",
("user-1", "2026-09-13T00:00:00+00:00"),
)
)
final = _migrate(database)
assert final == 1
assert _version(database) == 1
assert _counts(database)["children"] == 1
assert _counts(database)["admin_users"] == 1
finally:
_run(database.close())
def test_second_run_writes_nothing_to_the_file(tmp_path, caplog) -> None:
database = _open_db(tmp_path / "noop.db")
try:
_migrate(database)
db_file = tmp_path / "noop.db"
size_before = db_file.stat().st_size
mtime_before = db_file.stat().st_mtime_ns
_migrate(database)
# A no-op run must not open a write transaction: the main
# database file is byte-identical and untouched (mtime too), and
# no WAL checkpoint traffic dirtied it.
assert db_file.stat().st_size == size_before
assert db_file.stat().st_mtime_ns == mtime_before
finally:
_run(database.close())
def test_noop_run_logs_no_migration(caplog, tmp_path) -> None:
database = _open_db(tmp_path / "noop-log.db")
try:
_migrate(database)
caplog.clear()
with caplog.at_level(logging.INFO, logger="custom_components.nestquest.migrations"):
_migrate(database)
migrated = [
r for r in caplog.records if "migrated" in r.getMessage().lower()
]
assert migrated == []
finally:
_run(database.close())
def test_noop_run_is_logged_at_debug_level(caplog, tmp_path) -> None:
database = _open_db(tmp_path / "noop-debug.db")
try:
_migrate(database)
caplog.clear()
with caplog.at_level(logging.DEBUG, logger="custom_components.nestquest.migrations"):
_migrate(database)
assert any(
"already at version 1" in r.getMessage()
for r in caplog.records
)
finally:
_run(database.close())
# ---------------------------------------------------------------------------
# Failure atomicity: a failing migration leaves the prior version intact
# ---------------------------------------------------------------------------
def _failing_migrations(prior_statements: list[str]) -> list[list[str]]:
"""Two-migration list where migration 2 fails mid-application."""
return [
prior_statements,
[
"CREATE TABLE IF NOT EXISTS later_table (id INTEGER PRIMARY KEY)",
"CREATE TABLE broken (",
],
]
def test_failing_migration_rolls_back_to_prior_version(tmp_path) -> None:
database = _open_db(tmp_path / "rollback.db")
try:
# Manually simulate a database at version 1: apply migration 1's
# statements directly and stamp version 1 through the runner.
_migrate(database)
version_row = _run(
database.fetch_one(f"SELECT version FROM {VERSION_TABLE}")
)
assert version_row == (1,)
with pytest.raises(sqlite3.OperationalError):
_migrate(database, _failing_migrations(SCHEMA_V1_STATEMENTS))
# Version stamp unchanged, statements from the failed migration
# absent, and the v1 tables untouched.
assert _version(database) == 1
assert "later_table" not in _tables(database)
assert _counts(database)["children"] == 0
assert _counts(database)["completion_events"] == 0
finally:
_run(database.close())
def test_failing_migration_on_fresh_file_leaves_version_0(tmp_path) -> None:
database = _open_db(tmp_path / "fresh-rollback.db")
try:
# The fresh file still reaches migration 2 (the failing one):
# migration 1 commits, stamping version 1, before migration 2
# fails and rolls back. The run aborts at version 1 — the last
# fully-applied migration — not at version 0.
with pytest.raises(sqlite3.OperationalError):
_migrate(database, _failing_migrations(SCHEMA_V1_STATEMENTS))
assert _version(database) == 1
# Migration 2's tables must not survive the failed attempt.
assert "later_table" not in _tables(database)
# Migration 1's tables do: it committed before the failure.
assert "children" in _tables(database)
assert "task_instances" in _tables(database)
finally:
_run(database.close())
def test_failed_migration_can_be_retried_after_fix(tmp_path) -> None:
database = _open_db(tmp_path / "retry.db")
try:
_migrate(database)
broken = _failing_migrations(SCHEMA_V1_STATEMENTS)
with pytest.raises(sqlite3.OperationalError):
_migrate(database, broken)
# "Fix" the migration and retry: it now applies cleanly.
broken[1] = [
"CREATE TABLE IF NOT EXISTS later_table (id INTEGER PRIMARY KEY)"
]
final = _migrate(database, broken)
assert final == 2
assert _version(database) == 2
assert "later_table" in _tables(database)
finally:
_run(database.close())
def test_failing_migration_error_is_preserved(tmp_path) -> None:
database = _open_db(tmp_path / "error-preserved.db")
try:
with pytest.raises(sqlite3.OperationalError, match="incomplete input"):
_migrate(database, _failing_migrations(SCHEMA_V1_STATEMENTS))
finally:
_run(database.close())
# ---------------------------------------------------------------------------
# Version guard rails
# ---------------------------------------------------------------------------
def test_future_version_is_rejected(tmp_path) -> None:
database = _open_db(tmp_path / "future.db")
try:
_migrate(database)
_run(
database.execute(
f"UPDATE {VERSION_TABLE} SET version = ?", (99,)
)
)
with pytest.raises(RuntimeError, match="newer than"):
_migrate(database)
# Nothing was modified by the rejected run.
assert _version(database) == 99
finally:
_run(database.close())
def test_non_integer_version_raises_runtime_error(tmp_path) -> None:
database = _open_db(tmp_path / "string-version.db")
try:
_run(
database.execute(
f"CREATE TABLE {VERSION_TABLE} (version TEXT NOT NULL)"
)
)
_run(
database.execute(
f"INSERT INTO {VERSION_TABLE} (version) VALUES (?)",
("one",),
)
)
with pytest.raises(RuntimeError, match="not an integer"):
_run(read_schema_version(database))
finally:
_run(database.close())
def test_multiple_pending_migrations_apply_in_order(tmp_path) -> None:
database = _open_db(tmp_path / "multi-step.db")
try:
migrations = [
["CREATE TABLE IF NOT EXISTS step_a (id INTEGER PRIMARY KEY)"],
["CREATE TABLE IF NOT EXISTS step_b (id INTEGER PRIMARY KEY)"],
["CREATE TABLE IF NOT EXISTS step_c (id INTEGER PRIMARY KEY)"],
]
final = _run(apply_migrations(database, migrations))
assert final == 3
assert _version(database) == 3
assert {"step_a", "step_b", "step_c"} <= _tables(database)
finally:
_run(database.close())
def test_partial_pending_migrations_resume_from_current_version(
tmp_path,
) -> None:
database = _open_db(tmp_path / "resume.db")
try:
migrations = [
["CREATE TABLE IF NOT EXISTS step_a (id INTEGER PRIMARY KEY)"],
["CREATE TABLE IF NOT EXISTS step_b (id INTEGER PRIMARY KEY)"],
]
# Advance to version 1 only, by applying a truncated list of the
# same migrations: patch apply_migrations' latest by passing a
# single-migration list first, then the full list.
_run(apply_migrations(database, migrations[:1]))
assert _version(database) == 1
final = _run(apply_migrations(database, migrations))
assert final == 2
assert _version(database) == 2
assert "step_b" in _tables(database)
finally:
_run(database.close())