diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 5b5ed88..9ee7df7 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -96,7 +96,7 @@ Columns ending in `_ct` hold AES-256-GCM ciphertext (via `secretbox`). | `recipe_ingredients` | recipe_id, position, raw_text, name, qty, unit, note. | | `recipe_steps` | recipe_id, position, text. | | `photos` | source_url, local_path, width, height, sha256, fetched_at. | -| `dinners` | The meal record and its lifecycle: name, normalized_name, status (`suggested` / `quest_log` / `tried` / `regular` / `archived`), origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, recipe_url (a pasted recipe link awaiting grounding), photo_id, grounding_status (`pending` / `grounded` / `failed` / `manual`), grounding_error (why grounding failed), updated_by. | +| `dinners` | The meal record and its lifecycle: name, normalized_name, status (`suggested` / `quest_log` / `tried` / `regular` / `archived`), origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, recipe_url (a pasted recipe link awaiting grounding), photo_id, grounding_status (`pending` / `grounded` / `failed` / `manual`), grounding_error (why grounding failed), grounding_revision (raised when a pending dinner is renamed, so only the latest `ground_dinner` job may write it), updated_by. | | `suggestion_batches` | trigger, status, ai_run_id. | | `suggestions` | dinner_id, batch_id, reason, cited_regular_ids (JSON), status, decided_at. | | `week_plans` | week_start (unique), status (`draft` / `synced` / `changed`), ai_note, ai_run_id, calendar_synced_at. | @@ -187,8 +187,10 @@ automatic** (D-021). "Beef stew"); a link that names nothing is 400 — add it by name. A name is trimmed, at most 120 characters and must keep a word after `harness.Normalize`, which sets `normalized_name`. A name - `harness.Similar` to an unarchived dinner's, or a link one already has, - is 409 `duplicate_dinner`; archived names may be added again. + `harness.Similar` to an unarchived dinner's, or a link one already has + (the source_url of the recipe it is grounded in, else its pasted + `recipe_url`), is 409 `duplicate_dinner`; archived names may be added + again. `GET /api/v1/dinners?status=regular|quest_log` returns `{dinners: [...]}` ordered by normalized name, each with `photo_url` (`/media/photos/{photo_id}-400.jpg`, or null) and `grounding_status`; @@ -211,11 +213,22 @@ automatic** (D-021). `failed` with `grounding_error` — also the job's failure code — one of `no_recipe_found`, `recipe_unusable` (the pasted link), `search_not_configured`, `search_failed`, `ai_failed`, `interrupted` (the server stopped) or - `grounding_error`. It only writes a dinner still `pending`, so a fix-up - made meanwhile wins. At boot, after the job runner fails the jobs a - previous process left running, every dinner still `pending` with no - `ground_dinner` job queued or running is set `failed` with `interrupted`, - so no dinner stays pending with nothing left to ground it. + `grounding_error`. Each job carries the dinner's `grounding_revision` + from when it was enqueued; renaming a `pending` dinner clears its + `recipe_url`, raises the revision and enqueues a job for the new name. + A job for a dinner archived, no longer `pending` or at another revision + does no work, and a job only writes a dinner still unarchived, `pending` + and at its revision, so a fix-up made meanwhile wins and an earlier job + for an old name — queued or running — never decides the outcome. + Archiving or renaming a dinner also stops its running job where it is, + so no further search, page fetch, photo download or model call is made + for it. A + method, minutes_total or difficulty a member saved while the dinner was + `pending` is kept; grounding fills only those left empty. At boot, after + the job runner fails the jobs a previous process left running, every + unarchived dinner still `pending` with no `ground_dinner` job for its revision + queued or running is set `failed` with `interrupted`, so no dinner stays + pending with nothing left to ground it. `GET /api/v1/dinners/{id}/recipe-candidates` searches for the dinner's name and returns `{candidates: [{candidate_id, title, site_name, photo_url, minutes}]}`, at most 3, ranked as `Ground` @@ -229,8 +242,9 @@ automatic** (D-021). — no model runs — as a `recipes` row whose source_url is `mealquest:manual:{recipe_id}` and whose raw_jsonld is a schema.org Recipe built from it, and sets `manual` with its method and minutes, no - photo and the dinner's difficulty unchanged. It answers 200 with the - dinner; an invalid body is 400, an unknown dinner 404, a page that cannot + photo and the dinner's difficulty unchanged. Either clears the dinner's + pasted `recipe_url`: the recipe chosen replaces it. It answers 200 with + the dinner; an invalid body is 400, an unknown dinner 404, a page that cannot ground it 422 `ungroundable` (the dinner is unchanged), recipe search not set up 409 `not_configured`, a search or model failure 502 `search_failed` / `ai_failed` (only the reason code is logged, never the @@ -488,13 +502,24 @@ First run: while `household_settings.onboarding_completed_at` (read from `GET /api/v1/settings/household`) is null, every route redirects to `/onboarding`. Onboarding is a full screen outside the app shell, in three steps: the welcome (`/onboarding`), the Regulars (`/onboarding/regulars`, -mock 1h, adding and archiving through the dinners API optimistically) and +mock 1h, adding and archiving through the dinners API optimistically; Next +waits for three saved Regulars, and a common pick is not offered once a +Regular has a name `harness.Similar` to it — the PWA's copy of the name +rules in `web/src/dinners/names.ts` folds letters with +`web/src/dinners/letterFolds.json`, which `gen_folds.py` writes from the +same table as the server's, and runs the same cases, +`internal/ai/harness/testdata/dedupe_cases.json`, as the server's) and notifications (`/onboarding/notifications`: what MealQuest notifies about, the iOS install hint, "Turn on notifications" and "Maybe later"). Either button sends `PATCH /api/v1/settings/household` with `{"onboarding_completed": true}`; the server sets `onboarding_completed_at` to the time of that update (keeping the first time if it is already set; the field itself is read-only and `false` is refused) and the app goes Home. +Completing needs at least three Regulars (`household.MinRegulars`); with +fewer the server answers 400 `invalid_settings` naming +`onboarding_completed` and changes nothing. Step 3 can be reached with +fewer — by its URL, or after another member removes a Regular — so it then +shows the server's reason with an "Add Regulars" button back to step 2. Notification permission is never asked for on page load; until Feature 19 wires Web Push, "Turn on notifications" asks for nothing. Offline on the saved pages the settings cannot be read (the service worker never caches `/api/v1/settings`), diff --git a/internal/ai/harness/dedupe_test.go b/internal/ai/harness/dedupe_test.go index 99332d3..9f7ff2b 100644 --- a/internal/ai/harness/dedupe_test.go +++ b/internal/ai/harness/dedupe_test.go @@ -1,10 +1,70 @@ package harness import ( + "encoding/json" + "os" "testing" "unicode" + "unicode/utf8" ) +// TestSharedDedupeCases runs testdata/dedupe_cases.json, which the PWA's +// copy of Normalize and Similar (web/src/dinners/names.ts) runs too, so the +// two cannot drift apart on these names. +func TestSharedDedupeCases(t *testing.T) { + raw, err := os.ReadFile("testdata/dedupe_cases.json") + if err != nil { + t.Fatal(err) + } + var cases struct { + Normalize []struct{ Name, Want string } + Similar []struct { + A, B string + Want bool + } + } + if err := json.Unmarshal(raw, &cases); err != nil { + t.Fatal(err) + } + if len(cases.Normalize) == 0 || len(cases.Similar) == 0 { + t.Fatal("dedupe_cases.json has no cases") + } + for _, c := range cases.Normalize { + if got := Normalize(c.Name); got != c.Want { + t.Errorf("Normalize(%q) = %q, want %q", c.Name, got, c.Want) + } + } + for _, c := range cases.Similar { + if got := Similar(c.A, c.B); got != c.Want { + t.Errorf("Similar(%q, %q) = %v, want %v", c.A, c.B, got, c.Want) + } + } +} + +// TestLetterFoldsMatchPWA checks that the PWA's letter-fold table, which +// gen_folds.py writes beside folds_table.go, is letterFolds exactly; run +// go generate when it fails. +func TestLetterFoldsMatchPWA(t *testing.T) { + raw, err := os.ReadFile("../../../web/src/dinners/letterFolds.json") + if err != nil { + t.Fatal(err) + } + var pwa map[string]string + if err := json.Unmarshal(raw, &pwa); err != nil { + t.Fatal(err) + } + for r, want := range letterFolds { + if got, ok := pwa[string(r)]; !ok || got != want { + t.Errorf("PWA folds %U to %q (present %v), want %q", r, got, ok, want) + } + } + for k := range pwa { + if utf8.RuneCountInString(k) != 1 || letterFolds[[]rune(k)[0]] == "" { + t.Errorf("PWA folds %q, which letterFolds does not", k) + } + } +} + func TestNormalize(t *testing.T) { tests := []struct { name, want string diff --git a/internal/ai/harness/gen_folds.py b/internal/ai/harness/gen_folds.py index 6e907c4..4de6bb6 100644 --- a/internal/ai/harness/gen_folds.py +++ b/internal/ai/harness/gen_folds.py @@ -1,5 +1,8 @@ #!/usr/bin/env python3 -"""Generate folds_table.go, the letter-fold table used by harness.Normalize. +"""Generate folds_table.go, the letter-fold table used by harness.Normalize, +and its copy web/src/dinners/letterFolds.json, used by the PWA's +normalizeName (web/src/dinners/names.ts). TestLetterFoldsMatchPWA fails when +the two differ. Run from this directory with `go generate ./internal/ai/harness` (or `python3 gen_folds.py`; gofmt must be on PATH). It uses only the Python @@ -28,11 +31,14 @@ Hangul syllables are left out: their decomposition is into conjoining jamo, not combining marks. """ +import json import re import subprocess import sys import unicodedata +PWA_TABLE = "../../../web/src/dinners/letterFolds.json" + EXTRA = { "ß": "ss", "ẞ": "ss", "æ": "ae", "Æ": "ae", "œ": "oe", "Œ": "oe", "þ": "th", "Þ": "th", "ð": "d", "Ð": "d", "ı": "i", "ʼn": "n", @@ -110,6 +116,9 @@ def main(): with open("folds_table.go", "w", encoding="utf-8") as f: f.write("\n".join(out) + "\n") subprocess.run(["gofmt", "-w", "folds_table.go"], check=True) + with open(PWA_TABLE, "w", encoding="ascii") as f: + json.dump({chr(cp): folded for cp, folded in entries}, f, indent=2) + f.write("\n") if __name__ == "__main__": diff --git a/internal/ai/harness/testdata/dedupe_cases.json b/internal/ai/harness/testdata/dedupe_cases.json new file mode 100644 index 0000000..20c3aec --- /dev/null +++ b/internal/ai/harness/testdata/dedupe_cases.json @@ -0,0 +1,136 @@ +{ + "normalize": [ + { + "name": "Easy Chicken Tikka-Masala!", + "want": "chicken tikka masala" + }, + { + "name": " Homemade Chili ", + "want": "chili" + }, + { + "name": "Grandma's Stew", + "want": "grandmas stew" + }, + { + "name": "Grandma’s Stew", + "want": "grandmas stew" + }, + { + "name": "İskender Kebab", + "want": "iskender kebab" + }, + { + "name": "Paella à la Valenciana", + "want": "paella la valenciana" + }, + { + "name": "Crème Brûlée", + "want": "creme brulee" + }, + { + "name": "Crème Brûlée", + "want": "creme brulee" + }, + { + "name": "Smørrebrød", + "want": "smorrebrod" + }, + { + "name": "Pierogi z Łososiem", + "want": "pierogi z lososiem" + }, + { + "name": "Straße Würstchen", + "want": "strasse wurstchen" + }, + { + "name": "Œufs en Cocotte", + "want": "oeufs en cocotte" + }, + { + "name": "Æbleskiver", + "want": "aebleskiver" + }, + { + "name": "Phở Bò", + "want": "pho bo" + }, + { + "name": "The Best Quick Recipe", + "want": "" + }, + { + "name": "Chicken stir-fry", + "want": "chicken stir fry" + }, + { + "name": "7-layer dip", + "want": "7 layer dip" + }, + { + "name": "Ƈhili", + "want": "chili" + }, + { + "name": "Ꞔarbanzo Stew", + "want": "carbanzo stew" + }, + { + "name": "ΜΟΥΣΑΚΑΣ", + "want": "μουσακασ" + } + ], + "similar": [ + { + "a": "Homemade chili", + "b": "Chili", + "want": true + }, + { + "a": "Easy Spaghetti Bolognese", + "b": "spaghetti bolognese", + "want": true + }, + { + "a": "Chicken stir fry", + "b": "Chicken stir-fry", + "want": true + }, + { + "a": "Crème brûlée", + "b": "Creme brulee", + "want": true + }, + { + "a": "Smørrebrød", + "b": "smorrebrod", + "want": true + }, + { + "a": "Beef tacos", + "b": "Fish tacos", + "want": false + }, + { + "a": "Chili", + "b": "White chicken chili", + "want": false + }, + { + "a": "Pot roast", + "b": "Roast", + "want": false + }, + { + "a": "The best", + "b": "Easy recipe", + "want": false + }, + { + "a": "Ƈhili", + "b": "Chili", + "want": true + } + ] +} diff --git a/internal/db/db_test.go b/internal/db/db_test.go index 2f5f911..01f0a80 100644 --- a/internal/db/db_test.go +++ b/internal/db/db_test.go @@ -190,6 +190,7 @@ func TestDinners(t *testing.T) { Difficulty: str("medium"), Cuisine: sql.NullString{}, RecipeID: sql.NullString{}, + RecipeUrl: str("https://example.com/tacos"), PhotoID: str(photo.ID), GroundingStatus: "failed", UpdatedBy: str("sam@example.com"), @@ -267,7 +268,7 @@ func TestDinners(t *testing.T) { } var unarchivedIDs []string for _, d := range unarchived { - unarchivedIDs = append(unarchivedIDs, d.ID) + unarchivedIDs = append(unarchivedIDs, d.Dinner.ID) } assertIDs(t, "unarchived", unarchivedIDs, other.ID, third.ID) _, err = q.ArchiveDinner(ctx, db.ArchiveDinnerParams{ID: "missing", UpdatedAt: archivedAt}) diff --git a/internal/db/dinners.sql.go b/internal/db/dinners.sql.go index 049f318..ff1200e 100644 --- a/internal/db/dinners.sql.go +++ b/internal/db/dinners.sql.go @@ -16,7 +16,7 @@ SET status = 'archived', updated_by = ?, updated_at = ? WHERE id = ? -RETURNING id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error +RETURNING id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error, grounding_revision ` type ArchiveDinnerParams struct { @@ -48,6 +48,7 @@ func (q *Queries) ArchiveDinner(ctx context.Context, arg ArchiveDinnerParams) (D &i.UpdatedAt, &i.RecipeUrl, &i.GroundingError, + &i.GroundingRevision, ) return i, err } @@ -75,7 +76,7 @@ INSERT INTO dinners ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) -RETURNING id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error +RETURNING id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error, grounding_revision ` type CreateDinnerParams struct { @@ -138,6 +139,7 @@ func (q *Queries) CreateDinner(ctx context.Context, arg CreateDinnerParams) (Din &i.UpdatedAt, &i.RecipeUrl, &i.GroundingError, + &i.GroundingRevision, ) return i, err } @@ -148,11 +150,13 @@ SET grounding_status = 'failed', grounding_error = ?1, updated_at = ?2 WHERE grounding_status = 'pending' + AND status <> 'archived' AND NOT EXISTS ( SELECT 1 FROM jobs WHERE jobs.kind = ?3 AND jobs.status IN ('queued', 'running') AND json_extract(jobs.payload, '$.dinner_id') = dinners.id + AND json_extract(jobs.payload, '$.revision') = dinners.grounding_revision ) ` @@ -162,10 +166,11 @@ type FailOrphanedPendingDinnerGroundingsParams struct { Kind string } -// Fails every dinner still pending whose grounding job is no longer queued -// or running: the process stopped before the job, or before its handler -// recorded a result. Run at boot, after the job runner fails the jobs a -// previous process left running. +// Fails every unarchived dinner still pending whose grounding job for its +// current grounding_revision is no longer queued or running: the process +// stopped before the job, or before its handler recorded a result. An +// archived dinner is left as it is; archiving stopped its grounding. Run at +// boot, after the job runner fails the jobs a previous process left running. func (q *Queries) FailOrphanedPendingDinnerGroundings(ctx context.Context, arg FailOrphanedPendingDinnerGroundingsParams) (int64, error) { result, err := q.db.ExecContext(ctx, failOrphanedPendingDinnerGroundings, arg.GroundingError, arg.UpdatedAt, arg.Kind) if err != nil { @@ -179,25 +184,29 @@ UPDATE dinners SET grounding_status = 'failed', grounding_error = ?, updated_at = ? -WHERE id = ? AND grounding_status = 'pending' AND normalized_name = ? +WHERE id = ? + AND status <> 'archived' + AND grounding_status = 'pending' + AND grounding_revision = ? ` type FailPendingDinnerGroundingParams struct { - GroundingError sql.NullString - UpdatedAt string - ID string - NormalizedName string + GroundingError sql.NullString + UpdatedAt string + ID string + GroundingRevision int64 } -// Records why the background grounding of a dinner still pending under the -// normalized name it was grounded for failed; a dinner grounded some other -// way or renamed meanwhile is left as it is (0 rows). +// Records why the background grounding of an unarchived dinner still +// pending at the grounding_revision it was grounded for failed; a dinner +// grounded some other way, renamed or archived meanwhile is left as it is +// (0 rows). func (q *Queries) FailPendingDinnerGrounding(ctx context.Context, arg FailPendingDinnerGroundingParams) (int64, error) { result, err := q.db.ExecContext(ctx, failPendingDinnerGrounding, arg.GroundingError, arg.UpdatedAt, arg.ID, - arg.NormalizedName, + arg.GroundingRevision, ) if err != nil { return 0, err @@ -206,7 +215,7 @@ func (q *Queries) FailPendingDinnerGrounding(ctx context.Context, arg FailPendin } const getDinner = `-- name: GetDinner :one -SELECT id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error FROM dinners +SELECT id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error, grounding_revision FROM dinners WHERE id = ? ` @@ -232,6 +241,7 @@ func (q *Queries) GetDinner(ctx context.Context, id string) (Dinner, error) { &i.UpdatedAt, &i.RecipeUrl, &i.GroundingError, + &i.GroundingRevision, ) return i, err } @@ -239,6 +249,7 @@ func (q *Queries) GetDinner(ctx context.Context, id string) (Dinner, error) { const groundDinner = `-- name: GroundDinner :one UPDATE dinners SET recipe_id = ?, + recipe_url = NULL, photo_id = ?, method = ?, minutes_total = ?, @@ -248,7 +259,7 @@ SET recipe_id = ?, updated_by = ?, updated_at = ? WHERE id = ? -RETURNING id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error +RETURNING id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error, grounding_revision ` type GroundDinnerParams struct { @@ -263,8 +274,9 @@ type GroundDinnerParams struct { ID string } -// Records a dinner's grounding (grounded or manual) and clears any -// grounding_error. An unknown id returns sql.ErrNoRows. +// Records a dinner's grounding by hand (grounded or manual) and clears any +// grounding_error and pasted recipe_url: the recipe chosen replaces the +// link. An unknown id returns sql.ErrNoRows. func (q *Queries) GroundDinner(ctx context.Context, arg GroundDinnerParams) (Dinner, error) { row := q.db.QueryRowContext(ctx, groundDinner, arg.RecipeID, @@ -297,37 +309,43 @@ func (q *Queries) GroundDinner(ctx context.Context, arg GroundDinnerParams) (Din &i.UpdatedAt, &i.RecipeUrl, &i.GroundingError, + &i.GroundingRevision, ) return i, err } const groundPendingDinner = `-- name: GroundPendingDinner :execrows UPDATE dinners -SET recipe_id = ?, - photo_id = ?, - method = ?, - minutes_total = ?, - difficulty = ?, +SET recipe_id = ?1, + photo_id = ?2, + method = COALESCE(method, ?3), + minutes_total = COALESCE(minutes_total, ?4), + difficulty = COALESCE(difficulty, ?5), grounding_status = 'grounded', grounding_error = NULL, - updated_at = ? -WHERE id = ? AND grounding_status = 'pending' AND normalized_name = ? + updated_at = ?6 +WHERE id = ?7 + AND status <> 'archived' + AND grounding_status = 'pending' + AND grounding_revision = ?8 ` type GroundPendingDinnerParams struct { - RecipeID sql.NullString - PhotoID sql.NullString - Method sql.NullString - MinutesTotal sql.NullInt64 - Difficulty sql.NullString - UpdatedAt string - ID string - NormalizedName string + RecipeID sql.NullString + PhotoID sql.NullString + Method sql.NullString + MinutesTotal sql.NullInt64 + Difficulty sql.NullString + UpdatedAt string + ID string + GroundingRevision int64 } -// Records the background grounding of a dinner still pending under the -// normalized name it was grounded for; a dinner grounded some other way or -// renamed meanwhile is left as it is (0 rows). +// Records the background grounding of an unarchived dinner still pending at +// the grounding_revision it was grounded for; a dinner grounded some other +// way, renamed or archived meanwhile is left as it is (0 rows). A method, +// minutes_total or difficulty already set is kept: a pending dinner has +// them only when a member edited them while it was grounding. func (q *Queries) GroundPendingDinner(ctx context.Context, arg GroundPendingDinnerParams) (int64, error) { result, err := q.db.ExecContext(ctx, groundPendingDinner, arg.RecipeID, @@ -337,7 +355,7 @@ func (q *Queries) GroundPendingDinner(ctx context.Context, arg GroundPendingDinn arg.Difficulty, arg.UpdatedAt, arg.ID, - arg.NormalizedName, + arg.GroundingRevision, ) if err != nil { return 0, err @@ -346,7 +364,7 @@ func (q *Queries) GroundPendingDinner(ctx context.Context, arg GroundPendingDinn } const listDinnersByStatus = `-- name: ListDinnersByStatus :many -SELECT id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error FROM dinners +SELECT id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error, grounding_revision FROM dinners WHERE status = ? ORDER BY normalized_name, id ` @@ -379,6 +397,7 @@ func (q *Queries) ListDinnersByStatus(ctx context.Context, status string) ([]Din &i.UpdatedAt, &i.RecipeUrl, &i.GroundingError, + &i.GroundingRevision, ); err != nil { return nil, err } @@ -394,40 +413,50 @@ func (q *Queries) ListDinnersByStatus(ctx context.Context, status string) ([]Din } const listUnarchivedDinners = `-- name: ListUnarchivedDinners :many -SELECT id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error FROM dinners -WHERE status <> 'archived' -ORDER BY id +SELECT dinners.id, dinners.name, dinners.normalized_name, dinners.status, dinners.origin, dinners.method, dinners.minutes_total, dinners.difficulty, dinners.cuisine, dinners.is_favorite, dinners.recipe_id, dinners.photo_id, dinners.grounding_status, dinners.updated_by, dinners.created_at, dinners.updated_at, dinners.recipe_url, dinners.grounding_error, dinners.grounding_revision, recipes.source_url AS recipe_source_url +FROM dinners +LEFT JOIN recipes ON recipes.id = dinners.recipe_id +WHERE dinners.status <> 'archived' +ORDER BY dinners.id ` -// Every dinner a new or renamed dinner must not repeat. -func (q *Queries) ListUnarchivedDinners(ctx context.Context) ([]Dinner, error) { +type ListUnarchivedDinnersRow struct { + Dinner Dinner + RecipeSourceUrl sql.NullString +} + +// Every dinner a new or renamed dinner must not repeat, with the +// source_url of the recipe it is grounded in (NULL while it has none). +func (q *Queries) ListUnarchivedDinners(ctx context.Context) ([]ListUnarchivedDinnersRow, error) { rows, err := q.db.QueryContext(ctx, listUnarchivedDinners) if err != nil { return nil, err } defer rows.Close() - var items []Dinner + var items []ListUnarchivedDinnersRow for rows.Next() { - var i Dinner + var i ListUnarchivedDinnersRow if err := rows.Scan( - &i.ID, - &i.Name, - &i.NormalizedName, - &i.Status, - &i.Origin, - &i.Method, - &i.MinutesTotal, - &i.Difficulty, - &i.Cuisine, - &i.IsFavorite, - &i.RecipeID, - &i.PhotoID, - &i.GroundingStatus, - &i.UpdatedBy, - &i.CreatedAt, - &i.UpdatedAt, - &i.RecipeUrl, - &i.GroundingError, + &i.Dinner.ID, + &i.Dinner.Name, + &i.Dinner.NormalizedName, + &i.Dinner.Status, + &i.Dinner.Origin, + &i.Dinner.Method, + &i.Dinner.MinutesTotal, + &i.Dinner.Difficulty, + &i.Dinner.Cuisine, + &i.Dinner.IsFavorite, + &i.Dinner.RecipeID, + &i.Dinner.PhotoID, + &i.Dinner.GroundingStatus, + &i.Dinner.UpdatedBy, + &i.Dinner.CreatedAt, + &i.Dinner.UpdatedAt, + &i.Dinner.RecipeUrl, + &i.Dinner.GroundingError, + &i.Dinner.GroundingRevision, + &i.RecipeSourceUrl, ); err != nil { return nil, err } @@ -448,7 +477,7 @@ SET is_favorite = ?, updated_by = ?, updated_at = ? WHERE id = ? -RETURNING id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error +RETURNING id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error, grounding_revision ` type SetDinnerFavoriteParams struct { @@ -486,6 +515,7 @@ func (q *Queries) SetDinnerFavorite(ctx context.Context, arg SetDinnerFavoritePa &i.UpdatedAt, &i.RecipeUrl, &i.GroundingError, + &i.GroundingRevision, ) return i, err } @@ -496,7 +526,7 @@ SET is_favorite = 1 - is_favorite, updated_by = ?, updated_at = ? WHERE id = ? -RETURNING id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error +RETURNING id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error, grounding_revision ` type ToggleDinnerFavoriteParams struct { @@ -528,6 +558,7 @@ func (q *Queries) ToggleDinnerFavorite(ctx context.Context, arg ToggleDinnerFavo &i.UpdatedAt, &i.RecipeUrl, &i.GroundingError, + &i.GroundingRevision, ) return i, err } @@ -543,29 +574,33 @@ SET name = ?, difficulty = ?, cuisine = ?, recipe_id = ?, + recipe_url = ?, photo_id = ?, grounding_status = ?, + grounding_revision = ?, updated_by = ?, updated_at = ? WHERE id = ? -RETURNING id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error +RETURNING id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error, grounding_revision ` type UpdateDinnerParams struct { - Name string - NormalizedName string - Status string - Origin string - Method sql.NullString - MinutesTotal sql.NullInt64 - Difficulty sql.NullString - Cuisine sql.NullString - RecipeID sql.NullString - PhotoID sql.NullString - GroundingStatus string - UpdatedBy sql.NullString - UpdatedAt string - ID string + Name string + NormalizedName string + Status string + Origin string + Method sql.NullString + MinutesTotal sql.NullInt64 + Difficulty sql.NullString + Cuisine sql.NullString + RecipeID sql.NullString + RecipeUrl sql.NullString + PhotoID sql.NullString + GroundingStatus string + GroundingRevision int64 + UpdatedBy sql.NullString + UpdatedAt string + ID string } // An unknown id returns sql.ErrNoRows. @@ -580,8 +615,10 @@ func (q *Queries) UpdateDinner(ctx context.Context, arg UpdateDinnerParams) (Din arg.Difficulty, arg.Cuisine, arg.RecipeID, + arg.RecipeUrl, arg.PhotoID, arg.GroundingStatus, + arg.GroundingRevision, arg.UpdatedBy, arg.UpdatedAt, arg.ID, @@ -606,6 +643,7 @@ func (q *Queries) UpdateDinner(ctx context.Context, arg UpdateDinnerParams) (Din &i.UpdatedAt, &i.RecipeUrl, &i.GroundingError, + &i.GroundingRevision, ) return i, err } diff --git a/internal/db/models.go b/internal/db/models.go index ac5ef62..b026169 100644 --- a/internal/db/models.go +++ b/internal/db/models.go @@ -51,24 +51,25 @@ type ChatSetting struct { } type Dinner struct { - ID string - Name string - NormalizedName string - Status string - Origin string - Method sql.NullString - MinutesTotal sql.NullInt64 - Difficulty sql.NullString - Cuisine sql.NullString - IsFavorite int64 - RecipeID sql.NullString - PhotoID sql.NullString - GroundingStatus string - UpdatedBy sql.NullString - CreatedAt string - UpdatedAt string - RecipeUrl sql.NullString - GroundingError sql.NullString + ID string + Name string + NormalizedName string + Status string + Origin string + Method sql.NullString + MinutesTotal sql.NullInt64 + Difficulty sql.NullString + Cuisine sql.NullString + IsFavorite int64 + RecipeID sql.NullString + PhotoID sql.NullString + GroundingStatus string + UpdatedBy sql.NullString + CreatedAt string + UpdatedAt string + RecipeUrl sql.NullString + GroundingError sql.NullString + GroundingRevision int64 } type HouseholdSetting struct { diff --git a/internal/db/querier.go b/internal/db/querier.go index 2c6e15f..382e233 100644 --- a/internal/db/querier.go +++ b/internal/db/querier.go @@ -36,14 +36,16 @@ type Querier interface { DeleteRecipeIngredients(ctx context.Context, recipeID string) error DeleteRecipeSteps(ctx context.Context, recipeID string) error FailJob(ctx context.Context, arg FailJobParams) (int64, error) - // Fails every dinner still pending whose grounding job is no longer queued - // or running: the process stopped before the job, or before its handler - // recorded a result. Run at boot, after the job runner fails the jobs a - // previous process left running. + // Fails every unarchived dinner still pending whose grounding job for its + // current grounding_revision is no longer queued or running: the process + // stopped before the job, or before its handler recorded a result. An + // archived dinner is left as it is; archiving stopped its grounding. Run at + // boot, after the job runner fails the jobs a previous process left running. FailOrphanedPendingDinnerGroundings(ctx context.Context, arg FailOrphanedPendingDinnerGroundingsParams) (int64, error) - // Records why the background grounding of a dinner still pending under the - // normalized name it was grounded for failed; a dinner grounded some other - // way or renamed meanwhile is left as it is (0 rows). + // Records why the background grounding of an unarchived dinner still + // pending at the grounding_revision it was grounded for failed; a dinner + // grounded some other way, renamed or archived meanwhile is left as it is + // (0 rows). FailPendingDinnerGrounding(ctx context.Context, arg FailPendingDinnerGroundingParams) (int64, error) // Fails every job left running, at boot before any worker starts. FailRunningJobs(ctx context.Context, arg FailRunningJobsParams) (int64, error) @@ -65,12 +67,15 @@ type Querier interface { GetRecipeBySourceURL(ctx context.Context, sourceUrl string) (Recipe, error) // Returns sql.ErrNoRows until the row exists. GetRecipeSearchSettings(ctx context.Context) (RecipeSearchSetting, error) - // Records a dinner's grounding (grounded or manual) and clears any - // grounding_error. An unknown id returns sql.ErrNoRows. + // Records a dinner's grounding by hand (grounded or manual) and clears any + // grounding_error and pasted recipe_url: the recipe chosen replaces the + // link. An unknown id returns sql.ErrNoRows. GroundDinner(ctx context.Context, arg GroundDinnerParams) (Dinner, error) - // Records the background grounding of a dinner still pending under the - // normalized name it was grounded for; a dinner grounded some other way or - // renamed meanwhile is left as it is (0 rows). + // Records the background grounding of an unarchived dinner still pending at + // the grounding_revision it was grounded for; a dinner grounded some other + // way, renamed or archived meanwhile is left as it is (0 rows). A method, + // minutes_total or difficulty already set is kept: a pending dinner has + // them only when a member edited them while it was grounding. GroundPendingDinner(ctx context.Context, arg GroundPendingDinnerParams) (int64, error) ListDinnersByStatus(ctx context.Context, status string) ([]Dinner, error) // The planned dinners dated from from_date up to but not including @@ -81,8 +86,9 @@ type Querier interface { ListRecentDeclinedNames(ctx context.Context, limit int64) ([]string, error) ListRecipeIngredients(ctx context.Context, recipeID string) ([]RecipeIngredient, error) ListRecipeSteps(ctx context.Context, recipeID string) ([]RecipeStep, error) - // Every dinner a new or renamed dinner must not repeat. - ListUnarchivedDinners(ctx context.Context) ([]Dinner, error) + // Every dinner a new or renamed dinner must not repeat, with the + // source_url of the recipe it is grounded in (NULL while it has none). + ListUnarchivedDinners(ctx context.Context) ([]ListUnarchivedDinnersRow, error) // Stores a partial result of a running job. PublishJobResult(ctx context.Context, arg PublishJobResultParams) (int64, error) // An unknown id returns sql.ErrNoRows. diff --git a/internal/dinners/dinners.go b/internal/dinners/dinners.go index 731136a..605d675 100644 --- a/internal/dinners/dinners.go +++ b/internal/dinners/dinners.go @@ -11,9 +11,11 @@ import ( "errors" "fmt" "log/slog" + "math" "net/url" "slices" "strings" + "sync" "unicode" "unicode/utf8" @@ -130,6 +132,11 @@ type Service struct { grounder *recipes.Grounder logger *slog.Logger groundJob *jobs.Kind[GroundJob] + + // groundingsMu guards groundings: the KindGroundDinner jobs running, + // by dinner id, so archiving or renaming a dinner can stop them. + groundingsMu sync.Mutex + groundings map[string]map[*runningGrounding]struct{} } // NewService returns a service over st that grounds dinners with grounder @@ -146,7 +153,7 @@ func NewService(st *store.Store, grounder *recipes.Grounder, runner *jobs.Runner case logger == nil: return nil, errors.New("dinners: nil logger") } - s := &Service{st: st, grounder: grounder, logger: logger} + s := &Service{st: st, grounder: grounder, logger: logger, groundings: make(map[string]map[*runningGrounding]struct{})} kind, err := jobs.Register(runner, KindGroundDinner, s.runGroundJob, Reasons...) if err != nil { return nil, err @@ -176,7 +183,7 @@ func (s *Service) CreateRegular(ctx context.Context, r NewRegular, updatedBy str if err != nil { return Created{}, err } - jobID, err := s.enqueueGrounding(ctx, d.ID, harness.Normalize(d.Name)) + jobID, err := s.enqueueGrounding(ctx, d.ID, d.GroundingRevision) if err != nil { return Created{}, err } @@ -187,60 +194,59 @@ func (s *Service) CreateRegular(ctx context.Context, r NewRegular, updatedBy str } return Created{Dinner: fromRow(row)}, nil } - return Created{Dinner: d, GroundingJobID: jobID}, nil + return Created{Dinner: fromRow(d), GroundingJobID: jobID}, nil } -// enqueueGrounding enqueues the KindGroundDinner job for pending dinner id, -// whose normalized name is normalizedName, and returns its id. When the job -// cannot be enqueued, the dinner's grounding is failed with -// ReasonGroundingError, so it is not left pending with no job, and the id -// is nil. -func (s *Service) enqueueGrounding(ctx context.Context, id, normalizedName string) (*string, error) { - jobID, err := s.groundJob.Enqueue(ctx, GroundJob{DinnerID: id}) +// enqueueGrounding enqueues the KindGroundDinner job for pending dinner id +// at grounding_revision revision and returns its id. When the job cannot be +// enqueued, the dinner's grounding is failed with ReasonGroundingError, so +// it is not left pending with no job, and the id is nil. +func (s *Service) enqueueGrounding(ctx context.Context, id string, revision int64) (*string, error) { + jobID, err := s.groundJob.Enqueue(ctx, GroundJob{DinnerID: id, Revision: revision}) if err == nil { return &jobID, nil } // The error is a database error, never a dependency's text. s.logger.ErrorContext(ctx, "dinner grounding not enqueued", "dinner_id", id, "error", err) if _, err := db.New(s.st.Writer).FailPendingDinnerGrounding(context.WithoutCancel(ctx), db.FailPendingDinnerGroundingParams{ - GroundingError: sql.NullString{String: ReasonGroundingError, Valid: true}, - UpdatedAt: store.Now().String(), - ID: id, - NormalizedName: normalizedName, + GroundingError: sql.NullString{String: ReasonGroundingError, Valid: true}, + UpdatedAt: store.Now().String(), + ID: id, + GroundingRevision: revision, }); err != nil { return nil, fmt.Errorf("dinners: record failed grounding of %s: %w", id, err) } return nil, nil } -func (s *Service) createRegular(ctx context.Context, r NewRegular, updatedBy string) (Dinner, error) { +func (s *Service) createRegular(ctx context.Context, r NewRegular, updatedBy string) (db.Dinner, error) { if updatedBy == "" { - return Dinner{}, errors.New("dinners: create: no member to record as updated_by") + return db.Dinner{}, errors.New("dinners: create: no member to record as updated_by") } var name string var recipeURL sql.NullString switch { case r.Name != nil && r.RecipeURL != nil: - return Dinner{}, invalid("name", "give a name or a recipe_url, not both") + return db.Dinner{}, invalid("name", "give a name or a recipe_url, not both") case r.Name != nil: n, err := checkName(*r.Name) if err != nil { - return Dinner{}, err + return db.Dinner{}, err } name = n case r.RecipeURL != nil: u, err := checkRecipeURL(*r.RecipeURL) if err != nil { - return Dinner{}, err + return db.Dinner{}, err } n, ok := nameFromURL(u) if !ok { - return Dinner{}, invalid("recipe_url", "the link does not name a dinner; add it by name instead") + return db.Dinner{}, invalid("recipe_url", "the link does not name a dinner; add it by name instead") } name = n recipeURL = sql.NullString{String: u.String(), Valid: true} default: - return Dinner{}, invalid("name", "give a name or a recipe_url") + return db.Dinner{}, invalid("name", "give a name or a recipe_url") } var row db.Dinner @@ -269,9 +275,9 @@ func (s *Service) createRegular(ctx context.Context, r NewRegular, updatedBy str return nil }) if err != nil { - return Dinner{}, err + return db.Dinner{}, err } - return fromRow(row), nil + return row, nil } // List returns the dinners with status, which must be regular or @@ -294,8 +300,10 @@ func (s *Service) List(ctx context.Context, status string) ([]Dinner, error) { // Update applies p to dinner id, recording updatedBy. An invalid patch is a // *ValidationError and a new name repeating another unarchived dinner a // *DuplicateError; either changes nothing. Renaming a dinner still pending -// enqueues a fresh KindGroundDinner job for the new name: the job already -// running was for the old name, and its result no longer applies. +// clears its recipe_url, raises its grounding_revision and enqueues a fresh +// KindGroundDinner job that grounds the new name: a job queued or running +// for the old name or link carries the old revision, so it can no longer +// write the dinner, and one running is stopped. func (s *Service) Update(ctx context.Context, id string, p Patch, updatedBy string) (Dinner, error) { if updatedBy == "" { return Dinner{}, errors.New("dinners: update: no member to record as updated_by") @@ -317,20 +325,22 @@ func (s *Service) Update(ctx context.Context, id string, p Patch, updatedBy stri return fmt.Errorf("dinners: read %s: %w", id, err) } params := db.UpdateDinnerParams{ - ID: cur.ID, - Name: cur.Name, - NormalizedName: cur.NormalizedName, - Status: cur.Status, - Origin: cur.Origin, - Method: cur.Method, - MinutesTotal: cur.MinutesTotal, - Difficulty: cur.Difficulty, - Cuisine: cur.Cuisine, - RecipeID: cur.RecipeID, - PhotoID: cur.PhotoID, - GroundingStatus: cur.GroundingStatus, - UpdatedBy: sql.NullString{String: updatedBy, Valid: true}, - UpdatedAt: store.Now().String(), + ID: cur.ID, + Name: cur.Name, + NormalizedName: cur.NormalizedName, + Status: cur.Status, + Origin: cur.Origin, + Method: cur.Method, + MinutesTotal: cur.MinutesTotal, + Difficulty: cur.Difficulty, + Cuisine: cur.Cuisine, + RecipeID: cur.RecipeID, + RecipeUrl: cur.RecipeUrl, + PhotoID: cur.PhotoID, + GroundingStatus: cur.GroundingStatus, + GroundingRevision: cur.GroundingRevision, + UpdatedBy: sql.NullString{String: updatedBy, Valid: true}, + UpdatedAt: store.Now().String(), } if p.Name != nil { if err := checkUnique(ctx, q, cur.ID, name, sql.NullString{}); err != nil { @@ -338,6 +348,12 @@ func (s *Service) Update(ctx context.Context, id string, p Patch, updatedBy stri } params.Name = name params.NormalizedName = harness.Normalize(name) + if cur.GroundingStatus == GroundingPending && params.NormalizedName != cur.NormalizedName { + // The link was the old dinner's recipe; the new name is + // grounded by name, and a link can be given by hand. + params.RecipeUrl = sql.NullString{} + params.GroundingRevision++ + } } if p.Method != nil { params.Method = sql.NullString{String: *p.Method, Valid: true} @@ -352,14 +368,15 @@ func (s *Service) Update(ctx context.Context, id string, p Patch, updatedBy stri if err != nil { return fmt.Errorf("dinners: update %s: %w", id, err) } - regrounding = row.GroundingStatus == GroundingPending && row.NormalizedName != cur.NormalizedName + regrounding = row.GroundingRevision != cur.GroundingRevision return nil }) if err != nil { return Dinner{}, err } if regrounding { - jobID, err := s.enqueueGrounding(ctx, row.ID, row.NormalizedName) + s.stopGroundings(row.ID, row.GroundingRevision) + jobID, err := s.enqueueGrounding(ctx, row.ID, row.GroundingRevision) if err != nil { return Dinner{}, err } @@ -374,7 +391,8 @@ func (s *Service) Update(ctx context.Context, id string, p Patch, updatedBy stri } // Archive sets dinner id's status to archived, recording updatedBy. The row -// is kept, so its name still counts for Side quest dedupe. +// is kept, so its name still counts for Side quest dedupe. A +// KindGroundDinner job running for it is stopped. func (s *Service) Archive(ctx context.Context, id, updatedBy string) (Dinner, error) { if updatedBy == "" { return Dinner{}, errors.New("dinners: archive: no member to record as updated_by") @@ -390,6 +408,7 @@ func (s *Service) Archive(ctx context.Context, id, updatedBy string) (Dinner, er if err != nil { return Dinner{}, fmt.Errorf("dinners: archive %s: %w", id, err) } + s.stopGroundings(row.ID, math.MaxInt64) return fromRow(row), nil } @@ -507,18 +526,24 @@ func nameFromURL(u *url.URL) (name string, ok bool) { } // checkUnique returns a *DuplicateError when name is harness.Similar to an -// unarchived dinner other than self, or recipeURL is one's recipe link. +// unarchived dinner other than self, or recipeURL is one's recipe link: the +// source_url of the recipe it is grounded in, else the recipe_url still +// awaiting grounding. func checkUnique(ctx context.Context, q *db.Queries, self, name string, recipeURL sql.NullString) error { rows, err := q.ListUnarchivedDinners(ctx) if err != nil { return fmt.Errorf("dinners: list for dedupe: %w", err) } for _, row := range rows { - if row.ID == self { + if row.Dinner.ID == self { continue } - if harness.Similar(name, row.Name) || (recipeURL.Valid && row.RecipeUrl == recipeURL) { - return &DuplicateError{Existing: fromRow(row)} + link := row.Dinner.RecipeUrl + if row.RecipeSourceUrl.Valid { + link = row.RecipeSourceUrl + } + if harness.Similar(name, row.Dinner.Name) || (recipeURL.Valid && link == recipeURL) { + return &DuplicateError{Existing: fromRow(row.Dinner)} } } return nil diff --git a/internal/dinners/export_test.go b/internal/dinners/export_test.go index 9eb3511..ca06c60 100644 --- a/internal/dinners/export_test.go +++ b/internal/dinners/export_test.go @@ -1,6 +1,7 @@ package dinners import ( + "context" "testing" "time" ) @@ -12,3 +13,11 @@ func SetGroundTimeout(t *testing.T, d time.Duration) { groundTimeout = d t.Cleanup(func() { groundTimeout = old }) } + +// RunGroundJob runs the KindGroundDinner handler for p in the caller's +// goroutine, as the job runner would, and returns the grounding status it +// reports. +func (s *Service) RunGroundJob(ctx context.Context, p GroundJob) (string, error) { + r, err := s.runGroundJob(ctx, p, nil) + return r.GroundingStatus, err +} diff --git a/internal/dinners/grounding.go b/internal/dinners/grounding.go index b3ff4f7..9fd4115 100644 --- a/internal/dinners/grounding.go +++ b/internal/dinners/grounding.go @@ -76,14 +76,17 @@ var groundTimeout = GroundTimeout // runs past GroundTimeout; the dinner is left as it was. var ErrGroundTimeout = errors.New("dinners: grounding the recipe took too long") -// GroundJob is the KindGroundDinner payload. +// GroundJob is the KindGroundDinner payload: the dinner to ground and its +// grounding_revision when the job was enqueued. type GroundJob struct { DinnerID string `json:"dinner_id"` + Revision int64 `json:"revision"` } // groundResult is the KindGroundDinner result: the dinner's grounding -// status once the job ends. It is not grounded when the dinner was -// grounded by hand before the job finished. +// status once the job ends. It is not grounded by this job when the dinner +// was grounded by hand before the job finished, renamed (a later job +// grounds it) or archived; the job then reports the status it found. type groundResult struct { DinnerID string `json:"dinner_id"` GroundingStatus string `json:"grounding_status"` @@ -93,26 +96,38 @@ func (r groundResult) PublicJSON() ([]byte, error) { return json.Marshal(r) } // runGroundJob grounds a pending dinner as its creation asked: its // recipe_url with GroundURL, else its name with Ground. Success records -// the recipe, photo, method, minutes and difficulty with grounding_status -// grounded; failure records grounding_status failed with its Reason, which -// is also the job's failure code. Either is recorded only while the dinner -// is still pending under the normalized name read here: a dinner grounded -// by hand meanwhile is left as it is, and one renamed meanwhile is left -// pending for the job its rename enqueued. +// the recipe and photo with grounding_status grounded, and the recipe's +// method, minutes and difficulty where a member has not set them; failure +// records grounding_status failed with its Reason, which is also the job's +// failure code. A dinner archived, no longer pending, or at a +// grounding_revision other than p.Revision is not grounded at all, and +// either result is recorded only while the dinner is still unarchived, +// pending and at p.Revision: a dinner grounded by hand meanwhile is left as +// it is, and one renamed or archived meanwhile is left pending — for the +// job its rename enqueued, or for none. Archiving or renaming the dinner +// stops the grounding where it is, so no further search, fetch, photo +// download or model call is made for it. func (s *Service) runGroundJob(ctx context.Context, p GroundJob, _ func(groundResult) error) (groundResult, error) { + // Tracked before the dinner is read: an archive or rename committed + // after the read finds this job to stop. + groundCtx, done := s.trackGrounding(ctx, p.DinnerID, p.Revision) + defer done() row, err := db.New(s.st.Reader).GetDinner(ctx, p.DinnerID) if err != nil { return groundResult{}, fmt.Errorf("dinners: ground: read %s: %w", p.DinnerID, err) } - if row.GroundingStatus != GroundingPending { + if row.Status == StatusArchived || row.GroundingStatus != GroundingPending || row.GroundingRevision != p.Revision { return groundResult{DinnerID: row.ID, GroundingStatus: row.GroundingStatus}, nil } var gr recipes.Grounded if row.RecipeUrl.Valid { - gr, err = s.grounder.GroundURL(ctx, row.RecipeUrl.String) + gr, err = s.grounder.GroundURL(groundCtx, row.RecipeUrl.String) } else { - gr, err = s.grounder.Ground(ctx, row.Name, row.Name) + gr, err = s.grounder.Ground(groundCtx, row.Name, row.Name) + } + if err != nil && ctx.Err() == nil && errors.Is(context.Cause(groundCtx), errGroundingStopped) { + return s.currentGrounding(ctx, row.ID) } if err != nil { reason := reasonFor(err, row.RecipeUrl.Valid) @@ -124,10 +139,10 @@ func (s *Service) runGroundJob(ctx context.Context, p GroundJob, _ func(groundRe s.logger.InfoContext(ctx, "dinner grounding failed", "dinner_id", row.ID, "reason", reason) write := context.WithoutCancel(ctx) if _, failErr := db.New(s.st.Writer).FailPendingDinnerGrounding(write, db.FailPendingDinnerGroundingParams{ - GroundingError: sql.NullString{String: reason, Valid: true}, - UpdatedAt: store.Now().String(), - ID: row.ID, - NormalizedName: row.NormalizedName, + GroundingError: sql.NullString{String: reason, Valid: true}, + UpdatedAt: store.Now().String(), + ID: row.ID, + GroundingRevision: p.Revision, }); failErr != nil { return groundResult{}, fmt.Errorf("dinners: ground: record failure of %s: %w", row.ID, failErr) } @@ -138,32 +153,85 @@ func (s *Service) runGroundJob(ctx context.Context, p GroundJob, _ func(groundRe } n, err := db.New(s.st.Writer).GroundPendingDinner(ctx, db.GroundPendingDinnerParams{ - RecipeID: sql.NullString{String: gr.RecipeID, Valid: true}, - PhotoID: sql.NullString{String: gr.PhotoID, Valid: true}, - Method: sql.NullString{String: gr.Method, Valid: true}, - MinutesTotal: nullInt(gr.MinutesTotal), - Difficulty: sql.NullString{String: gr.Difficulty, Valid: true}, - UpdatedAt: store.Now().String(), - ID: row.ID, - NormalizedName: row.NormalizedName, + RecipeID: sql.NullString{String: gr.RecipeID, Valid: true}, + PhotoID: sql.NullString{String: gr.PhotoID, Valid: true}, + Method: sql.NullString{String: gr.Method, Valid: true}, + MinutesTotal: nullInt(gr.MinutesTotal), + Difficulty: sql.NullString{String: gr.Difficulty, Valid: true}, + UpdatedAt: store.Now().String(), + ID: row.ID, + GroundingRevision: p.Revision, }) if err != nil { return groundResult{}, fmt.Errorf("dinners: ground: record %s: %w", row.ID, err) } if n == 0 { - cur, err := db.New(s.st.Reader).GetDinner(ctx, row.ID) - if err != nil { - return groundResult{}, fmt.Errorf("dinners: ground: read %s: %w", row.ID, err) - } - return groundResult{DinnerID: cur.ID, GroundingStatus: cur.GroundingStatus}, nil + return s.currentGrounding(ctx, row.ID) } return groundResult{DinnerID: row.ID, GroundingStatus: GroundingGrounded}, nil } -// FailInterruptedGroundings fails, with ReasonInterrupted, every dinner -// still pending whose KindGroundDinner job is no longer queued or running: -// a previous process stopped before enqueuing the job or before its -// handler recorded a result. Call it once at boot, after the job runner's +// currentGrounding is the KindGroundDinner result of a job that did not +// ground dinner id: the grounding status the dinner has now. +func (s *Service) currentGrounding(ctx context.Context, id string) (groundResult, error) { + cur, err := db.New(s.st.Reader).GetDinner(ctx, id) + if err != nil { + return groundResult{}, fmt.Errorf("dinners: ground: read %s: %w", id, err) + } + return groundResult{DinnerID: cur.ID, GroundingStatus: cur.GroundingStatus}, nil +} + +// errGroundingStopped is the cause a running KindGroundDinner job's +// grounding is cancelled with when its dinner is archived or renamed. +var errGroundingStopped = errors.New("dinners: grounding stopped: dinner archived or renamed") + +// runningGrounding is a KindGroundDinner job running for a dinner at +// revision, with the cancel of its grounding. +type runningGrounding struct { + revision int64 + cancel context.CancelCauseFunc +} + +// trackGrounding returns a context for grounding dinnerID at revision that +// stopGroundings cancels, and the func to call when the job ends. +func (s *Service) trackGrounding(ctx context.Context, dinnerID string, revision int64) (context.Context, func()) { + groundCtx, cancel := context.WithCancelCause(ctx) + r := &runningGrounding{revision: revision, cancel: cancel} + s.groundingsMu.Lock() + if s.groundings[dinnerID] == nil { + s.groundings[dinnerID] = make(map[*runningGrounding]struct{}) + } + s.groundings[dinnerID][r] = struct{}{} + s.groundingsMu.Unlock() + return groundCtx, func() { + s.groundingsMu.Lock() + delete(s.groundings[dinnerID], r) + if len(s.groundings[dinnerID]) == 0 { + delete(s.groundings, dinnerID) + } + s.groundingsMu.Unlock() + cancel(nil) + } +} + +// stopGroundings cancels, with errGroundingStopped, the grounding of every +// KindGroundDinner job running for dinnerID at a revision below below. Call +// it once the change that makes those jobs stale is committed. +func (s *Service) stopGroundings(dinnerID string, below int64) { + s.groundingsMu.Lock() + defer s.groundingsMu.Unlock() + for r := range s.groundings[dinnerID] { + if r.revision < below { + r.cancel(errGroundingStopped) + } + } +} + +// FailInterruptedGroundings fails, with ReasonInterrupted, every unarchived +// dinner still pending whose KindGroundDinner job is no longer queued or +// running: a previous process stopped before enqueuing the job or before +// its handler recorded a result. An archived dinner is left pending, since +// archiving, not a restart, stopped its grounding. Call it once at boot, after the job runner's // Start has failed the jobs that process left running, so no dinner stays // pending with nothing left to ground it. It returns how many it failed. func (s *Service) FailInterruptedGroundings(ctx context.Context) (int64, error) { diff --git a/internal/dinners/grounding_test.go b/internal/dinners/grounding_test.go index 325082d..2e14546 100644 --- a/internal/dinners/grounding_test.go +++ b/internal/dinners/grounding_test.go @@ -2,11 +2,15 @@ package dinners_test import ( "context" + "database/sql" + "encoding/json" "errors" "io" + "io/fs" "log/slog" "slices" "testing" + "testing/fstest" "time" "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm" @@ -52,6 +56,13 @@ func newEnv(t *testing.T) *env { func newUnstartedEnv(t *testing.T) *env { t.Helper() dataDir := t.TempDir() + return newUnstartedEnvOn(t, openStore(t, dataDir), dataDir) +} + +// openStore opens the store in dataDir, closing it when the test ends. It +// is not migrated. +func openStore(t *testing.T, dataDir string) *store.Store { + t.Helper() st, err := store.Open(t.Context(), dataDir) if err != nil { t.Fatal(err) @@ -61,6 +72,13 @@ func newUnstartedEnv(t *testing.T) *env { t.Error(err) } }) + return st +} + +// newUnstartedEnvOn is newUnstartedEnv over st, the store in dataDir, +// which it migrates to the latest version. +func newUnstartedEnvOn(t *testing.T, st *store.Store, dataDir string) *env { + t.Helper() if err := store.Migrate(t.Context(), st.Writer, migrations.FS); err != nil { t.Fatal(err) } @@ -288,19 +306,16 @@ func TestManualFixWhilePendingIsKept(t *testing.T) { // TestRenameWhilePendingGroundsNewName: a Regular renamed while its job is // still grounding the old name never gets the old name's result, whether -// that job succeeds or fails; the job the rename enqueues grounds the new -// name. +// the old name would be found or not: the rename stops that job where it +// is, and the job the rename enqueues grounds the new name. func TestRenameWhilePendingGroundsNewName(t *testing.T) { t.Parallel() tests := []struct { name string oldName string - replies []llm.GenerateResult }{ - {"old name found", "Chicken curry", []llm.GenerateResult{ - recipestest.NormalizedReply("stovetop", 40), recipestest.NormalizedReply("oven", 150), - }}, - {"old name not found", "Unknown dish", []llm.GenerateResult{recipestest.NormalizedReply("oven", 150)}}, + {"old name found", "Chicken curry"}, + {"old name not found", "Unknown dish"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -308,9 +323,7 @@ func TestRenameWhilePendingGroundsNewName(t *testing.T) { e := newEnv(t) e.search.Results["Beef stew"] = []recipes.Candidate{{URL: stew}} e.search.Gate = make(chan struct{}) - for _, r := range tt.replies { - e.model.QueueGenerate(r) - } + e.model.QueueGenerate(recipestest.NormalizedReply("oven", 150)) c, err := e.svc.CreateRegular(t.Context(), dinners.NewRegular{Name: ptr(tt.oldName)}, member) if err != nil { @@ -325,12 +338,13 @@ func TestRenameWhilePendingGroundsNewName(t *testing.T) { if d.GroundingStatus != dinners.GroundingPending { t.Fatalf("renamed = %+v, want still pending", d) } + // The old name's job ends while its search is still held. + first := e.waitJob(t, *c.GroundingJobID) + if first.Status != jobs.StatusSucceeded || string(first.Result) != `{"dinner_id":"`+c.ID+`","grounding_status":"pending"}` { + t.Fatalf("old name's job = %+v (result %s), want succeeded leaving the dinner pending", first, first.Result) + } close(e.search.Gate) - first := e.waitJob(t, *c.GroundingJobID) - if string(first.Result) == `{"dinner_id":"`+c.ID+`","grounding_status":"grounded"}` { - t.Fatalf("old name's job reported grounded: %+v (result %s)", first, first.Result) - } row := e.waitGrounding(t, c.ID) if row.GroundingStatus != dinners.GroundingGrounded { t.Fatalf("dinner = %+v, want grounded for the new name", row) @@ -345,10 +359,222 @@ func TestRenameWhilePendingGroundsNewName(t *testing.T) { if got := e.search.Queries(); !slices.Equal(got, []string{tt.oldName, "Beef stew"}) { t.Errorf("searched %q, want the old name then the new", got) } + if got := len(e.model.Requests()); got != 1 { + t.Errorf("model called %d times, want once, for the new name only", got) + } }) } } +// TestRenameLinkWhilePendingGroundsNewName: a Regular added from a link and +// renamed while still pending drops the link and grounds its new name, not +// the old link's recipe and photo. +func TestRenameLinkWhilePendingGroundsNewName(t *testing.T) { + t.Parallel() + e := newUnstartedEnv(t) + e.search.Results["Beef stew"] = []recipes.Candidate{{URL: stew}} + e.model.QueueGenerate(recipestest.NormalizedReply("oven", 150)) + + c, err := e.svc.CreateRegular(t.Context(), dinners.NewRegular{RecipeURL: ptr(curryA)}, member) + if err != nil { + t.Fatal(err) + } + d, err := e.svc.Update(t.Context(), c.ID, dinners.Patch{Name: ptr("Beef stew")}, member) + if err != nil { + t.Fatal(err) + } + if d.GroundingStatus != dinners.GroundingPending || d.RecipeURL != nil { + t.Fatalf("renamed = %+v, want still pending with no recipe_url", d) + } + e.start(t) + + row := e.waitGrounding(t, c.ID) + if row.GroundingStatus != dinners.GroundingGrounded { + t.Fatalf("dinner = %+v, want grounded for the new name", row) + } + rec, err := db.New(e.st.Reader).GetRecipe(t.Context(), row.RecipeID.String) + if err != nil { + t.Fatal(err) + } + if rec.SourceUrl != stew || rec.PhotoID != row.PhotoID || row.Method.String != "oven" || row.MinutesTotal.Int64 != 150 { + t.Errorf("dinner = %+v recipe %s, want grounded in %s, not the old link %s", row, rec.SourceUrl, stew, curryA) + } + if got := e.search.Queries(); !slices.Equal(got, []string{"Beef stew"}) { + t.Errorf("searched %q, want only the new name", got) + } + if row.RecipeUrl.Valid { + t.Errorf("recipe_url = %q, want the old link dropped", row.RecipeUrl.String) + } + if _, err := db.New(e.st.Reader).GetRecipeBySourceURL(t.Context(), curryA); !errors.Is(err, sql.ErrNoRows) { + t.Errorf("old link's recipe lookup err = %v, want it never grounded (sql.ErrNoRows)", err) + } +} + +// queuedJob reads the payload of queued KindGroundDinner job id. +func (e *env) queuedJob(t *testing.T, id string) dinners.GroundJob { + t.Helper() + var payload string + if err := e.st.Reader.QueryRowContext(t.Context(), `SELECT payload FROM jobs WHERE id = ?`, id).Scan(&payload); err != nil { + t.Fatal(err) + } + var p dinners.GroundJob + if err := json.Unmarshal([]byte(payload), &p); err != nil { + t.Fatal(err) + } + return p +} + +// TestRenameBeforeJobStartsSkipsStaleJob: a Regular renamed before its +// first job starts is grounded only by the job its rename enqueued. The +// first job, though it would read the new name, does no work and writes +// nothing, so its failure cannot leave the dinner "Needs a recipe" while +// the rename's job finds a recipe. +func TestRenameBeforeJobStartsSkipsStaleJob(t *testing.T) { + t.Parallel() + e := newUnstartedEnv(t) + e.search.Results["Beef stew"] = []recipes.Candidate{{URL: stew}} + + c, err := e.svc.CreateRegular(t.Context(), dinners.NewRegular{Name: ptr("Chicken curry")}, member) + if err != nil { + t.Fatal(err) + } + first := e.queuedJob(t, *c.GroundingJobID) + if first.DinnerID != c.ID || first.Revision != e.dinner(t, c.ID).GroundingRevision { + t.Fatalf("first job = %+v, want dinner %s at its revision", first, c.ID) + } + if _, err := e.svc.Update(t.Context(), c.ID, dinners.Patch{Name: ptr("Beef stew")}, member); err != nil { + t.Fatal(err) + } + renamed := e.dinner(t, c.ID) + if renamed.GroundingRevision != first.Revision+1 { + t.Fatalf("revision after rename = %d, want %d", renamed.GroundingRevision, first.Revision+1) + } + + // The first job would fail if it searched. + e.search.Err = recipes.ErrSearchRateLimited + status, err := e.svc.RunGroundJob(t.Context(), first) + if err != nil || status != dinners.GroundingPending { + t.Fatalf("first job = %q, %v; want pending, no error", status, err) + } + if q := e.search.Queries(); len(q) != 0 { + t.Errorf("first job searched %q, want no work for a stale revision", q) + } + if row := e.dinner(t, c.ID); row != renamed { + t.Fatalf("first job wrote the dinner: %+v, was %+v", row, renamed) + } + + e.search.Err = nil + e.model.QueueGenerate(recipestest.NormalizedReply("oven", 150)) + status, err = e.svc.RunGroundJob(t.Context(), dinners.GroundJob{DinnerID: c.ID, Revision: renamed.GroundingRevision}) + if err != nil || status != dinners.GroundingGrounded { + t.Fatalf("rename's job = %q, %v; want grounded", status, err) + } + row := e.dinner(t, c.ID) + rec, err := db.New(e.st.Reader).GetRecipe(t.Context(), row.RecipeID.String) + if err != nil { + t.Fatal(err) + } + if row.GroundingStatus != dinners.GroundingGrounded || rec.SourceUrl != stew { + t.Errorf("dinner = %+v recipe %s, want grounded in %s", row, rec.SourceUrl, stew) + } + + // A stale job finishing last changes nothing either. + if status, err := e.svc.RunGroundJob(t.Context(), first); err != nil || status != dinners.GroundingGrounded { + t.Errorf("first job run last = %q, %v; want grounded reported, no error", status, err) + } + if after := e.dinner(t, c.ID); after != row { + t.Errorf("stale job changed the grounded dinner: %+v, was %+v", after, row) + } +} + +// TestEditWhilePendingIsKept: a method or minutes a member saves while a +// Regular is grounding is kept when grounding succeeds; what the member +// did not set comes from the recipe. +func TestEditWhilePendingIsKept(t *testing.T) { + t.Parallel() + e := newEnv(t) + e.search.Gate = make(chan struct{}) + e.model.QueueGenerate(recipestest.NormalizedReply("stovetop", 40)) + + c, err := e.svc.CreateRegular(t.Context(), dinners.NewRegular{Name: ptr("Chicken curry")}, member) + if err != nil { + t.Fatal(err) + } + e.waitQuery(t, "Chicken curry") + if _, err := e.svc.Update(t.Context(), c.ID, dinners.Patch{Method: ptr("grill"), MinutesTotal: ptr(15)}, member); err != nil { + t.Fatal(err) + } + close(e.search.Gate) + + row := e.waitGrounding(t, c.ID) + if row.GroundingStatus != dinners.GroundingGrounded || !row.RecipeID.Valid || !row.PhotoID.Valid { + t.Fatalf("dinner = %+v, want grounded", row) + } + if row.Method.String != "grill" || row.MinutesTotal.Int64 != 15 || row.Difficulty.String != "easy" { + t.Errorf("dinner = %s, %d minutes, %s; want the member's grill and 15 minutes, the recipe's easy", + row.Method.String, row.MinutesTotal.Int64, row.Difficulty.String) + } +} + +// TestArchiveWhilePendingStopsGrounding: a pending Regular archived before +// its job starts is not searched for, and one archived while its job runs +// is stopped there, with no page fetched or model called; neither ends up +// grounded. +func TestArchiveWhilePendingStopsGrounding(t *testing.T) { + t.Parallel() + t.Run("before the job starts", func(t *testing.T) { + t.Parallel() + e := newUnstartedEnv(t) + e.model.QueueGenerate(recipestest.NormalizedReply("stovetop", 40)) + c, err := e.svc.CreateRegular(t.Context(), dinners.NewRegular{Name: ptr("Chicken curry")}, member) + if err != nil { + t.Fatal(err) + } + if _, err := e.svc.Archive(t.Context(), c.ID, member); err != nil { + t.Fatal(err) + } + e.start(t) + if job := e.waitJob(t, *c.GroundingJobID); job.Status != jobs.StatusSucceeded { + t.Fatalf("job = %+v, want succeeded with nothing to do", job) + } + if q := e.search.Queries(); len(q) != 0 { + t.Errorf("searched %q for an archived dinner", q) + } + if row := e.dinner(t, c.ID); row.Status != dinners.StatusArchived || row.GroundingStatus == dinners.GroundingGrounded || row.RecipeID.Valid { + t.Errorf("dinner = %+v, want archived and not grounded", row) + } + }) + t.Run("while the job runs", func(t *testing.T) { + t.Parallel() + e := newEnv(t) + e.search.Gate = make(chan struct{}) + e.model.QueueGenerate(recipestest.NormalizedReply("stovetop", 40)) + c, err := e.svc.CreateRegular(t.Context(), dinners.NewRegular{Name: ptr("Chicken curry")}, member) + if err != nil { + t.Fatal(err) + } + e.waitQuery(t, "Chicken curry") + if _, err := e.svc.Archive(t.Context(), c.ID, member); err != nil { + t.Fatal(err) + } + // The job ends while its search is still held: archiving stopped it. + job := e.waitJob(t, *c.GroundingJobID) + close(e.search.Gate) + if job.Status != jobs.StatusSucceeded || string(job.Result) != `{"dinner_id":"`+c.ID+`","grounding_status":"pending"}` { + t.Fatalf("job = %+v (result %s), want succeeded leaving the dinner pending", job, job.Result) + } + if got := e.pages.Fetched(); len(got) != 0 { + t.Errorf("fetched %q for an archived dinner", got) + } + if got := len(e.model.Requests()); got != 0 { + t.Errorf("model called %d times for an archived dinner", got) + } + if row := e.dinner(t, c.ID); row.Status != dinners.StatusArchived || row.GroundingStatus == dinners.GroundingGrounded || row.RecipeID.Valid { + t.Errorf("dinner = %+v, want archived and not grounded", row) + } + }) +} + // TestRenameWhilePendingEnqueueFailure: a pending Regular renamed when the // job for its new name cannot be enqueued is failed, not left pending. func TestRenameWhilePendingEnqueueFailure(t *testing.T) { @@ -492,6 +718,59 @@ func TestFixViaCandidate(t *testing.T) { } } +// TestFixReplacesPendingLink: fixing a Regular added from a link that held +// no recipe drops that link, so it no longer blocks adding the link again, +// while the link of the recipe chosen does. +func TestFixReplacesPendingLink(t *testing.T) { + t.Parallel() + const lost = "https://gone.example/recipes/lamb-tagine" + tests := []struct { + name string + fix dinners.Fix + // linkTaken is the recipe link the fixed dinner now has, if any. + linkTaken string + }{ + {"candidate", dinners.Fix{CandidateID: ptr(curryB)}, curryB}, + {"manual", dinners.Fix{Manual: &dinners.ManualRecipe{Ingredients: []string{"1 lamb"}, Minutes: ptr(90), Method: ptr("oven")}}, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + e := newEnv(t) + c, err := e.svc.CreateRegular(t.Context(), dinners.NewRegular{RecipeURL: ptr(lost)}, member) + if err != nil { + t.Fatal(err) + } + if job := e.waitJob(t, *c.GroundingJobID); job.Status != jobs.StatusFailed { + t.Fatalf("job = %+v, want failed", job) + } + // Renamed, so only the link could repeat it. + if _, err := e.svc.Update(t.Context(), c.ID, dinners.Patch{Name: ptr("Weeknight dinner")}, member); err != nil { + t.Fatal(err) + } + e.model.QueueGenerate(recipestest.NormalizedReply("stovetop", 25)) + d, err := e.svc.Ground(t.Context(), c.ID, tt.fix, member) + if err != nil { + t.Fatal(err) + } + if d.RecipeURL != nil { + t.Errorf("recipe_url = %q after the fix, want the unusable link dropped", *d.RecipeURL) + } + + if _, err := e.svc.CreateRegular(t.Context(), dinners.NewRegular{RecipeURL: ptr(lost)}, member); err != nil { + t.Errorf("adding the dropped link again: %v, want added", err) + } + if tt.linkTaken != "" { + _, err := e.svc.CreateRegular(t.Context(), dinners.NewRegular{RecipeURL: ptr(tt.linkTaken)}, member) + var dup *dinners.DuplicateError + if !errors.As(err, &dup) || dup.Existing.ID != c.ID { + t.Errorf("adding the fixed dinner's recipe link: err = %v, want a duplicate of %s", err, c.ID) + } + } + }) + } +} + func TestFixViaRecipeURL(t *testing.T) { t.Parallel() e := newEnv(t) @@ -667,6 +946,126 @@ func TestRestartFailsInterruptedGrounding(t *testing.T) { } } +// TestRestartKeepsArchivedPendingDinner: a pending Regular archived while +// its job runs stays pending, its job succeeded; the boot cleanup leaves it +// as it is rather than failing it as interrupted. +func TestRestartKeepsArchivedPendingDinner(t *testing.T) { + t.Parallel() + e := newEnv(t) + e.search.Gate = make(chan struct{}) + c, err := e.svc.CreateRegular(t.Context(), dinners.NewRegular{Name: ptr("Chicken curry")}, member) + if err != nil { + t.Fatal(err) + } + e.waitQuery(t, "Chicken curry") + if _, err := e.svc.Archive(t.Context(), c.ID, member); err != nil { + t.Fatal(err) + } + job := e.waitJob(t, *c.GroundingJobID) + close(e.search.Gate) + if job.Status != jobs.StatusSucceeded { + t.Fatalf("job = %+v, want succeeded", job) + } + + // The database is now as the next boot finds it. + n, err := e.svc.FailInterruptedGroundings(t.Context()) + if err != nil { + t.Fatal(err) + } + if n != 0 { + t.Errorf("failed %d dinners, want 0", n) + } + if row := e.dinner(t, c.ID); row.Status != dinners.StatusArchived || + row.GroundingStatus != dinners.GroundingPending || row.GroundingError.Valid { + t.Errorf("dinner = %s, %s (%s), want archived and pending with no error", + row.Status, row.GroundingStatus, row.GroundingError.String) + } +} + +// TestUpgradeKeepsLegacyQueuedGrounding: a pending Regular whose +// ground_dinner job was queued before migration 0010, so its payload has +// no revision, is not failed as interrupted at the first boot after the +// upgrade; its job still grounds it. The upgrade starts either from 0009 +// or from a database that has already applied 0010. +func TestUpgradeKeepsLegacyQueuedGrounding(t *testing.T) { + t.Parallel() + for name, below := range map[string]string{ + "from 0009": "0010", + "from 0010": "0011", + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + upgradeWithLegacyQueuedGrounding(t, below) + }) + } +} + +// upgradeWithLegacyQueuedGrounding migrates a database to every migration +// named before below, adds a pending Regular with a queued ground_dinner +// job whose payload has no revision, then boots on the latest migrations. +func upgradeWithLegacyQueuedGrounding(t *testing.T, below string) { + ctx := t.Context() + dataDir := t.TempDir() + st := openStore(t, dataDir) + before := fstest.MapFS{} + files, err := fs.Glob(migrations.FS, "*.sql") + if err != nil { + t.Fatal(err) + } + for _, name := range files { + if name >= below { + continue + } + body, err := fs.ReadFile(migrations.FS, name) + if err != nil { + t.Fatal(err) + } + before[name] = &fstest.MapFile{Data: body} + } + if err := store.Migrate(ctx, st.Writer, before); err != nil { + t.Fatal(err) + } + now := store.Now().String() + if _, err := st.Writer.ExecContext(ctx, `INSERT INTO dinners + (id, name, normalized_name, status, origin, is_favorite, grounding_status, updated_by, created_at, updated_at) + VALUES ('dinner-1', 'Chicken curry', 'chicken curry', 'regular', 'regular', 0, 'pending', ?, ?, ?)`, + member, now, now); err != nil { + t.Fatal(err) + } + if _, err := st.Writer.ExecContext(ctx, `INSERT INTO jobs (id, kind, status, payload, created_at, updated_at) + VALUES ('job-1', ?, 'queued', '{"dinner_id":"dinner-1"}', ?, ?)`, + dinners.KindGroundDinner, now, now); err != nil { + t.Fatal(err) + } + + // Boot: migrate, start the runner, then fail interrupted groundings. The + // search is held, so the job cannot finish before the cleanup runs. + e := newUnstartedEnvOn(t, st, dataDir) + e.search.Gate = make(chan struct{}) + e.model.QueueGenerate(recipestest.NormalizedReply("stovetop", 40)) + if p := e.queuedJob(t, "job-1"); p != (dinners.GroundJob{DinnerID: "dinner-1", Revision: 0}) { + t.Errorf("migrated payload = %+v, want dinner-1 at revision 0", p) + } + e.start(t) + n, err := e.svc.FailInterruptedGroundings(ctx) + if err != nil { + t.Fatal(err) + } + if n != 0 { + t.Errorf("failed %d dinners, want 0", n) + } + if row := e.dinner(t, "dinner-1"); row.GroundingStatus != dinners.GroundingPending { + t.Fatalf("dinner = %s (%s), want still pending", row.GroundingStatus, row.GroundingError.String) + } + close(e.search.Gate) + if job := e.waitJob(t, "job-1"); job.Status != jobs.StatusSucceeded { + t.Fatalf("job = %+v", job) + } + if row := e.dinner(t, "dinner-1"); row.GroundingStatus != dinners.GroundingGrounded { + t.Errorf("dinner = %s, want grounded", row.GroundingStatus) + } +} + // TestCreateEnqueueFailureReturnsFailedDinner: a dinner saved when its job // cannot be enqueued is returned, with its id, failed and no job id. func TestCreateEnqueueFailureReturnsFailedDinner(t *testing.T) { diff --git a/internal/recipes/recipestest/recipestest.go b/internal/recipes/recipestest/recipestest.go index 46aa72a..9d22dfa 100644 --- a/internal/recipes/recipestest/recipestest.go +++ b/internal/recipes/recipestest/recipestest.go @@ -66,9 +66,15 @@ func (s *Search) Queries() []string { type Pages struct { Bodies map[string][]byte Gate chan struct{} + + mu sync.Mutex + fetched []string } func (p *Pages) Fetch(ctx context.Context, rawURL string) (recipes.Page, error) { + p.mu.Lock() + p.fetched = append(p.fetched, rawURL) + p.mu.Unlock() if p.Gate != nil { select { case <-p.Gate: @@ -83,6 +89,13 @@ func (p *Pages) Fetch(ctx context.Context, rawURL string) (recipes.Page, error) return recipes.Page{URL: rawURL, ContentType: "text/html", Body: body}, nil } +// Fetched returns every URL fetched, in order. +func (p *Pages) Fetched() []string { + p.mu.Lock() + defer p.mu.Unlock() + return append([]string(nil), p.fetched...) +} + // Images is a recipes.ImageFetcher serving JPEGs by URL; any other URL is // a 404. type Images struct { diff --git a/internal/store/migrate_test.go b/internal/store/migrate_test.go index 1c0c2a6..94b8ec3 100644 --- a/internal/store/migrate_test.go +++ b/internal/store/migrate_test.go @@ -175,8 +175,8 @@ func TestMigrateEmbeddedMigrations(t *testing.T) { if err := Migrate(context.Background(), s.Writer, migrations.FS); err != nil { t.Fatal(err) } - if got := versions(migrationRows(t, s)); got != "0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai,0006_ai_runs_status,0007_recipe_grounding,0008_dinner_recipe_url,0009_dinner_grounding_error" { - t.Errorf("versions = %q, want 0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai,0006_ai_runs_status,0007_recipe_grounding,0008_dinner_recipe_url,0009_dinner_grounding_error", got) + if got := versions(migrationRows(t, s)); got != "0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai,0006_ai_runs_status,0007_recipe_grounding,0008_dinner_recipe_url,0009_dinner_grounding_error,0010_dinner_grounding_revision,0011_ground_dinner_job_revision" { + t.Errorf("versions = %q, want 0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai,0006_ai_runs_status,0007_recipe_grounding,0008_dinner_recipe_url,0009_dinner_grounding_error,0010_dinner_grounding_revision,0011_ground_dinner_job_revision", got) } } @@ -196,8 +196,8 @@ func TestMigrateBootstrapThenEmbeddedInitApplies(t *testing.T) { if err := Migrate(ctx, s.Writer, migrations.FS); err != nil { t.Fatal(err) } - if got := versions(migrationRows(t, s)); got != "0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai,0006_ai_runs_status,0007_recipe_grounding,0008_dinner_recipe_url,0009_dinner_grounding_error" { - t.Errorf("versions = %q, want 0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai,0006_ai_runs_status,0007_recipe_grounding,0008_dinner_recipe_url,0009_dinner_grounding_error", got) + if got := versions(migrationRows(t, s)); got != "0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai,0006_ai_runs_status,0007_recipe_grounding,0008_dinner_recipe_url,0009_dinner_grounding_error,0010_dinner_grounding_revision,0011_ground_dinner_job_revision" { + t.Errorf("versions = %q, want 0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai,0006_ai_runs_status,0007_recipe_grounding,0008_dinner_recipe_url,0009_dinner_grounding_error,0010_dinner_grounding_revision,0011_ground_dinner_job_revision", got) } if !tableExists(t, s, "dinners") { t.Error("table from 0001_init was not created") diff --git a/migrations/0010_dinner_grounding_revision.sql b/migrations/0010_dinner_grounding_revision.sql new file mode 100644 index 0000000..5f13bd9 --- /dev/null +++ b/migrations/0010_dinner_grounding_revision.sql @@ -0,0 +1,7 @@ +-- 0010_dinner_grounding_revision: each ground_dinner job carries the +-- dinner's grounding_revision when it was enqueued, and records its result +-- only while the dinner still has that revision. Renaming a pending dinner +-- raises the revision before enqueuing the job for the new name, so an +-- earlier job — queued or running — can no longer write the dinner. + +ALTER TABLE dinners ADD COLUMN grounding_revision INTEGER NOT NULL DEFAULT 0; diff --git a/migrations/0011_ground_dinner_job_revision.sql b/migrations/0011_ground_dinner_job_revision.sql new file mode 100644 index 0000000..dd5de37 --- /dev/null +++ b/migrations/0011_ground_dinner_job_revision.sql @@ -0,0 +1,11 @@ +-- 0011_ground_dinner_job_revision: every dinner starts at grounding +-- revision 0 (0010), so a ground_dinner job enqueued before 0010, whose +-- payload has only dinner_id, is for revision 0. Recording it lets the boot +-- cleanup, which matches a pending dinner to its job by revision, see a job +-- still queued for its dinner. This is a migration of its own, not part of +-- 0010, so it also runs on a database that has already applied 0010. + +UPDATE jobs +SET payload = json_set(payload, '$.revision', 0) +WHERE kind = 'ground_dinner' + AND json_type(payload, '$.revision') IS NULL; diff --git a/migrations/queries/dinners.sql b/migrations/queries/dinners.sql index 118f439..e11dec1 100644 --- a/migrations/queries/dinners.sql +++ b/migrations/queries/dinners.sql @@ -25,10 +25,13 @@ SELECT COUNT(*) FROM dinners WHERE status = 'regular'; -- name: ListUnarchivedDinners :many --- Every dinner a new or renamed dinner must not repeat. -SELECT * FROM dinners -WHERE status <> 'archived' -ORDER BY id; +-- Every dinner a new or renamed dinner must not repeat, with the +-- source_url of the recipe it is grounded in (NULL while it has none). +SELECT sqlc.embed(dinners), recipes.source_url AS recipe_source_url +FROM dinners +LEFT JOIN recipes ON recipes.id = dinners.recipe_id +WHERE dinners.status <> 'archived' +ORDER BY dinners.id; -- name: UpdateDinner :one -- An unknown id returns sql.ErrNoRows. @@ -42,8 +45,10 @@ SET name = ?, difficulty = ?, cuisine = ?, recipe_id = ?, + recipe_url = ?, photo_id = ?, grounding_status = ?, + grounding_revision = ?, updated_by = ?, updated_at = ? WHERE id = ? @@ -77,10 +82,12 @@ WHERE id = ? RETURNING *; -- name: GroundDinner :one --- Records a dinner's grounding (grounded or manual) and clears any --- grounding_error. An unknown id returns sql.ErrNoRows. +-- Records a dinner's grounding by hand (grounded or manual) and clears any +-- grounding_error and pasted recipe_url: the recipe chosen replaces the +-- link. An unknown id returns sql.ErrNoRows. UPDATE dinners SET recipe_id = ?, + recipe_url = NULL, photo_id = ?, method = ?, minutes_total = ?, @@ -93,43 +100,55 @@ WHERE id = ? RETURNING *; -- name: GroundPendingDinner :execrows --- Records the background grounding of a dinner still pending under the --- normalized name it was grounded for; a dinner grounded some other way or --- renamed meanwhile is left as it is (0 rows). +-- Records the background grounding of an unarchived dinner still pending at +-- the grounding_revision it was grounded for; a dinner grounded some other +-- way, renamed or archived meanwhile is left as it is (0 rows). A method, +-- minutes_total or difficulty already set is kept: a pending dinner has +-- them only when a member edited them while it was grounding. UPDATE dinners -SET recipe_id = ?, - photo_id = ?, - method = ?, - minutes_total = ?, - difficulty = ?, +SET recipe_id = sqlc.arg(recipe_id), + photo_id = sqlc.arg(photo_id), + method = COALESCE(method, sqlc.arg(method)), + minutes_total = COALESCE(minutes_total, sqlc.arg(minutes_total)), + difficulty = COALESCE(difficulty, sqlc.arg(difficulty)), grounding_status = 'grounded', grounding_error = NULL, - updated_at = ? -WHERE id = ? AND grounding_status = 'pending' AND normalized_name = ?; + updated_at = sqlc.arg(updated_at) +WHERE id = sqlc.arg(id) + AND status <> 'archived' + AND grounding_status = 'pending' + AND grounding_revision = sqlc.arg(grounding_revision); -- name: FailPendingDinnerGrounding :execrows --- Records why the background grounding of a dinner still pending under the --- normalized name it was grounded for failed; a dinner grounded some other --- way or renamed meanwhile is left as it is (0 rows). +-- Records why the background grounding of an unarchived dinner still +-- pending at the grounding_revision it was grounded for failed; a dinner +-- grounded some other way, renamed or archived meanwhile is left as it is +-- (0 rows). UPDATE dinners SET grounding_status = 'failed', grounding_error = ?, updated_at = ? -WHERE id = ? AND grounding_status = 'pending' AND normalized_name = ?; +WHERE id = ? + AND status <> 'archived' + AND grounding_status = 'pending' + AND grounding_revision = ?; -- name: FailOrphanedPendingDinnerGroundings :execrows --- Fails every dinner still pending whose grounding job is no longer queued --- or running: the process stopped before the job, or before its handler --- recorded a result. Run at boot, after the job runner fails the jobs a --- previous process left running. +-- Fails every unarchived dinner still pending whose grounding job for its +-- current grounding_revision is no longer queued or running: the process +-- stopped before the job, or before its handler recorded a result. An +-- archived dinner is left as it is; archiving stopped its grounding. Run at +-- boot, after the job runner fails the jobs a previous process left running. UPDATE dinners SET grounding_status = 'failed', grounding_error = sqlc.arg(grounding_error), updated_at = sqlc.arg(updated_at) WHERE grounding_status = 'pending' + AND status <> 'archived' AND NOT EXISTS ( SELECT 1 FROM jobs WHERE jobs.kind = sqlc.arg(kind) AND jobs.status IN ('queued', 'running') AND json_extract(jobs.payload, '$.dinner_id') = dinners.id + AND json_extract(jobs.payload, '$.revision') = dinners.grounding_revision ); diff --git a/web/src/dinners/letterFolds.json b/web/src/dinners/letterFolds.json new file mode 100644 index 0000000..5be5eaf --- /dev/null +++ b/web/src/dinners/letterFolds.json @@ -0,0 +1,1167 @@ +{ + "\u00c0": "a", + "\u00c1": "a", + "\u00c2": "a", + "\u00c3": "a", + "\u00c4": "a", + "\u00c5": "a", + "\u00c6": "ae", + "\u00c7": "c", + "\u00c8": "e", + "\u00c9": "e", + "\u00ca": "e", + "\u00cb": "e", + "\u00cc": "i", + "\u00cd": "i", + "\u00ce": "i", + "\u00cf": "i", + "\u00d0": "d", + "\u00d1": "n", + "\u00d2": "o", + "\u00d3": "o", + "\u00d4": "o", + "\u00d5": "o", + "\u00d6": "o", + "\u00d8": "o", + "\u00d9": "u", + "\u00da": "u", + "\u00db": "u", + "\u00dc": "u", + "\u00dd": "y", + "\u00de": "th", + "\u00df": "ss", + "\u00e0": "a", + "\u00e1": "a", + "\u00e2": "a", + "\u00e3": "a", + "\u00e4": "a", + "\u00e5": "a", + "\u00e6": "ae", + "\u00e7": "c", + "\u00e8": "e", + "\u00e9": "e", + "\u00ea": "e", + "\u00eb": "e", + "\u00ec": "i", + "\u00ed": "i", + "\u00ee": "i", + "\u00ef": "i", + "\u00f0": "d", + "\u00f1": "n", + "\u00f2": "o", + "\u00f3": "o", + "\u00f4": "o", + "\u00f5": "o", + "\u00f6": "o", + "\u00f8": "o", + "\u00f9": "u", + "\u00fa": "u", + "\u00fb": "u", + "\u00fc": "u", + "\u00fd": "y", + "\u00fe": "th", + "\u00ff": "y", + "\u0100": "a", + "\u0101": "a", + "\u0102": "a", + "\u0103": "a", + "\u0104": "a", + "\u0105": "a", + "\u0106": "c", + "\u0107": "c", + "\u0108": "c", + "\u0109": "c", + "\u010a": "c", + "\u010b": "c", + "\u010c": "c", + "\u010d": "c", + "\u010e": "d", + "\u010f": "d", + "\u0110": "d", + "\u0111": "d", + "\u0112": "e", + "\u0113": "e", + "\u0114": "e", + "\u0115": "e", + "\u0116": "e", + "\u0117": "e", + "\u0118": "e", + "\u0119": "e", + "\u011a": "e", + "\u011b": "e", + "\u011c": "g", + "\u011d": "g", + "\u011e": "g", + "\u011f": "g", + "\u0120": "g", + "\u0121": "g", + "\u0122": "g", + "\u0123": "g", + "\u0124": "h", + "\u0125": "h", + "\u0126": "h", + "\u0127": "h", + "\u0128": "i", + "\u0129": "i", + "\u012a": "i", + "\u012b": "i", + "\u012c": "i", + "\u012d": "i", + "\u012e": "i", + "\u012f": "i", + "\u0130": "i", + "\u0131": "i", + "\u0132": "ij", + "\u0133": "ij", + "\u0134": "j", + "\u0135": "j", + "\u0136": "k", + "\u0137": "k", + "\u0139": "l", + "\u013a": "l", + "\u013b": "l", + "\u013c": "l", + "\u013d": "l", + "\u013e": "l", + "\u013f": "l", + "\u0140": "l", + "\u0141": "l", + "\u0142": "l", + "\u0143": "n", + "\u0144": "n", + "\u0145": "n", + "\u0146": "n", + "\u0147": "n", + "\u0148": "n", + "\u0149": "n", + "\u014c": "o", + "\u014d": "o", + "\u014e": "o", + "\u014f": "o", + "\u0150": "o", + "\u0151": "o", + "\u0152": "oe", + "\u0153": "oe", + "\u0154": "r", + "\u0155": "r", + "\u0156": "r", + "\u0157": "r", + "\u0158": "r", + "\u0159": "r", + "\u015a": "s", + "\u015b": "s", + "\u015c": "s", + "\u015d": "s", + "\u015e": "s", + "\u015f": "s", + "\u0160": "s", + "\u0161": "s", + "\u0162": "t", + "\u0163": "t", + "\u0164": "t", + "\u0165": "t", + "\u0166": "t", + "\u0167": "t", + "\u0168": "u", + "\u0169": "u", + "\u016a": "u", + "\u016b": "u", + "\u016c": "u", + "\u016d": "u", + "\u016e": "u", + "\u016f": "u", + "\u0170": "u", + "\u0171": "u", + "\u0172": "u", + "\u0173": "u", + "\u0174": "w", + "\u0175": "w", + "\u0176": "y", + "\u0177": "y", + "\u0178": "y", + "\u0179": "z", + "\u017a": "z", + "\u017b": "z", + "\u017c": "z", + "\u017d": "z", + "\u017e": "z", + "\u0180": "b", + "\u0181": "b", + "\u0182": "b", + "\u0183": "b", + "\u0187": "c", + "\u0188": "c", + "\u018a": "d", + "\u018b": "d", + "\u018c": "d", + "\u0191": "f", + "\u0192": "f", + "\u0193": "g", + "\u0197": "i", + "\u0198": "k", + "\u0199": "k", + "\u019a": "l", + "\u019d": "n", + "\u019e": "n", + "\u019f": "o", + "\u01a0": "o", + "\u01a1": "o", + "\u01a4": "p", + "\u01a5": "p", + "\u01ab": "t", + "\u01ac": "t", + "\u01ad": "t", + "\u01ae": "t", + "\u01af": "u", + "\u01b0": "u", + "\u01b2": "v", + "\u01b3": "y", + "\u01b4": "y", + "\u01b5": "z", + "\u01b6": "z", + "\u01c5": "d", + "\u01c8": "l", + "\u01cb": "n", + "\u01cd": "a", + "\u01ce": "a", + "\u01cf": "i", + "\u01d0": "i", + "\u01d1": "o", + "\u01d2": "o", + "\u01d3": "u", + "\u01d4": "u", + "\u01d5": "u", + "\u01d6": "u", + "\u01d7": "u", + "\u01d8": "u", + "\u01d9": "u", + "\u01da": "u", + "\u01db": "u", + "\u01dc": "u", + "\u01de": "a", + "\u01df": "a", + "\u01e0": "a", + "\u01e1": "a", + "\u01e2": "ae", + "\u01e3": "ae", + "\u01e4": "g", + "\u01e5": "g", + "\u01e6": "g", + "\u01e7": "g", + "\u01e8": "k", + "\u01e9": "k", + "\u01ea": "o", + "\u01eb": "o", + "\u01ec": "o", + "\u01ed": "o", + "\u01ee": "\u0292", + "\u01ef": "\u0292", + "\u01f0": "j", + "\u01f2": "d", + "\u01f4": "g", + "\u01f5": "g", + "\u01f8": "n", + "\u01f9": "n", + "\u01fa": "a", + "\u01fb": "a", + "\u01fc": "ae", + "\u01fd": "ae", + "\u01fe": "o", + "\u01ff": "o", + "\u0200": "a", + "\u0201": "a", + "\u0202": "a", + "\u0203": "a", + "\u0204": "e", + "\u0205": "e", + "\u0206": "e", + "\u0207": "e", + "\u0208": "i", + "\u0209": "i", + "\u020a": "i", + "\u020b": "i", + "\u020c": "o", + "\u020d": "o", + "\u020e": "o", + "\u020f": "o", + "\u0210": "r", + "\u0211": "r", + "\u0212": "r", + "\u0213": "r", + "\u0214": "u", + "\u0215": "u", + "\u0216": "u", + "\u0217": "u", + "\u0218": "s", + "\u0219": "s", + "\u021a": "t", + "\u021b": "t", + "\u021e": "h", + "\u021f": "h", + "\u0220": "n", + "\u0221": "d", + "\u0224": "z", + "\u0225": "z", + "\u0226": "a", + "\u0227": "a", + "\u0228": "e", + "\u0229": "e", + "\u022a": "o", + "\u022b": "o", + "\u022c": "o", + "\u022d": "o", + "\u022e": "o", + "\u022f": "o", + "\u0230": "o", + "\u0231": "o", + "\u0232": "y", + "\u0233": "y", + "\u0234": "l", + "\u0235": "n", + "\u0236": "t", + "\u023a": "a", + "\u023b": "c", + "\u023c": "c", + "\u023d": "l", + "\u023e": "t", + "\u023f": "s", + "\u0240": "z", + "\u0243": "b", + "\u0246": "e", + "\u0247": "e", + "\u0248": "j", + "\u0249": "j", + "\u024b": "q", + "\u024c": "r", + "\u024d": "r", + "\u024e": "y", + "\u024f": "y", + "\u0253": "b", + "\u0255": "c", + "\u0256": "d", + "\u0257": "d", + "\u0260": "g", + "\u0266": "h", + "\u0268": "i", + "\u026b": "l", + "\u026c": "l", + "\u026d": "l", + "\u0271": "m", + "\u0272": "n", + "\u0273": "n", + "\u027c": "r", + "\u027d": "r", + "\u027e": "r", + "\u0282": "s", + "\u0288": "t", + "\u028b": "v", + "\u0290": "z", + "\u0291": "z", + "\u029d": "j", + "\u02a0": "q", + "\u0386": "\u03b1", + "\u0388": "\u03b5", + "\u0389": "\u03b7", + "\u038a": "\u03b9", + "\u038c": "\u03bf", + "\u038e": "\u03c5", + "\u038f": "\u03c9", + "\u0390": "\u03b9", + "\u03aa": "\u03b9", + "\u03ab": "\u03c5", + "\u03ac": "\u03b1", + "\u03ad": "\u03b5", + "\u03ae": "\u03b7", + "\u03af": "\u03b9", + "\u03b0": "\u03c5", + "\u03ca": "\u03b9", + "\u03cb": "\u03c5", + "\u03cc": "\u03bf", + "\u03cd": "\u03c5", + "\u03ce": "\u03c9", + "\u03d3": "\u03d2", + "\u03d4": "\u03d2", + "\u0400": "\u0435", + "\u0401": "\u0435", + "\u0403": "\u0433", + "\u0407": "\u0456", + "\u040c": "\u043a", + "\u040d": "\u0438", + "\u040e": "\u0443", + "\u0419": "\u0438", + "\u0439": "\u0438", + "\u0450": "\u0435", + "\u0451": "\u0435", + "\u0453": "\u0433", + "\u0457": "\u0456", + "\u045c": "\u043a", + "\u045d": "\u0438", + "\u045e": "\u0443", + "\u0476": "\u0475", + "\u0477": "\u0475", + "\u04c1": "\u0436", + "\u04c2": "\u0436", + "\u04d0": "\u0430", + "\u04d1": "\u0430", + "\u04d2": "\u0430", + "\u04d3": "\u0430", + "\u04d6": "\u0435", + "\u04d7": "\u0435", + "\u04da": "\u04d9", + "\u04db": "\u04d9", + "\u04dc": "\u0436", + "\u04dd": "\u0436", + "\u04de": "\u0437", + "\u04df": "\u0437", + "\u04e2": "\u0438", + "\u04e3": "\u0438", + "\u04e4": "\u0438", + "\u04e5": "\u0438", + "\u04e6": "\u043e", + "\u04e7": "\u043e", + "\u04ea": "\u04e9", + "\u04eb": "\u04e9", + "\u04ec": "\u044d", + "\u04ed": "\u044d", + "\u04ee": "\u0443", + "\u04ef": "\u0443", + "\u04f0": "\u0443", + "\u04f1": "\u0443", + "\u04f2": "\u0443", + "\u04f3": "\u0443", + "\u04f4": "\u0447", + "\u04f5": "\u0447", + "\u04f8": "\u044b", + "\u04f9": "\u044b", + "\u0622": "\u0627", + "\u0623": "\u0627", + "\u0624": "\u0648", + "\u0625": "\u0627", + "\u0626": "\u064a", + "\u06c0": "\u06d5", + "\u06c2": "\u06c1", + "\u06d3": "\u06d2", + "\u0929": "\u0928", + "\u0931": "\u0930", + "\u0934": "\u0933", + "\u0958": "\u0915", + "\u0959": "\u0916", + "\u095a": "\u0917", + "\u095b": "\u091c", + "\u095c": "\u0921", + "\u095d": "\u0922", + "\u095e": "\u092b", + "\u095f": "\u092f", + "\u09dc": "\u09a1", + "\u09dd": "\u09a2", + "\u09df": "\u09af", + "\u0a33": "\u0a32", + "\u0a36": "\u0a38", + "\u0a59": "\u0a16", + "\u0a5a": "\u0a17", + "\u0a5b": "\u0a1c", + "\u0a5e": "\u0a2b", + "\u0b5c": "\u0b21", + "\u0b5d": "\u0b22", + "\u0f43": "\u0f42", + "\u0f4d": "\u0f4c", + "\u0f52": "\u0f51", + "\u0f57": "\u0f56", + "\u0f5c": "\u0f5b", + "\u0f69": "\u0f40", + "\u1026": "\u1025", + "\u1d6c": "b", + "\u1d6d": "d", + "\u1d6e": "f", + "\u1d6f": "m", + "\u1d70": "n", + "\u1d71": "p", + "\u1d72": "r", + "\u1d73": "r", + "\u1d74": "s", + "\u1d75": "t", + "\u1d76": "z", + "\u1d7d": "p", + "\u1d80": "b", + "\u1d81": "d", + "\u1d82": "f", + "\u1d83": "g", + "\u1d84": "k", + "\u1d85": "l", + "\u1d86": "m", + "\u1d87": "n", + "\u1d88": "p", + "\u1d89": "r", + "\u1d8a": "s", + "\u1d8c": "v", + "\u1d8d": "x", + "\u1d8e": "z", + "\u1d8f": "a", + "\u1d91": "d", + "\u1d92": "e", + "\u1d96": "i", + "\u1d99": "u", + "\u1e00": "a", + "\u1e01": "a", + "\u1e02": "b", + "\u1e03": "b", + "\u1e04": "b", + "\u1e05": "b", + "\u1e06": "b", + "\u1e07": "b", + "\u1e08": "c", + "\u1e09": "c", + "\u1e0a": "d", + "\u1e0b": "d", + "\u1e0c": "d", + "\u1e0d": "d", + "\u1e0e": "d", + "\u1e0f": "d", + "\u1e10": "d", + "\u1e11": "d", + "\u1e12": "d", + "\u1e13": "d", + "\u1e14": "e", + "\u1e15": "e", + "\u1e16": "e", + "\u1e17": "e", + "\u1e18": "e", + "\u1e19": "e", + "\u1e1a": "e", + "\u1e1b": "e", + "\u1e1c": "e", + "\u1e1d": "e", + "\u1e1e": "f", + "\u1e1f": "f", + "\u1e20": "g", + "\u1e21": "g", + "\u1e22": "h", + "\u1e23": "h", + "\u1e24": "h", + "\u1e25": "h", + "\u1e26": "h", + "\u1e27": "h", + "\u1e28": "h", + "\u1e29": "h", + "\u1e2a": "h", + "\u1e2b": "h", + "\u1e2c": "i", + "\u1e2d": "i", + "\u1e2e": "i", + "\u1e2f": "i", + "\u1e30": "k", + "\u1e31": "k", + "\u1e32": "k", + "\u1e33": "k", + "\u1e34": "k", + "\u1e35": "k", + "\u1e36": "l", + "\u1e37": "l", + "\u1e38": "l", + "\u1e39": "l", + "\u1e3a": "l", + "\u1e3b": "l", + "\u1e3c": "l", + "\u1e3d": "l", + "\u1e3e": "m", + "\u1e3f": "m", + "\u1e40": "m", + "\u1e41": "m", + "\u1e42": "m", + "\u1e43": "m", + "\u1e44": "n", + "\u1e45": "n", + "\u1e46": "n", + "\u1e47": "n", + "\u1e48": "n", + "\u1e49": "n", + "\u1e4a": "n", + "\u1e4b": "n", + "\u1e4c": "o", + "\u1e4d": "o", + "\u1e4e": "o", + "\u1e4f": "o", + "\u1e50": "o", + "\u1e51": "o", + "\u1e52": "o", + "\u1e53": "o", + "\u1e54": "p", + "\u1e55": "p", + "\u1e56": "p", + "\u1e57": "p", + "\u1e58": "r", + "\u1e59": "r", + "\u1e5a": "r", + "\u1e5b": "r", + "\u1e5c": "r", + "\u1e5d": "r", + "\u1e5e": "r", + "\u1e5f": "r", + "\u1e60": "s", + "\u1e61": "s", + "\u1e62": "s", + "\u1e63": "s", + "\u1e64": "s", + "\u1e65": "s", + "\u1e66": "s", + "\u1e67": "s", + "\u1e68": "s", + "\u1e69": "s", + "\u1e6a": "t", + "\u1e6b": "t", + "\u1e6c": "t", + "\u1e6d": "t", + "\u1e6e": "t", + "\u1e6f": "t", + "\u1e70": "t", + "\u1e71": "t", + "\u1e72": "u", + "\u1e73": "u", + "\u1e74": "u", + "\u1e75": "u", + "\u1e76": "u", + "\u1e77": "u", + "\u1e78": "u", + "\u1e79": "u", + "\u1e7a": "u", + "\u1e7b": "u", + "\u1e7c": "v", + "\u1e7d": "v", + "\u1e7e": "v", + "\u1e7f": "v", + "\u1e80": "w", + "\u1e81": "w", + "\u1e82": "w", + "\u1e83": "w", + "\u1e84": "w", + "\u1e85": "w", + "\u1e86": "w", + "\u1e87": "w", + "\u1e88": "w", + "\u1e89": "w", + "\u1e8a": "x", + "\u1e8b": "x", + "\u1e8c": "x", + "\u1e8d": "x", + "\u1e8e": "y", + "\u1e8f": "y", + "\u1e90": "z", + "\u1e91": "z", + "\u1e92": "z", + "\u1e93": "z", + "\u1e94": "z", + "\u1e95": "z", + "\u1e96": "h", + "\u1e97": "t", + "\u1e98": "w", + "\u1e99": "y", + "\u1e9a": "a", + "\u1e9b": "\u017f", + "\u1e9e": "ss", + "\u1ea0": "a", + "\u1ea1": "a", + "\u1ea2": "a", + "\u1ea3": "a", + "\u1ea4": "a", + "\u1ea5": "a", + "\u1ea6": "a", + "\u1ea7": "a", + "\u1ea8": "a", + "\u1ea9": "a", + "\u1eaa": "a", + "\u1eab": "a", + "\u1eac": "a", + "\u1ead": "a", + "\u1eae": "a", + "\u1eaf": "a", + "\u1eb0": "a", + "\u1eb1": "a", + "\u1eb2": "a", + "\u1eb3": "a", + "\u1eb4": "a", + "\u1eb5": "a", + "\u1eb6": "a", + "\u1eb7": "a", + "\u1eb8": "e", + "\u1eb9": "e", + "\u1eba": "e", + "\u1ebb": "e", + "\u1ebc": "e", + "\u1ebd": "e", + "\u1ebe": "e", + "\u1ebf": "e", + "\u1ec0": "e", + "\u1ec1": "e", + "\u1ec2": "e", + "\u1ec3": "e", + "\u1ec4": "e", + "\u1ec5": "e", + "\u1ec6": "e", + "\u1ec7": "e", + "\u1ec8": "i", + "\u1ec9": "i", + "\u1eca": "i", + "\u1ecb": "i", + "\u1ecc": "o", + "\u1ecd": "o", + "\u1ece": "o", + "\u1ecf": "o", + "\u1ed0": "o", + "\u1ed1": "o", + "\u1ed2": "o", + "\u1ed3": "o", + "\u1ed4": "o", + "\u1ed5": "o", + "\u1ed6": "o", + "\u1ed7": "o", + "\u1ed8": "o", + "\u1ed9": "o", + "\u1eda": "o", + "\u1edb": "o", + "\u1edc": "o", + "\u1edd": "o", + "\u1ede": "o", + "\u1edf": "o", + "\u1ee0": "o", + "\u1ee1": "o", + "\u1ee2": "o", + "\u1ee3": "o", + "\u1ee4": "u", + "\u1ee5": "u", + "\u1ee6": "u", + "\u1ee7": "u", + "\u1ee8": "u", + "\u1ee9": "u", + "\u1eea": "u", + "\u1eeb": "u", + "\u1eec": "u", + "\u1eed": "u", + "\u1eee": "u", + "\u1eef": "u", + "\u1ef0": "u", + "\u1ef1": "u", + "\u1ef2": "y", + "\u1ef3": "y", + "\u1ef4": "y", + "\u1ef5": "y", + "\u1ef6": "y", + "\u1ef7": "y", + "\u1ef8": "y", + "\u1ef9": "y", + "\u1efe": "y", + "\u1eff": "y", + "\u1f00": "\u03b1", + "\u1f01": "\u03b1", + "\u1f02": "\u03b1", + "\u1f03": "\u03b1", + "\u1f04": "\u03b1", + "\u1f05": "\u03b1", + "\u1f06": "\u03b1", + "\u1f07": "\u03b1", + "\u1f08": "\u03b1", + "\u1f09": "\u03b1", + "\u1f0a": "\u03b1", + "\u1f0b": "\u03b1", + "\u1f0c": "\u03b1", + "\u1f0d": "\u03b1", + "\u1f0e": "\u03b1", + "\u1f0f": "\u03b1", + "\u1f10": "\u03b5", + "\u1f11": "\u03b5", + "\u1f12": "\u03b5", + "\u1f13": "\u03b5", + "\u1f14": "\u03b5", + "\u1f15": "\u03b5", + "\u1f18": "\u03b5", + "\u1f19": "\u03b5", + "\u1f1a": "\u03b5", + "\u1f1b": "\u03b5", + "\u1f1c": "\u03b5", + "\u1f1d": "\u03b5", + "\u1f20": "\u03b7", + "\u1f21": "\u03b7", + "\u1f22": "\u03b7", + "\u1f23": "\u03b7", + "\u1f24": "\u03b7", + "\u1f25": "\u03b7", + "\u1f26": "\u03b7", + "\u1f27": "\u03b7", + "\u1f28": "\u03b7", + "\u1f29": "\u03b7", + "\u1f2a": "\u03b7", + "\u1f2b": "\u03b7", + "\u1f2c": "\u03b7", + "\u1f2d": "\u03b7", + "\u1f2e": "\u03b7", + "\u1f2f": "\u03b7", + "\u1f30": "\u03b9", + "\u1f31": "\u03b9", + "\u1f32": "\u03b9", + "\u1f33": "\u03b9", + "\u1f34": "\u03b9", + "\u1f35": "\u03b9", + "\u1f36": "\u03b9", + "\u1f37": "\u03b9", + "\u1f38": "\u03b9", + "\u1f39": "\u03b9", + "\u1f3a": "\u03b9", + "\u1f3b": "\u03b9", + "\u1f3c": "\u03b9", + "\u1f3d": "\u03b9", + "\u1f3e": "\u03b9", + "\u1f3f": "\u03b9", + "\u1f40": "\u03bf", + "\u1f41": "\u03bf", + "\u1f42": "\u03bf", + "\u1f43": "\u03bf", + "\u1f44": "\u03bf", + "\u1f45": "\u03bf", + "\u1f48": "\u03bf", + "\u1f49": "\u03bf", + "\u1f4a": "\u03bf", + "\u1f4b": "\u03bf", + "\u1f4c": "\u03bf", + "\u1f4d": "\u03bf", + "\u1f50": "\u03c5", + "\u1f51": "\u03c5", + "\u1f52": "\u03c5", + "\u1f53": "\u03c5", + "\u1f54": "\u03c5", + "\u1f55": "\u03c5", + "\u1f56": "\u03c5", + "\u1f57": "\u03c5", + "\u1f59": "\u03c5", + "\u1f5b": "\u03c5", + "\u1f5d": "\u03c5", + "\u1f5f": "\u03c5", + "\u1f60": "\u03c9", + "\u1f61": "\u03c9", + "\u1f62": "\u03c9", + "\u1f63": "\u03c9", + "\u1f64": "\u03c9", + "\u1f65": "\u03c9", + "\u1f66": "\u03c9", + "\u1f67": "\u03c9", + "\u1f68": "\u03c9", + "\u1f69": "\u03c9", + "\u1f6a": "\u03c9", + "\u1f6b": "\u03c9", + "\u1f6c": "\u03c9", + "\u1f6d": "\u03c9", + "\u1f6e": "\u03c9", + "\u1f6f": "\u03c9", + "\u1f70": "\u03b1", + "\u1f71": "\u03b1", + "\u1f72": "\u03b5", + "\u1f73": "\u03b5", + "\u1f74": "\u03b7", + "\u1f75": "\u03b7", + "\u1f76": "\u03b9", + "\u1f77": "\u03b9", + "\u1f78": "\u03bf", + "\u1f79": "\u03bf", + "\u1f7a": "\u03c5", + "\u1f7b": "\u03c5", + "\u1f7c": "\u03c9", + "\u1f7d": "\u03c9", + "\u1f80": "\u03b1", + "\u1f81": "\u03b1", + "\u1f82": "\u03b1", + "\u1f83": "\u03b1", + "\u1f84": "\u03b1", + "\u1f85": "\u03b1", + "\u1f86": "\u03b1", + "\u1f87": "\u03b1", + "\u1f88": "\u03b1", + "\u1f89": "\u03b1", + "\u1f8a": "\u03b1", + "\u1f8b": "\u03b1", + "\u1f8c": "\u03b1", + "\u1f8d": "\u03b1", + "\u1f8e": "\u03b1", + "\u1f8f": "\u03b1", + "\u1f90": "\u03b7", + "\u1f91": "\u03b7", + "\u1f92": "\u03b7", + "\u1f93": "\u03b7", + "\u1f94": "\u03b7", + "\u1f95": "\u03b7", + "\u1f96": "\u03b7", + "\u1f97": "\u03b7", + "\u1f98": "\u03b7", + "\u1f99": "\u03b7", + "\u1f9a": "\u03b7", + "\u1f9b": "\u03b7", + "\u1f9c": "\u03b7", + "\u1f9d": "\u03b7", + "\u1f9e": "\u03b7", + "\u1f9f": "\u03b7", + "\u1fa0": "\u03c9", + "\u1fa1": "\u03c9", + "\u1fa2": "\u03c9", + "\u1fa3": "\u03c9", + "\u1fa4": "\u03c9", + "\u1fa5": "\u03c9", + "\u1fa6": "\u03c9", + "\u1fa7": "\u03c9", + "\u1fa8": "\u03c9", + "\u1fa9": "\u03c9", + "\u1faa": "\u03c9", + "\u1fab": "\u03c9", + "\u1fac": "\u03c9", + "\u1fad": "\u03c9", + "\u1fae": "\u03c9", + "\u1faf": "\u03c9", + "\u1fb0": "\u03b1", + "\u1fb1": "\u03b1", + "\u1fb2": "\u03b1", + "\u1fb3": "\u03b1", + "\u1fb4": "\u03b1", + "\u1fb6": "\u03b1", + "\u1fb7": "\u03b1", + "\u1fb8": "\u03b1", + "\u1fb9": "\u03b1", + "\u1fba": "\u03b1", + "\u1fbb": "\u03b1", + "\u1fbc": "\u03b1", + "\u1fc2": "\u03b7", + "\u1fc3": "\u03b7", + "\u1fc4": "\u03b7", + "\u1fc6": "\u03b7", + "\u1fc7": "\u03b7", + "\u1fc8": "\u03b5", + "\u1fc9": "\u03b5", + "\u1fca": "\u03b7", + "\u1fcb": "\u03b7", + "\u1fcc": "\u03b7", + "\u1fd0": "\u03b9", + "\u1fd1": "\u03b9", + "\u1fd2": "\u03b9", + "\u1fd3": "\u03b9", + "\u1fd6": "\u03b9", + "\u1fd7": "\u03b9", + "\u1fd8": "\u03b9", + "\u1fd9": "\u03b9", + "\u1fda": "\u03b9", + "\u1fdb": "\u03b9", + "\u1fe0": "\u03c5", + "\u1fe1": "\u03c5", + "\u1fe2": "\u03c5", + "\u1fe3": "\u03c5", + "\u1fe4": "\u03c1", + "\u1fe5": "\u03c1", + "\u1fe6": "\u03c5", + "\u1fe7": "\u03c5", + "\u1fe8": "\u03c5", + "\u1fe9": "\u03c5", + "\u1fea": "\u03c5", + "\u1feb": "\u03c5", + "\u1fec": "\u03c1", + "\u1ff2": "\u03c9", + "\u1ff3": "\u03c9", + "\u1ff4": "\u03c9", + "\u1ff6": "\u03c9", + "\u1ff7": "\u03c9", + "\u1ff8": "\u03bf", + "\u1ff9": "\u03bf", + "\u1ffa": "\u03c9", + "\u1ffb": "\u03c9", + "\u1ffc": "\u03c9", + "\u212b": "a", + "\u2c60": "l", + "\u2c61": "l", + "\u2c62": "l", + "\u2c63": "p", + "\u2c64": "r", + "\u2c65": "a", + "\u2c66": "t", + "\u2c67": "h", + "\u2c68": "h", + "\u2c69": "k", + "\u2c6a": "k", + "\u2c6b": "z", + "\u2c6c": "z", + "\u2c6e": "m", + "\u2c71": "v", + "\u2c72": "w", + "\u2c73": "w", + "\u2c74": "v", + "\u2c78": "e", + "\u2c7a": "o", + "\u2c7e": "s", + "\u2c7f": "z", + "\u304c": "\u304b", + "\u304e": "\u304d", + "\u3050": "\u304f", + "\u3052": "\u3051", + "\u3054": "\u3053", + "\u3056": "\u3055", + "\u3058": "\u3057", + "\u305a": "\u3059", + "\u305c": "\u305b", + "\u305e": "\u305d", + "\u3060": "\u305f", + "\u3062": "\u3061", + "\u3065": "\u3064", + "\u3067": "\u3066", + "\u3069": "\u3068", + "\u3070": "\u306f", + "\u3071": "\u306f", + "\u3073": "\u3072", + "\u3074": "\u3072", + "\u3076": "\u3075", + "\u3077": "\u3075", + "\u3079": "\u3078", + "\u307a": "\u3078", + "\u307c": "\u307b", + "\u307d": "\u307b", + "\u3094": "\u3046", + "\u309e": "\u309d", + "\u30ac": "\u30ab", + "\u30ae": "\u30ad", + "\u30b0": "\u30af", + "\u30b2": "\u30b1", + "\u30b4": "\u30b3", + "\u30b6": "\u30b5", + "\u30b8": "\u30b7", + "\u30ba": "\u30b9", + "\u30bc": "\u30bb", + "\u30be": "\u30bd", + "\u30c0": "\u30bf", + "\u30c2": "\u30c1", + "\u30c5": "\u30c4", + "\u30c7": "\u30c6", + "\u30c9": "\u30c8", + "\u30d0": "\u30cf", + "\u30d1": "\u30cf", + "\u30d3": "\u30d2", + "\u30d4": "\u30d2", + "\u30d6": "\u30d5", + "\u30d7": "\u30d5", + "\u30d9": "\u30d8", + "\u30da": "\u30d8", + "\u30dc": "\u30db", + "\u30dd": "\u30db", + "\u30f4": "\u30a6", + "\u30f7": "\u30ef", + "\u30f8": "\u30f0", + "\u30f9": "\u30f1", + "\u30fa": "\u30f2", + "\u30fe": "\u30fd", + "\ua740": "k", + "\ua741": "k", + "\ua742": "k", + "\ua743": "k", + "\ua744": "k", + "\ua745": "k", + "\ua748": "l", + "\ua749": "l", + "\ua74a": "o", + "\ua74b": "o", + "\ua74c": "o", + "\ua74d": "o", + "\ua750": "p", + "\ua751": "p", + "\ua752": "p", + "\ua753": "p", + "\ua754": "p", + "\ua755": "p", + "\ua756": "q", + "\ua757": "q", + "\ua758": "q", + "\ua759": "q", + "\ua75e": "v", + "\ua75f": "v", + "\ua78e": "l", + "\ua790": "n", + "\ua791": "n", + "\ua792": "c", + "\ua793": "c", + "\ua794": "c", + "\ua795": "h", + "\ua796": "b", + "\ua797": "b", + "\ua798": "f", + "\ua799": "f", + "\ua7a0": "g", + "\ua7a1": "g", + "\ua7a2": "k", + "\ua7a3": "k", + "\ua7a4": "n", + "\ua7a5": "n", + "\ua7a6": "r", + "\ua7a7": "r", + "\ua7a8": "s", + "\ua7a9": "s", + "\ua7aa": "h", + "\ua7ad": "l", + "\ua7b2": "j", + "\ua7b8": "u", + "\ua7b9": "u", + "\ua7c4": "c", + "\ua7c5": "s", + "\ua7c6": "z", + "\ua7c7": "d", + "\ua7c8": "d", + "\ua7c9": "s", + "\ua7ca": "s", + "\uab34": "e", + "\uab37": "l", + "\uab38": "l", + "\uab39": "l", + "\uab3a": "m", + "\uab3b": "n", + "\uab49": "r", + "\uab4e": "u", + "\uab52": "u", + "\uab56": "x", + "\uab57": "x", + "\uab58": "x", + "\uab59": "x", + "\uab5a": "y", + "\ufb1d": "\u05d9", + "\ufb1f": "\u05f2", + "\ufb2a": "\u05e9", + "\ufb2b": "\u05e9", + "\ufb2c": "\u05e9", + "\ufb2d": "\u05e9", + "\ufb2e": "\u05d0", + "\ufb2f": "\u05d0", + "\ufb30": "\u05d0", + "\ufb31": "\u05d1", + "\ufb32": "\u05d2", + "\ufb33": "\u05d3", + "\ufb34": "\u05d4", + "\ufb35": "\u05d5", + "\ufb36": "\u05d6", + "\ufb38": "\u05d8", + "\ufb39": "\u05d9", + "\ufb3a": "\u05da", + "\ufb3b": "\u05db", + "\ufb3c": "\u05dc", + "\ufb3e": "\u05de", + "\ufb40": "\u05e0", + "\ufb41": "\u05e1", + "\ufb43": "\u05e3", + "\ufb44": "\u05e4", + "\ufb46": "\u05e6", + "\ufb47": "\u05e7", + "\ufb48": "\u05e8", + "\ufb49": "\u05e9", + "\ufb4a": "\u05ea", + "\ufb4b": "\u05d5", + "\ufb4c": "\u05d1", + "\ufb4d": "\u05db", + "\ufb4e": "\u05e4", + "\ud804\udc9a": "\ud804\udc99", + "\ud804\udc9c": "\ud804\udc9b", + "\ud804\udcab": "\ud804\udca5", + "\ud837\udf09": "t", + "\ud837\udf11": "l", + "\ud837\udf13": "l", + "\ud837\udf16": "r", + "\ud837\udf1a": "i", + "\ud837\udf1b": "o", + "\ud837\udf1d": "c", + "\ud837\udf1e": "s" +} diff --git a/web/src/dinners/names.test.ts b/web/src/dinners/names.test.ts new file mode 100644 index 0000000..f2677fd --- /dev/null +++ b/web/src/dinners/names.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, it } from 'vitest' +import cases from '../../../internal/ai/harness/testdata/dedupe_cases.json' +import { normalizeName, similarNames } from './names.ts' + +// The cases internal/ai/harness's tests run against Normalize and Similar. +describe('dinner names, as the server compares them', () => { + it('has cases to run', () => { + expect(cases.normalize.length).toBeGreaterThan(0) + expect(cases.similar.length).toBeGreaterThan(0) + }) + + it.each(cases.normalize)('normalizes "$name" to "$want"', ({ name, want }) => { + expect(normalizeName(name)).toBe(want) + }) + + it.each(cases.similar)('"$a" and "$b" similar: $want', ({ a, b, want }) => { + expect(similarNames(a, b)).toBe(want) + expect(similarNames(b, a)).toBe(want) + }) +}) diff --git a/web/src/dinners/names.ts b/web/src/dinners/names.ts new file mode 100644 index 0000000..52cc50d --- /dev/null +++ b/web/src/dinners/names.ts @@ -0,0 +1,58 @@ +// The PWA's copy of the server's dinner-name rules (internal/ai/harness +// Normalize and Similar), so a screen can tell beforehand that the dinners +// API would refuse a name as a duplicate. The server still decides. +// internal/ai/harness/testdata/dedupe_cases.json is run against both. + +import folds from './letterFolds.json' + +// SIMILAR_THRESHOLD is harness.SimilarThreshold. +const SIMILAR_THRESHOLD = 0.85 + +// fillerWords is harness's fillerWords. +const fillerWords = new Set(['easy', 'best', 'homemade', 'quick', 'simple', 'the', 'a', 'recipe', 'classic']) + +// letterFolds is harness's letterFolds: gen_folds.py writes both tables, +// and harness's TestLetterFoldsMatchPWA fails when they differ. +const letterFolds: Record = folds + +const combiningMark = /\p{Mn}/u +const wordChar = /[\p{L}\p{Nd}]/u + +// normalizeName is harness.Normalize: lowercased, accents dropped, letters +// folded, apostrophes removed, other punctuation turned into spaces and +// filler words dropped. "Easy Chicken Tikka-Masala!" is +// "chicken tikka masala". +export function normalizeName(name: string): string { + let out = '' + // Lowercased letter by letter, as Go's strings.ToLower does, so no + // letter is lowered by its context (a final Σ is σ, not ς). + for (const letter of name) { + for (const ch of letter.toLowerCase()) { + if (combiningMark.test(ch) || ch === "'" || ch === '’') continue + const fold = letterFolds[ch] + if (fold !== undefined) out += fold + else if (wordChar.test(ch)) out += ch + else out += ' ' + } + } + return out + .split(' ') + .filter((word) => word !== '' && !fillerWords.has(word)) + .join(' ') +} + +function words(name: string): Set { + const normalized = normalizeName(name) + return new Set(normalized === '' ? [] : normalized.split(' ')) +} + +// similarNames is harness.Similar: whether a and b name the same dinner, by +// the share of the words of their normalized forms they have in common. +export function similarNames(a: string, b: string): boolean { + const left = words(a) + const right = words(b) + if (left.size === 0 || right.size === 0) return false + let shared = 0 + for (const word of left) if (right.has(word)) shared++ + return shared / (left.size + right.size - shared) >= SIMILAR_THRESHOLD +} diff --git a/web/src/onboarding/NotificationsStep.test.tsx b/web/src/onboarding/NotificationsStep.test.tsx index a34c56c..e606aa9 100644 --- a/web/src/onboarding/NotificationsStep.test.tsx +++ b/web/src/onboarding/NotificationsStep.test.tsx @@ -100,6 +100,7 @@ async function renderStep(enableNotifications?: EnableNotifications) { element={} /> Home} /> + Regulars step} /> @@ -257,6 +258,41 @@ describe('onboarding step 3', () => { }) }) +describe('onboarding step 3 with too few Regulars', () => { + it('says why and offers a way back to add Regulars', async () => { + const server = await renderStep() + fireEvent.click(later()) + await vi.waitFor(() => expect(server.held).toHaveLength(1)) + server.held[0]!.answer( + jsonResponse(400, { + error: { + code: 'invalid_settings', + message: 'invalid household settings: onboarding_completed: add at least 3 Regulars first (have 2)', + retryable: false, + }, + }), + ) + + expect(await screen.findByRole('alert')).toHaveTextContent( + 'Could not finish setting up: invalid household settings: onboarding_completed: add at least 3 Regulars first (have 2)', + ) + const addRegulars = screen.getByRole('button', { name: 'Add Regulars' }) + expectMinHitArea(addRegulars) + fireEvent.click(addRegulars) + expect(await screen.findByRole('heading', { level: 1, name: 'Regulars step' })).toBeInTheDocument() + }) + + it('offers no way back for any other refusal', async () => { + const server = await renderStep() + fireEvent.click(later()) + await vi.waitFor(() => expect(server.held).toHaveLength(1)) + server.held[0]!.answer(errorResponse(500, 'internal')) + + expect(await screen.findByRole('alert')).toHaveTextContent('Could not finish setting up: internal message') + expect(screen.queryByRole('button', { name: 'Add Regulars' })).not.toBeInTheDocument() + }) +}) + describe('notificationsUnavailable', () => { it('answers "unavailable" without asking for permission', async () => { await expect(notificationsUnavailable()).resolves.toBe('unavailable') diff --git a/web/src/onboarding/NotificationsStep.tsx b/web/src/onboarding/NotificationsStep.tsx index d8d5ed0..f17ddd1 100644 --- a/web/src/onboarding/NotificationsStep.tsx +++ b/web/src/onboarding/NotificationsStep.tsx @@ -2,7 +2,7 @@ import { useState } from 'react' import { CalendarDays, ShoppingBasket, Sparkles, UtensilsCrossed, type LucideIcon } from 'lucide-react' import { useNavigate } from 'react-router' import { InstallHint } from '../pwa/InstallHint.tsx' -import { useCompleteOnboarding } from '../settings/api.ts' +import { needsMoreRegulars, useCompleteOnboarding } from '../settings/api.ts' import { Button } from '../ui/Button.tsx' import { Icon } from '../ui/Icon.tsx' import { OnboardingStep } from './OnboardingStep.tsx' @@ -50,12 +50,15 @@ export type NotificationsStepProps = { // NotificationsStep is onboarding step 3: what MealQuest notifies about, the // iOS install hint, and finishing onboarding. Either button records that -// onboarding is finished and goes Home. +// onboarding is finished and goes Home. When the server refuses because the +// household has fewer than three Regulars, the reason shows with a way back +// to step 2 to add them. export function NotificationsStep({ enableNotifications = notificationsUnavailable }: NotificationsStepProps) { const navigate = useNavigate() const complete = useCompleteOnboarding() const [enabling, setEnabling] = useState(false) const [problem, setProblem] = useState(null) + const [fewRegulars, setFewRegulars] = useState(false) const busy = enabling || complete.isPending async function finish() { @@ -63,13 +66,19 @@ export function NotificationsStep({ enableNotifications = notificationsUnavailab await complete.mutateAsync() } catch (error) { setProblem(`Could not finish setting up: ${(error as Error).message}`) + setFewRegulars(needsMoreRegulars(error as Error)) return } void navigate('/', { replace: true }) } - async function turnOn() { + function clearProblem() { setProblem(null) + setFewRegulars(false) + } + + async function turnOn() { + clearProblem() setEnabling(true) try { await enableNotifications() @@ -83,7 +92,7 @@ export function NotificationsStep({ enableNotifications = notificationsUnavailab } function later() { - setProblem(null) + clearProblem() void finish() } @@ -124,6 +133,11 @@ export function NotificationsStep({ enableNotifications = notificationsUnavailab {problem}

)} + {fewRegulars && ( + + )} ) } diff --git a/web/src/onboarding/Onboarding.module.css b/web/src/onboarding/Onboarding.module.css index 477e2da..ef2703c 100644 --- a/web/src/onboarding/Onboarding.module.css +++ b/web/src/onboarding/Onboarding.module.css @@ -229,6 +229,11 @@ color: var(--color-accent-700); } +/* Step 3: back to step 2 when the server wants more Regulars. */ +.recover { + margin-top: var(--space-3); +} + .status { margin-top: 10px; } diff --git a/web/src/onboarding/OnboardingPage.test.tsx b/web/src/onboarding/OnboardingPage.test.tsx index e6209d4..ec00b9d 100644 --- a/web/src/onboarding/OnboardingPage.test.tsx +++ b/web/src/onboarding/OnboardingPage.test.tsx @@ -146,6 +146,14 @@ describe('onboarding step 2 (mock 1h)', () => { expect(declaredStyle(pills[0]!).height).toBe('6px') }) + it('does not offer a common pick the server would refuse as a duplicate of a Regular', async () => { + await regularsStep([dinnerBody('d1', 'Homemade chili'), dinnerBody('d2', 'Easy Chicken Stir Fry')]) + + expect(screen.queryByRole('button', { name: 'Add Chili' })).not.toBeInTheDocument() + expect(screen.queryByRole('button', { name: 'Add Chicken stir-fry' })).not.toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Add Beef tacos' })).toBeInTheDocument() + }) + it('sizes the input pill, "+" button, picks and Next as the mock does', async () => { await regularsStep() expect(hitArea(input()).height).toBe(48) diff --git a/web/src/onboarding/RegularsStep.tsx b/web/src/onboarding/RegularsStep.tsx index c238670..8c991f3 100644 --- a/web/src/onboarding/RegularsStep.tsx +++ b/web/src/onboarding/RegularsStep.tsx @@ -2,6 +2,7 @@ import { useState, type FormEvent } from 'react' import { Plus } from 'lucide-react' import { useNavigate } from 'react-router' import { useAddRegular, useRegulars, useRemoveRegular, type Regular } from '../dinners/api.ts' +import { similarNames } from '../dinners/names.ts' import { useRefusals } from '../dinners/useRefusals.ts' import { Button } from '../ui/Button.tsx' import { Chip } from '../ui/Chip.tsx' @@ -16,7 +17,8 @@ import styles from './Onboarding.module.css' // household moves on. export const MIN_REGULARS = 3 -// commonPicks are offered as one-tap Regulars until the household has them. +// commonPicks are offered as one-tap Regulars until the household has them, +// or a Regular the server would count as the same dinner. export const commonPicks = [ 'Beef tacos', 'Spaghetti bolognese', @@ -28,10 +30,6 @@ export const commonPicks = [ 'Fried rice', ] -function sameName(a: string, b: string): boolean { - return a.trim().toLowerCase() === b.trim().toLowerCase() -} - // RegularsStep is onboarding step 2, handoff mock 1h: the household adds // the dinners it makes on repeat. Each add and remove shows at once and is // undone if the server refuses it. @@ -43,7 +41,7 @@ export function RegularsStep() { const [draft, setDraft] = useState('') const { refusals, track } = useRefusals() const list = regulars.data ?? [] - const picks = commonPicks.filter((pick) => !list.some((regular) => sameName(regular.name, pick))) + const picks = commonPicks.filter((pick) => !list.some((regular) => similarNames(regular.name, pick))) // Next waits for every add to be answered, so a refused add's error shows // here and an add the server takes back never counts towards the three. const adding = list.some((regular) => regular.pending) diff --git a/web/src/settings/api.ts b/web/src/settings/api.ts index b32b127..be27f9d 100644 --- a/web/src/settings/api.ts +++ b/web/src/settings/api.ts @@ -441,6 +441,18 @@ export function householdSettingsProblems(error: Error): HouseholdSettingsProble return problems } +// needsMoreRegulars reports whether error is the server refusing to finish +// onboarding because the household has fewer than three Regulars +// (internal/household MinRegulars) — the only refusal of +// {"onboarding_completed": true}. +export function needsMoreRegulars(error: Error): boolean { + return ( + error instanceof ApiError && + error.code === 'invalid_settings' && + error.message.startsWith(`${invalidHouseholdPrefix}onboarding_completed: `) + ) +} + // aiStatus is the AI assistant row's status line: the provider and the // model's display name, or that no model is chosen yet (the seed). export function aiStatus(settings: Pick): string {