From 13044879f62b3d17a87275f460becc6c368e185a Mon Sep 17 00:00:00 2001 From: dev_agent Date: Tue, 6 Oct 2026 19:05:26 -0400 Subject: [PATCH 1/4] Drop a pending Regular's link when it is renamed A Regular added from a recipe link stores recipe_url, and its grounding job grounds that link. Renaming the dinner while still pending kept the link, so the job queued for the new name still attached the old link's recipe and photo. UpdateDinner now sets recipe_url, and renaming a pending dinner clears it: the fresh job grounds the new name by search, and the household can give a link by hand to fix up the recipe if they want a specific one. Co-Authored-By: Claude Opus 5.5 --- internal/db/db_test.go | 1 + internal/db/dinners.sql.go | 3 +++ internal/dinners/dinners.go | 11 +++++++-- internal/dinners/grounding_test.go | 38 ++++++++++++++++++++++++++++++ migrations/queries/dinners.sql | 1 + 5 files changed, 52 insertions(+), 2 deletions(-) diff --git a/internal/db/db_test.go b/internal/db/db_test.go index 2f5f911..5ec14a0 100644 --- a/internal/db/db_test.go +++ b/internal/db/db_test.go @@ -190,6 +190,7 @@ func TestDinners(t *testing.T) { Difficulty: str("medium"), Cuisine: sql.NullString{}, RecipeID: sql.NullString{}, + RecipeUrl: str("https://example.com/tacos"), PhotoID: str(photo.ID), GroundingStatus: "failed", UpdatedBy: str("sam@example.com"), diff --git a/internal/db/dinners.sql.go b/internal/db/dinners.sql.go index 049f318..725dadc 100644 --- a/internal/db/dinners.sql.go +++ b/internal/db/dinners.sql.go @@ -543,6 +543,7 @@ SET name = ?, difficulty = ?, cuisine = ?, recipe_id = ?, + recipe_url = ?, photo_id = ?, grounding_status = ?, updated_by = ?, @@ -561,6 +562,7 @@ type UpdateDinnerParams struct { Difficulty sql.NullString Cuisine sql.NullString RecipeID sql.NullString + RecipeUrl sql.NullString PhotoID sql.NullString GroundingStatus string UpdatedBy sql.NullString @@ -580,6 +582,7 @@ func (q *Queries) UpdateDinner(ctx context.Context, arg UpdateDinnerParams) (Din arg.Difficulty, arg.Cuisine, arg.RecipeID, + arg.RecipeUrl, arg.PhotoID, arg.GroundingStatus, arg.UpdatedBy, diff --git a/internal/dinners/dinners.go b/internal/dinners/dinners.go index 731136a..a336bd8 100644 --- a/internal/dinners/dinners.go +++ b/internal/dinners/dinners.go @@ -294,8 +294,9 @@ func (s *Service) List(ctx context.Context, status string) ([]Dinner, error) { // Update applies p to dinner id, recording updatedBy. An invalid patch is a // *ValidationError and a new name repeating another unarchived dinner a // *DuplicateError; either changes nothing. Renaming a dinner still pending -// enqueues a fresh KindGroundDinner job for the new name: the job already -// running was for the old name, and its result no longer applies. +// clears its recipe_url and enqueues a fresh KindGroundDinner job that +// grounds the new name: the job already running was for the old name or +// link, and its result no longer applies. func (s *Service) Update(ctx context.Context, id string, p Patch, updatedBy string) (Dinner, error) { if updatedBy == "" { return Dinner{}, errors.New("dinners: update: no member to record as updated_by") @@ -327,6 +328,7 @@ func (s *Service) Update(ctx context.Context, id string, p Patch, updatedBy stri Difficulty: cur.Difficulty, Cuisine: cur.Cuisine, RecipeID: cur.RecipeID, + RecipeUrl: cur.RecipeUrl, PhotoID: cur.PhotoID, GroundingStatus: cur.GroundingStatus, UpdatedBy: sql.NullString{String: updatedBy, Valid: true}, @@ -338,6 +340,11 @@ func (s *Service) Update(ctx context.Context, id string, p Patch, updatedBy stri } params.Name = name params.NormalizedName = harness.Normalize(name) + if cur.GroundingStatus == GroundingPending && params.NormalizedName != cur.NormalizedName { + // The link was the old dinner's recipe; the new name is + // grounded by name, and a link can be given by hand. + params.RecipeUrl = sql.NullString{} + } } if p.Method != nil { params.Method = sql.NullString{String: *p.Method, Valid: true} diff --git a/internal/dinners/grounding_test.go b/internal/dinners/grounding_test.go index 325082d..928afea 100644 --- a/internal/dinners/grounding_test.go +++ b/internal/dinners/grounding_test.go @@ -349,6 +349,44 @@ func TestRenameWhilePendingGroundsNewName(t *testing.T) { } } +// TestRenameLinkWhilePendingGroundsNewName: a Regular added from a link and +// renamed while still pending drops the link and grounds its new name, not +// the old link's recipe and photo. +func TestRenameLinkWhilePendingGroundsNewName(t *testing.T) { + t.Parallel() + e := newUnstartedEnv(t) + e.search.Results["Beef stew"] = []recipes.Candidate{{URL: stew}} + e.model.QueueGenerate(recipestest.NormalizedReply("oven", 150)) + + c, err := e.svc.CreateRegular(t.Context(), dinners.NewRegular{RecipeURL: ptr(curryA)}, member) + if err != nil { + t.Fatal(err) + } + d, err := e.svc.Update(t.Context(), c.ID, dinners.Patch{Name: ptr("Beef stew")}, member) + if err != nil { + t.Fatal(err) + } + if d.GroundingStatus != dinners.GroundingPending || d.RecipeURL != nil { + t.Fatalf("renamed = %+v, want still pending with no recipe_url", d) + } + e.start(t) + + row := e.waitGrounding(t, c.ID) + if row.GroundingStatus != dinners.GroundingGrounded { + t.Fatalf("dinner = %+v, want grounded for the new name", row) + } + rec, err := db.New(e.st.Reader).GetRecipe(t.Context(), row.RecipeID.String) + if err != nil { + t.Fatal(err) + } + if rec.SourceUrl != stew || rec.PhotoID != row.PhotoID || row.Method.String != "oven" || row.MinutesTotal.Int64 != 150 { + t.Errorf("dinner = %+v recipe %s, want grounded in %s, not the old link %s", row, rec.SourceUrl, stew, curryA) + } + if got := e.search.Queries(); !slices.Equal(got, []string{"Beef stew"}) { + t.Errorf("searched %q, want only the new name", got) + } +} + // TestRenameWhilePendingEnqueueFailure: a pending Regular renamed when the // job for its new name cannot be enqueued is failed, not left pending. func TestRenameWhilePendingEnqueueFailure(t *testing.T) { diff --git a/migrations/queries/dinners.sql b/migrations/queries/dinners.sql index 118f439..a3782d8 100644 --- a/migrations/queries/dinners.sql +++ b/migrations/queries/dinners.sql @@ -42,6 +42,7 @@ SET name = ?, difficulty = ?, cuisine = ?, recipe_id = ?, + recipe_url = ?, photo_id = ?, grounding_status = ?, updated_by = ?, -- 2.55.0 From 837c9924d1eea2f7e95c5a1bc0ee62b52487655a Mon Sep 17 00:00:00 2001 From: dev_agent Date: Tue, 6 Oct 2026 19:24:47 -0400 Subject: [PATCH 2/4] =?UTF-8?q?Fix=20Feature=2012=20review=20findings=20F2?= =?UTF-8?q?=E2=80=93F8=20on=20Regular=20grounding=20and=20onboarding?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Grounding (F2–F5): - Each ground_dinner job carries the dinner's new grounding_revision (migration 0010). Renaming a pending Regular raises the revision, and a job only grounds and writes a dinner still at its revision, so a queued original job no longer grounds the new name or decides the outcome. - Background grounding keeps a method, minutes or difficulty a member saved while the Regular was pending, and fills only those left empty. - Grounding by hand clears the pasted recipe_url, and link deduplication uses the source_url of the attached recipe, else the pending link. - Archived dinners are not grounded: the job checks before any search, fetch or model call, and its final update requires an unarchived row. Onboarding (F6, F7): - Step 3 shows "Add Regulars" back to step 2 when the server refuses to finish with fewer than three Regulars. - Common picks are hidden by a copy of harness Normalize/Similar, checked against the same shared cases as the Go code. Docs (F8): ARCHITECTURE.md covers the three-Regular minimum, step-3 recovery, the grounding revision and the dedupe rules. F1 was fixed in 1304487; its test now also asserts the old link is never grounded and recipe_url stays cleared. Maestro task 5037ff86-181d-448e-927b-c2fd2258a25d. Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 42 +++- internal/ai/harness/dedupe_test.go | 35 +++ .../ai/harness/testdata/dedupe_cases.json | 119 ++++++++++ internal/db/db_test.go | 2 +- internal/db/dinners.sql.go | 197 +++++++++------- internal/db/models.go | 37 +-- internal/db/querier.go | 33 +-- internal/dinners/dinners.go | 100 ++++---- internal/dinners/export_test.go | 9 + internal/dinners/grounding.go | 50 ++-- internal/dinners/grounding_test.go | 217 ++++++++++++++++++ internal/store/migrate_test.go | 8 +- migrations/0010_dinner_grounding_revision.sql | 7 + migrations/queries/dinners.sql | 64 ++++-- web/src/dinners/names.test.ts | 20 ++ web/src/dinners/names.ts | 74 ++++++ web/src/onboarding/NotificationsStep.test.tsx | 36 +++ web/src/onboarding/NotificationsStep.tsx | 22 +- web/src/onboarding/Onboarding.module.css | 5 + web/src/onboarding/OnboardingPage.test.tsx | 8 + web/src/onboarding/RegularsStep.tsx | 10 +- web/src/settings/api.ts | 12 + 22 files changed, 875 insertions(+), 232 deletions(-) create mode 100644 internal/ai/harness/testdata/dedupe_cases.json create mode 100644 migrations/0010_dinner_grounding_revision.sql create mode 100644 web/src/dinners/names.test.ts create mode 100644 web/src/dinners/names.ts diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 5b5ed88..ee6f75b 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -96,7 +96,7 @@ Columns ending in `_ct` hold AES-256-GCM ciphertext (via `secretbox`). | `recipe_ingredients` | recipe_id, position, raw_text, name, qty, unit, note. | | `recipe_steps` | recipe_id, position, text. | | `photos` | source_url, local_path, width, height, sha256, fetched_at. | -| `dinners` | The meal record and its lifecycle: name, normalized_name, status (`suggested` / `quest_log` / `tried` / `regular` / `archived`), origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, recipe_url (a pasted recipe link awaiting grounding), photo_id, grounding_status (`pending` / `grounded` / `failed` / `manual`), grounding_error (why grounding failed), updated_by. | +| `dinners` | The meal record and its lifecycle: name, normalized_name, status (`suggested` / `quest_log` / `tried` / `regular` / `archived`), origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, recipe_url (a pasted recipe link awaiting grounding), photo_id, grounding_status (`pending` / `grounded` / `failed` / `manual`), grounding_error (why grounding failed), grounding_revision (raised when a pending dinner is renamed, so only the latest `ground_dinner` job may write it), updated_by. | | `suggestion_batches` | trigger, status, ai_run_id. | | `suggestions` | dinner_id, batch_id, reason, cited_regular_ids (JSON), status, decided_at. | | `week_plans` | week_start (unique), status (`draft` / `synced` / `changed`), ai_note, ai_run_id, calendar_synced_at. | @@ -187,8 +187,10 @@ automatic** (D-021). "Beef stew"); a link that names nothing is 400 — add it by name. A name is trimmed, at most 120 characters and must keep a word after `harness.Normalize`, which sets `normalized_name`. A name - `harness.Similar` to an unarchived dinner's, or a link one already has, - is 409 `duplicate_dinner`; archived names may be added again. + `harness.Similar` to an unarchived dinner's, or a link one already has + (the source_url of the recipe it is grounded in, else its pasted + `recipe_url`), is 409 `duplicate_dinner`; archived names may be added + again. `GET /api/v1/dinners?status=regular|quest_log` returns `{dinners: [...]}` ordered by normalized name, each with `photo_url` (`/media/photos/{photo_id}-400.jpg`, or null) and `grounding_status`; @@ -211,11 +213,19 @@ automatic** (D-021). `failed` with `grounding_error` — also the job's failure code — one of `no_recipe_found`, `recipe_unusable` (the pasted link), `search_not_configured`, `search_failed`, `ai_failed`, `interrupted` (the server stopped) or - `grounding_error`. It only writes a dinner still `pending`, so a fix-up - made meanwhile wins. At boot, after the job runner fails the jobs a - previous process left running, every dinner still `pending` with no - `ground_dinner` job queued or running is set `failed` with `interrupted`, - so no dinner stays pending with nothing left to ground it. + `grounding_error`. Each job carries the dinner's `grounding_revision` + from when it was enqueued; renaming a `pending` dinner clears its + `recipe_url`, raises the revision and enqueues a job for the new name. + A job for a dinner archived, no longer `pending` or at another revision + does no work, and a job only writes a dinner still unarchived, `pending` + and at its revision, so a fix-up made meanwhile wins and an earlier job + for an old name — queued or running — never decides the outcome. A + method, minutes_total or difficulty a member saved while the dinner was + `pending` is kept; grounding fills only those left empty. At boot, after + the job runner fails the jobs a previous process left running, every + dinner still `pending` with no `ground_dinner` job for its revision + queued or running is set `failed` with `interrupted`, so no dinner stays + pending with nothing left to ground it. `GET /api/v1/dinners/{id}/recipe-candidates` searches for the dinner's name and returns `{candidates: [{candidate_id, title, site_name, photo_url, minutes}]}`, at most 3, ranked as `Ground` @@ -229,8 +239,9 @@ automatic** (D-021). — no model runs — as a `recipes` row whose source_url is `mealquest:manual:{recipe_id}` and whose raw_jsonld is a schema.org Recipe built from it, and sets `manual` with its method and minutes, no - photo and the dinner's difficulty unchanged. It answers 200 with the - dinner; an invalid body is 400, an unknown dinner 404, a page that cannot + photo and the dinner's difficulty unchanged. Either clears the dinner's + pasted `recipe_url`: the recipe chosen replaces it. It answers 200 with + the dinner; an invalid body is 400, an unknown dinner 404, a page that cannot ground it 422 `ungroundable` (the dinner is unchanged), recipe search not set up 409 `not_configured`, a search or model failure 502 `search_failed` / `ai_failed` (only the reason code is logged, never the @@ -488,13 +499,22 @@ First run: while `household_settings.onboarding_completed_at` (read from `GET /api/v1/settings/household`) is null, every route redirects to `/onboarding`. Onboarding is a full screen outside the app shell, in three steps: the welcome (`/onboarding`), the Regulars (`/onboarding/regulars`, -mock 1h, adding and archiving through the dinners API optimistically) and +mock 1h, adding and archiving through the dinners API optimistically; Next +waits for three saved Regulars, and a common pick is not offered once a +Regular has a name `harness.Similar` to it — the PWA's copy of the name +rules in `web/src/dinners/names.ts` runs the same cases, +`internal/ai/harness/testdata/dedupe_cases.json`, as the server's) and notifications (`/onboarding/notifications`: what MealQuest notifies about, the iOS install hint, "Turn on notifications" and "Maybe later"). Either button sends `PATCH /api/v1/settings/household` with `{"onboarding_completed": true}`; the server sets `onboarding_completed_at` to the time of that update (keeping the first time if it is already set; the field itself is read-only and `false` is refused) and the app goes Home. +Completing needs at least three Regulars (`household.MinRegulars`); with +fewer the server answers 400 `invalid_settings` naming +`onboarding_completed` and changes nothing. Step 3 can be reached with +fewer — by its URL, or after another member removes a Regular — so it then +shows the server's reason with an "Add Regulars" button back to step 2. Notification permission is never asked for on page load; until Feature 19 wires Web Push, "Turn on notifications" asks for nothing. Offline on the saved pages the settings cannot be read (the service worker never caches `/api/v1/settings`), diff --git a/internal/ai/harness/dedupe_test.go b/internal/ai/harness/dedupe_test.go index 99332d3..2f64739 100644 --- a/internal/ai/harness/dedupe_test.go +++ b/internal/ai/harness/dedupe_test.go @@ -1,10 +1,45 @@ package harness import ( + "encoding/json" + "os" "testing" "unicode" ) +// TestSharedDedupeCases runs testdata/dedupe_cases.json, which the PWA's +// copy of Normalize and Similar (web/src/dinners/names.ts) runs too, so the +// two cannot drift apart on these names. +func TestSharedDedupeCases(t *testing.T) { + raw, err := os.ReadFile("testdata/dedupe_cases.json") + if err != nil { + t.Fatal(err) + } + var cases struct { + Normalize []struct{ Name, Want string } + Similar []struct { + A, B string + Want bool + } + } + if err := json.Unmarshal(raw, &cases); err != nil { + t.Fatal(err) + } + if len(cases.Normalize) == 0 || len(cases.Similar) == 0 { + t.Fatal("dedupe_cases.json has no cases") + } + for _, c := range cases.Normalize { + if got := Normalize(c.Name); got != c.Want { + t.Errorf("Normalize(%q) = %q, want %q", c.Name, got, c.Want) + } + } + for _, c := range cases.Similar { + if got := Similar(c.A, c.B); got != c.Want { + t.Errorf("Similar(%q, %q) = %v, want %v", c.A, c.B, got, c.Want) + } + } +} + func TestNormalize(t *testing.T) { tests := []struct { name, want string diff --git a/internal/ai/harness/testdata/dedupe_cases.json b/internal/ai/harness/testdata/dedupe_cases.json new file mode 100644 index 0000000..2488e8b --- /dev/null +++ b/internal/ai/harness/testdata/dedupe_cases.json @@ -0,0 +1,119 @@ +{ + "normalize": [ + { + "name": "Easy Chicken Tikka-Masala!", + "want": "chicken tikka masala" + }, + { + "name": " Homemade Chili ", + "want": "chili" + }, + { + "name": "Grandma's Stew", + "want": "grandmas stew" + }, + { + "name": "Grandma’s Stew", + "want": "grandmas stew" + }, + { + "name": "İskender Kebab", + "want": "iskender kebab" + }, + { + "name": "Paella à la Valenciana", + "want": "paella la valenciana" + }, + { + "name": "Crème Brûlée", + "want": "creme brulee" + }, + { + "name": "Crème Brûlée", + "want": "creme brulee" + }, + { + "name": "Smørrebrød", + "want": "smorrebrod" + }, + { + "name": "Pierogi z Łososiem", + "want": "pierogi z lososiem" + }, + { + "name": "Straße Würstchen", + "want": "strasse wurstchen" + }, + { + "name": "Œufs en Cocotte", + "want": "oeufs en cocotte" + }, + { + "name": "Æbleskiver", + "want": "aebleskiver" + }, + { + "name": "Phở Bò", + "want": "pho bo" + }, + { + "name": "The Best Quick Recipe", + "want": "" + }, + { + "name": "Chicken stir-fry", + "want": "chicken stir fry" + }, + { + "name": "7-layer dip", + "want": "7 layer dip" + } + ], + "similar": [ + { + "a": "Homemade chili", + "b": "Chili", + "want": true + }, + { + "a": "Easy Spaghetti Bolognese", + "b": "spaghetti bolognese", + "want": true + }, + { + "a": "Chicken stir fry", + "b": "Chicken stir-fry", + "want": true + }, + { + "a": "Crème brûlée", + "b": "Creme brulee", + "want": true + }, + { + "a": "Smørrebrød", + "b": "smorrebrod", + "want": true + }, + { + "a": "Beef tacos", + "b": "Fish tacos", + "want": false + }, + { + "a": "Chili", + "b": "White chicken chili", + "want": false + }, + { + "a": "Pot roast", + "b": "Roast", + "want": false + }, + { + "a": "The best", + "b": "Easy recipe", + "want": false + } + ] +} diff --git a/internal/db/db_test.go b/internal/db/db_test.go index 5ec14a0..01f0a80 100644 --- a/internal/db/db_test.go +++ b/internal/db/db_test.go @@ -268,7 +268,7 @@ func TestDinners(t *testing.T) { } var unarchivedIDs []string for _, d := range unarchived { - unarchivedIDs = append(unarchivedIDs, d.ID) + unarchivedIDs = append(unarchivedIDs, d.Dinner.ID) } assertIDs(t, "unarchived", unarchivedIDs, other.ID, third.ID) _, err = q.ArchiveDinner(ctx, db.ArchiveDinnerParams{ID: "missing", UpdatedAt: archivedAt}) diff --git a/internal/db/dinners.sql.go b/internal/db/dinners.sql.go index 725dadc..37c723a 100644 --- a/internal/db/dinners.sql.go +++ b/internal/db/dinners.sql.go @@ -16,7 +16,7 @@ SET status = 'archived', updated_by = ?, updated_at = ? WHERE id = ? -RETURNING id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error +RETURNING id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error, grounding_revision ` type ArchiveDinnerParams struct { @@ -48,6 +48,7 @@ func (q *Queries) ArchiveDinner(ctx context.Context, arg ArchiveDinnerParams) (D &i.UpdatedAt, &i.RecipeUrl, &i.GroundingError, + &i.GroundingRevision, ) return i, err } @@ -75,7 +76,7 @@ INSERT INTO dinners ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) -RETURNING id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error +RETURNING id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error, grounding_revision ` type CreateDinnerParams struct { @@ -138,6 +139,7 @@ func (q *Queries) CreateDinner(ctx context.Context, arg CreateDinnerParams) (Din &i.UpdatedAt, &i.RecipeUrl, &i.GroundingError, + &i.GroundingRevision, ) return i, err } @@ -153,6 +155,7 @@ WHERE grounding_status = 'pending' WHERE jobs.kind = ?3 AND jobs.status IN ('queued', 'running') AND json_extract(jobs.payload, '$.dinner_id') = dinners.id + AND json_extract(jobs.payload, '$.revision') = dinners.grounding_revision ) ` @@ -162,10 +165,10 @@ type FailOrphanedPendingDinnerGroundingsParams struct { Kind string } -// Fails every dinner still pending whose grounding job is no longer queued -// or running: the process stopped before the job, or before its handler -// recorded a result. Run at boot, after the job runner fails the jobs a -// previous process left running. +// Fails every dinner still pending whose grounding job for its current +// grounding_revision is no longer queued or running: the process stopped +// before the job, or before its handler recorded a result. Run at boot, +// after the job runner fails the jobs a previous process left running. func (q *Queries) FailOrphanedPendingDinnerGroundings(ctx context.Context, arg FailOrphanedPendingDinnerGroundingsParams) (int64, error) { result, err := q.db.ExecContext(ctx, failOrphanedPendingDinnerGroundings, arg.GroundingError, arg.UpdatedAt, arg.Kind) if err != nil { @@ -179,25 +182,29 @@ UPDATE dinners SET grounding_status = 'failed', grounding_error = ?, updated_at = ? -WHERE id = ? AND grounding_status = 'pending' AND normalized_name = ? +WHERE id = ? + AND status <> 'archived' + AND grounding_status = 'pending' + AND grounding_revision = ? ` type FailPendingDinnerGroundingParams struct { - GroundingError sql.NullString - UpdatedAt string - ID string - NormalizedName string + GroundingError sql.NullString + UpdatedAt string + ID string + GroundingRevision int64 } -// Records why the background grounding of a dinner still pending under the -// normalized name it was grounded for failed; a dinner grounded some other -// way or renamed meanwhile is left as it is (0 rows). +// Records why the background grounding of an unarchived dinner still +// pending at the grounding_revision it was grounded for failed; a dinner +// grounded some other way, renamed or archived meanwhile is left as it is +// (0 rows). func (q *Queries) FailPendingDinnerGrounding(ctx context.Context, arg FailPendingDinnerGroundingParams) (int64, error) { result, err := q.db.ExecContext(ctx, failPendingDinnerGrounding, arg.GroundingError, arg.UpdatedAt, arg.ID, - arg.NormalizedName, + arg.GroundingRevision, ) if err != nil { return 0, err @@ -206,7 +213,7 @@ func (q *Queries) FailPendingDinnerGrounding(ctx context.Context, arg FailPendin } const getDinner = `-- name: GetDinner :one -SELECT id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error FROM dinners +SELECT id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error, grounding_revision FROM dinners WHERE id = ? ` @@ -232,6 +239,7 @@ func (q *Queries) GetDinner(ctx context.Context, id string) (Dinner, error) { &i.UpdatedAt, &i.RecipeUrl, &i.GroundingError, + &i.GroundingRevision, ) return i, err } @@ -239,6 +247,7 @@ func (q *Queries) GetDinner(ctx context.Context, id string) (Dinner, error) { const groundDinner = `-- name: GroundDinner :one UPDATE dinners SET recipe_id = ?, + recipe_url = NULL, photo_id = ?, method = ?, minutes_total = ?, @@ -248,7 +257,7 @@ SET recipe_id = ?, updated_by = ?, updated_at = ? WHERE id = ? -RETURNING id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error +RETURNING id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error, grounding_revision ` type GroundDinnerParams struct { @@ -263,8 +272,9 @@ type GroundDinnerParams struct { ID string } -// Records a dinner's grounding (grounded or manual) and clears any -// grounding_error. An unknown id returns sql.ErrNoRows. +// Records a dinner's grounding by hand (grounded or manual) and clears any +// grounding_error and pasted recipe_url: the recipe chosen replaces the +// link. An unknown id returns sql.ErrNoRows. func (q *Queries) GroundDinner(ctx context.Context, arg GroundDinnerParams) (Dinner, error) { row := q.db.QueryRowContext(ctx, groundDinner, arg.RecipeID, @@ -297,37 +307,43 @@ func (q *Queries) GroundDinner(ctx context.Context, arg GroundDinnerParams) (Din &i.UpdatedAt, &i.RecipeUrl, &i.GroundingError, + &i.GroundingRevision, ) return i, err } const groundPendingDinner = `-- name: GroundPendingDinner :execrows UPDATE dinners -SET recipe_id = ?, - photo_id = ?, - method = ?, - minutes_total = ?, - difficulty = ?, +SET recipe_id = ?1, + photo_id = ?2, + method = COALESCE(method, ?3), + minutes_total = COALESCE(minutes_total, ?4), + difficulty = COALESCE(difficulty, ?5), grounding_status = 'grounded', grounding_error = NULL, - updated_at = ? -WHERE id = ? AND grounding_status = 'pending' AND normalized_name = ? + updated_at = ?6 +WHERE id = ?7 + AND status <> 'archived' + AND grounding_status = 'pending' + AND grounding_revision = ?8 ` type GroundPendingDinnerParams struct { - RecipeID sql.NullString - PhotoID sql.NullString - Method sql.NullString - MinutesTotal sql.NullInt64 - Difficulty sql.NullString - UpdatedAt string - ID string - NormalizedName string + RecipeID sql.NullString + PhotoID sql.NullString + Method sql.NullString + MinutesTotal sql.NullInt64 + Difficulty sql.NullString + UpdatedAt string + ID string + GroundingRevision int64 } -// Records the background grounding of a dinner still pending under the -// normalized name it was grounded for; a dinner grounded some other way or -// renamed meanwhile is left as it is (0 rows). +// Records the background grounding of an unarchived dinner still pending at +// the grounding_revision it was grounded for; a dinner grounded some other +// way, renamed or archived meanwhile is left as it is (0 rows). A method, +// minutes_total or difficulty already set is kept: a pending dinner has +// them only when a member edited them while it was grounding. func (q *Queries) GroundPendingDinner(ctx context.Context, arg GroundPendingDinnerParams) (int64, error) { result, err := q.db.ExecContext(ctx, groundPendingDinner, arg.RecipeID, @@ -337,7 +353,7 @@ func (q *Queries) GroundPendingDinner(ctx context.Context, arg GroundPendingDinn arg.Difficulty, arg.UpdatedAt, arg.ID, - arg.NormalizedName, + arg.GroundingRevision, ) if err != nil { return 0, err @@ -346,7 +362,7 @@ func (q *Queries) GroundPendingDinner(ctx context.Context, arg GroundPendingDinn } const listDinnersByStatus = `-- name: ListDinnersByStatus :many -SELECT id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error FROM dinners +SELECT id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error, grounding_revision FROM dinners WHERE status = ? ORDER BY normalized_name, id ` @@ -379,6 +395,7 @@ func (q *Queries) ListDinnersByStatus(ctx context.Context, status string) ([]Din &i.UpdatedAt, &i.RecipeUrl, &i.GroundingError, + &i.GroundingRevision, ); err != nil { return nil, err } @@ -394,40 +411,50 @@ func (q *Queries) ListDinnersByStatus(ctx context.Context, status string) ([]Din } const listUnarchivedDinners = `-- name: ListUnarchivedDinners :many -SELECT id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error FROM dinners -WHERE status <> 'archived' -ORDER BY id +SELECT dinners.id, dinners.name, dinners.normalized_name, dinners.status, dinners.origin, dinners.method, dinners.minutes_total, dinners.difficulty, dinners.cuisine, dinners.is_favorite, dinners.recipe_id, dinners.photo_id, dinners.grounding_status, dinners.updated_by, dinners.created_at, dinners.updated_at, dinners.recipe_url, dinners.grounding_error, dinners.grounding_revision, recipes.source_url AS recipe_source_url +FROM dinners +LEFT JOIN recipes ON recipes.id = dinners.recipe_id +WHERE dinners.status <> 'archived' +ORDER BY dinners.id ` -// Every dinner a new or renamed dinner must not repeat. -func (q *Queries) ListUnarchivedDinners(ctx context.Context) ([]Dinner, error) { +type ListUnarchivedDinnersRow struct { + Dinner Dinner + RecipeSourceUrl sql.NullString +} + +// Every dinner a new or renamed dinner must not repeat, with the +// source_url of the recipe it is grounded in (NULL while it has none). +func (q *Queries) ListUnarchivedDinners(ctx context.Context) ([]ListUnarchivedDinnersRow, error) { rows, err := q.db.QueryContext(ctx, listUnarchivedDinners) if err != nil { return nil, err } defer rows.Close() - var items []Dinner + var items []ListUnarchivedDinnersRow for rows.Next() { - var i Dinner + var i ListUnarchivedDinnersRow if err := rows.Scan( - &i.ID, - &i.Name, - &i.NormalizedName, - &i.Status, - &i.Origin, - &i.Method, - &i.MinutesTotal, - &i.Difficulty, - &i.Cuisine, - &i.IsFavorite, - &i.RecipeID, - &i.PhotoID, - &i.GroundingStatus, - &i.UpdatedBy, - &i.CreatedAt, - &i.UpdatedAt, - &i.RecipeUrl, - &i.GroundingError, + &i.Dinner.ID, + &i.Dinner.Name, + &i.Dinner.NormalizedName, + &i.Dinner.Status, + &i.Dinner.Origin, + &i.Dinner.Method, + &i.Dinner.MinutesTotal, + &i.Dinner.Difficulty, + &i.Dinner.Cuisine, + &i.Dinner.IsFavorite, + &i.Dinner.RecipeID, + &i.Dinner.PhotoID, + &i.Dinner.GroundingStatus, + &i.Dinner.UpdatedBy, + &i.Dinner.CreatedAt, + &i.Dinner.UpdatedAt, + &i.Dinner.RecipeUrl, + &i.Dinner.GroundingError, + &i.Dinner.GroundingRevision, + &i.RecipeSourceUrl, ); err != nil { return nil, err } @@ -448,7 +475,7 @@ SET is_favorite = ?, updated_by = ?, updated_at = ? WHERE id = ? -RETURNING id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error +RETURNING id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error, grounding_revision ` type SetDinnerFavoriteParams struct { @@ -486,6 +513,7 @@ func (q *Queries) SetDinnerFavorite(ctx context.Context, arg SetDinnerFavoritePa &i.UpdatedAt, &i.RecipeUrl, &i.GroundingError, + &i.GroundingRevision, ) return i, err } @@ -496,7 +524,7 @@ SET is_favorite = 1 - is_favorite, updated_by = ?, updated_at = ? WHERE id = ? -RETURNING id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error +RETURNING id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error, grounding_revision ` type ToggleDinnerFavoriteParams struct { @@ -528,6 +556,7 @@ func (q *Queries) ToggleDinnerFavorite(ctx context.Context, arg ToggleDinnerFavo &i.UpdatedAt, &i.RecipeUrl, &i.GroundingError, + &i.GroundingRevision, ) return i, err } @@ -546,28 +575,30 @@ SET name = ?, recipe_url = ?, photo_id = ?, grounding_status = ?, + grounding_revision = ?, updated_by = ?, updated_at = ? WHERE id = ? -RETURNING id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error +RETURNING id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error, grounding_revision ` type UpdateDinnerParams struct { - Name string - NormalizedName string - Status string - Origin string - Method sql.NullString - MinutesTotal sql.NullInt64 - Difficulty sql.NullString - Cuisine sql.NullString - RecipeID sql.NullString - RecipeUrl sql.NullString - PhotoID sql.NullString - GroundingStatus string - UpdatedBy sql.NullString - UpdatedAt string - ID string + Name string + NormalizedName string + Status string + Origin string + Method sql.NullString + MinutesTotal sql.NullInt64 + Difficulty sql.NullString + Cuisine sql.NullString + RecipeID sql.NullString + RecipeUrl sql.NullString + PhotoID sql.NullString + GroundingStatus string + GroundingRevision int64 + UpdatedBy sql.NullString + UpdatedAt string + ID string } // An unknown id returns sql.ErrNoRows. @@ -585,6 +616,7 @@ func (q *Queries) UpdateDinner(ctx context.Context, arg UpdateDinnerParams) (Din arg.RecipeUrl, arg.PhotoID, arg.GroundingStatus, + arg.GroundingRevision, arg.UpdatedBy, arg.UpdatedAt, arg.ID, @@ -609,6 +641,7 @@ func (q *Queries) UpdateDinner(ctx context.Context, arg UpdateDinnerParams) (Din &i.UpdatedAt, &i.RecipeUrl, &i.GroundingError, + &i.GroundingRevision, ) return i, err } diff --git a/internal/db/models.go b/internal/db/models.go index ac5ef62..b026169 100644 --- a/internal/db/models.go +++ b/internal/db/models.go @@ -51,24 +51,25 @@ type ChatSetting struct { } type Dinner struct { - ID string - Name string - NormalizedName string - Status string - Origin string - Method sql.NullString - MinutesTotal sql.NullInt64 - Difficulty sql.NullString - Cuisine sql.NullString - IsFavorite int64 - RecipeID sql.NullString - PhotoID sql.NullString - GroundingStatus string - UpdatedBy sql.NullString - CreatedAt string - UpdatedAt string - RecipeUrl sql.NullString - GroundingError sql.NullString + ID string + Name string + NormalizedName string + Status string + Origin string + Method sql.NullString + MinutesTotal sql.NullInt64 + Difficulty sql.NullString + Cuisine sql.NullString + IsFavorite int64 + RecipeID sql.NullString + PhotoID sql.NullString + GroundingStatus string + UpdatedBy sql.NullString + CreatedAt string + UpdatedAt string + RecipeUrl sql.NullString + GroundingError sql.NullString + GroundingRevision int64 } type HouseholdSetting struct { diff --git a/internal/db/querier.go b/internal/db/querier.go index 2c6e15f..c75f774 100644 --- a/internal/db/querier.go +++ b/internal/db/querier.go @@ -36,14 +36,15 @@ type Querier interface { DeleteRecipeIngredients(ctx context.Context, recipeID string) error DeleteRecipeSteps(ctx context.Context, recipeID string) error FailJob(ctx context.Context, arg FailJobParams) (int64, error) - // Fails every dinner still pending whose grounding job is no longer queued - // or running: the process stopped before the job, or before its handler - // recorded a result. Run at boot, after the job runner fails the jobs a - // previous process left running. + // Fails every dinner still pending whose grounding job for its current + // grounding_revision is no longer queued or running: the process stopped + // before the job, or before its handler recorded a result. Run at boot, + // after the job runner fails the jobs a previous process left running. FailOrphanedPendingDinnerGroundings(ctx context.Context, arg FailOrphanedPendingDinnerGroundingsParams) (int64, error) - // Records why the background grounding of a dinner still pending under the - // normalized name it was grounded for failed; a dinner grounded some other - // way or renamed meanwhile is left as it is (0 rows). + // Records why the background grounding of an unarchived dinner still + // pending at the grounding_revision it was grounded for failed; a dinner + // grounded some other way, renamed or archived meanwhile is left as it is + // (0 rows). FailPendingDinnerGrounding(ctx context.Context, arg FailPendingDinnerGroundingParams) (int64, error) // Fails every job left running, at boot before any worker starts. FailRunningJobs(ctx context.Context, arg FailRunningJobsParams) (int64, error) @@ -65,12 +66,15 @@ type Querier interface { GetRecipeBySourceURL(ctx context.Context, sourceUrl string) (Recipe, error) // Returns sql.ErrNoRows until the row exists. GetRecipeSearchSettings(ctx context.Context) (RecipeSearchSetting, error) - // Records a dinner's grounding (grounded or manual) and clears any - // grounding_error. An unknown id returns sql.ErrNoRows. + // Records a dinner's grounding by hand (grounded or manual) and clears any + // grounding_error and pasted recipe_url: the recipe chosen replaces the + // link. An unknown id returns sql.ErrNoRows. GroundDinner(ctx context.Context, arg GroundDinnerParams) (Dinner, error) - // Records the background grounding of a dinner still pending under the - // normalized name it was grounded for; a dinner grounded some other way or - // renamed meanwhile is left as it is (0 rows). + // Records the background grounding of an unarchived dinner still pending at + // the grounding_revision it was grounded for; a dinner grounded some other + // way, renamed or archived meanwhile is left as it is (0 rows). A method, + // minutes_total or difficulty already set is kept: a pending dinner has + // them only when a member edited them while it was grounding. GroundPendingDinner(ctx context.Context, arg GroundPendingDinnerParams) (int64, error) ListDinnersByStatus(ctx context.Context, status string) ([]Dinner, error) // The planned dinners dated from from_date up to but not including @@ -81,8 +85,9 @@ type Querier interface { ListRecentDeclinedNames(ctx context.Context, limit int64) ([]string, error) ListRecipeIngredients(ctx context.Context, recipeID string) ([]RecipeIngredient, error) ListRecipeSteps(ctx context.Context, recipeID string) ([]RecipeStep, error) - // Every dinner a new or renamed dinner must not repeat. - ListUnarchivedDinners(ctx context.Context) ([]Dinner, error) + // Every dinner a new or renamed dinner must not repeat, with the + // source_url of the recipe it is grounded in (NULL while it has none). + ListUnarchivedDinners(ctx context.Context) ([]ListUnarchivedDinnersRow, error) // Stores a partial result of a running job. PublishJobResult(ctx context.Context, arg PublishJobResultParams) (int64, error) // An unknown id returns sql.ErrNoRows. diff --git a/internal/dinners/dinners.go b/internal/dinners/dinners.go index a336bd8..7fac6f9 100644 --- a/internal/dinners/dinners.go +++ b/internal/dinners/dinners.go @@ -176,7 +176,7 @@ func (s *Service) CreateRegular(ctx context.Context, r NewRegular, updatedBy str if err != nil { return Created{}, err } - jobID, err := s.enqueueGrounding(ctx, d.ID, harness.Normalize(d.Name)) + jobID, err := s.enqueueGrounding(ctx, d.ID, d.GroundingRevision) if err != nil { return Created{}, err } @@ -187,60 +187,59 @@ func (s *Service) CreateRegular(ctx context.Context, r NewRegular, updatedBy str } return Created{Dinner: fromRow(row)}, nil } - return Created{Dinner: d, GroundingJobID: jobID}, nil + return Created{Dinner: fromRow(d), GroundingJobID: jobID}, nil } -// enqueueGrounding enqueues the KindGroundDinner job for pending dinner id, -// whose normalized name is normalizedName, and returns its id. When the job -// cannot be enqueued, the dinner's grounding is failed with -// ReasonGroundingError, so it is not left pending with no job, and the id -// is nil. -func (s *Service) enqueueGrounding(ctx context.Context, id, normalizedName string) (*string, error) { - jobID, err := s.groundJob.Enqueue(ctx, GroundJob{DinnerID: id}) +// enqueueGrounding enqueues the KindGroundDinner job for pending dinner id +// at grounding_revision revision and returns its id. When the job cannot be +// enqueued, the dinner's grounding is failed with ReasonGroundingError, so +// it is not left pending with no job, and the id is nil. +func (s *Service) enqueueGrounding(ctx context.Context, id string, revision int64) (*string, error) { + jobID, err := s.groundJob.Enqueue(ctx, GroundJob{DinnerID: id, Revision: revision}) if err == nil { return &jobID, nil } // The error is a database error, never a dependency's text. s.logger.ErrorContext(ctx, "dinner grounding not enqueued", "dinner_id", id, "error", err) if _, err := db.New(s.st.Writer).FailPendingDinnerGrounding(context.WithoutCancel(ctx), db.FailPendingDinnerGroundingParams{ - GroundingError: sql.NullString{String: ReasonGroundingError, Valid: true}, - UpdatedAt: store.Now().String(), - ID: id, - NormalizedName: normalizedName, + GroundingError: sql.NullString{String: ReasonGroundingError, Valid: true}, + UpdatedAt: store.Now().String(), + ID: id, + GroundingRevision: revision, }); err != nil { return nil, fmt.Errorf("dinners: record failed grounding of %s: %w", id, err) } return nil, nil } -func (s *Service) createRegular(ctx context.Context, r NewRegular, updatedBy string) (Dinner, error) { +func (s *Service) createRegular(ctx context.Context, r NewRegular, updatedBy string) (db.Dinner, error) { if updatedBy == "" { - return Dinner{}, errors.New("dinners: create: no member to record as updated_by") + return db.Dinner{}, errors.New("dinners: create: no member to record as updated_by") } var name string var recipeURL sql.NullString switch { case r.Name != nil && r.RecipeURL != nil: - return Dinner{}, invalid("name", "give a name or a recipe_url, not both") + return db.Dinner{}, invalid("name", "give a name or a recipe_url, not both") case r.Name != nil: n, err := checkName(*r.Name) if err != nil { - return Dinner{}, err + return db.Dinner{}, err } name = n case r.RecipeURL != nil: u, err := checkRecipeURL(*r.RecipeURL) if err != nil { - return Dinner{}, err + return db.Dinner{}, err } n, ok := nameFromURL(u) if !ok { - return Dinner{}, invalid("recipe_url", "the link does not name a dinner; add it by name instead") + return db.Dinner{}, invalid("recipe_url", "the link does not name a dinner; add it by name instead") } name = n recipeURL = sql.NullString{String: u.String(), Valid: true} default: - return Dinner{}, invalid("name", "give a name or a recipe_url") + return db.Dinner{}, invalid("name", "give a name or a recipe_url") } var row db.Dinner @@ -269,9 +268,9 @@ func (s *Service) createRegular(ctx context.Context, r NewRegular, updatedBy str return nil }) if err != nil { - return Dinner{}, err + return db.Dinner{}, err } - return fromRow(row), nil + return row, nil } // List returns the dinners with status, which must be regular or @@ -294,9 +293,10 @@ func (s *Service) List(ctx context.Context, status string) ([]Dinner, error) { // Update applies p to dinner id, recording updatedBy. An invalid patch is a // *ValidationError and a new name repeating another unarchived dinner a // *DuplicateError; either changes nothing. Renaming a dinner still pending -// clears its recipe_url and enqueues a fresh KindGroundDinner job that -// grounds the new name: the job already running was for the old name or -// link, and its result no longer applies. +// clears its recipe_url, raises its grounding_revision and enqueues a fresh +// KindGroundDinner job that grounds the new name: a job queued or running +// for the old name or link carries the old revision, so it can no longer +// write the dinner. func (s *Service) Update(ctx context.Context, id string, p Patch, updatedBy string) (Dinner, error) { if updatedBy == "" { return Dinner{}, errors.New("dinners: update: no member to record as updated_by") @@ -318,21 +318,22 @@ func (s *Service) Update(ctx context.Context, id string, p Patch, updatedBy stri return fmt.Errorf("dinners: read %s: %w", id, err) } params := db.UpdateDinnerParams{ - ID: cur.ID, - Name: cur.Name, - NormalizedName: cur.NormalizedName, - Status: cur.Status, - Origin: cur.Origin, - Method: cur.Method, - MinutesTotal: cur.MinutesTotal, - Difficulty: cur.Difficulty, - Cuisine: cur.Cuisine, - RecipeID: cur.RecipeID, - RecipeUrl: cur.RecipeUrl, - PhotoID: cur.PhotoID, - GroundingStatus: cur.GroundingStatus, - UpdatedBy: sql.NullString{String: updatedBy, Valid: true}, - UpdatedAt: store.Now().String(), + ID: cur.ID, + Name: cur.Name, + NormalizedName: cur.NormalizedName, + Status: cur.Status, + Origin: cur.Origin, + Method: cur.Method, + MinutesTotal: cur.MinutesTotal, + Difficulty: cur.Difficulty, + Cuisine: cur.Cuisine, + RecipeID: cur.RecipeID, + RecipeUrl: cur.RecipeUrl, + PhotoID: cur.PhotoID, + GroundingStatus: cur.GroundingStatus, + GroundingRevision: cur.GroundingRevision, + UpdatedBy: sql.NullString{String: updatedBy, Valid: true}, + UpdatedAt: store.Now().String(), } if p.Name != nil { if err := checkUnique(ctx, q, cur.ID, name, sql.NullString{}); err != nil { @@ -344,6 +345,7 @@ func (s *Service) Update(ctx context.Context, id string, p Patch, updatedBy stri // The link was the old dinner's recipe; the new name is // grounded by name, and a link can be given by hand. params.RecipeUrl = sql.NullString{} + params.GroundingRevision++ } } if p.Method != nil { @@ -359,14 +361,14 @@ func (s *Service) Update(ctx context.Context, id string, p Patch, updatedBy stri if err != nil { return fmt.Errorf("dinners: update %s: %w", id, err) } - regrounding = row.GroundingStatus == GroundingPending && row.NormalizedName != cur.NormalizedName + regrounding = row.GroundingRevision != cur.GroundingRevision return nil }) if err != nil { return Dinner{}, err } if regrounding { - jobID, err := s.enqueueGrounding(ctx, row.ID, row.NormalizedName) + jobID, err := s.enqueueGrounding(ctx, row.ID, row.GroundingRevision) if err != nil { return Dinner{}, err } @@ -514,18 +516,24 @@ func nameFromURL(u *url.URL) (name string, ok bool) { } // checkUnique returns a *DuplicateError when name is harness.Similar to an -// unarchived dinner other than self, or recipeURL is one's recipe link. +// unarchived dinner other than self, or recipeURL is one's recipe link: the +// source_url of the recipe it is grounded in, else the recipe_url still +// awaiting grounding. func checkUnique(ctx context.Context, q *db.Queries, self, name string, recipeURL sql.NullString) error { rows, err := q.ListUnarchivedDinners(ctx) if err != nil { return fmt.Errorf("dinners: list for dedupe: %w", err) } for _, row := range rows { - if row.ID == self { + if row.Dinner.ID == self { continue } - if harness.Similar(name, row.Name) || (recipeURL.Valid && row.RecipeUrl == recipeURL) { - return &DuplicateError{Existing: fromRow(row)} + link := row.Dinner.RecipeUrl + if row.RecipeSourceUrl.Valid { + link = row.RecipeSourceUrl + } + if harness.Similar(name, row.Dinner.Name) || (recipeURL.Valid && link == recipeURL) { + return &DuplicateError{Existing: fromRow(row.Dinner)} } } return nil diff --git a/internal/dinners/export_test.go b/internal/dinners/export_test.go index 9eb3511..ca06c60 100644 --- a/internal/dinners/export_test.go +++ b/internal/dinners/export_test.go @@ -1,6 +1,7 @@ package dinners import ( + "context" "testing" "time" ) @@ -12,3 +13,11 @@ func SetGroundTimeout(t *testing.T, d time.Duration) { groundTimeout = d t.Cleanup(func() { groundTimeout = old }) } + +// RunGroundJob runs the KindGroundDinner handler for p in the caller's +// goroutine, as the job runner would, and returns the grounding status it +// reports. +func (s *Service) RunGroundJob(ctx context.Context, p GroundJob) (string, error) { + r, err := s.runGroundJob(ctx, p, nil) + return r.GroundingStatus, err +} diff --git a/internal/dinners/grounding.go b/internal/dinners/grounding.go index b3ff4f7..4d43b13 100644 --- a/internal/dinners/grounding.go +++ b/internal/dinners/grounding.go @@ -76,14 +76,17 @@ var groundTimeout = GroundTimeout // runs past GroundTimeout; the dinner is left as it was. var ErrGroundTimeout = errors.New("dinners: grounding the recipe took too long") -// GroundJob is the KindGroundDinner payload. +// GroundJob is the KindGroundDinner payload: the dinner to ground and its +// grounding_revision when the job was enqueued. type GroundJob struct { DinnerID string `json:"dinner_id"` + Revision int64 `json:"revision"` } // groundResult is the KindGroundDinner result: the dinner's grounding -// status once the job ends. It is not grounded when the dinner was -// grounded by hand before the job finished. +// status once the job ends. It is not grounded by this job when the dinner +// was grounded by hand before the job finished, renamed (a later job +// grounds it) or archived; the job then reports the status it found. type groundResult struct { DinnerID string `json:"dinner_id"` GroundingStatus string `json:"grounding_status"` @@ -93,18 +96,21 @@ func (r groundResult) PublicJSON() ([]byte, error) { return json.Marshal(r) } // runGroundJob grounds a pending dinner as its creation asked: its // recipe_url with GroundURL, else its name with Ground. Success records -// the recipe, photo, method, minutes and difficulty with grounding_status -// grounded; failure records grounding_status failed with its Reason, which -// is also the job's failure code. Either is recorded only while the dinner -// is still pending under the normalized name read here: a dinner grounded -// by hand meanwhile is left as it is, and one renamed meanwhile is left -// pending for the job its rename enqueued. +// the recipe and photo with grounding_status grounded, and the recipe's +// method, minutes and difficulty where a member has not set them; failure +// records grounding_status failed with its Reason, which is also the job's +// failure code. A dinner archived, no longer pending, or at a +// grounding_revision other than p.Revision is not grounded at all, and +// either result is recorded only while the dinner is still unarchived, +// pending and at p.Revision: a dinner grounded by hand meanwhile is left as +// it is, and one renamed or archived meanwhile is left pending — for the +// job its rename enqueued, or for none. func (s *Service) runGroundJob(ctx context.Context, p GroundJob, _ func(groundResult) error) (groundResult, error) { row, err := db.New(s.st.Reader).GetDinner(ctx, p.DinnerID) if err != nil { return groundResult{}, fmt.Errorf("dinners: ground: read %s: %w", p.DinnerID, err) } - if row.GroundingStatus != GroundingPending { + if row.Status == StatusArchived || row.GroundingStatus != GroundingPending || row.GroundingRevision != p.Revision { return groundResult{DinnerID: row.ID, GroundingStatus: row.GroundingStatus}, nil } @@ -124,10 +130,10 @@ func (s *Service) runGroundJob(ctx context.Context, p GroundJob, _ func(groundRe s.logger.InfoContext(ctx, "dinner grounding failed", "dinner_id", row.ID, "reason", reason) write := context.WithoutCancel(ctx) if _, failErr := db.New(s.st.Writer).FailPendingDinnerGrounding(write, db.FailPendingDinnerGroundingParams{ - GroundingError: sql.NullString{String: reason, Valid: true}, - UpdatedAt: store.Now().String(), - ID: row.ID, - NormalizedName: row.NormalizedName, + GroundingError: sql.NullString{String: reason, Valid: true}, + UpdatedAt: store.Now().String(), + ID: row.ID, + GroundingRevision: p.Revision, }); failErr != nil { return groundResult{}, fmt.Errorf("dinners: ground: record failure of %s: %w", row.ID, failErr) } @@ -138,14 +144,14 @@ func (s *Service) runGroundJob(ctx context.Context, p GroundJob, _ func(groundRe } n, err := db.New(s.st.Writer).GroundPendingDinner(ctx, db.GroundPendingDinnerParams{ - RecipeID: sql.NullString{String: gr.RecipeID, Valid: true}, - PhotoID: sql.NullString{String: gr.PhotoID, Valid: true}, - Method: sql.NullString{String: gr.Method, Valid: true}, - MinutesTotal: nullInt(gr.MinutesTotal), - Difficulty: sql.NullString{String: gr.Difficulty, Valid: true}, - UpdatedAt: store.Now().String(), - ID: row.ID, - NormalizedName: row.NormalizedName, + RecipeID: sql.NullString{String: gr.RecipeID, Valid: true}, + PhotoID: sql.NullString{String: gr.PhotoID, Valid: true}, + Method: sql.NullString{String: gr.Method, Valid: true}, + MinutesTotal: nullInt(gr.MinutesTotal), + Difficulty: sql.NullString{String: gr.Difficulty, Valid: true}, + UpdatedAt: store.Now().String(), + ID: row.ID, + GroundingRevision: p.Revision, }) if err != nil { return groundResult{}, fmt.Errorf("dinners: ground: record %s: %w", row.ID, err) diff --git a/internal/dinners/grounding_test.go b/internal/dinners/grounding_test.go index 928afea..47f0f31 100644 --- a/internal/dinners/grounding_test.go +++ b/internal/dinners/grounding_test.go @@ -2,6 +2,8 @@ package dinners_test import ( "context" + "database/sql" + "encoding/json" "errors" "io" "log/slog" @@ -385,6 +387,168 @@ func TestRenameLinkWhilePendingGroundsNewName(t *testing.T) { if got := e.search.Queries(); !slices.Equal(got, []string{"Beef stew"}) { t.Errorf("searched %q, want only the new name", got) } + if row.RecipeUrl.Valid { + t.Errorf("recipe_url = %q, want the old link dropped", row.RecipeUrl.String) + } + if _, err := db.New(e.st.Reader).GetRecipeBySourceURL(t.Context(), curryA); !errors.Is(err, sql.ErrNoRows) { + t.Errorf("old link's recipe lookup err = %v, want it never grounded (sql.ErrNoRows)", err) + } +} + +// queuedJob reads the payload of queued KindGroundDinner job id. +func (e *env) queuedJob(t *testing.T, id string) dinners.GroundJob { + t.Helper() + var payload string + if err := e.st.Reader.QueryRowContext(t.Context(), `SELECT payload FROM jobs WHERE id = ?`, id).Scan(&payload); err != nil { + t.Fatal(err) + } + var p dinners.GroundJob + if err := json.Unmarshal([]byte(payload), &p); err != nil { + t.Fatal(err) + } + return p +} + +// TestRenameBeforeJobStartsSkipsStaleJob: a Regular renamed before its +// first job starts is grounded only by the job its rename enqueued. The +// first job, though it would read the new name, does no work and writes +// nothing, so its failure cannot leave the dinner "Needs a recipe" while +// the rename's job finds a recipe. +func TestRenameBeforeJobStartsSkipsStaleJob(t *testing.T) { + t.Parallel() + e := newUnstartedEnv(t) + e.search.Results["Beef stew"] = []recipes.Candidate{{URL: stew}} + + c, err := e.svc.CreateRegular(t.Context(), dinners.NewRegular{Name: ptr("Chicken curry")}, member) + if err != nil { + t.Fatal(err) + } + first := e.queuedJob(t, *c.GroundingJobID) + if first.DinnerID != c.ID || first.Revision != e.dinner(t, c.ID).GroundingRevision { + t.Fatalf("first job = %+v, want dinner %s at its revision", first, c.ID) + } + if _, err := e.svc.Update(t.Context(), c.ID, dinners.Patch{Name: ptr("Beef stew")}, member); err != nil { + t.Fatal(err) + } + renamed := e.dinner(t, c.ID) + if renamed.GroundingRevision != first.Revision+1 { + t.Fatalf("revision after rename = %d, want %d", renamed.GroundingRevision, first.Revision+1) + } + + // The first job would fail if it searched. + e.search.Err = recipes.ErrSearchRateLimited + status, err := e.svc.RunGroundJob(t.Context(), first) + if err != nil || status != dinners.GroundingPending { + t.Fatalf("first job = %q, %v; want pending, no error", status, err) + } + if q := e.search.Queries(); len(q) != 0 { + t.Errorf("first job searched %q, want no work for a stale revision", q) + } + if row := e.dinner(t, c.ID); row != renamed { + t.Fatalf("first job wrote the dinner: %+v, was %+v", row, renamed) + } + + e.search.Err = nil + e.model.QueueGenerate(recipestest.NormalizedReply("oven", 150)) + status, err = e.svc.RunGroundJob(t.Context(), dinners.GroundJob{DinnerID: c.ID, Revision: renamed.GroundingRevision}) + if err != nil || status != dinners.GroundingGrounded { + t.Fatalf("rename's job = %q, %v; want grounded", status, err) + } + row := e.dinner(t, c.ID) + rec, err := db.New(e.st.Reader).GetRecipe(t.Context(), row.RecipeID.String) + if err != nil { + t.Fatal(err) + } + if row.GroundingStatus != dinners.GroundingGrounded || rec.SourceUrl != stew { + t.Errorf("dinner = %+v recipe %s, want grounded in %s", row, rec.SourceUrl, stew) + } + + // A stale job finishing last changes nothing either. + if status, err := e.svc.RunGroundJob(t.Context(), first); err != nil || status != dinners.GroundingGrounded { + t.Errorf("first job run last = %q, %v; want grounded reported, no error", status, err) + } + if after := e.dinner(t, c.ID); after != row { + t.Errorf("stale job changed the grounded dinner: %+v, was %+v", after, row) + } +} + +// TestEditWhilePendingIsKept: a method or minutes a member saves while a +// Regular is grounding is kept when grounding succeeds; what the member +// did not set comes from the recipe. +func TestEditWhilePendingIsKept(t *testing.T) { + t.Parallel() + e := newEnv(t) + e.search.Gate = make(chan struct{}) + e.model.QueueGenerate(recipestest.NormalizedReply("stovetop", 40)) + + c, err := e.svc.CreateRegular(t.Context(), dinners.NewRegular{Name: ptr("Chicken curry")}, member) + if err != nil { + t.Fatal(err) + } + e.waitQuery(t, "Chicken curry") + if _, err := e.svc.Update(t.Context(), c.ID, dinners.Patch{Method: ptr("grill"), MinutesTotal: ptr(15)}, member); err != nil { + t.Fatal(err) + } + close(e.search.Gate) + + row := e.waitGrounding(t, c.ID) + if row.GroundingStatus != dinners.GroundingGrounded || !row.RecipeID.Valid || !row.PhotoID.Valid { + t.Fatalf("dinner = %+v, want grounded", row) + } + if row.Method.String != "grill" || row.MinutesTotal.Int64 != 15 || row.Difficulty.String != "easy" { + t.Errorf("dinner = %s, %d minutes, %s; want the member's grill and 15 minutes, the recipe's easy", + row.Method.String, row.MinutesTotal.Int64, row.Difficulty.String) + } +} + +// TestArchiveWhilePendingStopsGrounding: a pending Regular archived before +// its job starts is not searched for, and one archived while its job runs +// is not written; neither ends up grounded. +func TestArchiveWhilePendingStopsGrounding(t *testing.T) { + t.Parallel() + t.Run("before the job starts", func(t *testing.T) { + t.Parallel() + e := newUnstartedEnv(t) + e.model.QueueGenerate(recipestest.NormalizedReply("stovetop", 40)) + c, err := e.svc.CreateRegular(t.Context(), dinners.NewRegular{Name: ptr("Chicken curry")}, member) + if err != nil { + t.Fatal(err) + } + if _, err := e.svc.Archive(t.Context(), c.ID, member); err != nil { + t.Fatal(err) + } + e.start(t) + if job := e.waitJob(t, *c.GroundingJobID); job.Status != jobs.StatusSucceeded { + t.Fatalf("job = %+v, want succeeded with nothing to do", job) + } + if q := e.search.Queries(); len(q) != 0 { + t.Errorf("searched %q for an archived dinner", q) + } + if row := e.dinner(t, c.ID); row.Status != dinners.StatusArchived || row.GroundingStatus == dinners.GroundingGrounded || row.RecipeID.Valid { + t.Errorf("dinner = %+v, want archived and not grounded", row) + } + }) + t.Run("while the job runs", func(t *testing.T) { + t.Parallel() + e := newEnv(t) + e.search.Gate = make(chan struct{}) + e.model.QueueGenerate(recipestest.NormalizedReply("stovetop", 40)) + c, err := e.svc.CreateRegular(t.Context(), dinners.NewRegular{Name: ptr("Chicken curry")}, member) + if err != nil { + t.Fatal(err) + } + e.waitQuery(t, "Chicken curry") + if _, err := e.svc.Archive(t.Context(), c.ID, member); err != nil { + t.Fatal(err) + } + close(e.search.Gate) + if job := e.waitJob(t, *c.GroundingJobID); job.Status != jobs.StatusSucceeded { + t.Fatalf("job = %+v", job) + } + if row := e.dinner(t, c.ID); row.Status != dinners.StatusArchived || row.GroundingStatus == dinners.GroundingGrounded || row.RecipeID.Valid { + t.Errorf("dinner = %+v, want archived and not grounded", row) + } + }) } // TestRenameWhilePendingEnqueueFailure: a pending Regular renamed when the @@ -530,6 +694,59 @@ func TestFixViaCandidate(t *testing.T) { } } +// TestFixReplacesPendingLink: fixing a Regular added from a link that held +// no recipe drops that link, so it no longer blocks adding the link again, +// while the link of the recipe chosen does. +func TestFixReplacesPendingLink(t *testing.T) { + t.Parallel() + const lost = "https://gone.example/recipes/lamb-tagine" + tests := []struct { + name string + fix dinners.Fix + // linkTaken is the recipe link the fixed dinner now has, if any. + linkTaken string + }{ + {"candidate", dinners.Fix{CandidateID: ptr(curryB)}, curryB}, + {"manual", dinners.Fix{Manual: &dinners.ManualRecipe{Ingredients: []string{"1 lamb"}, Minutes: ptr(90), Method: ptr("oven")}}, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + e := newEnv(t) + c, err := e.svc.CreateRegular(t.Context(), dinners.NewRegular{RecipeURL: ptr(lost)}, member) + if err != nil { + t.Fatal(err) + } + if job := e.waitJob(t, *c.GroundingJobID); job.Status != jobs.StatusFailed { + t.Fatalf("job = %+v, want failed", job) + } + // Renamed, so only the link could repeat it. + if _, err := e.svc.Update(t.Context(), c.ID, dinners.Patch{Name: ptr("Weeknight dinner")}, member); err != nil { + t.Fatal(err) + } + e.model.QueueGenerate(recipestest.NormalizedReply("stovetop", 25)) + d, err := e.svc.Ground(t.Context(), c.ID, tt.fix, member) + if err != nil { + t.Fatal(err) + } + if d.RecipeURL != nil { + t.Errorf("recipe_url = %q after the fix, want the unusable link dropped", *d.RecipeURL) + } + + if _, err := e.svc.CreateRegular(t.Context(), dinners.NewRegular{RecipeURL: ptr(lost)}, member); err != nil { + t.Errorf("adding the dropped link again: %v, want added", err) + } + if tt.linkTaken != "" { + _, err := e.svc.CreateRegular(t.Context(), dinners.NewRegular{RecipeURL: ptr(tt.linkTaken)}, member) + var dup *dinners.DuplicateError + if !errors.As(err, &dup) || dup.Existing.ID != c.ID { + t.Errorf("adding the fixed dinner's recipe link: err = %v, want a duplicate of %s", err, c.ID) + } + } + }) + } +} + func TestFixViaRecipeURL(t *testing.T) { t.Parallel() e := newEnv(t) diff --git a/internal/store/migrate_test.go b/internal/store/migrate_test.go index 1c0c2a6..75b6491 100644 --- a/internal/store/migrate_test.go +++ b/internal/store/migrate_test.go @@ -175,8 +175,8 @@ func TestMigrateEmbeddedMigrations(t *testing.T) { if err := Migrate(context.Background(), s.Writer, migrations.FS); err != nil { t.Fatal(err) } - if got := versions(migrationRows(t, s)); got != "0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai,0006_ai_runs_status,0007_recipe_grounding,0008_dinner_recipe_url,0009_dinner_grounding_error" { - t.Errorf("versions = %q, want 0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai,0006_ai_runs_status,0007_recipe_grounding,0008_dinner_recipe_url,0009_dinner_grounding_error", got) + if got := versions(migrationRows(t, s)); got != "0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai,0006_ai_runs_status,0007_recipe_grounding,0008_dinner_recipe_url,0009_dinner_grounding_error,0010_dinner_grounding_revision" { + t.Errorf("versions = %q, want 0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai,0006_ai_runs_status,0007_recipe_grounding,0008_dinner_recipe_url,0009_dinner_grounding_error,0010_dinner_grounding_revision", got) } } @@ -196,8 +196,8 @@ func TestMigrateBootstrapThenEmbeddedInitApplies(t *testing.T) { if err := Migrate(ctx, s.Writer, migrations.FS); err != nil { t.Fatal(err) } - if got := versions(migrationRows(t, s)); got != "0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai,0006_ai_runs_status,0007_recipe_grounding,0008_dinner_recipe_url,0009_dinner_grounding_error" { - t.Errorf("versions = %q, want 0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai,0006_ai_runs_status,0007_recipe_grounding,0008_dinner_recipe_url,0009_dinner_grounding_error", got) + if got := versions(migrationRows(t, s)); got != "0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai,0006_ai_runs_status,0007_recipe_grounding,0008_dinner_recipe_url,0009_dinner_grounding_error,0010_dinner_grounding_revision" { + t.Errorf("versions = %q, want 0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai,0006_ai_runs_status,0007_recipe_grounding,0008_dinner_recipe_url,0009_dinner_grounding_error,0010_dinner_grounding_revision", got) } if !tableExists(t, s, "dinners") { t.Error("table from 0001_init was not created") diff --git a/migrations/0010_dinner_grounding_revision.sql b/migrations/0010_dinner_grounding_revision.sql new file mode 100644 index 0000000..5f13bd9 --- /dev/null +++ b/migrations/0010_dinner_grounding_revision.sql @@ -0,0 +1,7 @@ +-- 0010_dinner_grounding_revision: each ground_dinner job carries the +-- dinner's grounding_revision when it was enqueued, and records its result +-- only while the dinner still has that revision. Renaming a pending dinner +-- raises the revision before enqueuing the job for the new name, so an +-- earlier job — queued or running — can no longer write the dinner. + +ALTER TABLE dinners ADD COLUMN grounding_revision INTEGER NOT NULL DEFAULT 0; diff --git a/migrations/queries/dinners.sql b/migrations/queries/dinners.sql index a3782d8..90333a9 100644 --- a/migrations/queries/dinners.sql +++ b/migrations/queries/dinners.sql @@ -25,10 +25,13 @@ SELECT COUNT(*) FROM dinners WHERE status = 'regular'; -- name: ListUnarchivedDinners :many --- Every dinner a new or renamed dinner must not repeat. -SELECT * FROM dinners -WHERE status <> 'archived' -ORDER BY id; +-- Every dinner a new or renamed dinner must not repeat, with the +-- source_url of the recipe it is grounded in (NULL while it has none). +SELECT sqlc.embed(dinners), recipes.source_url AS recipe_source_url +FROM dinners +LEFT JOIN recipes ON recipes.id = dinners.recipe_id +WHERE dinners.status <> 'archived' +ORDER BY dinners.id; -- name: UpdateDinner :one -- An unknown id returns sql.ErrNoRows. @@ -45,6 +48,7 @@ SET name = ?, recipe_url = ?, photo_id = ?, grounding_status = ?, + grounding_revision = ?, updated_by = ?, updated_at = ? WHERE id = ? @@ -78,10 +82,12 @@ WHERE id = ? RETURNING *; -- name: GroundDinner :one --- Records a dinner's grounding (grounded or manual) and clears any --- grounding_error. An unknown id returns sql.ErrNoRows. +-- Records a dinner's grounding by hand (grounded or manual) and clears any +-- grounding_error and pasted recipe_url: the recipe chosen replaces the +-- link. An unknown id returns sql.ErrNoRows. UPDATE dinners SET recipe_id = ?, + recipe_url = NULL, photo_id = ?, method = ?, minutes_total = ?, @@ -94,35 +100,44 @@ WHERE id = ? RETURNING *; -- name: GroundPendingDinner :execrows --- Records the background grounding of a dinner still pending under the --- normalized name it was grounded for; a dinner grounded some other way or --- renamed meanwhile is left as it is (0 rows). +-- Records the background grounding of an unarchived dinner still pending at +-- the grounding_revision it was grounded for; a dinner grounded some other +-- way, renamed or archived meanwhile is left as it is (0 rows). A method, +-- minutes_total or difficulty already set is kept: a pending dinner has +-- them only when a member edited them while it was grounding. UPDATE dinners -SET recipe_id = ?, - photo_id = ?, - method = ?, - minutes_total = ?, - difficulty = ?, +SET recipe_id = sqlc.arg(recipe_id), + photo_id = sqlc.arg(photo_id), + method = COALESCE(method, sqlc.arg(method)), + minutes_total = COALESCE(minutes_total, sqlc.arg(minutes_total)), + difficulty = COALESCE(difficulty, sqlc.arg(difficulty)), grounding_status = 'grounded', grounding_error = NULL, - updated_at = ? -WHERE id = ? AND grounding_status = 'pending' AND normalized_name = ?; + updated_at = sqlc.arg(updated_at) +WHERE id = sqlc.arg(id) + AND status <> 'archived' + AND grounding_status = 'pending' + AND grounding_revision = sqlc.arg(grounding_revision); -- name: FailPendingDinnerGrounding :execrows --- Records why the background grounding of a dinner still pending under the --- normalized name it was grounded for failed; a dinner grounded some other --- way or renamed meanwhile is left as it is (0 rows). +-- Records why the background grounding of an unarchived dinner still +-- pending at the grounding_revision it was grounded for failed; a dinner +-- grounded some other way, renamed or archived meanwhile is left as it is +-- (0 rows). UPDATE dinners SET grounding_status = 'failed', grounding_error = ?, updated_at = ? -WHERE id = ? AND grounding_status = 'pending' AND normalized_name = ?; +WHERE id = ? + AND status <> 'archived' + AND grounding_status = 'pending' + AND grounding_revision = ?; -- name: FailOrphanedPendingDinnerGroundings :execrows --- Fails every dinner still pending whose grounding job is no longer queued --- or running: the process stopped before the job, or before its handler --- recorded a result. Run at boot, after the job runner fails the jobs a --- previous process left running. +-- Fails every dinner still pending whose grounding job for its current +-- grounding_revision is no longer queued or running: the process stopped +-- before the job, or before its handler recorded a result. Run at boot, +-- after the job runner fails the jobs a previous process left running. UPDATE dinners SET grounding_status = 'failed', grounding_error = sqlc.arg(grounding_error), @@ -133,4 +148,5 @@ WHERE grounding_status = 'pending' WHERE jobs.kind = sqlc.arg(kind) AND jobs.status IN ('queued', 'running') AND json_extract(jobs.payload, '$.dinner_id') = dinners.id + AND json_extract(jobs.payload, '$.revision') = dinners.grounding_revision ); diff --git a/web/src/dinners/names.test.ts b/web/src/dinners/names.test.ts new file mode 100644 index 0000000..f2677fd --- /dev/null +++ b/web/src/dinners/names.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, it } from 'vitest' +import cases from '../../../internal/ai/harness/testdata/dedupe_cases.json' +import { normalizeName, similarNames } from './names.ts' + +// The cases internal/ai/harness's tests run against Normalize and Similar. +describe('dinner names, as the server compares them', () => { + it('has cases to run', () => { + expect(cases.normalize.length).toBeGreaterThan(0) + expect(cases.similar.length).toBeGreaterThan(0) + }) + + it.each(cases.normalize)('normalizes "$name" to "$want"', ({ name, want }) => { + expect(normalizeName(name)).toBe(want) + }) + + it.each(cases.similar)('"$a" and "$b" similar: $want', ({ a, b, want }) => { + expect(similarNames(a, b)).toBe(want) + expect(similarNames(b, a)).toBe(want) + }) +}) diff --git a/web/src/dinners/names.ts b/web/src/dinners/names.ts new file mode 100644 index 0000000..b5afe11 --- /dev/null +++ b/web/src/dinners/names.ts @@ -0,0 +1,74 @@ +// The PWA's copy of the server's dinner-name rules (internal/ai/harness +// Normalize and Similar), so a screen can tell beforehand that the dinners +// API would refuse a name as a duplicate. The server still decides. +// internal/ai/harness/testdata/dedupe_cases.json is run against both. + +// SIMILAR_THRESHOLD is harness.SimilarThreshold. +const SIMILAR_THRESHOLD = 0.85 + +// fillerWords is harness's fillerWords. +const fillerWords = new Set(['easy', 'best', 'homemade', 'quick', 'simple', 'the', 'a', 'recipe', 'classic']) + +// letterFolds are the letters harness folds that canonical decomposition +// leaves whole: its ligatures and extra letters, and the common stroked +// Latin letters. Letters with accents fold by decomposing and dropping the +// marks, as harness's table does; rarer stroked letters are compared as +// they are. +const letterFolds: Record = { + ß: 'ss', + æ: 'ae', + œ: 'oe', + þ: 'th', + ð: 'd', + ı: 'i', + ʼn: 'n', + ij: 'ij', + ø: 'o', + đ: 'd', + ħ: 'h', + ŀ: 'l', + ł: 'l', + ŧ: 't', + ƀ: 'b', + ƶ: 'z', + ǥ: 'g', + ɨ: 'i', +} + +const combiningMark = /\p{Mn}/u +const wordChar = /[\p{L}\p{Nd}]/u + +// normalizeName is harness.Normalize: lowercased, accents dropped, letters +// folded, apostrophes removed, other punctuation turned into spaces and +// filler words dropped. "Easy Chicken Tikka-Masala!" is +// "chicken tikka masala". +export function normalizeName(name: string): string { + let out = '' + for (const ch of name.toLowerCase().normalize('NFD')) { + if (combiningMark.test(ch) || ch === "'" || ch === '’') continue + const fold = letterFolds[ch] + if (fold !== undefined) out += fold + else if (wordChar.test(ch)) out += ch + else out += ' ' + } + return out + .split(' ') + .filter((word) => word !== '' && !fillerWords.has(word)) + .join(' ') +} + +function words(name: string): Set { + const normalized = normalizeName(name) + return new Set(normalized === '' ? [] : normalized.split(' ')) +} + +// similarNames is harness.Similar: whether a and b name the same dinner, by +// the share of the words of their normalized forms they have in common. +export function similarNames(a: string, b: string): boolean { + const left = words(a) + const right = words(b) + if (left.size === 0 || right.size === 0) return false + let shared = 0 + for (const word of left) if (right.has(word)) shared++ + return shared / (left.size + right.size - shared) >= SIMILAR_THRESHOLD +} diff --git a/web/src/onboarding/NotificationsStep.test.tsx b/web/src/onboarding/NotificationsStep.test.tsx index a34c56c..e606aa9 100644 --- a/web/src/onboarding/NotificationsStep.test.tsx +++ b/web/src/onboarding/NotificationsStep.test.tsx @@ -100,6 +100,7 @@ async function renderStep(enableNotifications?: EnableNotifications) { element={} /> Home} /> + Regulars step} /> @@ -257,6 +258,41 @@ describe('onboarding step 3', () => { }) }) +describe('onboarding step 3 with too few Regulars', () => { + it('says why and offers a way back to add Regulars', async () => { + const server = await renderStep() + fireEvent.click(later()) + await vi.waitFor(() => expect(server.held).toHaveLength(1)) + server.held[0]!.answer( + jsonResponse(400, { + error: { + code: 'invalid_settings', + message: 'invalid household settings: onboarding_completed: add at least 3 Regulars first (have 2)', + retryable: false, + }, + }), + ) + + expect(await screen.findByRole('alert')).toHaveTextContent( + 'Could not finish setting up: invalid household settings: onboarding_completed: add at least 3 Regulars first (have 2)', + ) + const addRegulars = screen.getByRole('button', { name: 'Add Regulars' }) + expectMinHitArea(addRegulars) + fireEvent.click(addRegulars) + expect(await screen.findByRole('heading', { level: 1, name: 'Regulars step' })).toBeInTheDocument() + }) + + it('offers no way back for any other refusal', async () => { + const server = await renderStep() + fireEvent.click(later()) + await vi.waitFor(() => expect(server.held).toHaveLength(1)) + server.held[0]!.answer(errorResponse(500, 'internal')) + + expect(await screen.findByRole('alert')).toHaveTextContent('Could not finish setting up: internal message') + expect(screen.queryByRole('button', { name: 'Add Regulars' })).not.toBeInTheDocument() + }) +}) + describe('notificationsUnavailable', () => { it('answers "unavailable" without asking for permission', async () => { await expect(notificationsUnavailable()).resolves.toBe('unavailable') diff --git a/web/src/onboarding/NotificationsStep.tsx b/web/src/onboarding/NotificationsStep.tsx index d8d5ed0..f17ddd1 100644 --- a/web/src/onboarding/NotificationsStep.tsx +++ b/web/src/onboarding/NotificationsStep.tsx @@ -2,7 +2,7 @@ import { useState } from 'react' import { CalendarDays, ShoppingBasket, Sparkles, UtensilsCrossed, type LucideIcon } from 'lucide-react' import { useNavigate } from 'react-router' import { InstallHint } from '../pwa/InstallHint.tsx' -import { useCompleteOnboarding } from '../settings/api.ts' +import { needsMoreRegulars, useCompleteOnboarding } from '../settings/api.ts' import { Button } from '../ui/Button.tsx' import { Icon } from '../ui/Icon.tsx' import { OnboardingStep } from './OnboardingStep.tsx' @@ -50,12 +50,15 @@ export type NotificationsStepProps = { // NotificationsStep is onboarding step 3: what MealQuest notifies about, the // iOS install hint, and finishing onboarding. Either button records that -// onboarding is finished and goes Home. +// onboarding is finished and goes Home. When the server refuses because the +// household has fewer than three Regulars, the reason shows with a way back +// to step 2 to add them. export function NotificationsStep({ enableNotifications = notificationsUnavailable }: NotificationsStepProps) { const navigate = useNavigate() const complete = useCompleteOnboarding() const [enabling, setEnabling] = useState(false) const [problem, setProblem] = useState(null) + const [fewRegulars, setFewRegulars] = useState(false) const busy = enabling || complete.isPending async function finish() { @@ -63,13 +66,19 @@ export function NotificationsStep({ enableNotifications = notificationsUnavailab await complete.mutateAsync() } catch (error) { setProblem(`Could not finish setting up: ${(error as Error).message}`) + setFewRegulars(needsMoreRegulars(error as Error)) return } void navigate('/', { replace: true }) } - async function turnOn() { + function clearProblem() { setProblem(null) + setFewRegulars(false) + } + + async function turnOn() { + clearProblem() setEnabling(true) try { await enableNotifications() @@ -83,7 +92,7 @@ export function NotificationsStep({ enableNotifications = notificationsUnavailab } function later() { - setProblem(null) + clearProblem() void finish() } @@ -124,6 +133,11 @@ export function NotificationsStep({ enableNotifications = notificationsUnavailab {problem}

)} + {fewRegulars && ( + + )} ) } diff --git a/web/src/onboarding/Onboarding.module.css b/web/src/onboarding/Onboarding.module.css index 477e2da..ef2703c 100644 --- a/web/src/onboarding/Onboarding.module.css +++ b/web/src/onboarding/Onboarding.module.css @@ -229,6 +229,11 @@ color: var(--color-accent-700); } +/* Step 3: back to step 2 when the server wants more Regulars. */ +.recover { + margin-top: var(--space-3); +} + .status { margin-top: 10px; } diff --git a/web/src/onboarding/OnboardingPage.test.tsx b/web/src/onboarding/OnboardingPage.test.tsx index e6209d4..ec00b9d 100644 --- a/web/src/onboarding/OnboardingPage.test.tsx +++ b/web/src/onboarding/OnboardingPage.test.tsx @@ -146,6 +146,14 @@ describe('onboarding step 2 (mock 1h)', () => { expect(declaredStyle(pills[0]!).height).toBe('6px') }) + it('does not offer a common pick the server would refuse as a duplicate of a Regular', async () => { + await regularsStep([dinnerBody('d1', 'Homemade chili'), dinnerBody('d2', 'Easy Chicken Stir Fry')]) + + expect(screen.queryByRole('button', { name: 'Add Chili' })).not.toBeInTheDocument() + expect(screen.queryByRole('button', { name: 'Add Chicken stir-fry' })).not.toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Add Beef tacos' })).toBeInTheDocument() + }) + it('sizes the input pill, "+" button, picks and Next as the mock does', async () => { await regularsStep() expect(hitArea(input()).height).toBe(48) diff --git a/web/src/onboarding/RegularsStep.tsx b/web/src/onboarding/RegularsStep.tsx index c238670..8c991f3 100644 --- a/web/src/onboarding/RegularsStep.tsx +++ b/web/src/onboarding/RegularsStep.tsx @@ -2,6 +2,7 @@ import { useState, type FormEvent } from 'react' import { Plus } from 'lucide-react' import { useNavigate } from 'react-router' import { useAddRegular, useRegulars, useRemoveRegular, type Regular } from '../dinners/api.ts' +import { similarNames } from '../dinners/names.ts' import { useRefusals } from '../dinners/useRefusals.ts' import { Button } from '../ui/Button.tsx' import { Chip } from '../ui/Chip.tsx' @@ -16,7 +17,8 @@ import styles from './Onboarding.module.css' // household moves on. export const MIN_REGULARS = 3 -// commonPicks are offered as one-tap Regulars until the household has them. +// commonPicks are offered as one-tap Regulars until the household has them, +// or a Regular the server would count as the same dinner. export const commonPicks = [ 'Beef tacos', 'Spaghetti bolognese', @@ -28,10 +30,6 @@ export const commonPicks = [ 'Fried rice', ] -function sameName(a: string, b: string): boolean { - return a.trim().toLowerCase() === b.trim().toLowerCase() -} - // RegularsStep is onboarding step 2, handoff mock 1h: the household adds // the dinners it makes on repeat. Each add and remove shows at once and is // undone if the server refuses it. @@ -43,7 +41,7 @@ export function RegularsStep() { const [draft, setDraft] = useState('') const { refusals, track } = useRefusals() const list = regulars.data ?? [] - const picks = commonPicks.filter((pick) => !list.some((regular) => sameName(regular.name, pick))) + const picks = commonPicks.filter((pick) => !list.some((regular) => similarNames(regular.name, pick))) // Next waits for every add to be answered, so a refused add's error shows // here and an add the server takes back never counts towards the three. const adding = list.some((regular) => regular.pending) diff --git a/web/src/settings/api.ts b/web/src/settings/api.ts index b32b127..be27f9d 100644 --- a/web/src/settings/api.ts +++ b/web/src/settings/api.ts @@ -441,6 +441,18 @@ export function householdSettingsProblems(error: Error): HouseholdSettingsProble return problems } +// needsMoreRegulars reports whether error is the server refusing to finish +// onboarding because the household has fewer than three Regulars +// (internal/household MinRegulars) — the only refusal of +// {"onboarding_completed": true}. +export function needsMoreRegulars(error: Error): boolean { + return ( + error instanceof ApiError && + error.code === 'invalid_settings' && + error.message.startsWith(`${invalidHouseholdPrefix}onboarding_completed: `) + ) +} + // aiStatus is the AI assistant row's status line: the provider and the // model's display name, or that no model is chosen yet (the seed). export function aiStatus(settings: Pick): string { -- 2.55.0 From dff58d491f953606fd45983d958f456bffa90282 Mon Sep 17 00:00:00 2001 From: dev_agent Date: Tue, 6 Oct 2026 19:40:34 -0400 Subject: [PATCH 3/4] =?UTF-8?q?Fix=20review=20findings=20R1=E2=80=93R3=20o?= =?UTF-8?q?n=20Regular=20grounding=20and=20name=20folding?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit R1: Migration 0010 now writes revision 0 into every ground_dinner job payload that has none. Jobs queued before the upgrade carry only dinner_id, so the boot cleanup, which matches a pending dinner to its job by revision, failed those dinners as interrupted while their jobs were still queued. The migration runs before the job runner starts. A new test migrates a database to 0009, adds a pending Regular with a legacy queued job, upgrades and boots, and checks the dinner stays pending and is then grounded. R2: Archiving a dinner, or renaming a pending one, now cancels the grounding of any ground_dinner job running for it at an older revision, so no further search, page fetch, photo download or model call is made. The job is registered before it reads the dinner, so an archive or rename committed after that read always finds it. A stopped job succeeds and reports the dinner's current grounding status. The archive and rename tests now check that the job ends while its search is still held, with no page fetched and no model call for the stale name. R3: gen_folds.py writes web/src/dinners/letterFolds.json from the same table as folds_table.go, and names.ts folds through it instead of a hand-copied subset. names.ts also lowercases letter by letter, as Go's strings.ToLower does, rather than by context. TestLetterFoldsMatchPWA fails when the two tables differ. The shared cases add "Ƈhili", "Ꞔarbanzo Stew" and "ΜΟΥΣΑΚΑΣ" (final sigma). Maestro task 5037ff86-181d-448e-927b-c2fd2258a25d. Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 9 +- internal/ai/harness/dedupe_test.go | 25 + internal/ai/harness/gen_folds.py | 11 +- .../ai/harness/testdata/dedupe_cases.json | 17 + internal/dinners/dinners.go | 16 +- internal/dinners/grounding.go | 77 +- internal/dinners/grounding_test.go | 126 +- internal/recipes/recipestest/recipestest.go | 13 + migrations/0010_dinner_grounding_revision.sql | 9 + web/src/dinners/letterFolds.json | 1167 +++++++++++++++++ web/src/dinners/names.ts | 46 +- 11 files changed, 1454 insertions(+), 62 deletions(-) create mode 100644 web/src/dinners/letterFolds.json diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index ee6f75b..ad22749 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -219,7 +219,10 @@ automatic** (D-021). A job for a dinner archived, no longer `pending` or at another revision does no work, and a job only writes a dinner still unarchived, `pending` and at its revision, so a fix-up made meanwhile wins and an earlier job - for an old name — queued or running — never decides the outcome. A + for an old name — queued or running — never decides the outcome. + Archiving or renaming a dinner also stops its running job where it is, + so no further search, page fetch, photo download or model call is made + for it. A method, minutes_total or difficulty a member saved while the dinner was `pending` is kept; grounding fills only those left empty. At boot, after the job runner fails the jobs a previous process left running, every @@ -502,7 +505,9 @@ steps: the welcome (`/onboarding`), the Regulars (`/onboarding/regulars`, mock 1h, adding and archiving through the dinners API optimistically; Next waits for three saved Regulars, and a common pick is not offered once a Regular has a name `harness.Similar` to it — the PWA's copy of the name -rules in `web/src/dinners/names.ts` runs the same cases, +rules in `web/src/dinners/names.ts` folds letters with +`web/src/dinners/letterFolds.json`, which `gen_folds.py` writes from the +same table as the server's, and runs the same cases, `internal/ai/harness/testdata/dedupe_cases.json`, as the server's) and notifications (`/onboarding/notifications`: what MealQuest notifies about, the iOS install hint, "Turn on notifications" and "Maybe later"). Either diff --git a/internal/ai/harness/dedupe_test.go b/internal/ai/harness/dedupe_test.go index 2f64739..9f7ff2b 100644 --- a/internal/ai/harness/dedupe_test.go +++ b/internal/ai/harness/dedupe_test.go @@ -5,6 +5,7 @@ import ( "os" "testing" "unicode" + "unicode/utf8" ) // TestSharedDedupeCases runs testdata/dedupe_cases.json, which the PWA's @@ -40,6 +41,30 @@ func TestSharedDedupeCases(t *testing.T) { } } +// TestLetterFoldsMatchPWA checks that the PWA's letter-fold table, which +// gen_folds.py writes beside folds_table.go, is letterFolds exactly; run +// go generate when it fails. +func TestLetterFoldsMatchPWA(t *testing.T) { + raw, err := os.ReadFile("../../../web/src/dinners/letterFolds.json") + if err != nil { + t.Fatal(err) + } + var pwa map[string]string + if err := json.Unmarshal(raw, &pwa); err != nil { + t.Fatal(err) + } + for r, want := range letterFolds { + if got, ok := pwa[string(r)]; !ok || got != want { + t.Errorf("PWA folds %U to %q (present %v), want %q", r, got, ok, want) + } + } + for k := range pwa { + if utf8.RuneCountInString(k) != 1 || letterFolds[[]rune(k)[0]] == "" { + t.Errorf("PWA folds %q, which letterFolds does not", k) + } + } +} + func TestNormalize(t *testing.T) { tests := []struct { name, want string diff --git a/internal/ai/harness/gen_folds.py b/internal/ai/harness/gen_folds.py index 6e907c4..4de6bb6 100644 --- a/internal/ai/harness/gen_folds.py +++ b/internal/ai/harness/gen_folds.py @@ -1,5 +1,8 @@ #!/usr/bin/env python3 -"""Generate folds_table.go, the letter-fold table used by harness.Normalize. +"""Generate folds_table.go, the letter-fold table used by harness.Normalize, +and its copy web/src/dinners/letterFolds.json, used by the PWA's +normalizeName (web/src/dinners/names.ts). TestLetterFoldsMatchPWA fails when +the two differ. Run from this directory with `go generate ./internal/ai/harness` (or `python3 gen_folds.py`; gofmt must be on PATH). It uses only the Python @@ -28,11 +31,14 @@ Hangul syllables are left out: their decomposition is into conjoining jamo, not combining marks. """ +import json import re import subprocess import sys import unicodedata +PWA_TABLE = "../../../web/src/dinners/letterFolds.json" + EXTRA = { "ß": "ss", "ẞ": "ss", "æ": "ae", "Æ": "ae", "œ": "oe", "Œ": "oe", "þ": "th", "Þ": "th", "ð": "d", "Ð": "d", "ı": "i", "ʼn": "n", @@ -110,6 +116,9 @@ def main(): with open("folds_table.go", "w", encoding="utf-8") as f: f.write("\n".join(out) + "\n") subprocess.run(["gofmt", "-w", "folds_table.go"], check=True) + with open(PWA_TABLE, "w", encoding="ascii") as f: + json.dump({chr(cp): folded for cp, folded in entries}, f, indent=2) + f.write("\n") if __name__ == "__main__": diff --git a/internal/ai/harness/testdata/dedupe_cases.json b/internal/ai/harness/testdata/dedupe_cases.json index 2488e8b..20c3aec 100644 --- a/internal/ai/harness/testdata/dedupe_cases.json +++ b/internal/ai/harness/testdata/dedupe_cases.json @@ -67,6 +67,18 @@ { "name": "7-layer dip", "want": "7 layer dip" + }, + { + "name": "Ƈhili", + "want": "chili" + }, + { + "name": "Ꞔarbanzo Stew", + "want": "carbanzo stew" + }, + { + "name": "ΜΟΥΣΑΚΑΣ", + "want": "μουσακασ" } ], "similar": [ @@ -114,6 +126,11 @@ "a": "The best", "b": "Easy recipe", "want": false + }, + { + "a": "Ƈhili", + "b": "Chili", + "want": true } ] } diff --git a/internal/dinners/dinners.go b/internal/dinners/dinners.go index 7fac6f9..605d675 100644 --- a/internal/dinners/dinners.go +++ b/internal/dinners/dinners.go @@ -11,9 +11,11 @@ import ( "errors" "fmt" "log/slog" + "math" "net/url" "slices" "strings" + "sync" "unicode" "unicode/utf8" @@ -130,6 +132,11 @@ type Service struct { grounder *recipes.Grounder logger *slog.Logger groundJob *jobs.Kind[GroundJob] + + // groundingsMu guards groundings: the KindGroundDinner jobs running, + // by dinner id, so archiving or renaming a dinner can stop them. + groundingsMu sync.Mutex + groundings map[string]map[*runningGrounding]struct{} } // NewService returns a service over st that grounds dinners with grounder @@ -146,7 +153,7 @@ func NewService(st *store.Store, grounder *recipes.Grounder, runner *jobs.Runner case logger == nil: return nil, errors.New("dinners: nil logger") } - s := &Service{st: st, grounder: grounder, logger: logger} + s := &Service{st: st, grounder: grounder, logger: logger, groundings: make(map[string]map[*runningGrounding]struct{})} kind, err := jobs.Register(runner, KindGroundDinner, s.runGroundJob, Reasons...) if err != nil { return nil, err @@ -296,7 +303,7 @@ func (s *Service) List(ctx context.Context, status string) ([]Dinner, error) { // clears its recipe_url, raises its grounding_revision and enqueues a fresh // KindGroundDinner job that grounds the new name: a job queued or running // for the old name or link carries the old revision, so it can no longer -// write the dinner. +// write the dinner, and one running is stopped. func (s *Service) Update(ctx context.Context, id string, p Patch, updatedBy string) (Dinner, error) { if updatedBy == "" { return Dinner{}, errors.New("dinners: update: no member to record as updated_by") @@ -368,6 +375,7 @@ func (s *Service) Update(ctx context.Context, id string, p Patch, updatedBy stri return Dinner{}, err } if regrounding { + s.stopGroundings(row.ID, row.GroundingRevision) jobID, err := s.enqueueGrounding(ctx, row.ID, row.GroundingRevision) if err != nil { return Dinner{}, err @@ -383,7 +391,8 @@ func (s *Service) Update(ctx context.Context, id string, p Patch, updatedBy stri } // Archive sets dinner id's status to archived, recording updatedBy. The row -// is kept, so its name still counts for Side quest dedupe. +// is kept, so its name still counts for Side quest dedupe. A +// KindGroundDinner job running for it is stopped. func (s *Service) Archive(ctx context.Context, id, updatedBy string) (Dinner, error) { if updatedBy == "" { return Dinner{}, errors.New("dinners: archive: no member to record as updated_by") @@ -399,6 +408,7 @@ func (s *Service) Archive(ctx context.Context, id, updatedBy string) (Dinner, er if err != nil { return Dinner{}, fmt.Errorf("dinners: archive %s: %w", id, err) } + s.stopGroundings(row.ID, math.MaxInt64) return fromRow(row), nil } diff --git a/internal/dinners/grounding.go b/internal/dinners/grounding.go index 4d43b13..a57bb71 100644 --- a/internal/dinners/grounding.go +++ b/internal/dinners/grounding.go @@ -104,8 +104,14 @@ func (r groundResult) PublicJSON() ([]byte, error) { return json.Marshal(r) } // either result is recorded only while the dinner is still unarchived, // pending and at p.Revision: a dinner grounded by hand meanwhile is left as // it is, and one renamed or archived meanwhile is left pending — for the -// job its rename enqueued, or for none. +// job its rename enqueued, or for none. Archiving or renaming the dinner +// stops the grounding where it is, so no further search, fetch, photo +// download or model call is made for it. func (s *Service) runGroundJob(ctx context.Context, p GroundJob, _ func(groundResult) error) (groundResult, error) { + // Tracked before the dinner is read: an archive or rename committed + // after the read finds this job to stop. + groundCtx, done := s.trackGrounding(ctx, p.DinnerID, p.Revision) + defer done() row, err := db.New(s.st.Reader).GetDinner(ctx, p.DinnerID) if err != nil { return groundResult{}, fmt.Errorf("dinners: ground: read %s: %w", p.DinnerID, err) @@ -116,9 +122,12 @@ func (s *Service) runGroundJob(ctx context.Context, p GroundJob, _ func(groundRe var gr recipes.Grounded if row.RecipeUrl.Valid { - gr, err = s.grounder.GroundURL(ctx, row.RecipeUrl.String) + gr, err = s.grounder.GroundURL(groundCtx, row.RecipeUrl.String) } else { - gr, err = s.grounder.Ground(ctx, row.Name, row.Name) + gr, err = s.grounder.Ground(groundCtx, row.Name, row.Name) + } + if err != nil && ctx.Err() == nil && errors.Is(context.Cause(groundCtx), errGroundingStopped) { + return s.currentGrounding(ctx, row.ID) } if err != nil { reason := reasonFor(err, row.RecipeUrl.Valid) @@ -157,15 +166,67 @@ func (s *Service) runGroundJob(ctx context.Context, p GroundJob, _ func(groundRe return groundResult{}, fmt.Errorf("dinners: ground: record %s: %w", row.ID, err) } if n == 0 { - cur, err := db.New(s.st.Reader).GetDinner(ctx, row.ID) - if err != nil { - return groundResult{}, fmt.Errorf("dinners: ground: read %s: %w", row.ID, err) - } - return groundResult{DinnerID: cur.ID, GroundingStatus: cur.GroundingStatus}, nil + return s.currentGrounding(ctx, row.ID) } return groundResult{DinnerID: row.ID, GroundingStatus: GroundingGrounded}, nil } +// currentGrounding is the KindGroundDinner result of a job that did not +// ground dinner id: the grounding status the dinner has now. +func (s *Service) currentGrounding(ctx context.Context, id string) (groundResult, error) { + cur, err := db.New(s.st.Reader).GetDinner(ctx, id) + if err != nil { + return groundResult{}, fmt.Errorf("dinners: ground: read %s: %w", id, err) + } + return groundResult{DinnerID: cur.ID, GroundingStatus: cur.GroundingStatus}, nil +} + +// errGroundingStopped is the cause a running KindGroundDinner job's +// grounding is cancelled with when its dinner is archived or renamed. +var errGroundingStopped = errors.New("dinners: grounding stopped: dinner archived or renamed") + +// runningGrounding is a KindGroundDinner job running for a dinner at +// revision, with the cancel of its grounding. +type runningGrounding struct { + revision int64 + cancel context.CancelCauseFunc +} + +// trackGrounding returns a context for grounding dinnerID at revision that +// stopGroundings cancels, and the func to call when the job ends. +func (s *Service) trackGrounding(ctx context.Context, dinnerID string, revision int64) (context.Context, func()) { + groundCtx, cancel := context.WithCancelCause(ctx) + r := &runningGrounding{revision: revision, cancel: cancel} + s.groundingsMu.Lock() + if s.groundings[dinnerID] == nil { + s.groundings[dinnerID] = make(map[*runningGrounding]struct{}) + } + s.groundings[dinnerID][r] = struct{}{} + s.groundingsMu.Unlock() + return groundCtx, func() { + s.groundingsMu.Lock() + delete(s.groundings[dinnerID], r) + if len(s.groundings[dinnerID]) == 0 { + delete(s.groundings, dinnerID) + } + s.groundingsMu.Unlock() + cancel(nil) + } +} + +// stopGroundings cancels, with errGroundingStopped, the grounding of every +// KindGroundDinner job running for dinnerID at a revision below below. Call +// it once the change that makes those jobs stale is committed. +func (s *Service) stopGroundings(dinnerID string, below int64) { + s.groundingsMu.Lock() + defer s.groundingsMu.Unlock() + for r := range s.groundings[dinnerID] { + if r.revision < below { + r.cancel(errGroundingStopped) + } + } +} + // FailInterruptedGroundings fails, with ReasonInterrupted, every dinner // still pending whose KindGroundDinner job is no longer queued or running: // a previous process stopped before enqueuing the job or before its diff --git a/internal/dinners/grounding_test.go b/internal/dinners/grounding_test.go index 47f0f31..ba90e3e 100644 --- a/internal/dinners/grounding_test.go +++ b/internal/dinners/grounding_test.go @@ -6,9 +6,11 @@ import ( "encoding/json" "errors" "io" + "io/fs" "log/slog" "slices" "testing" + "testing/fstest" "time" "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm" @@ -54,6 +56,13 @@ func newEnv(t *testing.T) *env { func newUnstartedEnv(t *testing.T) *env { t.Helper() dataDir := t.TempDir() + return newUnstartedEnvOn(t, openStore(t, dataDir), dataDir) +} + +// openStore opens the store in dataDir, closing it when the test ends. It +// is not migrated. +func openStore(t *testing.T, dataDir string) *store.Store { + t.Helper() st, err := store.Open(t.Context(), dataDir) if err != nil { t.Fatal(err) @@ -63,6 +72,13 @@ func newUnstartedEnv(t *testing.T) *env { t.Error(err) } }) + return st +} + +// newUnstartedEnvOn is newUnstartedEnv over st, the store in dataDir, +// which it migrates to the latest version. +func newUnstartedEnvOn(t *testing.T, st *store.Store, dataDir string) *env { + t.Helper() if err := store.Migrate(t.Context(), st.Writer, migrations.FS); err != nil { t.Fatal(err) } @@ -290,19 +306,16 @@ func TestManualFixWhilePendingIsKept(t *testing.T) { // TestRenameWhilePendingGroundsNewName: a Regular renamed while its job is // still grounding the old name never gets the old name's result, whether -// that job succeeds or fails; the job the rename enqueues grounds the new -// name. +// the old name would be found or not: the rename stops that job where it +// is, and the job the rename enqueues grounds the new name. func TestRenameWhilePendingGroundsNewName(t *testing.T) { t.Parallel() tests := []struct { name string oldName string - replies []llm.GenerateResult }{ - {"old name found", "Chicken curry", []llm.GenerateResult{ - recipestest.NormalizedReply("stovetop", 40), recipestest.NormalizedReply("oven", 150), - }}, - {"old name not found", "Unknown dish", []llm.GenerateResult{recipestest.NormalizedReply("oven", 150)}}, + {"old name found", "Chicken curry"}, + {"old name not found", "Unknown dish"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -310,9 +323,7 @@ func TestRenameWhilePendingGroundsNewName(t *testing.T) { e := newEnv(t) e.search.Results["Beef stew"] = []recipes.Candidate{{URL: stew}} e.search.Gate = make(chan struct{}) - for _, r := range tt.replies { - e.model.QueueGenerate(r) - } + e.model.QueueGenerate(recipestest.NormalizedReply("oven", 150)) c, err := e.svc.CreateRegular(t.Context(), dinners.NewRegular{Name: ptr(tt.oldName)}, member) if err != nil { @@ -327,12 +338,13 @@ func TestRenameWhilePendingGroundsNewName(t *testing.T) { if d.GroundingStatus != dinners.GroundingPending { t.Fatalf("renamed = %+v, want still pending", d) } + // The old name's job ends while its search is still held. + first := e.waitJob(t, *c.GroundingJobID) + if first.Status != jobs.StatusSucceeded || string(first.Result) != `{"dinner_id":"`+c.ID+`","grounding_status":"pending"}` { + t.Fatalf("old name's job = %+v (result %s), want succeeded leaving the dinner pending", first, first.Result) + } close(e.search.Gate) - first := e.waitJob(t, *c.GroundingJobID) - if string(first.Result) == `{"dinner_id":"`+c.ID+`","grounding_status":"grounded"}` { - t.Fatalf("old name's job reported grounded: %+v (result %s)", first, first.Result) - } row := e.waitGrounding(t, c.ID) if row.GroundingStatus != dinners.GroundingGrounded { t.Fatalf("dinner = %+v, want grounded for the new name", row) @@ -347,6 +359,9 @@ func TestRenameWhilePendingGroundsNewName(t *testing.T) { if got := e.search.Queries(); !slices.Equal(got, []string{tt.oldName, "Beef stew"}) { t.Errorf("searched %q, want the old name then the new", got) } + if got := len(e.model.Requests()); got != 1 { + t.Errorf("model called %d times, want once, for the new name only", got) + } }) } } @@ -503,7 +518,8 @@ func TestEditWhilePendingIsKept(t *testing.T) { // TestArchiveWhilePendingStopsGrounding: a pending Regular archived before // its job starts is not searched for, and one archived while its job runs -// is not written; neither ends up grounded. +// is stopped there, with no page fetched or model called; neither ends up +// grounded. func TestArchiveWhilePendingStopsGrounding(t *testing.T) { t.Parallel() t.Run("before the job starts", func(t *testing.T) { @@ -541,9 +557,17 @@ func TestArchiveWhilePendingStopsGrounding(t *testing.T) { if _, err := e.svc.Archive(t.Context(), c.ID, member); err != nil { t.Fatal(err) } + // The job ends while its search is still held: archiving stopped it. + job := e.waitJob(t, *c.GroundingJobID) close(e.search.Gate) - if job := e.waitJob(t, *c.GroundingJobID); job.Status != jobs.StatusSucceeded { - t.Fatalf("job = %+v", job) + if job.Status != jobs.StatusSucceeded || string(job.Result) != `{"dinner_id":"`+c.ID+`","grounding_status":"pending"}` { + t.Fatalf("job = %+v (result %s), want succeeded leaving the dinner pending", job, job.Result) + } + if got := e.pages.Fetched(); len(got) != 0 { + t.Errorf("fetched %q for an archived dinner", got) + } + if got := len(e.model.Requests()); got != 0 { + t.Errorf("model called %d times for an archived dinner", got) } if row := e.dinner(t, c.ID); row.Status != dinners.StatusArchived || row.GroundingStatus == dinners.GroundingGrounded || row.RecipeID.Valid { t.Errorf("dinner = %+v, want archived and not grounded", row) @@ -922,6 +946,74 @@ func TestRestartFailsInterruptedGrounding(t *testing.T) { } } +// TestUpgradeKeepsLegacyQueuedGrounding: a pending Regular whose +// ground_dinner job was queued before migration 0010, so its payload has +// no revision, is not failed as interrupted at the first boot after the +// upgrade; its job still grounds it. +func TestUpgradeKeepsLegacyQueuedGrounding(t *testing.T) { + t.Parallel() + ctx := t.Context() + dataDir := t.TempDir() + st := openStore(t, dataDir) + before := fstest.MapFS{} + files, err := fs.Glob(migrations.FS, "*.sql") + if err != nil { + t.Fatal(err) + } + for _, name := range files { + if name >= "0010" { + continue + } + body, err := fs.ReadFile(migrations.FS, name) + if err != nil { + t.Fatal(err) + } + before[name] = &fstest.MapFile{Data: body} + } + if err := store.Migrate(ctx, st.Writer, before); err != nil { + t.Fatal(err) + } + now := store.Now().String() + if _, err := st.Writer.ExecContext(ctx, `INSERT INTO dinners + (id, name, normalized_name, status, origin, is_favorite, grounding_status, updated_by, created_at, updated_at) + VALUES ('dinner-1', 'Chicken curry', 'chicken curry', 'regular', 'regular', 0, 'pending', ?, ?, ?)`, + member, now, now); err != nil { + t.Fatal(err) + } + if _, err := st.Writer.ExecContext(ctx, `INSERT INTO jobs (id, kind, status, payload, created_at, updated_at) + VALUES ('job-1', ?, 'queued', '{"dinner_id":"dinner-1"}', ?, ?)`, + dinners.KindGroundDinner, now, now); err != nil { + t.Fatal(err) + } + + // Boot: migrate, start the runner, then fail interrupted groundings. The + // search is held, so the job cannot finish before the cleanup runs. + e := newUnstartedEnvOn(t, st, dataDir) + e.search.Gate = make(chan struct{}) + e.model.QueueGenerate(recipestest.NormalizedReply("stovetop", 40)) + if p := e.queuedJob(t, "job-1"); p != (dinners.GroundJob{DinnerID: "dinner-1", Revision: 0}) { + t.Errorf("migrated payload = %+v, want dinner-1 at revision 0", p) + } + e.start(t) + n, err := e.svc.FailInterruptedGroundings(ctx) + if err != nil { + t.Fatal(err) + } + if n != 0 { + t.Errorf("failed %d dinners, want 0", n) + } + if row := e.dinner(t, "dinner-1"); row.GroundingStatus != dinners.GroundingPending { + t.Fatalf("dinner = %s (%s), want still pending", row.GroundingStatus, row.GroundingError.String) + } + close(e.search.Gate) + if job := e.waitJob(t, "job-1"); job.Status != jobs.StatusSucceeded { + t.Fatalf("job = %+v", job) + } + if row := e.dinner(t, "dinner-1"); row.GroundingStatus != dinners.GroundingGrounded { + t.Errorf("dinner = %s, want grounded", row.GroundingStatus) + } +} + // TestCreateEnqueueFailureReturnsFailedDinner: a dinner saved when its job // cannot be enqueued is returned, with its id, failed and no job id. func TestCreateEnqueueFailureReturnsFailedDinner(t *testing.T) { diff --git a/internal/recipes/recipestest/recipestest.go b/internal/recipes/recipestest/recipestest.go index 46aa72a..9d22dfa 100644 --- a/internal/recipes/recipestest/recipestest.go +++ b/internal/recipes/recipestest/recipestest.go @@ -66,9 +66,15 @@ func (s *Search) Queries() []string { type Pages struct { Bodies map[string][]byte Gate chan struct{} + + mu sync.Mutex + fetched []string } func (p *Pages) Fetch(ctx context.Context, rawURL string) (recipes.Page, error) { + p.mu.Lock() + p.fetched = append(p.fetched, rawURL) + p.mu.Unlock() if p.Gate != nil { select { case <-p.Gate: @@ -83,6 +89,13 @@ func (p *Pages) Fetch(ctx context.Context, rawURL string) (recipes.Page, error) return recipes.Page{URL: rawURL, ContentType: "text/html", Body: body}, nil } +// Fetched returns every URL fetched, in order. +func (p *Pages) Fetched() []string { + p.mu.Lock() + defer p.mu.Unlock() + return append([]string(nil), p.fetched...) +} + // Images is a recipes.ImageFetcher serving JPEGs by URL; any other URL is // a 404. type Images struct { diff --git a/migrations/0010_dinner_grounding_revision.sql b/migrations/0010_dinner_grounding_revision.sql index 5f13bd9..c2d4fc1 100644 --- a/migrations/0010_dinner_grounding_revision.sql +++ b/migrations/0010_dinner_grounding_revision.sql @@ -5,3 +5,12 @@ -- earlier job — queued or running — can no longer write the dinner. ALTER TABLE dinners ADD COLUMN grounding_revision INTEGER NOT NULL DEFAULT 0; + +-- Every dinner starts at revision 0, so a ground_dinner job enqueued +-- before this migration, whose payload has only dinner_id, is for revision +-- 0. Recording it lets the boot cleanup, which matches a pending dinner to +-- its job by revision, see a job still queued for its dinner. +UPDATE jobs +SET payload = json_set(payload, '$.revision', 0) +WHERE kind = 'ground_dinner' + AND json_type(payload, '$.revision') IS NULL; diff --git a/web/src/dinners/letterFolds.json b/web/src/dinners/letterFolds.json new file mode 100644 index 0000000..5be5eaf --- /dev/null +++ b/web/src/dinners/letterFolds.json @@ -0,0 +1,1167 @@ +{ + "\u00c0": "a", + "\u00c1": "a", + "\u00c2": "a", + "\u00c3": "a", + "\u00c4": "a", + "\u00c5": "a", + "\u00c6": "ae", + "\u00c7": "c", + "\u00c8": "e", + "\u00c9": "e", + "\u00ca": "e", + "\u00cb": "e", + "\u00cc": "i", + "\u00cd": "i", + "\u00ce": "i", + "\u00cf": "i", + "\u00d0": "d", + "\u00d1": "n", + "\u00d2": "o", + "\u00d3": "o", + "\u00d4": "o", + "\u00d5": "o", + "\u00d6": "o", + "\u00d8": "o", + "\u00d9": "u", + "\u00da": "u", + "\u00db": "u", + "\u00dc": "u", + "\u00dd": "y", + "\u00de": "th", + "\u00df": "ss", + "\u00e0": "a", + "\u00e1": "a", + "\u00e2": "a", + "\u00e3": "a", + "\u00e4": "a", + "\u00e5": "a", + "\u00e6": "ae", + "\u00e7": "c", + "\u00e8": "e", + "\u00e9": "e", + "\u00ea": "e", + "\u00eb": "e", + "\u00ec": "i", + "\u00ed": "i", + "\u00ee": "i", + "\u00ef": "i", + "\u00f0": "d", + "\u00f1": "n", + "\u00f2": "o", + "\u00f3": "o", + "\u00f4": "o", + "\u00f5": "o", + "\u00f6": "o", + "\u00f8": "o", + "\u00f9": "u", + "\u00fa": "u", + "\u00fb": "u", + "\u00fc": "u", + "\u00fd": "y", + "\u00fe": "th", + "\u00ff": "y", + "\u0100": "a", + "\u0101": "a", + "\u0102": "a", + "\u0103": "a", + "\u0104": "a", + "\u0105": "a", + "\u0106": "c", + "\u0107": "c", + "\u0108": "c", + "\u0109": "c", + "\u010a": "c", + "\u010b": "c", + "\u010c": "c", + "\u010d": "c", + "\u010e": "d", + "\u010f": "d", + "\u0110": "d", + "\u0111": "d", + "\u0112": "e", + "\u0113": "e", + "\u0114": "e", + "\u0115": "e", + "\u0116": "e", + "\u0117": "e", + "\u0118": "e", + "\u0119": "e", + "\u011a": "e", + "\u011b": "e", + "\u011c": "g", + "\u011d": "g", + "\u011e": "g", + "\u011f": "g", + "\u0120": "g", + "\u0121": "g", + "\u0122": "g", + "\u0123": "g", + "\u0124": "h", + "\u0125": "h", + "\u0126": "h", + "\u0127": "h", + "\u0128": "i", + "\u0129": "i", + "\u012a": "i", + "\u012b": "i", + "\u012c": "i", + "\u012d": "i", + "\u012e": "i", + "\u012f": "i", + "\u0130": "i", + "\u0131": "i", + "\u0132": "ij", + "\u0133": "ij", + "\u0134": "j", + "\u0135": "j", + "\u0136": "k", + "\u0137": "k", + "\u0139": "l", + "\u013a": "l", + "\u013b": "l", + "\u013c": "l", + "\u013d": "l", + "\u013e": "l", + "\u013f": "l", + "\u0140": "l", + "\u0141": "l", + "\u0142": "l", + "\u0143": "n", + "\u0144": "n", + "\u0145": "n", + "\u0146": "n", + "\u0147": "n", + "\u0148": "n", + "\u0149": "n", + "\u014c": "o", + "\u014d": "o", + "\u014e": "o", + "\u014f": "o", + "\u0150": "o", + "\u0151": "o", + "\u0152": "oe", + "\u0153": "oe", + "\u0154": "r", + "\u0155": "r", + "\u0156": "r", + "\u0157": "r", + "\u0158": "r", + "\u0159": "r", + "\u015a": "s", + "\u015b": "s", + "\u015c": "s", + "\u015d": "s", + "\u015e": "s", + "\u015f": "s", + "\u0160": "s", + "\u0161": "s", + "\u0162": "t", + "\u0163": "t", + "\u0164": "t", + "\u0165": "t", + "\u0166": "t", + "\u0167": "t", + "\u0168": "u", + "\u0169": "u", + "\u016a": "u", + "\u016b": "u", + "\u016c": "u", + "\u016d": "u", + "\u016e": "u", + "\u016f": "u", + "\u0170": "u", + "\u0171": "u", + "\u0172": "u", + "\u0173": "u", + "\u0174": "w", + "\u0175": "w", + "\u0176": "y", + "\u0177": "y", + "\u0178": "y", + "\u0179": "z", + "\u017a": "z", + "\u017b": "z", + "\u017c": "z", + "\u017d": "z", + "\u017e": "z", + "\u0180": "b", + "\u0181": "b", + "\u0182": "b", + "\u0183": "b", + "\u0187": "c", + "\u0188": "c", + "\u018a": "d", + "\u018b": "d", + "\u018c": "d", + "\u0191": "f", + "\u0192": "f", + "\u0193": "g", + "\u0197": "i", + "\u0198": "k", + "\u0199": "k", + "\u019a": "l", + "\u019d": "n", + "\u019e": "n", + "\u019f": "o", + "\u01a0": "o", + "\u01a1": "o", + "\u01a4": "p", + "\u01a5": "p", + "\u01ab": "t", + "\u01ac": "t", + "\u01ad": "t", + "\u01ae": "t", + "\u01af": "u", + "\u01b0": "u", + "\u01b2": "v", + "\u01b3": "y", + "\u01b4": "y", + "\u01b5": "z", + "\u01b6": "z", + "\u01c5": "d", + "\u01c8": "l", + "\u01cb": "n", + "\u01cd": "a", + "\u01ce": "a", + "\u01cf": "i", + "\u01d0": "i", + "\u01d1": "o", + "\u01d2": "o", + "\u01d3": "u", + "\u01d4": "u", + "\u01d5": "u", + "\u01d6": "u", + "\u01d7": "u", + "\u01d8": "u", + "\u01d9": "u", + "\u01da": "u", + "\u01db": "u", + "\u01dc": "u", + "\u01de": "a", + "\u01df": "a", + "\u01e0": "a", + "\u01e1": "a", + "\u01e2": "ae", + "\u01e3": "ae", + "\u01e4": "g", + "\u01e5": "g", + "\u01e6": "g", + "\u01e7": "g", + "\u01e8": "k", + "\u01e9": "k", + "\u01ea": "o", + "\u01eb": "o", + "\u01ec": "o", + "\u01ed": "o", + "\u01ee": "\u0292", + "\u01ef": "\u0292", + "\u01f0": "j", + "\u01f2": "d", + "\u01f4": "g", + "\u01f5": "g", + "\u01f8": "n", + "\u01f9": "n", + "\u01fa": "a", + "\u01fb": "a", + "\u01fc": "ae", + "\u01fd": "ae", + "\u01fe": "o", + "\u01ff": "o", + "\u0200": "a", + "\u0201": "a", + "\u0202": "a", + "\u0203": "a", + "\u0204": "e", + "\u0205": "e", + "\u0206": "e", + "\u0207": "e", + "\u0208": "i", + "\u0209": "i", + "\u020a": "i", + "\u020b": "i", + "\u020c": "o", + "\u020d": "o", + "\u020e": "o", + "\u020f": "o", + "\u0210": "r", + "\u0211": "r", + "\u0212": "r", + "\u0213": "r", + "\u0214": "u", + "\u0215": "u", + "\u0216": "u", + "\u0217": "u", + "\u0218": "s", + "\u0219": "s", + "\u021a": "t", + "\u021b": "t", + "\u021e": "h", + "\u021f": "h", + "\u0220": "n", + "\u0221": "d", + "\u0224": "z", + "\u0225": "z", + "\u0226": "a", + "\u0227": "a", + "\u0228": "e", + "\u0229": "e", + "\u022a": "o", + "\u022b": "o", + "\u022c": "o", + "\u022d": "o", + "\u022e": "o", + "\u022f": "o", + "\u0230": "o", + "\u0231": "o", + "\u0232": "y", + "\u0233": "y", + "\u0234": "l", + "\u0235": "n", + "\u0236": "t", + "\u023a": "a", + "\u023b": "c", + "\u023c": "c", + "\u023d": "l", + "\u023e": "t", + "\u023f": "s", + "\u0240": "z", + "\u0243": "b", + "\u0246": "e", + "\u0247": "e", + "\u0248": "j", + "\u0249": "j", + "\u024b": "q", + "\u024c": "r", + "\u024d": "r", + "\u024e": "y", + "\u024f": "y", + "\u0253": "b", + "\u0255": "c", + "\u0256": "d", + "\u0257": "d", + "\u0260": "g", + "\u0266": "h", + "\u0268": "i", + "\u026b": "l", + "\u026c": "l", + "\u026d": "l", + "\u0271": "m", + "\u0272": "n", + "\u0273": "n", + "\u027c": "r", + "\u027d": "r", + "\u027e": "r", + "\u0282": "s", + "\u0288": "t", + "\u028b": "v", + "\u0290": "z", + "\u0291": "z", + "\u029d": "j", + "\u02a0": "q", + "\u0386": "\u03b1", + "\u0388": "\u03b5", + "\u0389": "\u03b7", + "\u038a": "\u03b9", + "\u038c": "\u03bf", + "\u038e": "\u03c5", + "\u038f": "\u03c9", + "\u0390": "\u03b9", + "\u03aa": "\u03b9", + "\u03ab": "\u03c5", + "\u03ac": "\u03b1", + "\u03ad": "\u03b5", + "\u03ae": "\u03b7", + "\u03af": "\u03b9", + "\u03b0": "\u03c5", + "\u03ca": "\u03b9", + "\u03cb": "\u03c5", + "\u03cc": "\u03bf", + "\u03cd": "\u03c5", + "\u03ce": "\u03c9", + "\u03d3": "\u03d2", + "\u03d4": "\u03d2", + "\u0400": "\u0435", + "\u0401": "\u0435", + "\u0403": "\u0433", + "\u0407": "\u0456", + "\u040c": "\u043a", + "\u040d": "\u0438", + "\u040e": "\u0443", + "\u0419": "\u0438", + "\u0439": "\u0438", + "\u0450": "\u0435", + "\u0451": "\u0435", + "\u0453": "\u0433", + "\u0457": "\u0456", + "\u045c": "\u043a", + "\u045d": "\u0438", + "\u045e": "\u0443", + "\u0476": "\u0475", + "\u0477": "\u0475", + "\u04c1": "\u0436", + "\u04c2": "\u0436", + "\u04d0": "\u0430", + "\u04d1": "\u0430", + "\u04d2": "\u0430", + "\u04d3": "\u0430", + "\u04d6": "\u0435", + "\u04d7": "\u0435", + "\u04da": "\u04d9", + "\u04db": "\u04d9", + "\u04dc": "\u0436", + "\u04dd": "\u0436", + "\u04de": "\u0437", + "\u04df": "\u0437", + "\u04e2": "\u0438", + "\u04e3": "\u0438", + "\u04e4": "\u0438", + "\u04e5": "\u0438", + "\u04e6": "\u043e", + "\u04e7": "\u043e", + "\u04ea": "\u04e9", + "\u04eb": "\u04e9", + "\u04ec": "\u044d", + "\u04ed": "\u044d", + "\u04ee": "\u0443", + "\u04ef": "\u0443", + "\u04f0": "\u0443", + "\u04f1": "\u0443", + "\u04f2": "\u0443", + "\u04f3": "\u0443", + "\u04f4": "\u0447", + "\u04f5": "\u0447", + "\u04f8": "\u044b", + "\u04f9": "\u044b", + "\u0622": "\u0627", + "\u0623": "\u0627", + "\u0624": "\u0648", + "\u0625": "\u0627", + "\u0626": "\u064a", + "\u06c0": "\u06d5", + "\u06c2": "\u06c1", + "\u06d3": "\u06d2", + "\u0929": "\u0928", + "\u0931": "\u0930", + "\u0934": "\u0933", + "\u0958": "\u0915", + "\u0959": "\u0916", + "\u095a": "\u0917", + "\u095b": "\u091c", + "\u095c": "\u0921", + "\u095d": "\u0922", + "\u095e": "\u092b", + "\u095f": "\u092f", + "\u09dc": "\u09a1", + "\u09dd": "\u09a2", + "\u09df": "\u09af", + "\u0a33": "\u0a32", + "\u0a36": "\u0a38", + "\u0a59": "\u0a16", + "\u0a5a": "\u0a17", + "\u0a5b": "\u0a1c", + "\u0a5e": "\u0a2b", + "\u0b5c": "\u0b21", + "\u0b5d": "\u0b22", + "\u0f43": "\u0f42", + "\u0f4d": "\u0f4c", + "\u0f52": "\u0f51", + "\u0f57": "\u0f56", + "\u0f5c": "\u0f5b", + "\u0f69": "\u0f40", + "\u1026": "\u1025", + "\u1d6c": "b", + "\u1d6d": "d", + "\u1d6e": "f", + "\u1d6f": "m", + "\u1d70": "n", + "\u1d71": "p", + "\u1d72": "r", + "\u1d73": "r", + "\u1d74": "s", + "\u1d75": "t", + "\u1d76": "z", + "\u1d7d": "p", + "\u1d80": "b", + "\u1d81": "d", + "\u1d82": "f", + "\u1d83": "g", + "\u1d84": "k", + "\u1d85": "l", + "\u1d86": "m", + "\u1d87": "n", + "\u1d88": "p", + "\u1d89": "r", + "\u1d8a": "s", + "\u1d8c": "v", + "\u1d8d": "x", + "\u1d8e": "z", + "\u1d8f": "a", + "\u1d91": "d", + "\u1d92": "e", + "\u1d96": "i", + "\u1d99": "u", + "\u1e00": "a", + "\u1e01": "a", + "\u1e02": "b", + "\u1e03": "b", + "\u1e04": "b", + "\u1e05": "b", + "\u1e06": "b", + "\u1e07": "b", + "\u1e08": "c", + "\u1e09": "c", + "\u1e0a": "d", + "\u1e0b": "d", + "\u1e0c": "d", + "\u1e0d": "d", + "\u1e0e": "d", + "\u1e0f": "d", + "\u1e10": "d", + "\u1e11": "d", + "\u1e12": "d", + "\u1e13": "d", + "\u1e14": "e", + "\u1e15": "e", + "\u1e16": "e", + "\u1e17": "e", + "\u1e18": "e", + "\u1e19": "e", + "\u1e1a": "e", + "\u1e1b": "e", + "\u1e1c": "e", + "\u1e1d": "e", + "\u1e1e": "f", + "\u1e1f": "f", + "\u1e20": "g", + "\u1e21": "g", + "\u1e22": "h", + "\u1e23": "h", + "\u1e24": "h", + "\u1e25": "h", + "\u1e26": "h", + "\u1e27": "h", + "\u1e28": "h", + "\u1e29": "h", + "\u1e2a": "h", + "\u1e2b": "h", + "\u1e2c": "i", + "\u1e2d": "i", + "\u1e2e": "i", + "\u1e2f": "i", + "\u1e30": "k", + "\u1e31": "k", + "\u1e32": "k", + "\u1e33": "k", + "\u1e34": "k", + "\u1e35": "k", + "\u1e36": "l", + "\u1e37": "l", + "\u1e38": "l", + "\u1e39": "l", + "\u1e3a": "l", + "\u1e3b": "l", + "\u1e3c": "l", + "\u1e3d": "l", + "\u1e3e": "m", + "\u1e3f": "m", + "\u1e40": "m", + "\u1e41": "m", + "\u1e42": "m", + "\u1e43": "m", + "\u1e44": "n", + "\u1e45": "n", + "\u1e46": "n", + "\u1e47": "n", + "\u1e48": "n", + "\u1e49": "n", + "\u1e4a": "n", + "\u1e4b": "n", + "\u1e4c": "o", + "\u1e4d": "o", + "\u1e4e": "o", + "\u1e4f": "o", + "\u1e50": "o", + "\u1e51": "o", + "\u1e52": "o", + "\u1e53": "o", + "\u1e54": "p", + "\u1e55": "p", + "\u1e56": "p", + "\u1e57": "p", + "\u1e58": "r", + "\u1e59": "r", + "\u1e5a": "r", + "\u1e5b": "r", + "\u1e5c": "r", + "\u1e5d": "r", + "\u1e5e": "r", + "\u1e5f": "r", + "\u1e60": "s", + "\u1e61": "s", + "\u1e62": "s", + "\u1e63": "s", + "\u1e64": "s", + "\u1e65": "s", + "\u1e66": "s", + "\u1e67": "s", + "\u1e68": "s", + "\u1e69": "s", + "\u1e6a": "t", + "\u1e6b": "t", + "\u1e6c": "t", + "\u1e6d": "t", + "\u1e6e": "t", + "\u1e6f": "t", + "\u1e70": "t", + "\u1e71": "t", + "\u1e72": "u", + "\u1e73": "u", + "\u1e74": "u", + "\u1e75": "u", + "\u1e76": "u", + "\u1e77": "u", + "\u1e78": "u", + "\u1e79": "u", + "\u1e7a": "u", + "\u1e7b": "u", + "\u1e7c": "v", + "\u1e7d": "v", + "\u1e7e": "v", + "\u1e7f": "v", + "\u1e80": "w", + "\u1e81": "w", + "\u1e82": "w", + "\u1e83": "w", + "\u1e84": "w", + "\u1e85": "w", + "\u1e86": "w", + "\u1e87": "w", + "\u1e88": "w", + "\u1e89": "w", + "\u1e8a": "x", + "\u1e8b": "x", + "\u1e8c": "x", + "\u1e8d": "x", + "\u1e8e": "y", + "\u1e8f": "y", + "\u1e90": "z", + "\u1e91": "z", + "\u1e92": "z", + "\u1e93": "z", + "\u1e94": "z", + "\u1e95": "z", + "\u1e96": "h", + "\u1e97": "t", + "\u1e98": "w", + "\u1e99": "y", + "\u1e9a": "a", + "\u1e9b": "\u017f", + "\u1e9e": "ss", + "\u1ea0": "a", + "\u1ea1": "a", + "\u1ea2": "a", + "\u1ea3": "a", + "\u1ea4": "a", + "\u1ea5": "a", + "\u1ea6": "a", + "\u1ea7": "a", + "\u1ea8": "a", + "\u1ea9": "a", + "\u1eaa": "a", + "\u1eab": "a", + "\u1eac": "a", + "\u1ead": "a", + "\u1eae": "a", + "\u1eaf": "a", + "\u1eb0": "a", + "\u1eb1": "a", + "\u1eb2": "a", + "\u1eb3": "a", + "\u1eb4": "a", + "\u1eb5": "a", + "\u1eb6": "a", + "\u1eb7": "a", + "\u1eb8": "e", + "\u1eb9": "e", + "\u1eba": "e", + "\u1ebb": "e", + "\u1ebc": "e", + "\u1ebd": "e", + "\u1ebe": "e", + "\u1ebf": "e", + "\u1ec0": "e", + "\u1ec1": "e", + "\u1ec2": "e", + "\u1ec3": "e", + "\u1ec4": "e", + "\u1ec5": "e", + "\u1ec6": "e", + "\u1ec7": "e", + "\u1ec8": "i", + "\u1ec9": "i", + "\u1eca": "i", + "\u1ecb": "i", + "\u1ecc": "o", + "\u1ecd": "o", + "\u1ece": "o", + "\u1ecf": "o", + "\u1ed0": "o", + "\u1ed1": "o", + "\u1ed2": "o", + "\u1ed3": "o", + "\u1ed4": "o", + "\u1ed5": "o", + "\u1ed6": "o", + "\u1ed7": "o", + "\u1ed8": "o", + "\u1ed9": "o", + "\u1eda": "o", + "\u1edb": "o", + "\u1edc": "o", + "\u1edd": "o", + "\u1ede": "o", + "\u1edf": "o", + "\u1ee0": "o", + "\u1ee1": "o", + "\u1ee2": "o", + "\u1ee3": "o", + "\u1ee4": "u", + "\u1ee5": "u", + "\u1ee6": "u", + "\u1ee7": "u", + "\u1ee8": "u", + "\u1ee9": "u", + "\u1eea": "u", + "\u1eeb": "u", + "\u1eec": "u", + "\u1eed": "u", + "\u1eee": "u", + "\u1eef": "u", + "\u1ef0": "u", + "\u1ef1": "u", + "\u1ef2": "y", + "\u1ef3": "y", + "\u1ef4": "y", + "\u1ef5": "y", + "\u1ef6": "y", + "\u1ef7": "y", + "\u1ef8": "y", + "\u1ef9": "y", + "\u1efe": "y", + "\u1eff": "y", + "\u1f00": "\u03b1", + "\u1f01": "\u03b1", + "\u1f02": "\u03b1", + "\u1f03": "\u03b1", + "\u1f04": "\u03b1", + "\u1f05": "\u03b1", + "\u1f06": "\u03b1", + "\u1f07": "\u03b1", + "\u1f08": "\u03b1", + "\u1f09": "\u03b1", + "\u1f0a": "\u03b1", + "\u1f0b": "\u03b1", + "\u1f0c": "\u03b1", + "\u1f0d": "\u03b1", + "\u1f0e": "\u03b1", + "\u1f0f": "\u03b1", + "\u1f10": "\u03b5", + "\u1f11": "\u03b5", + "\u1f12": "\u03b5", + "\u1f13": "\u03b5", + "\u1f14": "\u03b5", + "\u1f15": "\u03b5", + "\u1f18": "\u03b5", + "\u1f19": "\u03b5", + "\u1f1a": "\u03b5", + "\u1f1b": "\u03b5", + "\u1f1c": "\u03b5", + "\u1f1d": "\u03b5", + "\u1f20": "\u03b7", + "\u1f21": "\u03b7", + "\u1f22": "\u03b7", + "\u1f23": "\u03b7", + "\u1f24": "\u03b7", + "\u1f25": "\u03b7", + "\u1f26": "\u03b7", + "\u1f27": "\u03b7", + "\u1f28": "\u03b7", + "\u1f29": "\u03b7", + "\u1f2a": "\u03b7", + "\u1f2b": "\u03b7", + "\u1f2c": "\u03b7", + "\u1f2d": "\u03b7", + "\u1f2e": "\u03b7", + "\u1f2f": "\u03b7", + "\u1f30": "\u03b9", + "\u1f31": "\u03b9", + "\u1f32": "\u03b9", + "\u1f33": "\u03b9", + "\u1f34": "\u03b9", + "\u1f35": "\u03b9", + "\u1f36": "\u03b9", + "\u1f37": "\u03b9", + "\u1f38": "\u03b9", + "\u1f39": "\u03b9", + "\u1f3a": "\u03b9", + "\u1f3b": "\u03b9", + "\u1f3c": "\u03b9", + "\u1f3d": "\u03b9", + "\u1f3e": "\u03b9", + "\u1f3f": "\u03b9", + "\u1f40": "\u03bf", + "\u1f41": "\u03bf", + "\u1f42": "\u03bf", + "\u1f43": "\u03bf", + "\u1f44": "\u03bf", + "\u1f45": "\u03bf", + "\u1f48": "\u03bf", + "\u1f49": "\u03bf", + "\u1f4a": "\u03bf", + "\u1f4b": "\u03bf", + "\u1f4c": "\u03bf", + "\u1f4d": "\u03bf", + "\u1f50": "\u03c5", + "\u1f51": "\u03c5", + "\u1f52": "\u03c5", + "\u1f53": "\u03c5", + "\u1f54": "\u03c5", + "\u1f55": "\u03c5", + "\u1f56": "\u03c5", + "\u1f57": "\u03c5", + "\u1f59": "\u03c5", + "\u1f5b": "\u03c5", + "\u1f5d": "\u03c5", + "\u1f5f": "\u03c5", + "\u1f60": "\u03c9", + "\u1f61": "\u03c9", + "\u1f62": "\u03c9", + "\u1f63": "\u03c9", + "\u1f64": "\u03c9", + "\u1f65": "\u03c9", + "\u1f66": "\u03c9", + "\u1f67": "\u03c9", + "\u1f68": "\u03c9", + "\u1f69": "\u03c9", + "\u1f6a": "\u03c9", + "\u1f6b": "\u03c9", + "\u1f6c": "\u03c9", + "\u1f6d": "\u03c9", + "\u1f6e": "\u03c9", + "\u1f6f": "\u03c9", + "\u1f70": "\u03b1", + "\u1f71": "\u03b1", + "\u1f72": "\u03b5", + "\u1f73": "\u03b5", + "\u1f74": "\u03b7", + "\u1f75": "\u03b7", + "\u1f76": "\u03b9", + "\u1f77": "\u03b9", + "\u1f78": "\u03bf", + "\u1f79": "\u03bf", + "\u1f7a": "\u03c5", + "\u1f7b": "\u03c5", + "\u1f7c": "\u03c9", + "\u1f7d": "\u03c9", + "\u1f80": "\u03b1", + "\u1f81": "\u03b1", + "\u1f82": "\u03b1", + "\u1f83": "\u03b1", + "\u1f84": "\u03b1", + "\u1f85": "\u03b1", + "\u1f86": "\u03b1", + "\u1f87": "\u03b1", + "\u1f88": "\u03b1", + "\u1f89": "\u03b1", + "\u1f8a": "\u03b1", + "\u1f8b": "\u03b1", + "\u1f8c": "\u03b1", + "\u1f8d": "\u03b1", + "\u1f8e": "\u03b1", + "\u1f8f": "\u03b1", + "\u1f90": "\u03b7", + "\u1f91": "\u03b7", + "\u1f92": "\u03b7", + "\u1f93": "\u03b7", + "\u1f94": "\u03b7", + "\u1f95": "\u03b7", + "\u1f96": "\u03b7", + "\u1f97": "\u03b7", + "\u1f98": "\u03b7", + "\u1f99": "\u03b7", + "\u1f9a": "\u03b7", + "\u1f9b": "\u03b7", + "\u1f9c": "\u03b7", + "\u1f9d": "\u03b7", + "\u1f9e": "\u03b7", + "\u1f9f": "\u03b7", + "\u1fa0": "\u03c9", + "\u1fa1": "\u03c9", + "\u1fa2": "\u03c9", + "\u1fa3": "\u03c9", + "\u1fa4": "\u03c9", + "\u1fa5": "\u03c9", + "\u1fa6": "\u03c9", + "\u1fa7": "\u03c9", + "\u1fa8": "\u03c9", + "\u1fa9": "\u03c9", + "\u1faa": "\u03c9", + "\u1fab": "\u03c9", + "\u1fac": "\u03c9", + "\u1fad": "\u03c9", + "\u1fae": "\u03c9", + "\u1faf": "\u03c9", + "\u1fb0": "\u03b1", + "\u1fb1": "\u03b1", + "\u1fb2": "\u03b1", + "\u1fb3": "\u03b1", + "\u1fb4": "\u03b1", + "\u1fb6": "\u03b1", + "\u1fb7": "\u03b1", + "\u1fb8": "\u03b1", + "\u1fb9": "\u03b1", + "\u1fba": "\u03b1", + "\u1fbb": "\u03b1", + "\u1fbc": "\u03b1", + "\u1fc2": "\u03b7", + "\u1fc3": "\u03b7", + "\u1fc4": "\u03b7", + "\u1fc6": "\u03b7", + "\u1fc7": "\u03b7", + "\u1fc8": "\u03b5", + "\u1fc9": "\u03b5", + "\u1fca": "\u03b7", + "\u1fcb": "\u03b7", + "\u1fcc": "\u03b7", + "\u1fd0": "\u03b9", + "\u1fd1": "\u03b9", + "\u1fd2": "\u03b9", + "\u1fd3": "\u03b9", + "\u1fd6": "\u03b9", + "\u1fd7": "\u03b9", + "\u1fd8": "\u03b9", + "\u1fd9": "\u03b9", + "\u1fda": "\u03b9", + "\u1fdb": "\u03b9", + "\u1fe0": "\u03c5", + "\u1fe1": "\u03c5", + "\u1fe2": "\u03c5", + "\u1fe3": "\u03c5", + "\u1fe4": "\u03c1", + "\u1fe5": "\u03c1", + "\u1fe6": "\u03c5", + "\u1fe7": "\u03c5", + "\u1fe8": "\u03c5", + "\u1fe9": "\u03c5", + "\u1fea": "\u03c5", + "\u1feb": "\u03c5", + "\u1fec": "\u03c1", + "\u1ff2": "\u03c9", + "\u1ff3": "\u03c9", + "\u1ff4": "\u03c9", + "\u1ff6": "\u03c9", + "\u1ff7": "\u03c9", + "\u1ff8": "\u03bf", + "\u1ff9": "\u03bf", + "\u1ffa": "\u03c9", + "\u1ffb": "\u03c9", + "\u1ffc": "\u03c9", + "\u212b": "a", + "\u2c60": "l", + "\u2c61": "l", + "\u2c62": "l", + "\u2c63": "p", + "\u2c64": "r", + "\u2c65": "a", + "\u2c66": "t", + "\u2c67": "h", + "\u2c68": "h", + "\u2c69": "k", + "\u2c6a": "k", + "\u2c6b": "z", + "\u2c6c": "z", + "\u2c6e": "m", + "\u2c71": "v", + "\u2c72": "w", + "\u2c73": "w", + "\u2c74": "v", + "\u2c78": "e", + "\u2c7a": "o", + "\u2c7e": "s", + "\u2c7f": "z", + "\u304c": "\u304b", + "\u304e": "\u304d", + "\u3050": "\u304f", + "\u3052": "\u3051", + "\u3054": "\u3053", + "\u3056": "\u3055", + "\u3058": "\u3057", + "\u305a": "\u3059", + "\u305c": "\u305b", + "\u305e": "\u305d", + "\u3060": "\u305f", + "\u3062": "\u3061", + "\u3065": "\u3064", + "\u3067": "\u3066", + "\u3069": "\u3068", + "\u3070": "\u306f", + "\u3071": "\u306f", + "\u3073": "\u3072", + "\u3074": "\u3072", + "\u3076": "\u3075", + "\u3077": "\u3075", + "\u3079": "\u3078", + "\u307a": "\u3078", + "\u307c": "\u307b", + "\u307d": "\u307b", + "\u3094": "\u3046", + "\u309e": "\u309d", + "\u30ac": "\u30ab", + "\u30ae": "\u30ad", + "\u30b0": "\u30af", + "\u30b2": "\u30b1", + "\u30b4": "\u30b3", + "\u30b6": "\u30b5", + "\u30b8": "\u30b7", + "\u30ba": "\u30b9", + "\u30bc": "\u30bb", + "\u30be": "\u30bd", + "\u30c0": "\u30bf", + "\u30c2": "\u30c1", + "\u30c5": "\u30c4", + "\u30c7": "\u30c6", + "\u30c9": "\u30c8", + "\u30d0": "\u30cf", + "\u30d1": "\u30cf", + "\u30d3": "\u30d2", + "\u30d4": "\u30d2", + "\u30d6": "\u30d5", + "\u30d7": "\u30d5", + "\u30d9": "\u30d8", + "\u30da": "\u30d8", + "\u30dc": "\u30db", + "\u30dd": "\u30db", + "\u30f4": "\u30a6", + "\u30f7": "\u30ef", + "\u30f8": "\u30f0", + "\u30f9": "\u30f1", + "\u30fa": "\u30f2", + "\u30fe": "\u30fd", + "\ua740": "k", + "\ua741": "k", + "\ua742": "k", + "\ua743": "k", + "\ua744": "k", + "\ua745": "k", + "\ua748": "l", + "\ua749": "l", + "\ua74a": "o", + "\ua74b": "o", + "\ua74c": "o", + "\ua74d": "o", + "\ua750": "p", + "\ua751": "p", + "\ua752": "p", + "\ua753": "p", + "\ua754": "p", + "\ua755": "p", + "\ua756": "q", + "\ua757": "q", + "\ua758": "q", + "\ua759": "q", + "\ua75e": "v", + "\ua75f": "v", + "\ua78e": "l", + "\ua790": "n", + "\ua791": "n", + "\ua792": "c", + "\ua793": "c", + "\ua794": "c", + "\ua795": "h", + "\ua796": "b", + "\ua797": "b", + "\ua798": "f", + "\ua799": "f", + "\ua7a0": "g", + "\ua7a1": "g", + "\ua7a2": "k", + "\ua7a3": "k", + "\ua7a4": "n", + "\ua7a5": "n", + "\ua7a6": "r", + "\ua7a7": "r", + "\ua7a8": "s", + "\ua7a9": "s", + "\ua7aa": "h", + "\ua7ad": "l", + "\ua7b2": "j", + "\ua7b8": "u", + "\ua7b9": "u", + "\ua7c4": "c", + "\ua7c5": "s", + "\ua7c6": "z", + "\ua7c7": "d", + "\ua7c8": "d", + "\ua7c9": "s", + "\ua7ca": "s", + "\uab34": "e", + "\uab37": "l", + "\uab38": "l", + "\uab39": "l", + "\uab3a": "m", + "\uab3b": "n", + "\uab49": "r", + "\uab4e": "u", + "\uab52": "u", + "\uab56": "x", + "\uab57": "x", + "\uab58": "x", + "\uab59": "x", + "\uab5a": "y", + "\ufb1d": "\u05d9", + "\ufb1f": "\u05f2", + "\ufb2a": "\u05e9", + "\ufb2b": "\u05e9", + "\ufb2c": "\u05e9", + "\ufb2d": "\u05e9", + "\ufb2e": "\u05d0", + "\ufb2f": "\u05d0", + "\ufb30": "\u05d0", + "\ufb31": "\u05d1", + "\ufb32": "\u05d2", + "\ufb33": "\u05d3", + "\ufb34": "\u05d4", + "\ufb35": "\u05d5", + "\ufb36": "\u05d6", + "\ufb38": "\u05d8", + "\ufb39": "\u05d9", + "\ufb3a": "\u05da", + "\ufb3b": "\u05db", + "\ufb3c": "\u05dc", + "\ufb3e": "\u05de", + "\ufb40": "\u05e0", + "\ufb41": "\u05e1", + "\ufb43": "\u05e3", + "\ufb44": "\u05e4", + "\ufb46": "\u05e6", + "\ufb47": "\u05e7", + "\ufb48": "\u05e8", + "\ufb49": "\u05e9", + "\ufb4a": "\u05ea", + "\ufb4b": "\u05d5", + "\ufb4c": "\u05d1", + "\ufb4d": "\u05db", + "\ufb4e": "\u05e4", + "\ud804\udc9a": "\ud804\udc99", + "\ud804\udc9c": "\ud804\udc9b", + "\ud804\udcab": "\ud804\udca5", + "\ud837\udf09": "t", + "\ud837\udf11": "l", + "\ud837\udf13": "l", + "\ud837\udf16": "r", + "\ud837\udf1a": "i", + "\ud837\udf1b": "o", + "\ud837\udf1d": "c", + "\ud837\udf1e": "s" +} diff --git a/web/src/dinners/names.ts b/web/src/dinners/names.ts index b5afe11..52cc50d 100644 --- a/web/src/dinners/names.ts +++ b/web/src/dinners/names.ts @@ -3,37 +3,17 @@ // API would refuse a name as a duplicate. The server still decides. // internal/ai/harness/testdata/dedupe_cases.json is run against both. +import folds from './letterFolds.json' + // SIMILAR_THRESHOLD is harness.SimilarThreshold. const SIMILAR_THRESHOLD = 0.85 // fillerWords is harness's fillerWords. const fillerWords = new Set(['easy', 'best', 'homemade', 'quick', 'simple', 'the', 'a', 'recipe', 'classic']) -// letterFolds are the letters harness folds that canonical decomposition -// leaves whole: its ligatures and extra letters, and the common stroked -// Latin letters. Letters with accents fold by decomposing and dropping the -// marks, as harness's table does; rarer stroked letters are compared as -// they are. -const letterFolds: Record = { - ß: 'ss', - æ: 'ae', - œ: 'oe', - þ: 'th', - ð: 'd', - ı: 'i', - ʼn: 'n', - ij: 'ij', - ø: 'o', - đ: 'd', - ħ: 'h', - ŀ: 'l', - ł: 'l', - ŧ: 't', - ƀ: 'b', - ƶ: 'z', - ǥ: 'g', - ɨ: 'i', -} +// letterFolds is harness's letterFolds: gen_folds.py writes both tables, +// and harness's TestLetterFoldsMatchPWA fails when they differ. +const letterFolds: Record = folds const combiningMark = /\p{Mn}/u const wordChar = /[\p{L}\p{Nd}]/u @@ -44,12 +24,16 @@ const wordChar = /[\p{L}\p{Nd}]/u // "chicken tikka masala". export function normalizeName(name: string): string { let out = '' - for (const ch of name.toLowerCase().normalize('NFD')) { - if (combiningMark.test(ch) || ch === "'" || ch === '’') continue - const fold = letterFolds[ch] - if (fold !== undefined) out += fold - else if (wordChar.test(ch)) out += ch - else out += ' ' + // Lowercased letter by letter, as Go's strings.ToLower does, so no + // letter is lowered by its context (a final Σ is σ, not ς). + for (const letter of name) { + for (const ch of letter.toLowerCase()) { + if (combiningMark.test(ch) || ch === "'" || ch === '’') continue + const fold = letterFolds[ch] + if (fold !== undefined) out += fold + else if (wordChar.test(ch)) out += ch + else out += ' ' + } } return out .split(' ') -- 2.55.0 From 35ceb3985b4e891368021daf718898bb994ad8a2 Mon Sep 17 00:00:00 2001 From: dev_agent Date: Tue, 6 Oct 2026 19:49:06 -0400 Subject: [PATCH 4/4] Move legacy job repair to migration 0011 and keep archived dinners pending R1-A: Migration 0010 is back to its round-2 content, the ALTER TABLE only, since a database that already applied 0010 never runs it again. The repair that writes revision 0 into ground_dinner job payloads without one is now migration 0011, applied by the same boot Migrate call, so before the job runner starts and before FailInterruptedGroundings. TestUpgradeKeepsLegacyQueuedGrounding now upgrades from 0009 and from a database with 0010 already recorded, each with a pending Regular and a queued legacy job; both keep the dinner pending through the boot cleanup and then ground it. R2-A: The boot cleanup no longer fails archived dinners. Archiving stops a running job and leaves the dinner pending, which the cleanup relabelled failed with interrupted. TestRestartKeepsArchivedPendingDinner archives a pending Regular while its job runs and checks the cleanup leaves it pending with no error. Maestro task 5037ff86-181d-448e-927b-c2fd2258a25d. Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 2 +- internal/db/dinners.sql.go | 10 ++-- internal/db/querier.go | 9 +-- internal/dinners/grounding.go | 9 +-- internal/dinners/grounding_test.go | 56 ++++++++++++++++++- internal/store/migrate_test.go | 8 +-- migrations/0010_dinner_grounding_revision.sql | 9 --- .../0011_ground_dinner_job_revision.sql | 11 ++++ migrations/queries/dinners.sql | 10 ++-- 9 files changed, 92 insertions(+), 32 deletions(-) create mode 100644 migrations/0011_ground_dinner_job_revision.sql diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index ad22749..9ee7df7 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -226,7 +226,7 @@ automatic** (D-021). method, minutes_total or difficulty a member saved while the dinner was `pending` is kept; grounding fills only those left empty. At boot, after the job runner fails the jobs a previous process left running, every - dinner still `pending` with no `ground_dinner` job for its revision + unarchived dinner still `pending` with no `ground_dinner` job for its revision queued or running is set `failed` with `interrupted`, so no dinner stays pending with nothing left to ground it. `GET /api/v1/dinners/{id}/recipe-candidates` diff --git a/internal/db/dinners.sql.go b/internal/db/dinners.sql.go index 37c723a..ff1200e 100644 --- a/internal/db/dinners.sql.go +++ b/internal/db/dinners.sql.go @@ -150,6 +150,7 @@ SET grounding_status = 'failed', grounding_error = ?1, updated_at = ?2 WHERE grounding_status = 'pending' + AND status <> 'archived' AND NOT EXISTS ( SELECT 1 FROM jobs WHERE jobs.kind = ?3 @@ -165,10 +166,11 @@ type FailOrphanedPendingDinnerGroundingsParams struct { Kind string } -// Fails every dinner still pending whose grounding job for its current -// grounding_revision is no longer queued or running: the process stopped -// before the job, or before its handler recorded a result. Run at boot, -// after the job runner fails the jobs a previous process left running. +// Fails every unarchived dinner still pending whose grounding job for its +// current grounding_revision is no longer queued or running: the process +// stopped before the job, or before its handler recorded a result. An +// archived dinner is left as it is; archiving stopped its grounding. Run at +// boot, after the job runner fails the jobs a previous process left running. func (q *Queries) FailOrphanedPendingDinnerGroundings(ctx context.Context, arg FailOrphanedPendingDinnerGroundingsParams) (int64, error) { result, err := q.db.ExecContext(ctx, failOrphanedPendingDinnerGroundings, arg.GroundingError, arg.UpdatedAt, arg.Kind) if err != nil { diff --git a/internal/db/querier.go b/internal/db/querier.go index c75f774..382e233 100644 --- a/internal/db/querier.go +++ b/internal/db/querier.go @@ -36,10 +36,11 @@ type Querier interface { DeleteRecipeIngredients(ctx context.Context, recipeID string) error DeleteRecipeSteps(ctx context.Context, recipeID string) error FailJob(ctx context.Context, arg FailJobParams) (int64, error) - // Fails every dinner still pending whose grounding job for its current - // grounding_revision is no longer queued or running: the process stopped - // before the job, or before its handler recorded a result. Run at boot, - // after the job runner fails the jobs a previous process left running. + // Fails every unarchived dinner still pending whose grounding job for its + // current grounding_revision is no longer queued or running: the process + // stopped before the job, or before its handler recorded a result. An + // archived dinner is left as it is; archiving stopped its grounding. Run at + // boot, after the job runner fails the jobs a previous process left running. FailOrphanedPendingDinnerGroundings(ctx context.Context, arg FailOrphanedPendingDinnerGroundingsParams) (int64, error) // Records why the background grounding of an unarchived dinner still // pending at the grounding_revision it was grounded for failed; a dinner diff --git a/internal/dinners/grounding.go b/internal/dinners/grounding.go index a57bb71..9fd4115 100644 --- a/internal/dinners/grounding.go +++ b/internal/dinners/grounding.go @@ -227,10 +227,11 @@ func (s *Service) stopGroundings(dinnerID string, below int64) { } } -// FailInterruptedGroundings fails, with ReasonInterrupted, every dinner -// still pending whose KindGroundDinner job is no longer queued or running: -// a previous process stopped before enqueuing the job or before its -// handler recorded a result. Call it once at boot, after the job runner's +// FailInterruptedGroundings fails, with ReasonInterrupted, every unarchived +// dinner still pending whose KindGroundDinner job is no longer queued or +// running: a previous process stopped before enqueuing the job or before +// its handler recorded a result. An archived dinner is left pending, since +// archiving, not a restart, stopped its grounding. Call it once at boot, after the job runner's // Start has failed the jobs that process left running, so no dinner stays // pending with nothing left to ground it. It returns how many it failed. func (s *Service) FailInterruptedGroundings(ctx context.Context) (int64, error) { diff --git a/internal/dinners/grounding_test.go b/internal/dinners/grounding_test.go index ba90e3e..2e14546 100644 --- a/internal/dinners/grounding_test.go +++ b/internal/dinners/grounding_test.go @@ -946,12 +946,64 @@ func TestRestartFailsInterruptedGrounding(t *testing.T) { } } +// TestRestartKeepsArchivedPendingDinner: a pending Regular archived while +// its job runs stays pending, its job succeeded; the boot cleanup leaves it +// as it is rather than failing it as interrupted. +func TestRestartKeepsArchivedPendingDinner(t *testing.T) { + t.Parallel() + e := newEnv(t) + e.search.Gate = make(chan struct{}) + c, err := e.svc.CreateRegular(t.Context(), dinners.NewRegular{Name: ptr("Chicken curry")}, member) + if err != nil { + t.Fatal(err) + } + e.waitQuery(t, "Chicken curry") + if _, err := e.svc.Archive(t.Context(), c.ID, member); err != nil { + t.Fatal(err) + } + job := e.waitJob(t, *c.GroundingJobID) + close(e.search.Gate) + if job.Status != jobs.StatusSucceeded { + t.Fatalf("job = %+v, want succeeded", job) + } + + // The database is now as the next boot finds it. + n, err := e.svc.FailInterruptedGroundings(t.Context()) + if err != nil { + t.Fatal(err) + } + if n != 0 { + t.Errorf("failed %d dinners, want 0", n) + } + if row := e.dinner(t, c.ID); row.Status != dinners.StatusArchived || + row.GroundingStatus != dinners.GroundingPending || row.GroundingError.Valid { + t.Errorf("dinner = %s, %s (%s), want archived and pending with no error", + row.Status, row.GroundingStatus, row.GroundingError.String) + } +} + // TestUpgradeKeepsLegacyQueuedGrounding: a pending Regular whose // ground_dinner job was queued before migration 0010, so its payload has // no revision, is not failed as interrupted at the first boot after the -// upgrade; its job still grounds it. +// upgrade; its job still grounds it. The upgrade starts either from 0009 +// or from a database that has already applied 0010. func TestUpgradeKeepsLegacyQueuedGrounding(t *testing.T) { t.Parallel() + for name, below := range map[string]string{ + "from 0009": "0010", + "from 0010": "0011", + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + upgradeWithLegacyQueuedGrounding(t, below) + }) + } +} + +// upgradeWithLegacyQueuedGrounding migrates a database to every migration +// named before below, adds a pending Regular with a queued ground_dinner +// job whose payload has no revision, then boots on the latest migrations. +func upgradeWithLegacyQueuedGrounding(t *testing.T, below string) { ctx := t.Context() dataDir := t.TempDir() st := openStore(t, dataDir) @@ -961,7 +1013,7 @@ func TestUpgradeKeepsLegacyQueuedGrounding(t *testing.T) { t.Fatal(err) } for _, name := range files { - if name >= "0010" { + if name >= below { continue } body, err := fs.ReadFile(migrations.FS, name) diff --git a/internal/store/migrate_test.go b/internal/store/migrate_test.go index 75b6491..94b8ec3 100644 --- a/internal/store/migrate_test.go +++ b/internal/store/migrate_test.go @@ -175,8 +175,8 @@ func TestMigrateEmbeddedMigrations(t *testing.T) { if err := Migrate(context.Background(), s.Writer, migrations.FS); err != nil { t.Fatal(err) } - if got := versions(migrationRows(t, s)); got != "0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai,0006_ai_runs_status,0007_recipe_grounding,0008_dinner_recipe_url,0009_dinner_grounding_error,0010_dinner_grounding_revision" { - t.Errorf("versions = %q, want 0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai,0006_ai_runs_status,0007_recipe_grounding,0008_dinner_recipe_url,0009_dinner_grounding_error,0010_dinner_grounding_revision", got) + if got := versions(migrationRows(t, s)); got != "0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai,0006_ai_runs_status,0007_recipe_grounding,0008_dinner_recipe_url,0009_dinner_grounding_error,0010_dinner_grounding_revision,0011_ground_dinner_job_revision" { + t.Errorf("versions = %q, want 0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai,0006_ai_runs_status,0007_recipe_grounding,0008_dinner_recipe_url,0009_dinner_grounding_error,0010_dinner_grounding_revision,0011_ground_dinner_job_revision", got) } } @@ -196,8 +196,8 @@ func TestMigrateBootstrapThenEmbeddedInitApplies(t *testing.T) { if err := Migrate(ctx, s.Writer, migrations.FS); err != nil { t.Fatal(err) } - if got := versions(migrationRows(t, s)); got != "0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai,0006_ai_runs_status,0007_recipe_grounding,0008_dinner_recipe_url,0009_dinner_grounding_error,0010_dinner_grounding_revision" { - t.Errorf("versions = %q, want 0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai,0006_ai_runs_status,0007_recipe_grounding,0008_dinner_recipe_url,0009_dinner_grounding_error,0010_dinner_grounding_revision", got) + if got := versions(migrationRows(t, s)); got != "0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai,0006_ai_runs_status,0007_recipe_grounding,0008_dinner_recipe_url,0009_dinner_grounding_error,0010_dinner_grounding_revision,0011_ground_dinner_job_revision" { + t.Errorf("versions = %q, want 0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai,0006_ai_runs_status,0007_recipe_grounding,0008_dinner_recipe_url,0009_dinner_grounding_error,0010_dinner_grounding_revision,0011_ground_dinner_job_revision", got) } if !tableExists(t, s, "dinners") { t.Error("table from 0001_init was not created") diff --git a/migrations/0010_dinner_grounding_revision.sql b/migrations/0010_dinner_grounding_revision.sql index c2d4fc1..5f13bd9 100644 --- a/migrations/0010_dinner_grounding_revision.sql +++ b/migrations/0010_dinner_grounding_revision.sql @@ -5,12 +5,3 @@ -- earlier job — queued or running — can no longer write the dinner. ALTER TABLE dinners ADD COLUMN grounding_revision INTEGER NOT NULL DEFAULT 0; - --- Every dinner starts at revision 0, so a ground_dinner job enqueued --- before this migration, whose payload has only dinner_id, is for revision --- 0. Recording it lets the boot cleanup, which matches a pending dinner to --- its job by revision, see a job still queued for its dinner. -UPDATE jobs -SET payload = json_set(payload, '$.revision', 0) -WHERE kind = 'ground_dinner' - AND json_type(payload, '$.revision') IS NULL; diff --git a/migrations/0011_ground_dinner_job_revision.sql b/migrations/0011_ground_dinner_job_revision.sql new file mode 100644 index 0000000..dd5de37 --- /dev/null +++ b/migrations/0011_ground_dinner_job_revision.sql @@ -0,0 +1,11 @@ +-- 0011_ground_dinner_job_revision: every dinner starts at grounding +-- revision 0 (0010), so a ground_dinner job enqueued before 0010, whose +-- payload has only dinner_id, is for revision 0. Recording it lets the boot +-- cleanup, which matches a pending dinner to its job by revision, see a job +-- still queued for its dinner. This is a migration of its own, not part of +-- 0010, so it also runs on a database that has already applied 0010. + +UPDATE jobs +SET payload = json_set(payload, '$.revision', 0) +WHERE kind = 'ground_dinner' + AND json_type(payload, '$.revision') IS NULL; diff --git a/migrations/queries/dinners.sql b/migrations/queries/dinners.sql index 90333a9..e11dec1 100644 --- a/migrations/queries/dinners.sql +++ b/migrations/queries/dinners.sql @@ -134,15 +134,17 @@ WHERE id = ? AND grounding_revision = ?; -- name: FailOrphanedPendingDinnerGroundings :execrows --- Fails every dinner still pending whose grounding job for its current --- grounding_revision is no longer queued or running: the process stopped --- before the job, or before its handler recorded a result. Run at boot, --- after the job runner fails the jobs a previous process left running. +-- Fails every unarchived dinner still pending whose grounding job for its +-- current grounding_revision is no longer queued or running: the process +-- stopped before the job, or before its handler recorded a result. An +-- archived dinner is left as it is; archiving stopped its grounding. Run at +-- boot, after the job runner fails the jobs a previous process left running. UPDATE dinners SET grounding_status = 'failed', grounding_error = sqlc.arg(grounding_error), updated_at = sqlc.arg(updated_at) WHERE grounding_status = 'pending' + AND status <> 'archived' AND NOT EXISTS ( SELECT 1 FROM jobs WHERE jobs.kind = sqlc.arg(kind) -- 2.55.0