From dff58d491f953606fd45983d958f456bffa90282 Mon Sep 17 00:00:00 2001 From: dev_agent Date: Tue, 6 Oct 2026 19:40:34 -0400 Subject: [PATCH] =?UTF-8?q?Fix=20review=20findings=20R1=E2=80=93R3=20on=20?= =?UTF-8?q?Regular=20grounding=20and=20name=20folding?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit R1: Migration 0010 now writes revision 0 into every ground_dinner job payload that has none. Jobs queued before the upgrade carry only dinner_id, so the boot cleanup, which matches a pending dinner to its job by revision, failed those dinners as interrupted while their jobs were still queued. The migration runs before the job runner starts. A new test migrates a database to 0009, adds a pending Regular with a legacy queued job, upgrades and boots, and checks the dinner stays pending and is then grounded. R2: Archiving a dinner, or renaming a pending one, now cancels the grounding of any ground_dinner job running for it at an older revision, so no further search, page fetch, photo download or model call is made. The job is registered before it reads the dinner, so an archive or rename committed after that read always finds it. A stopped job succeeds and reports the dinner's current grounding status. The archive and rename tests now check that the job ends while its search is still held, with no page fetched and no model call for the stale name. R3: gen_folds.py writes web/src/dinners/letterFolds.json from the same table as folds_table.go, and names.ts folds through it instead of a hand-copied subset. names.ts also lowercases letter by letter, as Go's strings.ToLower does, rather than by context. TestLetterFoldsMatchPWA fails when the two tables differ. The shared cases add "Ƈhili", "Ꞔarbanzo Stew" and "ΜΟΥΣΑΚΑΣ" (final sigma). Maestro task 5037ff86-181d-448e-927b-c2fd2258a25d. Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 9 +- internal/ai/harness/dedupe_test.go | 25 + internal/ai/harness/gen_folds.py | 11 +- .../ai/harness/testdata/dedupe_cases.json | 17 + internal/dinners/dinners.go | 16 +- internal/dinners/grounding.go | 77 +- internal/dinners/grounding_test.go | 126 +- internal/recipes/recipestest/recipestest.go | 13 + migrations/0010_dinner_grounding_revision.sql | 9 + web/src/dinners/letterFolds.json | 1167 +++++++++++++++++ web/src/dinners/names.ts | 46 +- 11 files changed, 1454 insertions(+), 62 deletions(-) create mode 100644 web/src/dinners/letterFolds.json diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index ee6f75b..ad22749 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -219,7 +219,10 @@ automatic** (D-021). A job for a dinner archived, no longer `pending` or at another revision does no work, and a job only writes a dinner still unarchived, `pending` and at its revision, so a fix-up made meanwhile wins and an earlier job - for an old name — queued or running — never decides the outcome. A + for an old name — queued or running — never decides the outcome. + Archiving or renaming a dinner also stops its running job where it is, + so no further search, page fetch, photo download or model call is made + for it. A method, minutes_total or difficulty a member saved while the dinner was `pending` is kept; grounding fills only those left empty. At boot, after the job runner fails the jobs a previous process left running, every @@ -502,7 +505,9 @@ steps: the welcome (`/onboarding`), the Regulars (`/onboarding/regulars`, mock 1h, adding and archiving through the dinners API optimistically; Next waits for three saved Regulars, and a common pick is not offered once a Regular has a name `harness.Similar` to it — the PWA's copy of the name -rules in `web/src/dinners/names.ts` runs the same cases, +rules in `web/src/dinners/names.ts` folds letters with +`web/src/dinners/letterFolds.json`, which `gen_folds.py` writes from the +same table as the server's, and runs the same cases, `internal/ai/harness/testdata/dedupe_cases.json`, as the server's) and notifications (`/onboarding/notifications`: what MealQuest notifies about, the iOS install hint, "Turn on notifications" and "Maybe later"). Either diff --git a/internal/ai/harness/dedupe_test.go b/internal/ai/harness/dedupe_test.go index 2f64739..9f7ff2b 100644 --- a/internal/ai/harness/dedupe_test.go +++ b/internal/ai/harness/dedupe_test.go @@ -5,6 +5,7 @@ import ( "os" "testing" "unicode" + "unicode/utf8" ) // TestSharedDedupeCases runs testdata/dedupe_cases.json, which the PWA's @@ -40,6 +41,30 @@ func TestSharedDedupeCases(t *testing.T) { } } +// TestLetterFoldsMatchPWA checks that the PWA's letter-fold table, which +// gen_folds.py writes beside folds_table.go, is letterFolds exactly; run +// go generate when it fails. +func TestLetterFoldsMatchPWA(t *testing.T) { + raw, err := os.ReadFile("../../../web/src/dinners/letterFolds.json") + if err != nil { + t.Fatal(err) + } + var pwa map[string]string + if err := json.Unmarshal(raw, &pwa); err != nil { + t.Fatal(err) + } + for r, want := range letterFolds { + if got, ok := pwa[string(r)]; !ok || got != want { + t.Errorf("PWA folds %U to %q (present %v), want %q", r, got, ok, want) + } + } + for k := range pwa { + if utf8.RuneCountInString(k) != 1 || letterFolds[[]rune(k)[0]] == "" { + t.Errorf("PWA folds %q, which letterFolds does not", k) + } + } +} + func TestNormalize(t *testing.T) { tests := []struct { name, want string diff --git a/internal/ai/harness/gen_folds.py b/internal/ai/harness/gen_folds.py index 6e907c4..4de6bb6 100644 --- a/internal/ai/harness/gen_folds.py +++ b/internal/ai/harness/gen_folds.py @@ -1,5 +1,8 @@ #!/usr/bin/env python3 -"""Generate folds_table.go, the letter-fold table used by harness.Normalize. +"""Generate folds_table.go, the letter-fold table used by harness.Normalize, +and its copy web/src/dinners/letterFolds.json, used by the PWA's +normalizeName (web/src/dinners/names.ts). TestLetterFoldsMatchPWA fails when +the two differ. Run from this directory with `go generate ./internal/ai/harness` (or `python3 gen_folds.py`; gofmt must be on PATH). It uses only the Python @@ -28,11 +31,14 @@ Hangul syllables are left out: their decomposition is into conjoining jamo, not combining marks. """ +import json import re import subprocess import sys import unicodedata +PWA_TABLE = "../../../web/src/dinners/letterFolds.json" + EXTRA = { "ß": "ss", "ẞ": "ss", "æ": "ae", "Æ": "ae", "œ": "oe", "Œ": "oe", "þ": "th", "Þ": "th", "ð": "d", "Ð": "d", "ı": "i", "ʼn": "n", @@ -110,6 +116,9 @@ def main(): with open("folds_table.go", "w", encoding="utf-8") as f: f.write("\n".join(out) + "\n") subprocess.run(["gofmt", "-w", "folds_table.go"], check=True) + with open(PWA_TABLE, "w", encoding="ascii") as f: + json.dump({chr(cp): folded for cp, folded in entries}, f, indent=2) + f.write("\n") if __name__ == "__main__": diff --git a/internal/ai/harness/testdata/dedupe_cases.json b/internal/ai/harness/testdata/dedupe_cases.json index 2488e8b..20c3aec 100644 --- a/internal/ai/harness/testdata/dedupe_cases.json +++ b/internal/ai/harness/testdata/dedupe_cases.json @@ -67,6 +67,18 @@ { "name": "7-layer dip", "want": "7 layer dip" + }, + { + "name": "Ƈhili", + "want": "chili" + }, + { + "name": "Ꞔarbanzo Stew", + "want": "carbanzo stew" + }, + { + "name": "ΜΟΥΣΑΚΑΣ", + "want": "μουσακασ" } ], "similar": [ @@ -114,6 +126,11 @@ "a": "The best", "b": "Easy recipe", "want": false + }, + { + "a": "Ƈhili", + "b": "Chili", + "want": true } ] } diff --git a/internal/dinners/dinners.go b/internal/dinners/dinners.go index 7fac6f9..605d675 100644 --- a/internal/dinners/dinners.go +++ b/internal/dinners/dinners.go @@ -11,9 +11,11 @@ import ( "errors" "fmt" "log/slog" + "math" "net/url" "slices" "strings" + "sync" "unicode" "unicode/utf8" @@ -130,6 +132,11 @@ type Service struct { grounder *recipes.Grounder logger *slog.Logger groundJob *jobs.Kind[GroundJob] + + // groundingsMu guards groundings: the KindGroundDinner jobs running, + // by dinner id, so archiving or renaming a dinner can stop them. + groundingsMu sync.Mutex + groundings map[string]map[*runningGrounding]struct{} } // NewService returns a service over st that grounds dinners with grounder @@ -146,7 +153,7 @@ func NewService(st *store.Store, grounder *recipes.Grounder, runner *jobs.Runner case logger == nil: return nil, errors.New("dinners: nil logger") } - s := &Service{st: st, grounder: grounder, logger: logger} + s := &Service{st: st, grounder: grounder, logger: logger, groundings: make(map[string]map[*runningGrounding]struct{})} kind, err := jobs.Register(runner, KindGroundDinner, s.runGroundJob, Reasons...) if err != nil { return nil, err @@ -296,7 +303,7 @@ func (s *Service) List(ctx context.Context, status string) ([]Dinner, error) { // clears its recipe_url, raises its grounding_revision and enqueues a fresh // KindGroundDinner job that grounds the new name: a job queued or running // for the old name or link carries the old revision, so it can no longer -// write the dinner. +// write the dinner, and one running is stopped. func (s *Service) Update(ctx context.Context, id string, p Patch, updatedBy string) (Dinner, error) { if updatedBy == "" { return Dinner{}, errors.New("dinners: update: no member to record as updated_by") @@ -368,6 +375,7 @@ func (s *Service) Update(ctx context.Context, id string, p Patch, updatedBy stri return Dinner{}, err } if regrounding { + s.stopGroundings(row.ID, row.GroundingRevision) jobID, err := s.enqueueGrounding(ctx, row.ID, row.GroundingRevision) if err != nil { return Dinner{}, err @@ -383,7 +391,8 @@ func (s *Service) Update(ctx context.Context, id string, p Patch, updatedBy stri } // Archive sets dinner id's status to archived, recording updatedBy. The row -// is kept, so its name still counts for Side quest dedupe. +// is kept, so its name still counts for Side quest dedupe. A +// KindGroundDinner job running for it is stopped. func (s *Service) Archive(ctx context.Context, id, updatedBy string) (Dinner, error) { if updatedBy == "" { return Dinner{}, errors.New("dinners: archive: no member to record as updated_by") @@ -399,6 +408,7 @@ func (s *Service) Archive(ctx context.Context, id, updatedBy string) (Dinner, er if err != nil { return Dinner{}, fmt.Errorf("dinners: archive %s: %w", id, err) } + s.stopGroundings(row.ID, math.MaxInt64) return fromRow(row), nil } diff --git a/internal/dinners/grounding.go b/internal/dinners/grounding.go index 4d43b13..a57bb71 100644 --- a/internal/dinners/grounding.go +++ b/internal/dinners/grounding.go @@ -104,8 +104,14 @@ func (r groundResult) PublicJSON() ([]byte, error) { return json.Marshal(r) } // either result is recorded only while the dinner is still unarchived, // pending and at p.Revision: a dinner grounded by hand meanwhile is left as // it is, and one renamed or archived meanwhile is left pending — for the -// job its rename enqueued, or for none. +// job its rename enqueued, or for none. Archiving or renaming the dinner +// stops the grounding where it is, so no further search, fetch, photo +// download or model call is made for it. func (s *Service) runGroundJob(ctx context.Context, p GroundJob, _ func(groundResult) error) (groundResult, error) { + // Tracked before the dinner is read: an archive or rename committed + // after the read finds this job to stop. + groundCtx, done := s.trackGrounding(ctx, p.DinnerID, p.Revision) + defer done() row, err := db.New(s.st.Reader).GetDinner(ctx, p.DinnerID) if err != nil { return groundResult{}, fmt.Errorf("dinners: ground: read %s: %w", p.DinnerID, err) @@ -116,9 +122,12 @@ func (s *Service) runGroundJob(ctx context.Context, p GroundJob, _ func(groundRe var gr recipes.Grounded if row.RecipeUrl.Valid { - gr, err = s.grounder.GroundURL(ctx, row.RecipeUrl.String) + gr, err = s.grounder.GroundURL(groundCtx, row.RecipeUrl.String) } else { - gr, err = s.grounder.Ground(ctx, row.Name, row.Name) + gr, err = s.grounder.Ground(groundCtx, row.Name, row.Name) + } + if err != nil && ctx.Err() == nil && errors.Is(context.Cause(groundCtx), errGroundingStopped) { + return s.currentGrounding(ctx, row.ID) } if err != nil { reason := reasonFor(err, row.RecipeUrl.Valid) @@ -157,15 +166,67 @@ func (s *Service) runGroundJob(ctx context.Context, p GroundJob, _ func(groundRe return groundResult{}, fmt.Errorf("dinners: ground: record %s: %w", row.ID, err) } if n == 0 { - cur, err := db.New(s.st.Reader).GetDinner(ctx, row.ID) - if err != nil { - return groundResult{}, fmt.Errorf("dinners: ground: read %s: %w", row.ID, err) - } - return groundResult{DinnerID: cur.ID, GroundingStatus: cur.GroundingStatus}, nil + return s.currentGrounding(ctx, row.ID) } return groundResult{DinnerID: row.ID, GroundingStatus: GroundingGrounded}, nil } +// currentGrounding is the KindGroundDinner result of a job that did not +// ground dinner id: the grounding status the dinner has now. +func (s *Service) currentGrounding(ctx context.Context, id string) (groundResult, error) { + cur, err := db.New(s.st.Reader).GetDinner(ctx, id) + if err != nil { + return groundResult{}, fmt.Errorf("dinners: ground: read %s: %w", id, err) + } + return groundResult{DinnerID: cur.ID, GroundingStatus: cur.GroundingStatus}, nil +} + +// errGroundingStopped is the cause a running KindGroundDinner job's +// grounding is cancelled with when its dinner is archived or renamed. +var errGroundingStopped = errors.New("dinners: grounding stopped: dinner archived or renamed") + +// runningGrounding is a KindGroundDinner job running for a dinner at +// revision, with the cancel of its grounding. +type runningGrounding struct { + revision int64 + cancel context.CancelCauseFunc +} + +// trackGrounding returns a context for grounding dinnerID at revision that +// stopGroundings cancels, and the func to call when the job ends. +func (s *Service) trackGrounding(ctx context.Context, dinnerID string, revision int64) (context.Context, func()) { + groundCtx, cancel := context.WithCancelCause(ctx) + r := &runningGrounding{revision: revision, cancel: cancel} + s.groundingsMu.Lock() + if s.groundings[dinnerID] == nil { + s.groundings[dinnerID] = make(map[*runningGrounding]struct{}) + } + s.groundings[dinnerID][r] = struct{}{} + s.groundingsMu.Unlock() + return groundCtx, func() { + s.groundingsMu.Lock() + delete(s.groundings[dinnerID], r) + if len(s.groundings[dinnerID]) == 0 { + delete(s.groundings, dinnerID) + } + s.groundingsMu.Unlock() + cancel(nil) + } +} + +// stopGroundings cancels, with errGroundingStopped, the grounding of every +// KindGroundDinner job running for dinnerID at a revision below below. Call +// it once the change that makes those jobs stale is committed. +func (s *Service) stopGroundings(dinnerID string, below int64) { + s.groundingsMu.Lock() + defer s.groundingsMu.Unlock() + for r := range s.groundings[dinnerID] { + if r.revision < below { + r.cancel(errGroundingStopped) + } + } +} + // FailInterruptedGroundings fails, with ReasonInterrupted, every dinner // still pending whose KindGroundDinner job is no longer queued or running: // a previous process stopped before enqueuing the job or before its diff --git a/internal/dinners/grounding_test.go b/internal/dinners/grounding_test.go index 47f0f31..ba90e3e 100644 --- a/internal/dinners/grounding_test.go +++ b/internal/dinners/grounding_test.go @@ -6,9 +6,11 @@ import ( "encoding/json" "errors" "io" + "io/fs" "log/slog" "slices" "testing" + "testing/fstest" "time" "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm" @@ -54,6 +56,13 @@ func newEnv(t *testing.T) *env { func newUnstartedEnv(t *testing.T) *env { t.Helper() dataDir := t.TempDir() + return newUnstartedEnvOn(t, openStore(t, dataDir), dataDir) +} + +// openStore opens the store in dataDir, closing it when the test ends. It +// is not migrated. +func openStore(t *testing.T, dataDir string) *store.Store { + t.Helper() st, err := store.Open(t.Context(), dataDir) if err != nil { t.Fatal(err) @@ -63,6 +72,13 @@ func newUnstartedEnv(t *testing.T) *env { t.Error(err) } }) + return st +} + +// newUnstartedEnvOn is newUnstartedEnv over st, the store in dataDir, +// which it migrates to the latest version. +func newUnstartedEnvOn(t *testing.T, st *store.Store, dataDir string) *env { + t.Helper() if err := store.Migrate(t.Context(), st.Writer, migrations.FS); err != nil { t.Fatal(err) } @@ -290,19 +306,16 @@ func TestManualFixWhilePendingIsKept(t *testing.T) { // TestRenameWhilePendingGroundsNewName: a Regular renamed while its job is // still grounding the old name never gets the old name's result, whether -// that job succeeds or fails; the job the rename enqueues grounds the new -// name. +// the old name would be found or not: the rename stops that job where it +// is, and the job the rename enqueues grounds the new name. func TestRenameWhilePendingGroundsNewName(t *testing.T) { t.Parallel() tests := []struct { name string oldName string - replies []llm.GenerateResult }{ - {"old name found", "Chicken curry", []llm.GenerateResult{ - recipestest.NormalizedReply("stovetop", 40), recipestest.NormalizedReply("oven", 150), - }}, - {"old name not found", "Unknown dish", []llm.GenerateResult{recipestest.NormalizedReply("oven", 150)}}, + {"old name found", "Chicken curry"}, + {"old name not found", "Unknown dish"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -310,9 +323,7 @@ func TestRenameWhilePendingGroundsNewName(t *testing.T) { e := newEnv(t) e.search.Results["Beef stew"] = []recipes.Candidate{{URL: stew}} e.search.Gate = make(chan struct{}) - for _, r := range tt.replies { - e.model.QueueGenerate(r) - } + e.model.QueueGenerate(recipestest.NormalizedReply("oven", 150)) c, err := e.svc.CreateRegular(t.Context(), dinners.NewRegular{Name: ptr(tt.oldName)}, member) if err != nil { @@ -327,12 +338,13 @@ func TestRenameWhilePendingGroundsNewName(t *testing.T) { if d.GroundingStatus != dinners.GroundingPending { t.Fatalf("renamed = %+v, want still pending", d) } + // The old name's job ends while its search is still held. + first := e.waitJob(t, *c.GroundingJobID) + if first.Status != jobs.StatusSucceeded || string(first.Result) != `{"dinner_id":"`+c.ID+`","grounding_status":"pending"}` { + t.Fatalf("old name's job = %+v (result %s), want succeeded leaving the dinner pending", first, first.Result) + } close(e.search.Gate) - first := e.waitJob(t, *c.GroundingJobID) - if string(first.Result) == `{"dinner_id":"`+c.ID+`","grounding_status":"grounded"}` { - t.Fatalf("old name's job reported grounded: %+v (result %s)", first, first.Result) - } row := e.waitGrounding(t, c.ID) if row.GroundingStatus != dinners.GroundingGrounded { t.Fatalf("dinner = %+v, want grounded for the new name", row) @@ -347,6 +359,9 @@ func TestRenameWhilePendingGroundsNewName(t *testing.T) { if got := e.search.Queries(); !slices.Equal(got, []string{tt.oldName, "Beef stew"}) { t.Errorf("searched %q, want the old name then the new", got) } + if got := len(e.model.Requests()); got != 1 { + t.Errorf("model called %d times, want once, for the new name only", got) + } }) } } @@ -503,7 +518,8 @@ func TestEditWhilePendingIsKept(t *testing.T) { // TestArchiveWhilePendingStopsGrounding: a pending Regular archived before // its job starts is not searched for, and one archived while its job runs -// is not written; neither ends up grounded. +// is stopped there, with no page fetched or model called; neither ends up +// grounded. func TestArchiveWhilePendingStopsGrounding(t *testing.T) { t.Parallel() t.Run("before the job starts", func(t *testing.T) { @@ -541,9 +557,17 @@ func TestArchiveWhilePendingStopsGrounding(t *testing.T) { if _, err := e.svc.Archive(t.Context(), c.ID, member); err != nil { t.Fatal(err) } + // The job ends while its search is still held: archiving stopped it. + job := e.waitJob(t, *c.GroundingJobID) close(e.search.Gate) - if job := e.waitJob(t, *c.GroundingJobID); job.Status != jobs.StatusSucceeded { - t.Fatalf("job = %+v", job) + if job.Status != jobs.StatusSucceeded || string(job.Result) != `{"dinner_id":"`+c.ID+`","grounding_status":"pending"}` { + t.Fatalf("job = %+v (result %s), want succeeded leaving the dinner pending", job, job.Result) + } + if got := e.pages.Fetched(); len(got) != 0 { + t.Errorf("fetched %q for an archived dinner", got) + } + if got := len(e.model.Requests()); got != 0 { + t.Errorf("model called %d times for an archived dinner", got) } if row := e.dinner(t, c.ID); row.Status != dinners.StatusArchived || row.GroundingStatus == dinners.GroundingGrounded || row.RecipeID.Valid { t.Errorf("dinner = %+v, want archived and not grounded", row) @@ -922,6 +946,74 @@ func TestRestartFailsInterruptedGrounding(t *testing.T) { } } +// TestUpgradeKeepsLegacyQueuedGrounding: a pending Regular whose +// ground_dinner job was queued before migration 0010, so its payload has +// no revision, is not failed as interrupted at the first boot after the +// upgrade; its job still grounds it. +func TestUpgradeKeepsLegacyQueuedGrounding(t *testing.T) { + t.Parallel() + ctx := t.Context() + dataDir := t.TempDir() + st := openStore(t, dataDir) + before := fstest.MapFS{} + files, err := fs.Glob(migrations.FS, "*.sql") + if err != nil { + t.Fatal(err) + } + for _, name := range files { + if name >= "0010" { + continue + } + body, err := fs.ReadFile(migrations.FS, name) + if err != nil { + t.Fatal(err) + } + before[name] = &fstest.MapFile{Data: body} + } + if err := store.Migrate(ctx, st.Writer, before); err != nil { + t.Fatal(err) + } + now := store.Now().String() + if _, err := st.Writer.ExecContext(ctx, `INSERT INTO dinners + (id, name, normalized_name, status, origin, is_favorite, grounding_status, updated_by, created_at, updated_at) + VALUES ('dinner-1', 'Chicken curry', 'chicken curry', 'regular', 'regular', 0, 'pending', ?, ?, ?)`, + member, now, now); err != nil { + t.Fatal(err) + } + if _, err := st.Writer.ExecContext(ctx, `INSERT INTO jobs (id, kind, status, payload, created_at, updated_at) + VALUES ('job-1', ?, 'queued', '{"dinner_id":"dinner-1"}', ?, ?)`, + dinners.KindGroundDinner, now, now); err != nil { + t.Fatal(err) + } + + // Boot: migrate, start the runner, then fail interrupted groundings. The + // search is held, so the job cannot finish before the cleanup runs. + e := newUnstartedEnvOn(t, st, dataDir) + e.search.Gate = make(chan struct{}) + e.model.QueueGenerate(recipestest.NormalizedReply("stovetop", 40)) + if p := e.queuedJob(t, "job-1"); p != (dinners.GroundJob{DinnerID: "dinner-1", Revision: 0}) { + t.Errorf("migrated payload = %+v, want dinner-1 at revision 0", p) + } + e.start(t) + n, err := e.svc.FailInterruptedGroundings(ctx) + if err != nil { + t.Fatal(err) + } + if n != 0 { + t.Errorf("failed %d dinners, want 0", n) + } + if row := e.dinner(t, "dinner-1"); row.GroundingStatus != dinners.GroundingPending { + t.Fatalf("dinner = %s (%s), want still pending", row.GroundingStatus, row.GroundingError.String) + } + close(e.search.Gate) + if job := e.waitJob(t, "job-1"); job.Status != jobs.StatusSucceeded { + t.Fatalf("job = %+v", job) + } + if row := e.dinner(t, "dinner-1"); row.GroundingStatus != dinners.GroundingGrounded { + t.Errorf("dinner = %s, want grounded", row.GroundingStatus) + } +} + // TestCreateEnqueueFailureReturnsFailedDinner: a dinner saved when its job // cannot be enqueued is returned, with its id, failed and no job id. func TestCreateEnqueueFailureReturnsFailedDinner(t *testing.T) { diff --git a/internal/recipes/recipestest/recipestest.go b/internal/recipes/recipestest/recipestest.go index 46aa72a..9d22dfa 100644 --- a/internal/recipes/recipestest/recipestest.go +++ b/internal/recipes/recipestest/recipestest.go @@ -66,9 +66,15 @@ func (s *Search) Queries() []string { type Pages struct { Bodies map[string][]byte Gate chan struct{} + + mu sync.Mutex + fetched []string } func (p *Pages) Fetch(ctx context.Context, rawURL string) (recipes.Page, error) { + p.mu.Lock() + p.fetched = append(p.fetched, rawURL) + p.mu.Unlock() if p.Gate != nil { select { case <-p.Gate: @@ -83,6 +89,13 @@ func (p *Pages) Fetch(ctx context.Context, rawURL string) (recipes.Page, error) return recipes.Page{URL: rawURL, ContentType: "text/html", Body: body}, nil } +// Fetched returns every URL fetched, in order. +func (p *Pages) Fetched() []string { + p.mu.Lock() + defer p.mu.Unlock() + return append([]string(nil), p.fetched...) +} + // Images is a recipes.ImageFetcher serving JPEGs by URL; any other URL is // a 404. type Images struct { diff --git a/migrations/0010_dinner_grounding_revision.sql b/migrations/0010_dinner_grounding_revision.sql index 5f13bd9..c2d4fc1 100644 --- a/migrations/0010_dinner_grounding_revision.sql +++ b/migrations/0010_dinner_grounding_revision.sql @@ -5,3 +5,12 @@ -- earlier job — queued or running — can no longer write the dinner. ALTER TABLE dinners ADD COLUMN grounding_revision INTEGER NOT NULL DEFAULT 0; + +-- Every dinner starts at revision 0, so a ground_dinner job enqueued +-- before this migration, whose payload has only dinner_id, is for revision +-- 0. Recording it lets the boot cleanup, which matches a pending dinner to +-- its job by revision, see a job still queued for its dinner. +UPDATE jobs +SET payload = json_set(payload, '$.revision', 0) +WHERE kind = 'ground_dinner' + AND json_type(payload, '$.revision') IS NULL; diff --git a/web/src/dinners/letterFolds.json b/web/src/dinners/letterFolds.json new file mode 100644 index 0000000..5be5eaf --- /dev/null +++ b/web/src/dinners/letterFolds.json @@ -0,0 +1,1167 @@ +{ + "\u00c0": "a", + "\u00c1": "a", + "\u00c2": "a", + "\u00c3": "a", + "\u00c4": "a", + "\u00c5": "a", + "\u00c6": "ae", + "\u00c7": "c", + "\u00c8": "e", + "\u00c9": "e", + "\u00ca": "e", + "\u00cb": "e", + "\u00cc": "i", + "\u00cd": "i", + "\u00ce": "i", + "\u00cf": "i", + "\u00d0": "d", + "\u00d1": "n", + "\u00d2": "o", + "\u00d3": "o", + "\u00d4": "o", + "\u00d5": "o", + "\u00d6": "o", + "\u00d8": "o", + "\u00d9": "u", + "\u00da": "u", + "\u00db": "u", + "\u00dc": "u", + "\u00dd": "y", + "\u00de": "th", + "\u00df": "ss", + "\u00e0": "a", + "\u00e1": "a", + "\u00e2": "a", + "\u00e3": "a", + "\u00e4": "a", + "\u00e5": "a", + "\u00e6": "ae", + "\u00e7": "c", + "\u00e8": "e", + "\u00e9": "e", + "\u00ea": "e", + "\u00eb": "e", + "\u00ec": "i", + "\u00ed": "i", + "\u00ee": "i", + "\u00ef": "i", + "\u00f0": "d", + "\u00f1": "n", + "\u00f2": "o", + "\u00f3": "o", + "\u00f4": "o", + "\u00f5": "o", + "\u00f6": "o", + "\u00f8": "o", + "\u00f9": "u", + "\u00fa": "u", + "\u00fb": "u", + "\u00fc": "u", + "\u00fd": "y", + "\u00fe": "th", + "\u00ff": "y", + "\u0100": "a", + "\u0101": "a", + "\u0102": "a", + "\u0103": "a", + "\u0104": "a", + "\u0105": "a", + "\u0106": "c", + "\u0107": "c", + "\u0108": "c", + "\u0109": "c", + "\u010a": "c", + "\u010b": "c", + "\u010c": "c", + "\u010d": "c", + "\u010e": "d", + "\u010f": "d", + "\u0110": "d", + "\u0111": "d", + "\u0112": "e", + "\u0113": "e", + "\u0114": "e", + "\u0115": "e", + "\u0116": "e", + "\u0117": "e", + "\u0118": "e", + "\u0119": "e", + "\u011a": "e", + "\u011b": "e", + "\u011c": "g", + "\u011d": "g", + "\u011e": "g", + "\u011f": "g", + "\u0120": "g", + "\u0121": "g", + "\u0122": "g", + "\u0123": "g", + "\u0124": "h", + "\u0125": "h", + "\u0126": "h", + "\u0127": "h", + "\u0128": "i", + "\u0129": "i", + "\u012a": "i", + "\u012b": "i", + "\u012c": "i", + "\u012d": "i", + "\u012e": "i", + "\u012f": "i", + "\u0130": "i", + "\u0131": "i", + "\u0132": "ij", + "\u0133": "ij", + "\u0134": "j", + "\u0135": "j", + "\u0136": "k", + "\u0137": "k", + "\u0139": "l", + "\u013a": "l", + "\u013b": "l", + "\u013c": "l", + "\u013d": "l", + "\u013e": "l", + "\u013f": "l", + "\u0140": "l", + "\u0141": "l", + "\u0142": "l", + "\u0143": "n", + "\u0144": "n", + "\u0145": "n", + "\u0146": "n", + "\u0147": "n", + "\u0148": "n", + "\u0149": "n", + "\u014c": "o", + "\u014d": "o", + "\u014e": "o", + "\u014f": "o", + "\u0150": "o", + "\u0151": "o", + "\u0152": "oe", + "\u0153": "oe", + "\u0154": "r", + "\u0155": "r", + "\u0156": "r", + "\u0157": "r", + "\u0158": "r", + "\u0159": "r", + "\u015a": "s", + "\u015b": "s", + "\u015c": "s", + "\u015d": "s", + "\u015e": "s", + "\u015f": "s", + "\u0160": "s", + "\u0161": "s", + "\u0162": "t", + "\u0163": "t", + "\u0164": "t", + "\u0165": "t", + "\u0166": "t", + "\u0167": "t", + "\u0168": "u", + "\u0169": "u", + "\u016a": "u", + "\u016b": "u", + "\u016c": "u", + "\u016d": "u", + "\u016e": "u", + "\u016f": "u", + "\u0170": "u", + "\u0171": "u", + "\u0172": "u", + "\u0173": "u", + "\u0174": "w", + "\u0175": "w", + "\u0176": "y", + "\u0177": "y", + "\u0178": "y", + "\u0179": "z", + "\u017a": "z", + "\u017b": "z", + "\u017c": "z", + "\u017d": "z", + "\u017e": "z", + "\u0180": "b", + "\u0181": "b", + "\u0182": "b", + "\u0183": "b", + "\u0187": "c", + "\u0188": "c", + "\u018a": "d", + "\u018b": "d", + "\u018c": "d", + "\u0191": "f", + "\u0192": "f", + "\u0193": "g", + "\u0197": "i", + "\u0198": "k", + "\u0199": "k", + "\u019a": "l", + "\u019d": "n", + "\u019e": "n", + "\u019f": "o", + "\u01a0": "o", + "\u01a1": "o", + "\u01a4": "p", + "\u01a5": "p", + "\u01ab": "t", + "\u01ac": "t", + "\u01ad": "t", + "\u01ae": "t", + "\u01af": "u", + "\u01b0": "u", + "\u01b2": "v", + "\u01b3": "y", + "\u01b4": "y", + "\u01b5": "z", + "\u01b6": "z", + "\u01c5": "d", + "\u01c8": "l", + "\u01cb": "n", + "\u01cd": "a", + "\u01ce": "a", + "\u01cf": "i", + "\u01d0": "i", + "\u01d1": "o", + "\u01d2": "o", + "\u01d3": "u", + "\u01d4": "u", + "\u01d5": "u", + "\u01d6": "u", + "\u01d7": "u", + "\u01d8": "u", + "\u01d9": "u", + "\u01da": "u", + "\u01db": "u", + "\u01dc": "u", + "\u01de": "a", + "\u01df": "a", + "\u01e0": "a", + "\u01e1": "a", + "\u01e2": "ae", + "\u01e3": "ae", + "\u01e4": "g", + "\u01e5": "g", + "\u01e6": "g", + "\u01e7": "g", + "\u01e8": "k", + "\u01e9": "k", + "\u01ea": "o", + "\u01eb": "o", + "\u01ec": "o", + "\u01ed": "o", + "\u01ee": "\u0292", + "\u01ef": "\u0292", + "\u01f0": "j", + "\u01f2": "d", + "\u01f4": "g", + "\u01f5": "g", + "\u01f8": "n", + "\u01f9": "n", + "\u01fa": "a", + "\u01fb": "a", + "\u01fc": "ae", + "\u01fd": "ae", + "\u01fe": "o", + "\u01ff": "o", + "\u0200": "a", + "\u0201": "a", + "\u0202": "a", + "\u0203": "a", + "\u0204": "e", + "\u0205": "e", + "\u0206": "e", + "\u0207": "e", + "\u0208": "i", + "\u0209": "i", + "\u020a": "i", + "\u020b": "i", + "\u020c": "o", + "\u020d": "o", + "\u020e": "o", + "\u020f": "o", + "\u0210": "r", + "\u0211": "r", + "\u0212": "r", + "\u0213": "r", + "\u0214": "u", + "\u0215": "u", + "\u0216": "u", + "\u0217": "u", + "\u0218": "s", + "\u0219": "s", + "\u021a": "t", + "\u021b": "t", + "\u021e": "h", + "\u021f": "h", + "\u0220": "n", + "\u0221": "d", + "\u0224": "z", + "\u0225": "z", + "\u0226": "a", + "\u0227": "a", + "\u0228": "e", + "\u0229": "e", + "\u022a": "o", + "\u022b": "o", + "\u022c": "o", + "\u022d": "o", + "\u022e": "o", + "\u022f": "o", + "\u0230": "o", + "\u0231": "o", + "\u0232": "y", + "\u0233": "y", + "\u0234": "l", + "\u0235": "n", + "\u0236": "t", + "\u023a": "a", + "\u023b": "c", + "\u023c": "c", + "\u023d": "l", + "\u023e": "t", + "\u023f": "s", + "\u0240": "z", + "\u0243": "b", + "\u0246": "e", + "\u0247": "e", + "\u0248": "j", + "\u0249": "j", + "\u024b": "q", + "\u024c": "r", + "\u024d": "r", + "\u024e": "y", + "\u024f": "y", + "\u0253": "b", + "\u0255": "c", + "\u0256": "d", + "\u0257": "d", + "\u0260": "g", + "\u0266": "h", + "\u0268": "i", + "\u026b": "l", + "\u026c": "l", + "\u026d": "l", + "\u0271": "m", + "\u0272": "n", + "\u0273": "n", + "\u027c": "r", + "\u027d": "r", + "\u027e": "r", + "\u0282": "s", + "\u0288": "t", + "\u028b": "v", + "\u0290": "z", + "\u0291": "z", + "\u029d": "j", + "\u02a0": "q", + "\u0386": "\u03b1", + "\u0388": "\u03b5", + "\u0389": "\u03b7", + "\u038a": "\u03b9", + "\u038c": "\u03bf", + "\u038e": "\u03c5", + "\u038f": "\u03c9", + "\u0390": "\u03b9", + "\u03aa": "\u03b9", + "\u03ab": "\u03c5", + "\u03ac": "\u03b1", + "\u03ad": "\u03b5", + "\u03ae": "\u03b7", + "\u03af": "\u03b9", + "\u03b0": "\u03c5", + "\u03ca": "\u03b9", + "\u03cb": "\u03c5", + "\u03cc": "\u03bf", + "\u03cd": "\u03c5", + "\u03ce": "\u03c9", + "\u03d3": "\u03d2", + "\u03d4": "\u03d2", + "\u0400": "\u0435", + "\u0401": "\u0435", + "\u0403": "\u0433", + "\u0407": "\u0456", + "\u040c": "\u043a", + "\u040d": "\u0438", + "\u040e": "\u0443", + "\u0419": "\u0438", + "\u0439": "\u0438", + "\u0450": "\u0435", + "\u0451": "\u0435", + "\u0453": "\u0433", + "\u0457": "\u0456", + "\u045c": "\u043a", + "\u045d": "\u0438", + "\u045e": "\u0443", + "\u0476": "\u0475", + "\u0477": "\u0475", + "\u04c1": "\u0436", + "\u04c2": "\u0436", + "\u04d0": "\u0430", + "\u04d1": "\u0430", + "\u04d2": "\u0430", + "\u04d3": "\u0430", + "\u04d6": "\u0435", + "\u04d7": "\u0435", + "\u04da": "\u04d9", + "\u04db": "\u04d9", + "\u04dc": "\u0436", + "\u04dd": "\u0436", + "\u04de": "\u0437", + "\u04df": "\u0437", + "\u04e2": "\u0438", + "\u04e3": "\u0438", + "\u04e4": "\u0438", + "\u04e5": "\u0438", + "\u04e6": "\u043e", + "\u04e7": "\u043e", + "\u04ea": "\u04e9", + "\u04eb": "\u04e9", + "\u04ec": "\u044d", + "\u04ed": "\u044d", + "\u04ee": "\u0443", + "\u04ef": "\u0443", + "\u04f0": "\u0443", + "\u04f1": "\u0443", + "\u04f2": "\u0443", + "\u04f3": "\u0443", + "\u04f4": "\u0447", + "\u04f5": "\u0447", + "\u04f8": "\u044b", + "\u04f9": "\u044b", + "\u0622": "\u0627", + "\u0623": "\u0627", + "\u0624": "\u0648", + "\u0625": "\u0627", + "\u0626": "\u064a", + "\u06c0": "\u06d5", + "\u06c2": "\u06c1", + "\u06d3": "\u06d2", + "\u0929": "\u0928", + "\u0931": "\u0930", + "\u0934": "\u0933", + "\u0958": "\u0915", + "\u0959": "\u0916", + "\u095a": "\u0917", + "\u095b": "\u091c", + "\u095c": "\u0921", + "\u095d": "\u0922", + "\u095e": "\u092b", + "\u095f": "\u092f", + "\u09dc": "\u09a1", + "\u09dd": "\u09a2", + "\u09df": "\u09af", + "\u0a33": "\u0a32", + "\u0a36": "\u0a38", + "\u0a59": "\u0a16", + "\u0a5a": "\u0a17", + "\u0a5b": "\u0a1c", + "\u0a5e": "\u0a2b", + "\u0b5c": "\u0b21", + "\u0b5d": "\u0b22", + "\u0f43": "\u0f42", + "\u0f4d": "\u0f4c", + "\u0f52": "\u0f51", + "\u0f57": "\u0f56", + "\u0f5c": "\u0f5b", + "\u0f69": "\u0f40", + "\u1026": "\u1025", + "\u1d6c": "b", + "\u1d6d": "d", + "\u1d6e": "f", + "\u1d6f": "m", + "\u1d70": "n", + "\u1d71": "p", + "\u1d72": "r", + "\u1d73": "r", + "\u1d74": "s", + "\u1d75": "t", + "\u1d76": "z", + "\u1d7d": "p", + "\u1d80": "b", + "\u1d81": "d", + "\u1d82": "f", + "\u1d83": "g", + "\u1d84": "k", + "\u1d85": "l", + "\u1d86": "m", + "\u1d87": "n", + "\u1d88": "p", + "\u1d89": "r", + "\u1d8a": "s", + "\u1d8c": "v", + "\u1d8d": "x", + "\u1d8e": "z", + "\u1d8f": "a", + "\u1d91": "d", + "\u1d92": "e", + "\u1d96": "i", + "\u1d99": "u", + "\u1e00": "a", + "\u1e01": "a", + "\u1e02": "b", + "\u1e03": "b", + "\u1e04": "b", + "\u1e05": "b", + "\u1e06": "b", + "\u1e07": "b", + "\u1e08": "c", + "\u1e09": "c", + "\u1e0a": "d", + "\u1e0b": "d", + "\u1e0c": "d", + "\u1e0d": "d", + "\u1e0e": "d", + "\u1e0f": "d", + "\u1e10": "d", + "\u1e11": "d", + "\u1e12": "d", + "\u1e13": "d", + "\u1e14": "e", + "\u1e15": "e", + "\u1e16": "e", + "\u1e17": "e", + "\u1e18": "e", + "\u1e19": "e", + "\u1e1a": "e", + "\u1e1b": "e", + "\u1e1c": "e", + "\u1e1d": "e", + "\u1e1e": "f", + "\u1e1f": "f", + "\u1e20": "g", + "\u1e21": "g", + "\u1e22": "h", + "\u1e23": "h", + "\u1e24": "h", + "\u1e25": "h", + "\u1e26": "h", + "\u1e27": "h", + "\u1e28": "h", + "\u1e29": "h", + "\u1e2a": "h", + "\u1e2b": "h", + "\u1e2c": "i", + "\u1e2d": "i", + "\u1e2e": "i", + "\u1e2f": "i", + "\u1e30": "k", + "\u1e31": "k", + "\u1e32": "k", + "\u1e33": "k", + "\u1e34": "k", + "\u1e35": "k", + "\u1e36": "l", + "\u1e37": "l", + "\u1e38": "l", + "\u1e39": "l", + "\u1e3a": "l", + "\u1e3b": "l", + "\u1e3c": "l", + "\u1e3d": "l", + "\u1e3e": "m", + "\u1e3f": "m", + "\u1e40": "m", + "\u1e41": "m", + "\u1e42": "m", + "\u1e43": "m", + "\u1e44": "n", + "\u1e45": "n", + "\u1e46": "n", + "\u1e47": "n", + "\u1e48": "n", + "\u1e49": "n", + "\u1e4a": "n", + "\u1e4b": "n", + "\u1e4c": "o", + "\u1e4d": "o", + "\u1e4e": "o", + "\u1e4f": "o", + "\u1e50": "o", + "\u1e51": "o", + "\u1e52": "o", + "\u1e53": "o", + "\u1e54": "p", + "\u1e55": "p", + "\u1e56": "p", + "\u1e57": "p", + "\u1e58": "r", + "\u1e59": "r", + "\u1e5a": "r", + "\u1e5b": "r", + "\u1e5c": "r", + "\u1e5d": "r", + "\u1e5e": "r", + "\u1e5f": "r", + "\u1e60": "s", + "\u1e61": "s", + "\u1e62": "s", + "\u1e63": "s", + "\u1e64": "s", + "\u1e65": "s", + "\u1e66": "s", + "\u1e67": "s", + "\u1e68": "s", + "\u1e69": "s", + "\u1e6a": "t", + "\u1e6b": "t", + "\u1e6c": "t", + "\u1e6d": "t", + "\u1e6e": "t", + "\u1e6f": "t", + "\u1e70": "t", + "\u1e71": "t", + "\u1e72": "u", + "\u1e73": "u", + "\u1e74": "u", + "\u1e75": "u", + "\u1e76": "u", + "\u1e77": "u", + "\u1e78": "u", + "\u1e79": "u", + "\u1e7a": "u", + "\u1e7b": "u", + "\u1e7c": "v", + "\u1e7d": "v", + "\u1e7e": "v", + "\u1e7f": "v", + "\u1e80": "w", + "\u1e81": "w", + "\u1e82": "w", + "\u1e83": "w", + "\u1e84": "w", + "\u1e85": "w", + "\u1e86": "w", + "\u1e87": "w", + "\u1e88": "w", + "\u1e89": "w", + "\u1e8a": "x", + "\u1e8b": "x", + "\u1e8c": "x", + "\u1e8d": "x", + "\u1e8e": "y", + "\u1e8f": "y", + "\u1e90": "z", + "\u1e91": "z", + "\u1e92": "z", + "\u1e93": "z", + "\u1e94": "z", + "\u1e95": "z", + "\u1e96": "h", + "\u1e97": "t", + "\u1e98": "w", + "\u1e99": "y", + "\u1e9a": "a", + "\u1e9b": "\u017f", + "\u1e9e": "ss", + "\u1ea0": "a", + "\u1ea1": "a", + "\u1ea2": "a", + "\u1ea3": "a", + "\u1ea4": "a", + "\u1ea5": "a", + "\u1ea6": "a", + "\u1ea7": "a", + "\u1ea8": "a", + "\u1ea9": "a", + "\u1eaa": "a", + "\u1eab": "a", + "\u1eac": "a", + "\u1ead": "a", + "\u1eae": "a", + "\u1eaf": "a", + "\u1eb0": "a", + "\u1eb1": "a", + "\u1eb2": "a", + "\u1eb3": "a", + "\u1eb4": "a", + "\u1eb5": "a", + "\u1eb6": "a", + "\u1eb7": "a", + "\u1eb8": "e", + "\u1eb9": "e", + "\u1eba": "e", + "\u1ebb": "e", + "\u1ebc": "e", + "\u1ebd": "e", + "\u1ebe": "e", + "\u1ebf": "e", + "\u1ec0": "e", + "\u1ec1": "e", + "\u1ec2": "e", + "\u1ec3": "e", + "\u1ec4": "e", + "\u1ec5": "e", + "\u1ec6": "e", + "\u1ec7": "e", + "\u1ec8": "i", + "\u1ec9": "i", + "\u1eca": "i", + "\u1ecb": "i", + "\u1ecc": "o", + "\u1ecd": "o", + "\u1ece": "o", + "\u1ecf": "o", + "\u1ed0": "o", + "\u1ed1": "o", + "\u1ed2": "o", + "\u1ed3": "o", + "\u1ed4": "o", + "\u1ed5": "o", + "\u1ed6": "o", + "\u1ed7": "o", + "\u1ed8": "o", + "\u1ed9": "o", + "\u1eda": "o", + "\u1edb": "o", + "\u1edc": "o", + "\u1edd": "o", + "\u1ede": "o", + "\u1edf": "o", + "\u1ee0": "o", + "\u1ee1": "o", + "\u1ee2": "o", + "\u1ee3": "o", + "\u1ee4": "u", + "\u1ee5": "u", + "\u1ee6": "u", + "\u1ee7": "u", + "\u1ee8": "u", + "\u1ee9": "u", + "\u1eea": "u", + "\u1eeb": "u", + "\u1eec": "u", + "\u1eed": "u", + "\u1eee": "u", + "\u1eef": "u", + "\u1ef0": "u", + "\u1ef1": "u", + "\u1ef2": "y", + "\u1ef3": "y", + "\u1ef4": "y", + "\u1ef5": "y", + "\u1ef6": "y", + "\u1ef7": "y", + "\u1ef8": "y", + "\u1ef9": "y", + "\u1efe": "y", + "\u1eff": "y", + "\u1f00": "\u03b1", + "\u1f01": "\u03b1", + "\u1f02": "\u03b1", + "\u1f03": "\u03b1", + "\u1f04": "\u03b1", + "\u1f05": "\u03b1", + "\u1f06": "\u03b1", + "\u1f07": "\u03b1", + "\u1f08": "\u03b1", + "\u1f09": "\u03b1", + "\u1f0a": "\u03b1", + "\u1f0b": "\u03b1", + "\u1f0c": "\u03b1", + "\u1f0d": "\u03b1", + "\u1f0e": "\u03b1", + "\u1f0f": "\u03b1", + "\u1f10": "\u03b5", + "\u1f11": "\u03b5", + "\u1f12": "\u03b5", + "\u1f13": "\u03b5", + "\u1f14": "\u03b5", + "\u1f15": "\u03b5", + "\u1f18": "\u03b5", + "\u1f19": "\u03b5", + "\u1f1a": "\u03b5", + "\u1f1b": "\u03b5", + "\u1f1c": "\u03b5", + "\u1f1d": "\u03b5", + "\u1f20": "\u03b7", + "\u1f21": "\u03b7", + "\u1f22": "\u03b7", + "\u1f23": "\u03b7", + "\u1f24": "\u03b7", + "\u1f25": "\u03b7", + "\u1f26": "\u03b7", + "\u1f27": "\u03b7", + "\u1f28": "\u03b7", + "\u1f29": "\u03b7", + "\u1f2a": "\u03b7", + "\u1f2b": "\u03b7", + "\u1f2c": "\u03b7", + "\u1f2d": "\u03b7", + "\u1f2e": "\u03b7", + "\u1f2f": "\u03b7", + "\u1f30": "\u03b9", + "\u1f31": "\u03b9", + "\u1f32": "\u03b9", + "\u1f33": "\u03b9", + "\u1f34": "\u03b9", + "\u1f35": "\u03b9", + "\u1f36": "\u03b9", + "\u1f37": "\u03b9", + "\u1f38": "\u03b9", + "\u1f39": "\u03b9", + "\u1f3a": "\u03b9", + "\u1f3b": "\u03b9", + "\u1f3c": "\u03b9", + "\u1f3d": "\u03b9", + "\u1f3e": "\u03b9", + "\u1f3f": "\u03b9", + "\u1f40": "\u03bf", + "\u1f41": "\u03bf", + "\u1f42": "\u03bf", + "\u1f43": "\u03bf", + "\u1f44": "\u03bf", + "\u1f45": "\u03bf", + "\u1f48": "\u03bf", + "\u1f49": "\u03bf", + "\u1f4a": "\u03bf", + "\u1f4b": "\u03bf", + "\u1f4c": "\u03bf", + "\u1f4d": "\u03bf", + "\u1f50": "\u03c5", + "\u1f51": "\u03c5", + "\u1f52": "\u03c5", + "\u1f53": "\u03c5", + "\u1f54": "\u03c5", + "\u1f55": "\u03c5", + "\u1f56": "\u03c5", + "\u1f57": "\u03c5", + "\u1f59": "\u03c5", + "\u1f5b": "\u03c5", + "\u1f5d": "\u03c5", + "\u1f5f": "\u03c5", + "\u1f60": "\u03c9", + "\u1f61": "\u03c9", + "\u1f62": "\u03c9", + "\u1f63": "\u03c9", + "\u1f64": "\u03c9", + "\u1f65": "\u03c9", + "\u1f66": "\u03c9", + "\u1f67": "\u03c9", + "\u1f68": "\u03c9", + "\u1f69": "\u03c9", + "\u1f6a": "\u03c9", + "\u1f6b": "\u03c9", + "\u1f6c": "\u03c9", + "\u1f6d": "\u03c9", + "\u1f6e": "\u03c9", + "\u1f6f": "\u03c9", + "\u1f70": "\u03b1", + "\u1f71": "\u03b1", + "\u1f72": "\u03b5", + "\u1f73": "\u03b5", + "\u1f74": "\u03b7", + "\u1f75": "\u03b7", + "\u1f76": "\u03b9", + "\u1f77": "\u03b9", + "\u1f78": "\u03bf", + "\u1f79": "\u03bf", + "\u1f7a": "\u03c5", + "\u1f7b": "\u03c5", + "\u1f7c": "\u03c9", + "\u1f7d": "\u03c9", + "\u1f80": "\u03b1", + "\u1f81": "\u03b1", + "\u1f82": "\u03b1", + "\u1f83": "\u03b1", + "\u1f84": "\u03b1", + "\u1f85": "\u03b1", + "\u1f86": "\u03b1", + "\u1f87": "\u03b1", + "\u1f88": "\u03b1", + "\u1f89": "\u03b1", + "\u1f8a": "\u03b1", + "\u1f8b": "\u03b1", + "\u1f8c": "\u03b1", + "\u1f8d": "\u03b1", + "\u1f8e": "\u03b1", + "\u1f8f": "\u03b1", + "\u1f90": "\u03b7", + "\u1f91": "\u03b7", + "\u1f92": "\u03b7", + "\u1f93": "\u03b7", + "\u1f94": "\u03b7", + "\u1f95": "\u03b7", + "\u1f96": "\u03b7", + "\u1f97": "\u03b7", + "\u1f98": "\u03b7", + "\u1f99": "\u03b7", + "\u1f9a": "\u03b7", + "\u1f9b": "\u03b7", + "\u1f9c": "\u03b7", + "\u1f9d": "\u03b7", + "\u1f9e": "\u03b7", + "\u1f9f": "\u03b7", + "\u1fa0": "\u03c9", + "\u1fa1": "\u03c9", + "\u1fa2": "\u03c9", + "\u1fa3": "\u03c9", + "\u1fa4": "\u03c9", + "\u1fa5": "\u03c9", + "\u1fa6": "\u03c9", + "\u1fa7": "\u03c9", + "\u1fa8": "\u03c9", + "\u1fa9": "\u03c9", + "\u1faa": "\u03c9", + "\u1fab": "\u03c9", + "\u1fac": "\u03c9", + "\u1fad": "\u03c9", + "\u1fae": "\u03c9", + "\u1faf": "\u03c9", + "\u1fb0": "\u03b1", + "\u1fb1": "\u03b1", + "\u1fb2": "\u03b1", + "\u1fb3": "\u03b1", + "\u1fb4": "\u03b1", + "\u1fb6": "\u03b1", + "\u1fb7": "\u03b1", + "\u1fb8": "\u03b1", + "\u1fb9": "\u03b1", + "\u1fba": "\u03b1", + "\u1fbb": "\u03b1", + "\u1fbc": "\u03b1", + "\u1fc2": "\u03b7", + "\u1fc3": "\u03b7", + "\u1fc4": "\u03b7", + "\u1fc6": "\u03b7", + "\u1fc7": "\u03b7", + "\u1fc8": "\u03b5", + "\u1fc9": "\u03b5", + "\u1fca": "\u03b7", + "\u1fcb": "\u03b7", + "\u1fcc": "\u03b7", + "\u1fd0": "\u03b9", + "\u1fd1": "\u03b9", + "\u1fd2": "\u03b9", + "\u1fd3": "\u03b9", + "\u1fd6": "\u03b9", + "\u1fd7": "\u03b9", + "\u1fd8": "\u03b9", + "\u1fd9": "\u03b9", + "\u1fda": "\u03b9", + "\u1fdb": "\u03b9", + "\u1fe0": "\u03c5", + "\u1fe1": "\u03c5", + "\u1fe2": "\u03c5", + "\u1fe3": "\u03c5", + "\u1fe4": "\u03c1", + "\u1fe5": "\u03c1", + "\u1fe6": "\u03c5", + "\u1fe7": "\u03c5", + "\u1fe8": "\u03c5", + "\u1fe9": "\u03c5", + "\u1fea": "\u03c5", + "\u1feb": "\u03c5", + "\u1fec": "\u03c1", + "\u1ff2": "\u03c9", + "\u1ff3": "\u03c9", + "\u1ff4": "\u03c9", + "\u1ff6": "\u03c9", + "\u1ff7": "\u03c9", + "\u1ff8": "\u03bf", + "\u1ff9": "\u03bf", + "\u1ffa": "\u03c9", + "\u1ffb": "\u03c9", + "\u1ffc": "\u03c9", + "\u212b": "a", + "\u2c60": "l", + "\u2c61": "l", + "\u2c62": "l", + "\u2c63": "p", + "\u2c64": "r", + "\u2c65": "a", + "\u2c66": "t", + "\u2c67": "h", + "\u2c68": "h", + "\u2c69": "k", + "\u2c6a": "k", + "\u2c6b": "z", + "\u2c6c": "z", + "\u2c6e": "m", + "\u2c71": "v", + "\u2c72": "w", + "\u2c73": "w", + "\u2c74": "v", + "\u2c78": "e", + "\u2c7a": "o", + "\u2c7e": "s", + "\u2c7f": "z", + "\u304c": "\u304b", + "\u304e": "\u304d", + "\u3050": "\u304f", + "\u3052": "\u3051", + "\u3054": "\u3053", + "\u3056": "\u3055", + "\u3058": "\u3057", + "\u305a": "\u3059", + "\u305c": "\u305b", + "\u305e": "\u305d", + "\u3060": "\u305f", + "\u3062": "\u3061", + "\u3065": "\u3064", + "\u3067": "\u3066", + "\u3069": "\u3068", + "\u3070": "\u306f", + "\u3071": "\u306f", + "\u3073": "\u3072", + "\u3074": "\u3072", + "\u3076": "\u3075", + "\u3077": "\u3075", + "\u3079": "\u3078", + "\u307a": "\u3078", + "\u307c": "\u307b", + "\u307d": "\u307b", + "\u3094": "\u3046", + "\u309e": "\u309d", + "\u30ac": "\u30ab", + "\u30ae": "\u30ad", + "\u30b0": "\u30af", + "\u30b2": "\u30b1", + "\u30b4": "\u30b3", + "\u30b6": "\u30b5", + "\u30b8": "\u30b7", + "\u30ba": "\u30b9", + "\u30bc": "\u30bb", + "\u30be": "\u30bd", + "\u30c0": "\u30bf", + "\u30c2": "\u30c1", + "\u30c5": "\u30c4", + "\u30c7": "\u30c6", + "\u30c9": "\u30c8", + "\u30d0": "\u30cf", + "\u30d1": "\u30cf", + "\u30d3": "\u30d2", + "\u30d4": "\u30d2", + "\u30d6": "\u30d5", + "\u30d7": "\u30d5", + "\u30d9": "\u30d8", + "\u30da": "\u30d8", + "\u30dc": "\u30db", + "\u30dd": "\u30db", + "\u30f4": "\u30a6", + "\u30f7": "\u30ef", + "\u30f8": "\u30f0", + "\u30f9": "\u30f1", + "\u30fa": "\u30f2", + "\u30fe": "\u30fd", + "\ua740": "k", + "\ua741": "k", + "\ua742": "k", + "\ua743": "k", + "\ua744": "k", + "\ua745": "k", + "\ua748": "l", + "\ua749": "l", + "\ua74a": "o", + "\ua74b": "o", + "\ua74c": "o", + "\ua74d": "o", + "\ua750": "p", + "\ua751": "p", + "\ua752": "p", + "\ua753": "p", + "\ua754": "p", + "\ua755": "p", + "\ua756": "q", + "\ua757": "q", + "\ua758": "q", + "\ua759": "q", + "\ua75e": "v", + "\ua75f": "v", + "\ua78e": "l", + "\ua790": "n", + "\ua791": "n", + "\ua792": "c", + "\ua793": "c", + "\ua794": "c", + "\ua795": "h", + "\ua796": "b", + "\ua797": "b", + "\ua798": "f", + "\ua799": "f", + "\ua7a0": "g", + "\ua7a1": "g", + "\ua7a2": "k", + "\ua7a3": "k", + "\ua7a4": "n", + "\ua7a5": "n", + "\ua7a6": "r", + "\ua7a7": "r", + "\ua7a8": "s", + "\ua7a9": "s", + "\ua7aa": "h", + "\ua7ad": "l", + "\ua7b2": "j", + "\ua7b8": "u", + "\ua7b9": "u", + "\ua7c4": "c", + "\ua7c5": "s", + "\ua7c6": "z", + "\ua7c7": "d", + "\ua7c8": "d", + "\ua7c9": "s", + "\ua7ca": "s", + "\uab34": "e", + "\uab37": "l", + "\uab38": "l", + "\uab39": "l", + "\uab3a": "m", + "\uab3b": "n", + "\uab49": "r", + "\uab4e": "u", + "\uab52": "u", + "\uab56": "x", + "\uab57": "x", + "\uab58": "x", + "\uab59": "x", + "\uab5a": "y", + "\ufb1d": "\u05d9", + "\ufb1f": "\u05f2", + "\ufb2a": "\u05e9", + "\ufb2b": "\u05e9", + "\ufb2c": "\u05e9", + "\ufb2d": "\u05e9", + "\ufb2e": "\u05d0", + "\ufb2f": "\u05d0", + "\ufb30": "\u05d0", + "\ufb31": "\u05d1", + "\ufb32": "\u05d2", + "\ufb33": "\u05d3", + "\ufb34": "\u05d4", + "\ufb35": "\u05d5", + "\ufb36": "\u05d6", + "\ufb38": "\u05d8", + "\ufb39": "\u05d9", + "\ufb3a": "\u05da", + "\ufb3b": "\u05db", + "\ufb3c": "\u05dc", + "\ufb3e": "\u05de", + "\ufb40": "\u05e0", + "\ufb41": "\u05e1", + "\ufb43": "\u05e3", + "\ufb44": "\u05e4", + "\ufb46": "\u05e6", + "\ufb47": "\u05e7", + "\ufb48": "\u05e8", + "\ufb49": "\u05e9", + "\ufb4a": "\u05ea", + "\ufb4b": "\u05d5", + "\ufb4c": "\u05d1", + "\ufb4d": "\u05db", + "\ufb4e": "\u05e4", + "\ud804\udc9a": "\ud804\udc99", + "\ud804\udc9c": "\ud804\udc9b", + "\ud804\udcab": "\ud804\udca5", + "\ud837\udf09": "t", + "\ud837\udf11": "l", + "\ud837\udf13": "l", + "\ud837\udf16": "r", + "\ud837\udf1a": "i", + "\ud837\udf1b": "o", + "\ud837\udf1d": "c", + "\ud837\udf1e": "s" +} diff --git a/web/src/dinners/names.ts b/web/src/dinners/names.ts index b5afe11..52cc50d 100644 --- a/web/src/dinners/names.ts +++ b/web/src/dinners/names.ts @@ -3,37 +3,17 @@ // API would refuse a name as a duplicate. The server still decides. // internal/ai/harness/testdata/dedupe_cases.json is run against both. +import folds from './letterFolds.json' + // SIMILAR_THRESHOLD is harness.SimilarThreshold. const SIMILAR_THRESHOLD = 0.85 // fillerWords is harness's fillerWords. const fillerWords = new Set(['easy', 'best', 'homemade', 'quick', 'simple', 'the', 'a', 'recipe', 'classic']) -// letterFolds are the letters harness folds that canonical decomposition -// leaves whole: its ligatures and extra letters, and the common stroked -// Latin letters. Letters with accents fold by decomposing and dropping the -// marks, as harness's table does; rarer stroked letters are compared as -// they are. -const letterFolds: Record = { - ß: 'ss', - æ: 'ae', - œ: 'oe', - þ: 'th', - ð: 'd', - ı: 'i', - ʼn: 'n', - ij: 'ij', - ø: 'o', - đ: 'd', - ħ: 'h', - ŀ: 'l', - ł: 'l', - ŧ: 't', - ƀ: 'b', - ƶ: 'z', - ǥ: 'g', - ɨ: 'i', -} +// letterFolds is harness's letterFolds: gen_folds.py writes both tables, +// and harness's TestLetterFoldsMatchPWA fails when they differ. +const letterFolds: Record = folds const combiningMark = /\p{Mn}/u const wordChar = /[\p{L}\p{Nd}]/u @@ -44,12 +24,16 @@ const wordChar = /[\p{L}\p{Nd}]/u // "chicken tikka masala". export function normalizeName(name: string): string { let out = '' - for (const ch of name.toLowerCase().normalize('NFD')) { - if (combiningMark.test(ch) || ch === "'" || ch === '’') continue - const fold = letterFolds[ch] - if (fold !== undefined) out += fold - else if (wordChar.test(ch)) out += ch - else out += ' ' + // Lowercased letter by letter, as Go's strings.ToLower does, so no + // letter is lowered by its context (a final Σ is σ, not ς). + for (const letter of name) { + for (const ch of letter.toLowerCase()) { + if (combiningMark.test(ch) || ch === "'" || ch === '’') continue + const fold = letterFolds[ch] + if (fold !== undefined) out += fold + else if (wordChar.test(ch)) out += ch + else out += ' ' + } } return out .split(' ')