diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index ee6f75b..ad22749 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -219,7 +219,10 @@ automatic** (D-021). A job for a dinner archived, no longer `pending` or at another revision does no work, and a job only writes a dinner still unarchived, `pending` and at its revision, so a fix-up made meanwhile wins and an earlier job - for an old name — queued or running — never decides the outcome. A + for an old name — queued or running — never decides the outcome. + Archiving or renaming a dinner also stops its running job where it is, + so no further search, page fetch, photo download or model call is made + for it. A method, minutes_total or difficulty a member saved while the dinner was `pending` is kept; grounding fills only those left empty. At boot, after the job runner fails the jobs a previous process left running, every @@ -502,7 +505,9 @@ steps: the welcome (`/onboarding`), the Regulars (`/onboarding/regulars`, mock 1h, adding and archiving through the dinners API optimistically; Next waits for three saved Regulars, and a common pick is not offered once a Regular has a name `harness.Similar` to it — the PWA's copy of the name -rules in `web/src/dinners/names.ts` runs the same cases, +rules in `web/src/dinners/names.ts` folds letters with +`web/src/dinners/letterFolds.json`, which `gen_folds.py` writes from the +same table as the server's, and runs the same cases, `internal/ai/harness/testdata/dedupe_cases.json`, as the server's) and notifications (`/onboarding/notifications`: what MealQuest notifies about, the iOS install hint, "Turn on notifications" and "Maybe later"). Either diff --git a/internal/ai/harness/dedupe_test.go b/internal/ai/harness/dedupe_test.go index 2f64739..9f7ff2b 100644 --- a/internal/ai/harness/dedupe_test.go +++ b/internal/ai/harness/dedupe_test.go @@ -5,6 +5,7 @@ import ( "os" "testing" "unicode" + "unicode/utf8" ) // TestSharedDedupeCases runs testdata/dedupe_cases.json, which the PWA's @@ -40,6 +41,30 @@ func TestSharedDedupeCases(t *testing.T) { } } +// TestLetterFoldsMatchPWA checks that the PWA's letter-fold table, which +// gen_folds.py writes beside folds_table.go, is letterFolds exactly; run +// go generate when it fails. +func TestLetterFoldsMatchPWA(t *testing.T) { + raw, err := os.ReadFile("../../../web/src/dinners/letterFolds.json") + if err != nil { + t.Fatal(err) + } + var pwa map[string]string + if err := json.Unmarshal(raw, &pwa); err != nil { + t.Fatal(err) + } + for r, want := range letterFolds { + if got, ok := pwa[string(r)]; !ok || got != want { + t.Errorf("PWA folds %U to %q (present %v), want %q", r, got, ok, want) + } + } + for k := range pwa { + if utf8.RuneCountInString(k) != 1 || letterFolds[[]rune(k)[0]] == "" { + t.Errorf("PWA folds %q, which letterFolds does not", k) + } + } +} + func TestNormalize(t *testing.T) { tests := []struct { name, want string diff --git a/internal/ai/harness/gen_folds.py b/internal/ai/harness/gen_folds.py index 6e907c4..4de6bb6 100644 --- a/internal/ai/harness/gen_folds.py +++ b/internal/ai/harness/gen_folds.py @@ -1,5 +1,8 @@ #!/usr/bin/env python3 -"""Generate folds_table.go, the letter-fold table used by harness.Normalize. +"""Generate folds_table.go, the letter-fold table used by harness.Normalize, +and its copy web/src/dinners/letterFolds.json, used by the PWA's +normalizeName (web/src/dinners/names.ts). TestLetterFoldsMatchPWA fails when +the two differ. Run from this directory with `go generate ./internal/ai/harness` (or `python3 gen_folds.py`; gofmt must be on PATH). It uses only the Python @@ -28,11 +31,14 @@ Hangul syllables are left out: their decomposition is into conjoining jamo, not combining marks. """ +import json import re import subprocess import sys import unicodedata +PWA_TABLE = "../../../web/src/dinners/letterFolds.json" + EXTRA = { "ß": "ss", "ẞ": "ss", "æ": "ae", "Æ": "ae", "œ": "oe", "Œ": "oe", "þ": "th", "Þ": "th", "ð": "d", "Ð": "d", "ı": "i", "ʼn": "n", @@ -110,6 +116,9 @@ def main(): with open("folds_table.go", "w", encoding="utf-8") as f: f.write("\n".join(out) + "\n") subprocess.run(["gofmt", "-w", "folds_table.go"], check=True) + with open(PWA_TABLE, "w", encoding="ascii") as f: + json.dump({chr(cp): folded for cp, folded in entries}, f, indent=2) + f.write("\n") if __name__ == "__main__": diff --git a/internal/ai/harness/testdata/dedupe_cases.json b/internal/ai/harness/testdata/dedupe_cases.json index 2488e8b..20c3aec 100644 --- a/internal/ai/harness/testdata/dedupe_cases.json +++ b/internal/ai/harness/testdata/dedupe_cases.json @@ -67,6 +67,18 @@ { "name": "7-layer dip", "want": "7 layer dip" + }, + { + "name": "Ƈhili", + "want": "chili" + }, + { + "name": "Ꞔarbanzo Stew", + "want": "carbanzo stew" + }, + { + "name": "ΜΟΥΣΑΚΑΣ", + "want": "μουσακασ" } ], "similar": [ @@ -114,6 +126,11 @@ "a": "The best", "b": "Easy recipe", "want": false + }, + { + "a": "Ƈhili", + "b": "Chili", + "want": true } ] } diff --git a/internal/dinners/dinners.go b/internal/dinners/dinners.go index 7fac6f9..605d675 100644 --- a/internal/dinners/dinners.go +++ b/internal/dinners/dinners.go @@ -11,9 +11,11 @@ import ( "errors" "fmt" "log/slog" + "math" "net/url" "slices" "strings" + "sync" "unicode" "unicode/utf8" @@ -130,6 +132,11 @@ type Service struct { grounder *recipes.Grounder logger *slog.Logger groundJob *jobs.Kind[GroundJob] + + // groundingsMu guards groundings: the KindGroundDinner jobs running, + // by dinner id, so archiving or renaming a dinner can stop them. + groundingsMu sync.Mutex + groundings map[string]map[*runningGrounding]struct{} } // NewService returns a service over st that grounds dinners with grounder @@ -146,7 +153,7 @@ func NewService(st *store.Store, grounder *recipes.Grounder, runner *jobs.Runner case logger == nil: return nil, errors.New("dinners: nil logger") } - s := &Service{st: st, grounder: grounder, logger: logger} + s := &Service{st: st, grounder: grounder, logger: logger, groundings: make(map[string]map[*runningGrounding]struct{})} kind, err := jobs.Register(runner, KindGroundDinner, s.runGroundJob, Reasons...) if err != nil { return nil, err @@ -296,7 +303,7 @@ func (s *Service) List(ctx context.Context, status string) ([]Dinner, error) { // clears its recipe_url, raises its grounding_revision and enqueues a fresh // KindGroundDinner job that grounds the new name: a job queued or running // for the old name or link carries the old revision, so it can no longer -// write the dinner. +// write the dinner, and one running is stopped. func (s *Service) Update(ctx context.Context, id string, p Patch, updatedBy string) (Dinner, error) { if updatedBy == "" { return Dinner{}, errors.New("dinners: update: no member to record as updated_by") @@ -368,6 +375,7 @@ func (s *Service) Update(ctx context.Context, id string, p Patch, updatedBy stri return Dinner{}, err } if regrounding { + s.stopGroundings(row.ID, row.GroundingRevision) jobID, err := s.enqueueGrounding(ctx, row.ID, row.GroundingRevision) if err != nil { return Dinner{}, err @@ -383,7 +391,8 @@ func (s *Service) Update(ctx context.Context, id string, p Patch, updatedBy stri } // Archive sets dinner id's status to archived, recording updatedBy. The row -// is kept, so its name still counts for Side quest dedupe. +// is kept, so its name still counts for Side quest dedupe. A +// KindGroundDinner job running for it is stopped. func (s *Service) Archive(ctx context.Context, id, updatedBy string) (Dinner, error) { if updatedBy == "" { return Dinner{}, errors.New("dinners: archive: no member to record as updated_by") @@ -399,6 +408,7 @@ func (s *Service) Archive(ctx context.Context, id, updatedBy string) (Dinner, er if err != nil { return Dinner{}, fmt.Errorf("dinners: archive %s: %w", id, err) } + s.stopGroundings(row.ID, math.MaxInt64) return fromRow(row), nil } diff --git a/internal/dinners/grounding.go b/internal/dinners/grounding.go index 4d43b13..a57bb71 100644 --- a/internal/dinners/grounding.go +++ b/internal/dinners/grounding.go @@ -104,8 +104,14 @@ func (r groundResult) PublicJSON() ([]byte, error) { return json.Marshal(r) } // either result is recorded only while the dinner is still unarchived, // pending and at p.Revision: a dinner grounded by hand meanwhile is left as // it is, and one renamed or archived meanwhile is left pending — for the -// job its rename enqueued, or for none. +// job its rename enqueued, or for none. Archiving or renaming the dinner +// stops the grounding where it is, so no further search, fetch, photo +// download or model call is made for it. func (s *Service) runGroundJob(ctx context.Context, p GroundJob, _ func(groundResult) error) (groundResult, error) { + // Tracked before the dinner is read: an archive or rename committed + // after the read finds this job to stop. + groundCtx, done := s.trackGrounding(ctx, p.DinnerID, p.Revision) + defer done() row, err := db.New(s.st.Reader).GetDinner(ctx, p.DinnerID) if err != nil { return groundResult{}, fmt.Errorf("dinners: ground: read %s: %w", p.DinnerID, err) @@ -116,9 +122,12 @@ func (s *Service) runGroundJob(ctx context.Context, p GroundJob, _ func(groundRe var gr recipes.Grounded if row.RecipeUrl.Valid { - gr, err = s.grounder.GroundURL(ctx, row.RecipeUrl.String) + gr, err = s.grounder.GroundURL(groundCtx, row.RecipeUrl.String) } else { - gr, err = s.grounder.Ground(ctx, row.Name, row.Name) + gr, err = s.grounder.Ground(groundCtx, row.Name, row.Name) + } + if err != nil && ctx.Err() == nil && errors.Is(context.Cause(groundCtx), errGroundingStopped) { + return s.currentGrounding(ctx, row.ID) } if err != nil { reason := reasonFor(err, row.RecipeUrl.Valid) @@ -157,15 +166,67 @@ func (s *Service) runGroundJob(ctx context.Context, p GroundJob, _ func(groundRe return groundResult{}, fmt.Errorf("dinners: ground: record %s: %w", row.ID, err) } if n == 0 { - cur, err := db.New(s.st.Reader).GetDinner(ctx, row.ID) - if err != nil { - return groundResult{}, fmt.Errorf("dinners: ground: read %s: %w", row.ID, err) - } - return groundResult{DinnerID: cur.ID, GroundingStatus: cur.GroundingStatus}, nil + return s.currentGrounding(ctx, row.ID) } return groundResult{DinnerID: row.ID, GroundingStatus: GroundingGrounded}, nil } +// currentGrounding is the KindGroundDinner result of a job that did not +// ground dinner id: the grounding status the dinner has now. +func (s *Service) currentGrounding(ctx context.Context, id string) (groundResult, error) { + cur, err := db.New(s.st.Reader).GetDinner(ctx, id) + if err != nil { + return groundResult{}, fmt.Errorf("dinners: ground: read %s: %w", id, err) + } + return groundResult{DinnerID: cur.ID, GroundingStatus: cur.GroundingStatus}, nil +} + +// errGroundingStopped is the cause a running KindGroundDinner job's +// grounding is cancelled with when its dinner is archived or renamed. +var errGroundingStopped = errors.New("dinners: grounding stopped: dinner archived or renamed") + +// runningGrounding is a KindGroundDinner job running for a dinner at +// revision, with the cancel of its grounding. +type runningGrounding struct { + revision int64 + cancel context.CancelCauseFunc +} + +// trackGrounding returns a context for grounding dinnerID at revision that +// stopGroundings cancels, and the func to call when the job ends. +func (s *Service) trackGrounding(ctx context.Context, dinnerID string, revision int64) (context.Context, func()) { + groundCtx, cancel := context.WithCancelCause(ctx) + r := &runningGrounding{revision: revision, cancel: cancel} + s.groundingsMu.Lock() + if s.groundings[dinnerID] == nil { + s.groundings[dinnerID] = make(map[*runningGrounding]struct{}) + } + s.groundings[dinnerID][r] = struct{}{} + s.groundingsMu.Unlock() + return groundCtx, func() { + s.groundingsMu.Lock() + delete(s.groundings[dinnerID], r) + if len(s.groundings[dinnerID]) == 0 { + delete(s.groundings, dinnerID) + } + s.groundingsMu.Unlock() + cancel(nil) + } +} + +// stopGroundings cancels, with errGroundingStopped, the grounding of every +// KindGroundDinner job running for dinnerID at a revision below below. Call +// it once the change that makes those jobs stale is committed. +func (s *Service) stopGroundings(dinnerID string, below int64) { + s.groundingsMu.Lock() + defer s.groundingsMu.Unlock() + for r := range s.groundings[dinnerID] { + if r.revision < below { + r.cancel(errGroundingStopped) + } + } +} + // FailInterruptedGroundings fails, with ReasonInterrupted, every dinner // still pending whose KindGroundDinner job is no longer queued or running: // a previous process stopped before enqueuing the job or before its diff --git a/internal/dinners/grounding_test.go b/internal/dinners/grounding_test.go index 47f0f31..ba90e3e 100644 --- a/internal/dinners/grounding_test.go +++ b/internal/dinners/grounding_test.go @@ -6,9 +6,11 @@ import ( "encoding/json" "errors" "io" + "io/fs" "log/slog" "slices" "testing" + "testing/fstest" "time" "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm" @@ -54,6 +56,13 @@ func newEnv(t *testing.T) *env { func newUnstartedEnv(t *testing.T) *env { t.Helper() dataDir := t.TempDir() + return newUnstartedEnvOn(t, openStore(t, dataDir), dataDir) +} + +// openStore opens the store in dataDir, closing it when the test ends. It +// is not migrated. +func openStore(t *testing.T, dataDir string) *store.Store { + t.Helper() st, err := store.Open(t.Context(), dataDir) if err != nil { t.Fatal(err) @@ -63,6 +72,13 @@ func newUnstartedEnv(t *testing.T) *env { t.Error(err) } }) + return st +} + +// newUnstartedEnvOn is newUnstartedEnv over st, the store in dataDir, +// which it migrates to the latest version. +func newUnstartedEnvOn(t *testing.T, st *store.Store, dataDir string) *env { + t.Helper() if err := store.Migrate(t.Context(), st.Writer, migrations.FS); err != nil { t.Fatal(err) } @@ -290,19 +306,16 @@ func TestManualFixWhilePendingIsKept(t *testing.T) { // TestRenameWhilePendingGroundsNewName: a Regular renamed while its job is // still grounding the old name never gets the old name's result, whether -// that job succeeds or fails; the job the rename enqueues grounds the new -// name. +// the old name would be found or not: the rename stops that job where it +// is, and the job the rename enqueues grounds the new name. func TestRenameWhilePendingGroundsNewName(t *testing.T) { t.Parallel() tests := []struct { name string oldName string - replies []llm.GenerateResult }{ - {"old name found", "Chicken curry", []llm.GenerateResult{ - recipestest.NormalizedReply("stovetop", 40), recipestest.NormalizedReply("oven", 150), - }}, - {"old name not found", "Unknown dish", []llm.GenerateResult{recipestest.NormalizedReply("oven", 150)}}, + {"old name found", "Chicken curry"}, + {"old name not found", "Unknown dish"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -310,9 +323,7 @@ func TestRenameWhilePendingGroundsNewName(t *testing.T) { e := newEnv(t) e.search.Results["Beef stew"] = []recipes.Candidate{{URL: stew}} e.search.Gate = make(chan struct{}) - for _, r := range tt.replies { - e.model.QueueGenerate(r) - } + e.model.QueueGenerate(recipestest.NormalizedReply("oven", 150)) c, err := e.svc.CreateRegular(t.Context(), dinners.NewRegular{Name: ptr(tt.oldName)}, member) if err != nil { @@ -327,12 +338,13 @@ func TestRenameWhilePendingGroundsNewName(t *testing.T) { if d.GroundingStatus != dinners.GroundingPending { t.Fatalf("renamed = %+v, want still pending", d) } + // The old name's job ends while its search is still held. + first := e.waitJob(t, *c.GroundingJobID) + if first.Status != jobs.StatusSucceeded || string(first.Result) != `{"dinner_id":"`+c.ID+`","grounding_status":"pending"}` { + t.Fatalf("old name's job = %+v (result %s), want succeeded leaving the dinner pending", first, first.Result) + } close(e.search.Gate) - first := e.waitJob(t, *c.GroundingJobID) - if string(first.Result) == `{"dinner_id":"`+c.ID+`","grounding_status":"grounded"}` { - t.Fatalf("old name's job reported grounded: %+v (result %s)", first, first.Result) - } row := e.waitGrounding(t, c.ID) if row.GroundingStatus != dinners.GroundingGrounded { t.Fatalf("dinner = %+v, want grounded for the new name", row) @@ -347,6 +359,9 @@ func TestRenameWhilePendingGroundsNewName(t *testing.T) { if got := e.search.Queries(); !slices.Equal(got, []string{tt.oldName, "Beef stew"}) { t.Errorf("searched %q, want the old name then the new", got) } + if got := len(e.model.Requests()); got != 1 { + t.Errorf("model called %d times, want once, for the new name only", got) + } }) } } @@ -503,7 +518,8 @@ func TestEditWhilePendingIsKept(t *testing.T) { // TestArchiveWhilePendingStopsGrounding: a pending Regular archived before // its job starts is not searched for, and one archived while its job runs -// is not written; neither ends up grounded. +// is stopped there, with no page fetched or model called; neither ends up +// grounded. func TestArchiveWhilePendingStopsGrounding(t *testing.T) { t.Parallel() t.Run("before the job starts", func(t *testing.T) { @@ -541,9 +557,17 @@ func TestArchiveWhilePendingStopsGrounding(t *testing.T) { if _, err := e.svc.Archive(t.Context(), c.ID, member); err != nil { t.Fatal(err) } + // The job ends while its search is still held: archiving stopped it. + job := e.waitJob(t, *c.GroundingJobID) close(e.search.Gate) - if job := e.waitJob(t, *c.GroundingJobID); job.Status != jobs.StatusSucceeded { - t.Fatalf("job = %+v", job) + if job.Status != jobs.StatusSucceeded || string(job.Result) != `{"dinner_id":"`+c.ID+`","grounding_status":"pending"}` { + t.Fatalf("job = %+v (result %s), want succeeded leaving the dinner pending", job, job.Result) + } + if got := e.pages.Fetched(); len(got) != 0 { + t.Errorf("fetched %q for an archived dinner", got) + } + if got := len(e.model.Requests()); got != 0 { + t.Errorf("model called %d times for an archived dinner", got) } if row := e.dinner(t, c.ID); row.Status != dinners.StatusArchived || row.GroundingStatus == dinners.GroundingGrounded || row.RecipeID.Valid { t.Errorf("dinner = %+v, want archived and not grounded", row) @@ -922,6 +946,74 @@ func TestRestartFailsInterruptedGrounding(t *testing.T) { } } +// TestUpgradeKeepsLegacyQueuedGrounding: a pending Regular whose +// ground_dinner job was queued before migration 0010, so its payload has +// no revision, is not failed as interrupted at the first boot after the +// upgrade; its job still grounds it. +func TestUpgradeKeepsLegacyQueuedGrounding(t *testing.T) { + t.Parallel() + ctx := t.Context() + dataDir := t.TempDir() + st := openStore(t, dataDir) + before := fstest.MapFS{} + files, err := fs.Glob(migrations.FS, "*.sql") + if err != nil { + t.Fatal(err) + } + for _, name := range files { + if name >= "0010" { + continue + } + body, err := fs.ReadFile(migrations.FS, name) + if err != nil { + t.Fatal(err) + } + before[name] = &fstest.MapFile{Data: body} + } + if err := store.Migrate(ctx, st.Writer, before); err != nil { + t.Fatal(err) + } + now := store.Now().String() + if _, err := st.Writer.ExecContext(ctx, `INSERT INTO dinners + (id, name, normalized_name, status, origin, is_favorite, grounding_status, updated_by, created_at, updated_at) + VALUES ('dinner-1', 'Chicken curry', 'chicken curry', 'regular', 'regular', 0, 'pending', ?, ?, ?)`, + member, now, now); err != nil { + t.Fatal(err) + } + if _, err := st.Writer.ExecContext(ctx, `INSERT INTO jobs (id, kind, status, payload, created_at, updated_at) + VALUES ('job-1', ?, 'queued', '{"dinner_id":"dinner-1"}', ?, ?)`, + dinners.KindGroundDinner, now, now); err != nil { + t.Fatal(err) + } + + // Boot: migrate, start the runner, then fail interrupted groundings. The + // search is held, so the job cannot finish before the cleanup runs. + e := newUnstartedEnvOn(t, st, dataDir) + e.search.Gate = make(chan struct{}) + e.model.QueueGenerate(recipestest.NormalizedReply("stovetop", 40)) + if p := e.queuedJob(t, "job-1"); p != (dinners.GroundJob{DinnerID: "dinner-1", Revision: 0}) { + t.Errorf("migrated payload = %+v, want dinner-1 at revision 0", p) + } + e.start(t) + n, err := e.svc.FailInterruptedGroundings(ctx) + if err != nil { + t.Fatal(err) + } + if n != 0 { + t.Errorf("failed %d dinners, want 0", n) + } + if row := e.dinner(t, "dinner-1"); row.GroundingStatus != dinners.GroundingPending { + t.Fatalf("dinner = %s (%s), want still pending", row.GroundingStatus, row.GroundingError.String) + } + close(e.search.Gate) + if job := e.waitJob(t, "job-1"); job.Status != jobs.StatusSucceeded { + t.Fatalf("job = %+v", job) + } + if row := e.dinner(t, "dinner-1"); row.GroundingStatus != dinners.GroundingGrounded { + t.Errorf("dinner = %s, want grounded", row.GroundingStatus) + } +} + // TestCreateEnqueueFailureReturnsFailedDinner: a dinner saved when its job // cannot be enqueued is returned, with its id, failed and no job id. func TestCreateEnqueueFailureReturnsFailedDinner(t *testing.T) { diff --git a/internal/recipes/recipestest/recipestest.go b/internal/recipes/recipestest/recipestest.go index 46aa72a..9d22dfa 100644 --- a/internal/recipes/recipestest/recipestest.go +++ b/internal/recipes/recipestest/recipestest.go @@ -66,9 +66,15 @@ func (s *Search) Queries() []string { type Pages struct { Bodies map[string][]byte Gate chan struct{} + + mu sync.Mutex + fetched []string } func (p *Pages) Fetch(ctx context.Context, rawURL string) (recipes.Page, error) { + p.mu.Lock() + p.fetched = append(p.fetched, rawURL) + p.mu.Unlock() if p.Gate != nil { select { case <-p.Gate: @@ -83,6 +89,13 @@ func (p *Pages) Fetch(ctx context.Context, rawURL string) (recipes.Page, error) return recipes.Page{URL: rawURL, ContentType: "text/html", Body: body}, nil } +// Fetched returns every URL fetched, in order. +func (p *Pages) Fetched() []string { + p.mu.Lock() + defer p.mu.Unlock() + return append([]string(nil), p.fetched...) +} + // Images is a recipes.ImageFetcher serving JPEGs by URL; any other URL is // a 404. type Images struct { diff --git a/migrations/0010_dinner_grounding_revision.sql b/migrations/0010_dinner_grounding_revision.sql index 5f13bd9..c2d4fc1 100644 --- a/migrations/0010_dinner_grounding_revision.sql +++ b/migrations/0010_dinner_grounding_revision.sql @@ -5,3 +5,12 @@ -- earlier job — queued or running — can no longer write the dinner. ALTER TABLE dinners ADD COLUMN grounding_revision INTEGER NOT NULL DEFAULT 0; + +-- Every dinner starts at revision 0, so a ground_dinner job enqueued +-- before this migration, whose payload has only dinner_id, is for revision +-- 0. Recording it lets the boot cleanup, which matches a pending dinner to +-- its job by revision, see a job still queued for its dinner. +UPDATE jobs +SET payload = json_set(payload, '$.revision', 0) +WHERE kind = 'ground_dinner' + AND json_type(payload, '$.revision') IS NULL; diff --git a/web/src/dinners/letterFolds.json b/web/src/dinners/letterFolds.json new file mode 100644 index 0000000..5be5eaf --- /dev/null +++ b/web/src/dinners/letterFolds.json @@ -0,0 +1,1167 @@ +{ + "\u00c0": "a", + "\u00c1": "a", + "\u00c2": "a", + "\u00c3": "a", + "\u00c4": "a", + "\u00c5": "a", + "\u00c6": "ae", + "\u00c7": "c", + "\u00c8": "e", + "\u00c9": "e", + "\u00ca": "e", + "\u00cb": "e", + "\u00cc": "i", + "\u00cd": "i", + "\u00ce": "i", + "\u00cf": "i", + "\u00d0": "d", + "\u00d1": "n", + "\u00d2": "o", + "\u00d3": "o", + "\u00d4": "o", + "\u00d5": "o", + "\u00d6": "o", + "\u00d8": "o", + "\u00d9": "u", + "\u00da": "u", + "\u00db": "u", + "\u00dc": "u", + "\u00dd": "y", + "\u00de": "th", + "\u00df": "ss", + "\u00e0": "a", + "\u00e1": "a", + "\u00e2": "a", + "\u00e3": "a", + "\u00e4": "a", + "\u00e5": "a", + "\u00e6": "ae", + "\u00e7": "c", + "\u00e8": "e", + "\u00e9": "e", + "\u00ea": "e", + "\u00eb": "e", + "\u00ec": "i", + "\u00ed": "i", + "\u00ee": "i", + "\u00ef": "i", + "\u00f0": "d", + "\u00f1": "n", + "\u00f2": "o", + "\u00f3": "o", + "\u00f4": "o", + "\u00f5": "o", + "\u00f6": "o", + "\u00f8": "o", + "\u00f9": "u", + "\u00fa": "u", + "\u00fb": "u", + "\u00fc": "u", + "\u00fd": "y", + "\u00fe": "th", + "\u00ff": "y", + "\u0100": "a", + "\u0101": "a", + "\u0102": "a", + "\u0103": "a", + "\u0104": "a", + "\u0105": "a", + "\u0106": "c", + "\u0107": "c", + "\u0108": "c", + "\u0109": "c", + "\u010a": "c", + "\u010b": "c", + "\u010c": "c", + "\u010d": "c", + "\u010e": "d", + "\u010f": "d", + "\u0110": "d", + "\u0111": "d", + "\u0112": "e", + "\u0113": "e", + "\u0114": "e", + "\u0115": "e", + "\u0116": "e", + "\u0117": "e", + "\u0118": "e", + "\u0119": "e", + "\u011a": "e", + "\u011b": "e", + "\u011c": "g", + "\u011d": "g", + "\u011e": "g", + "\u011f": "g", + "\u0120": "g", + "\u0121": "g", + "\u0122": "g", + "\u0123": "g", + "\u0124": "h", + "\u0125": "h", + "\u0126": "h", + "\u0127": "h", + "\u0128": "i", + "\u0129": "i", + "\u012a": "i", + "\u012b": "i", + "\u012c": "i", + "\u012d": "i", + "\u012e": "i", + "\u012f": "i", + "\u0130": "i", + "\u0131": "i", + "\u0132": "ij", + "\u0133": "ij", + "\u0134": "j", + "\u0135": "j", + "\u0136": "k", + "\u0137": "k", + "\u0139": "l", + "\u013a": "l", + "\u013b": "l", + "\u013c": "l", + "\u013d": "l", + "\u013e": "l", + "\u013f": "l", + "\u0140": "l", + "\u0141": "l", + "\u0142": "l", + "\u0143": "n", + "\u0144": "n", + "\u0145": "n", + "\u0146": "n", + "\u0147": "n", + "\u0148": "n", + "\u0149": "n", + "\u014c": "o", + "\u014d": "o", + "\u014e": "o", + "\u014f": "o", + "\u0150": "o", + "\u0151": "o", + "\u0152": "oe", + "\u0153": "oe", + "\u0154": "r", + "\u0155": "r", + "\u0156": "r", + "\u0157": "r", + "\u0158": "r", + "\u0159": "r", + "\u015a": "s", + "\u015b": "s", + "\u015c": "s", + "\u015d": "s", + "\u015e": "s", + "\u015f": "s", + "\u0160": "s", + "\u0161": "s", + "\u0162": "t", + "\u0163": "t", + "\u0164": "t", + "\u0165": "t", + "\u0166": "t", + "\u0167": "t", + "\u0168": "u", + "\u0169": "u", + "\u016a": "u", + "\u016b": "u", + "\u016c": "u", + "\u016d": "u", + "\u016e": "u", + "\u016f": "u", + "\u0170": "u", + "\u0171": "u", + "\u0172": "u", + "\u0173": "u", + "\u0174": "w", + "\u0175": "w", + "\u0176": "y", + "\u0177": "y", + "\u0178": "y", + "\u0179": "z", + "\u017a": "z", + "\u017b": "z", + "\u017c": "z", + "\u017d": "z", + "\u017e": "z", + "\u0180": "b", + "\u0181": "b", + "\u0182": "b", + "\u0183": "b", + "\u0187": "c", + "\u0188": "c", + "\u018a": "d", + "\u018b": "d", + "\u018c": "d", + "\u0191": "f", + "\u0192": "f", + "\u0193": "g", + "\u0197": "i", + "\u0198": "k", + "\u0199": "k", + "\u019a": "l", + "\u019d": "n", + "\u019e": "n", + "\u019f": "o", + "\u01a0": "o", + "\u01a1": "o", + "\u01a4": "p", + "\u01a5": "p", + "\u01ab": "t", + "\u01ac": "t", + "\u01ad": "t", + "\u01ae": "t", + "\u01af": "u", + "\u01b0": "u", + "\u01b2": "v", + "\u01b3": "y", + "\u01b4": "y", + "\u01b5": "z", + "\u01b6": "z", + "\u01c5": "d", + "\u01c8": "l", + "\u01cb": "n", + "\u01cd": "a", + "\u01ce": "a", + "\u01cf": "i", + "\u01d0": "i", + "\u01d1": "o", + "\u01d2": "o", + "\u01d3": "u", + "\u01d4": "u", + "\u01d5": "u", + "\u01d6": "u", + "\u01d7": "u", + "\u01d8": "u", + "\u01d9": "u", + "\u01da": "u", + "\u01db": "u", + "\u01dc": "u", + "\u01de": "a", + "\u01df": "a", + "\u01e0": "a", + "\u01e1": "a", + "\u01e2": "ae", + "\u01e3": "ae", + "\u01e4": "g", + "\u01e5": "g", + "\u01e6": "g", + "\u01e7": "g", + "\u01e8": "k", + "\u01e9": "k", + "\u01ea": "o", + "\u01eb": "o", + "\u01ec": "o", + "\u01ed": "o", + "\u01ee": "\u0292", + "\u01ef": "\u0292", + "\u01f0": "j", + "\u01f2": "d", + "\u01f4": "g", + "\u01f5": "g", + "\u01f8": "n", + "\u01f9": "n", + "\u01fa": "a", + "\u01fb": "a", + "\u01fc": "ae", + "\u01fd": "ae", + "\u01fe": "o", + "\u01ff": "o", + "\u0200": "a", + "\u0201": "a", + "\u0202": "a", + "\u0203": "a", + "\u0204": "e", + "\u0205": "e", + "\u0206": "e", + "\u0207": "e", + "\u0208": "i", + "\u0209": "i", + "\u020a": "i", + "\u020b": "i", + "\u020c": "o", + "\u020d": "o", + "\u020e": "o", + "\u020f": "o", + "\u0210": "r", + "\u0211": "r", + "\u0212": "r", + "\u0213": "r", + "\u0214": "u", + "\u0215": "u", + "\u0216": "u", + "\u0217": "u", + "\u0218": "s", + "\u0219": "s", + "\u021a": "t", + "\u021b": "t", + "\u021e": "h", + "\u021f": "h", + "\u0220": "n", + "\u0221": "d", + "\u0224": "z", + "\u0225": "z", + "\u0226": "a", + "\u0227": "a", + "\u0228": "e", + "\u0229": "e", + "\u022a": "o", + "\u022b": "o", + "\u022c": "o", + "\u022d": "o", + "\u022e": "o", + "\u022f": "o", + "\u0230": "o", + "\u0231": "o", + "\u0232": "y", + "\u0233": "y", + "\u0234": "l", + "\u0235": "n", + "\u0236": "t", + "\u023a": "a", + "\u023b": "c", + "\u023c": "c", + "\u023d": "l", + "\u023e": "t", + "\u023f": "s", + "\u0240": "z", + "\u0243": "b", + "\u0246": "e", + "\u0247": "e", + "\u0248": "j", + "\u0249": "j", + "\u024b": "q", + "\u024c": "r", + "\u024d": "r", + "\u024e": "y", + "\u024f": "y", + "\u0253": "b", + "\u0255": "c", + "\u0256": "d", + "\u0257": "d", + "\u0260": "g", + "\u0266": "h", + "\u0268": "i", + "\u026b": "l", + "\u026c": "l", + "\u026d": "l", + "\u0271": "m", + "\u0272": "n", + "\u0273": "n", + "\u027c": "r", + "\u027d": "r", + "\u027e": "r", + "\u0282": "s", + "\u0288": "t", + "\u028b": "v", + "\u0290": "z", + "\u0291": "z", + "\u029d": "j", + "\u02a0": "q", + "\u0386": "\u03b1", + "\u0388": "\u03b5", + "\u0389": "\u03b7", + "\u038a": "\u03b9", + "\u038c": "\u03bf", + "\u038e": "\u03c5", + "\u038f": "\u03c9", + "\u0390": "\u03b9", + "\u03aa": "\u03b9", + "\u03ab": "\u03c5", + "\u03ac": "\u03b1", + "\u03ad": "\u03b5", + "\u03ae": "\u03b7", + "\u03af": "\u03b9", + "\u03b0": "\u03c5", + "\u03ca": "\u03b9", + "\u03cb": "\u03c5", + "\u03cc": "\u03bf", + "\u03cd": "\u03c5", + "\u03ce": "\u03c9", + "\u03d3": "\u03d2", + "\u03d4": "\u03d2", + "\u0400": "\u0435", + "\u0401": "\u0435", + "\u0403": "\u0433", + "\u0407": "\u0456", + "\u040c": "\u043a", + "\u040d": "\u0438", + "\u040e": "\u0443", + "\u0419": "\u0438", + "\u0439": "\u0438", + "\u0450": "\u0435", + "\u0451": "\u0435", + "\u0453": "\u0433", + "\u0457": "\u0456", + "\u045c": "\u043a", + "\u045d": "\u0438", + "\u045e": "\u0443", + "\u0476": "\u0475", + "\u0477": "\u0475", + "\u04c1": "\u0436", + "\u04c2": "\u0436", + "\u04d0": "\u0430", + "\u04d1": "\u0430", + "\u04d2": "\u0430", + "\u04d3": "\u0430", + "\u04d6": "\u0435", + "\u04d7": "\u0435", + "\u04da": "\u04d9", + "\u04db": "\u04d9", + "\u04dc": "\u0436", + "\u04dd": "\u0436", + "\u04de": "\u0437", + "\u04df": "\u0437", + "\u04e2": "\u0438", + "\u04e3": "\u0438", + "\u04e4": "\u0438", + "\u04e5": "\u0438", + "\u04e6": "\u043e", + "\u04e7": "\u043e", + "\u04ea": "\u04e9", + "\u04eb": "\u04e9", + "\u04ec": "\u044d", + "\u04ed": "\u044d", + "\u04ee": "\u0443", + "\u04ef": "\u0443", + "\u04f0": "\u0443", + "\u04f1": "\u0443", + "\u04f2": "\u0443", + "\u04f3": "\u0443", + "\u04f4": "\u0447", + "\u04f5": "\u0447", + "\u04f8": "\u044b", + "\u04f9": "\u044b", + "\u0622": "\u0627", + "\u0623": "\u0627", + "\u0624": "\u0648", + "\u0625": "\u0627", + "\u0626": "\u064a", + "\u06c0": "\u06d5", + "\u06c2": "\u06c1", + "\u06d3": "\u06d2", + "\u0929": "\u0928", + "\u0931": "\u0930", + "\u0934": "\u0933", + "\u0958": "\u0915", + "\u0959": "\u0916", + "\u095a": "\u0917", + "\u095b": "\u091c", + "\u095c": "\u0921", + "\u095d": "\u0922", + "\u095e": "\u092b", + "\u095f": "\u092f", + "\u09dc": "\u09a1", + "\u09dd": "\u09a2", + "\u09df": "\u09af", + "\u0a33": "\u0a32", + "\u0a36": "\u0a38", + "\u0a59": "\u0a16", + "\u0a5a": "\u0a17", + "\u0a5b": "\u0a1c", + "\u0a5e": "\u0a2b", + "\u0b5c": "\u0b21", + "\u0b5d": "\u0b22", + "\u0f43": "\u0f42", + "\u0f4d": "\u0f4c", + "\u0f52": "\u0f51", + "\u0f57": "\u0f56", + "\u0f5c": "\u0f5b", + "\u0f69": "\u0f40", + "\u1026": "\u1025", + "\u1d6c": "b", + "\u1d6d": "d", + "\u1d6e": "f", + "\u1d6f": "m", + "\u1d70": "n", + "\u1d71": "p", + "\u1d72": "r", + "\u1d73": "r", + "\u1d74": "s", + "\u1d75": "t", + "\u1d76": "z", + "\u1d7d": "p", + "\u1d80": "b", + "\u1d81": "d", + "\u1d82": "f", + "\u1d83": "g", + "\u1d84": "k", + "\u1d85": "l", + "\u1d86": "m", + "\u1d87": "n", + "\u1d88": "p", + "\u1d89": "r", + "\u1d8a": "s", + "\u1d8c": "v", + "\u1d8d": "x", + "\u1d8e": "z", + "\u1d8f": "a", + "\u1d91": "d", + "\u1d92": "e", + "\u1d96": "i", + "\u1d99": "u", + "\u1e00": "a", + "\u1e01": "a", + "\u1e02": "b", + "\u1e03": "b", + "\u1e04": "b", + "\u1e05": "b", + "\u1e06": "b", + "\u1e07": "b", + "\u1e08": "c", + "\u1e09": "c", + "\u1e0a": "d", + "\u1e0b": "d", + "\u1e0c": "d", + "\u1e0d": "d", + "\u1e0e": "d", + "\u1e0f": "d", + "\u1e10": "d", + "\u1e11": "d", + "\u1e12": "d", + "\u1e13": "d", + "\u1e14": "e", + "\u1e15": "e", + "\u1e16": "e", + "\u1e17": "e", + "\u1e18": "e", + "\u1e19": "e", + "\u1e1a": "e", + "\u1e1b": "e", + "\u1e1c": "e", + "\u1e1d": "e", + "\u1e1e": "f", + "\u1e1f": "f", + "\u1e20": "g", + "\u1e21": "g", + "\u1e22": "h", + "\u1e23": "h", + "\u1e24": "h", + "\u1e25": "h", + "\u1e26": "h", + "\u1e27": "h", + "\u1e28": "h", + "\u1e29": "h", + "\u1e2a": "h", + "\u1e2b": "h", + "\u1e2c": "i", + "\u1e2d": "i", + "\u1e2e": "i", + "\u1e2f": "i", + "\u1e30": "k", + "\u1e31": "k", + "\u1e32": "k", + "\u1e33": "k", + "\u1e34": "k", + "\u1e35": "k", + "\u1e36": "l", + "\u1e37": "l", + "\u1e38": "l", + "\u1e39": "l", + "\u1e3a": "l", + "\u1e3b": "l", + "\u1e3c": "l", + "\u1e3d": "l", + "\u1e3e": "m", + "\u1e3f": "m", + "\u1e40": "m", + "\u1e41": "m", + "\u1e42": "m", + "\u1e43": "m", + "\u1e44": "n", + "\u1e45": "n", + "\u1e46": "n", + "\u1e47": "n", + "\u1e48": "n", + "\u1e49": "n", + "\u1e4a": "n", + "\u1e4b": "n", + "\u1e4c": "o", + "\u1e4d": "o", + "\u1e4e": "o", + "\u1e4f": "o", + "\u1e50": "o", + "\u1e51": "o", + "\u1e52": "o", + "\u1e53": "o", + "\u1e54": "p", + "\u1e55": "p", + "\u1e56": "p", + "\u1e57": "p", + "\u1e58": "r", + "\u1e59": "r", + "\u1e5a": "r", + "\u1e5b": "r", + "\u1e5c": "r", + "\u1e5d": "r", + "\u1e5e": "r", + "\u1e5f": "r", + "\u1e60": "s", + "\u1e61": "s", + "\u1e62": "s", + "\u1e63": "s", + "\u1e64": "s", + "\u1e65": "s", + "\u1e66": "s", + "\u1e67": "s", + "\u1e68": "s", + "\u1e69": "s", + "\u1e6a": "t", + "\u1e6b": "t", + "\u1e6c": "t", + "\u1e6d": "t", + "\u1e6e": "t", + "\u1e6f": "t", + "\u1e70": "t", + "\u1e71": "t", + "\u1e72": "u", + "\u1e73": "u", + "\u1e74": "u", + "\u1e75": "u", + "\u1e76": "u", + "\u1e77": "u", + "\u1e78": "u", + "\u1e79": "u", + "\u1e7a": "u", + "\u1e7b": "u", + "\u1e7c": "v", + "\u1e7d": "v", + "\u1e7e": "v", + "\u1e7f": "v", + "\u1e80": "w", + "\u1e81": "w", + "\u1e82": "w", + "\u1e83": "w", + "\u1e84": "w", + "\u1e85": "w", + "\u1e86": "w", + "\u1e87": "w", + "\u1e88": "w", + "\u1e89": "w", + "\u1e8a": "x", + "\u1e8b": "x", + "\u1e8c": "x", + "\u1e8d": "x", + "\u1e8e": "y", + "\u1e8f": "y", + "\u1e90": "z", + "\u1e91": "z", + "\u1e92": "z", + "\u1e93": "z", + "\u1e94": "z", + "\u1e95": "z", + "\u1e96": "h", + "\u1e97": "t", + "\u1e98": "w", + "\u1e99": "y", + "\u1e9a": "a", + "\u1e9b": "\u017f", + "\u1e9e": "ss", + "\u1ea0": "a", + "\u1ea1": "a", + "\u1ea2": "a", + "\u1ea3": "a", + "\u1ea4": "a", + "\u1ea5": "a", + "\u1ea6": "a", + "\u1ea7": "a", + "\u1ea8": "a", + "\u1ea9": "a", + "\u1eaa": "a", + "\u1eab": "a", + "\u1eac": "a", + "\u1ead": "a", + "\u1eae": "a", + "\u1eaf": "a", + "\u1eb0": "a", + "\u1eb1": "a", + "\u1eb2": "a", + "\u1eb3": "a", + "\u1eb4": "a", + "\u1eb5": "a", + "\u1eb6": "a", + "\u1eb7": "a", + "\u1eb8": "e", + "\u1eb9": "e", + "\u1eba": "e", + "\u1ebb": "e", + "\u1ebc": "e", + "\u1ebd": "e", + "\u1ebe": "e", + "\u1ebf": "e", + "\u1ec0": "e", + "\u1ec1": "e", + "\u1ec2": "e", + "\u1ec3": "e", + "\u1ec4": "e", + "\u1ec5": "e", + "\u1ec6": "e", + "\u1ec7": "e", + "\u1ec8": "i", + "\u1ec9": "i", + "\u1eca": "i", + "\u1ecb": "i", + "\u1ecc": "o", + "\u1ecd": "o", + "\u1ece": "o", + "\u1ecf": "o", + "\u1ed0": "o", + "\u1ed1": "o", + "\u1ed2": "o", + "\u1ed3": "o", + "\u1ed4": "o", + "\u1ed5": "o", + "\u1ed6": "o", + "\u1ed7": "o", + "\u1ed8": "o", + "\u1ed9": "o", + "\u1eda": "o", + "\u1edb": "o", + "\u1edc": "o", + "\u1edd": "o", + "\u1ede": "o", + "\u1edf": "o", + "\u1ee0": "o", + "\u1ee1": "o", + "\u1ee2": "o", + "\u1ee3": "o", + "\u1ee4": "u", + "\u1ee5": "u", + "\u1ee6": "u", + "\u1ee7": "u", + "\u1ee8": "u", + "\u1ee9": "u", + "\u1eea": "u", + "\u1eeb": "u", + "\u1eec": "u", + "\u1eed": "u", + "\u1eee": "u", + "\u1eef": "u", + "\u1ef0": "u", + "\u1ef1": "u", + "\u1ef2": "y", + "\u1ef3": "y", + "\u1ef4": "y", + "\u1ef5": "y", + "\u1ef6": "y", + "\u1ef7": "y", + "\u1ef8": "y", + "\u1ef9": "y", + "\u1efe": "y", + "\u1eff": "y", + "\u1f00": "\u03b1", + "\u1f01": "\u03b1", + "\u1f02": "\u03b1", + "\u1f03": "\u03b1", + "\u1f04": "\u03b1", + "\u1f05": "\u03b1", + "\u1f06": "\u03b1", + "\u1f07": "\u03b1", + "\u1f08": "\u03b1", + "\u1f09": "\u03b1", + "\u1f0a": "\u03b1", + "\u1f0b": "\u03b1", + "\u1f0c": "\u03b1", + "\u1f0d": "\u03b1", + "\u1f0e": "\u03b1", + "\u1f0f": "\u03b1", + "\u1f10": "\u03b5", + "\u1f11": "\u03b5", + "\u1f12": "\u03b5", + "\u1f13": "\u03b5", + "\u1f14": "\u03b5", + "\u1f15": "\u03b5", + "\u1f18": "\u03b5", + "\u1f19": "\u03b5", + "\u1f1a": "\u03b5", + "\u1f1b": "\u03b5", + "\u1f1c": "\u03b5", + "\u1f1d": "\u03b5", + "\u1f20": "\u03b7", + "\u1f21": "\u03b7", + "\u1f22": "\u03b7", + "\u1f23": "\u03b7", + "\u1f24": "\u03b7", + "\u1f25": "\u03b7", + "\u1f26": "\u03b7", + "\u1f27": "\u03b7", + "\u1f28": "\u03b7", + "\u1f29": "\u03b7", + "\u1f2a": "\u03b7", + "\u1f2b": "\u03b7", + "\u1f2c": "\u03b7", + "\u1f2d": "\u03b7", + "\u1f2e": "\u03b7", + "\u1f2f": "\u03b7", + "\u1f30": "\u03b9", + "\u1f31": "\u03b9", + "\u1f32": "\u03b9", + "\u1f33": "\u03b9", + "\u1f34": "\u03b9", + "\u1f35": "\u03b9", + "\u1f36": "\u03b9", + "\u1f37": "\u03b9", + "\u1f38": "\u03b9", + "\u1f39": "\u03b9", + "\u1f3a": "\u03b9", + "\u1f3b": "\u03b9", + "\u1f3c": "\u03b9", + "\u1f3d": "\u03b9", + "\u1f3e": "\u03b9", + "\u1f3f": "\u03b9", + "\u1f40": "\u03bf", + "\u1f41": "\u03bf", + "\u1f42": "\u03bf", + "\u1f43": "\u03bf", + "\u1f44": "\u03bf", + "\u1f45": "\u03bf", + "\u1f48": "\u03bf", + "\u1f49": "\u03bf", + "\u1f4a": "\u03bf", + "\u1f4b": "\u03bf", + "\u1f4c": "\u03bf", + "\u1f4d": "\u03bf", + "\u1f50": "\u03c5", + "\u1f51": "\u03c5", + "\u1f52": "\u03c5", + "\u1f53": "\u03c5", + "\u1f54": "\u03c5", + "\u1f55": "\u03c5", + "\u1f56": "\u03c5", + "\u1f57": "\u03c5", + "\u1f59": "\u03c5", + "\u1f5b": "\u03c5", + "\u1f5d": "\u03c5", + "\u1f5f": "\u03c5", + "\u1f60": "\u03c9", + "\u1f61": "\u03c9", + "\u1f62": "\u03c9", + "\u1f63": "\u03c9", + "\u1f64": "\u03c9", + "\u1f65": "\u03c9", + "\u1f66": "\u03c9", + "\u1f67": "\u03c9", + "\u1f68": "\u03c9", + "\u1f69": "\u03c9", + "\u1f6a": "\u03c9", + "\u1f6b": "\u03c9", + "\u1f6c": "\u03c9", + "\u1f6d": "\u03c9", + "\u1f6e": "\u03c9", + "\u1f6f": "\u03c9", + "\u1f70": "\u03b1", + "\u1f71": "\u03b1", + "\u1f72": "\u03b5", + "\u1f73": "\u03b5", + "\u1f74": "\u03b7", + "\u1f75": "\u03b7", + "\u1f76": "\u03b9", + "\u1f77": "\u03b9", + "\u1f78": "\u03bf", + "\u1f79": "\u03bf", + "\u1f7a": "\u03c5", + "\u1f7b": "\u03c5", + "\u1f7c": "\u03c9", + "\u1f7d": "\u03c9", + "\u1f80": "\u03b1", + "\u1f81": "\u03b1", + "\u1f82": "\u03b1", + "\u1f83": "\u03b1", + "\u1f84": "\u03b1", + "\u1f85": "\u03b1", + "\u1f86": "\u03b1", + "\u1f87": "\u03b1", + "\u1f88": "\u03b1", + "\u1f89": "\u03b1", + "\u1f8a": "\u03b1", + "\u1f8b": "\u03b1", + "\u1f8c": "\u03b1", + "\u1f8d": "\u03b1", + "\u1f8e": "\u03b1", + "\u1f8f": "\u03b1", + "\u1f90": "\u03b7", + "\u1f91": "\u03b7", + "\u1f92": "\u03b7", + "\u1f93": "\u03b7", + "\u1f94": "\u03b7", + "\u1f95": "\u03b7", + "\u1f96": "\u03b7", + "\u1f97": "\u03b7", + "\u1f98": "\u03b7", + "\u1f99": "\u03b7", + "\u1f9a": "\u03b7", + "\u1f9b": "\u03b7", + "\u1f9c": "\u03b7", + "\u1f9d": "\u03b7", + "\u1f9e": "\u03b7", + "\u1f9f": "\u03b7", + "\u1fa0": "\u03c9", + "\u1fa1": "\u03c9", + "\u1fa2": "\u03c9", + "\u1fa3": "\u03c9", + "\u1fa4": "\u03c9", + "\u1fa5": "\u03c9", + "\u1fa6": "\u03c9", + "\u1fa7": "\u03c9", + "\u1fa8": "\u03c9", + "\u1fa9": "\u03c9", + "\u1faa": "\u03c9", + "\u1fab": "\u03c9", + "\u1fac": "\u03c9", + "\u1fad": "\u03c9", + "\u1fae": "\u03c9", + "\u1faf": "\u03c9", + "\u1fb0": "\u03b1", + "\u1fb1": "\u03b1", + "\u1fb2": "\u03b1", + "\u1fb3": "\u03b1", + "\u1fb4": "\u03b1", + "\u1fb6": "\u03b1", + "\u1fb7": "\u03b1", + "\u1fb8": "\u03b1", + "\u1fb9": "\u03b1", + "\u1fba": "\u03b1", + "\u1fbb": "\u03b1", + "\u1fbc": "\u03b1", + "\u1fc2": "\u03b7", + "\u1fc3": "\u03b7", + "\u1fc4": "\u03b7", + "\u1fc6": "\u03b7", + "\u1fc7": "\u03b7", + "\u1fc8": "\u03b5", + "\u1fc9": "\u03b5", + "\u1fca": "\u03b7", + "\u1fcb": "\u03b7", + "\u1fcc": "\u03b7", + "\u1fd0": "\u03b9", + "\u1fd1": "\u03b9", + "\u1fd2": "\u03b9", + "\u1fd3": "\u03b9", + "\u1fd6": "\u03b9", + "\u1fd7": "\u03b9", + "\u1fd8": "\u03b9", + "\u1fd9": "\u03b9", + "\u1fda": "\u03b9", + "\u1fdb": "\u03b9", + "\u1fe0": "\u03c5", + "\u1fe1": "\u03c5", + "\u1fe2": "\u03c5", + "\u1fe3": "\u03c5", + "\u1fe4": "\u03c1", + "\u1fe5": "\u03c1", + "\u1fe6": "\u03c5", + "\u1fe7": "\u03c5", + "\u1fe8": "\u03c5", + "\u1fe9": "\u03c5", + "\u1fea": "\u03c5", + "\u1feb": "\u03c5", + "\u1fec": "\u03c1", + "\u1ff2": "\u03c9", + "\u1ff3": "\u03c9", + "\u1ff4": "\u03c9", + "\u1ff6": "\u03c9", + "\u1ff7": "\u03c9", + "\u1ff8": "\u03bf", + "\u1ff9": "\u03bf", + "\u1ffa": "\u03c9", + "\u1ffb": "\u03c9", + "\u1ffc": "\u03c9", + "\u212b": "a", + "\u2c60": "l", + "\u2c61": "l", + "\u2c62": "l", + "\u2c63": "p", + "\u2c64": "r", + "\u2c65": "a", + "\u2c66": "t", + "\u2c67": "h", + "\u2c68": "h", + "\u2c69": "k", + "\u2c6a": "k", + "\u2c6b": "z", + "\u2c6c": "z", + "\u2c6e": "m", + "\u2c71": "v", + "\u2c72": "w", + "\u2c73": "w", + "\u2c74": "v", + "\u2c78": "e", + "\u2c7a": "o", + "\u2c7e": "s", + "\u2c7f": "z", + "\u304c": "\u304b", + "\u304e": "\u304d", + "\u3050": "\u304f", + "\u3052": "\u3051", + "\u3054": "\u3053", + "\u3056": "\u3055", + "\u3058": "\u3057", + "\u305a": "\u3059", + "\u305c": "\u305b", + "\u305e": "\u305d", + "\u3060": "\u305f", + "\u3062": "\u3061", + "\u3065": "\u3064", + "\u3067": "\u3066", + "\u3069": "\u3068", + "\u3070": "\u306f", + "\u3071": "\u306f", + "\u3073": "\u3072", + "\u3074": "\u3072", + "\u3076": "\u3075", + "\u3077": "\u3075", + "\u3079": "\u3078", + "\u307a": "\u3078", + "\u307c": "\u307b", + "\u307d": "\u307b", + "\u3094": "\u3046", + "\u309e": "\u309d", + "\u30ac": "\u30ab", + "\u30ae": "\u30ad", + "\u30b0": "\u30af", + "\u30b2": "\u30b1", + "\u30b4": "\u30b3", + "\u30b6": "\u30b5", + "\u30b8": "\u30b7", + "\u30ba": "\u30b9", + "\u30bc": "\u30bb", + "\u30be": "\u30bd", + "\u30c0": "\u30bf", + "\u30c2": "\u30c1", + "\u30c5": "\u30c4", + "\u30c7": "\u30c6", + "\u30c9": "\u30c8", + "\u30d0": "\u30cf", + "\u30d1": "\u30cf", + "\u30d3": "\u30d2", + "\u30d4": "\u30d2", + "\u30d6": "\u30d5", + "\u30d7": "\u30d5", + "\u30d9": "\u30d8", + "\u30da": "\u30d8", + "\u30dc": "\u30db", + "\u30dd": "\u30db", + "\u30f4": "\u30a6", + "\u30f7": "\u30ef", + "\u30f8": "\u30f0", + "\u30f9": "\u30f1", + "\u30fa": "\u30f2", + "\u30fe": "\u30fd", + "\ua740": "k", + "\ua741": "k", + "\ua742": "k", + "\ua743": "k", + "\ua744": "k", + "\ua745": "k", + "\ua748": "l", + "\ua749": "l", + "\ua74a": "o", + "\ua74b": "o", + "\ua74c": "o", + "\ua74d": "o", + "\ua750": "p", + "\ua751": "p", + "\ua752": "p", + "\ua753": "p", + "\ua754": "p", + "\ua755": "p", + "\ua756": "q", + "\ua757": "q", + "\ua758": "q", + "\ua759": "q", + "\ua75e": "v", + "\ua75f": "v", + "\ua78e": "l", + "\ua790": "n", + "\ua791": "n", + "\ua792": "c", + "\ua793": "c", + "\ua794": "c", + "\ua795": "h", + "\ua796": "b", + "\ua797": "b", + "\ua798": "f", + "\ua799": "f", + "\ua7a0": "g", + "\ua7a1": "g", + "\ua7a2": "k", + "\ua7a3": "k", + "\ua7a4": "n", + "\ua7a5": "n", + "\ua7a6": "r", + "\ua7a7": "r", + "\ua7a8": "s", + "\ua7a9": "s", + "\ua7aa": "h", + "\ua7ad": "l", + "\ua7b2": "j", + "\ua7b8": "u", + "\ua7b9": "u", + "\ua7c4": "c", + "\ua7c5": "s", + "\ua7c6": "z", + "\ua7c7": "d", + "\ua7c8": "d", + "\ua7c9": "s", + "\ua7ca": "s", + "\uab34": "e", + "\uab37": "l", + "\uab38": "l", + "\uab39": "l", + "\uab3a": "m", + "\uab3b": "n", + "\uab49": "r", + "\uab4e": "u", + "\uab52": "u", + "\uab56": "x", + "\uab57": "x", + "\uab58": "x", + "\uab59": "x", + "\uab5a": "y", + "\ufb1d": "\u05d9", + "\ufb1f": "\u05f2", + "\ufb2a": "\u05e9", + "\ufb2b": "\u05e9", + "\ufb2c": "\u05e9", + "\ufb2d": "\u05e9", + "\ufb2e": "\u05d0", + "\ufb2f": "\u05d0", + "\ufb30": "\u05d0", + "\ufb31": "\u05d1", + "\ufb32": "\u05d2", + "\ufb33": "\u05d3", + "\ufb34": "\u05d4", + "\ufb35": "\u05d5", + "\ufb36": "\u05d6", + "\ufb38": "\u05d8", + "\ufb39": "\u05d9", + "\ufb3a": "\u05da", + "\ufb3b": "\u05db", + "\ufb3c": "\u05dc", + "\ufb3e": "\u05de", + "\ufb40": "\u05e0", + "\ufb41": "\u05e1", + "\ufb43": "\u05e3", + "\ufb44": "\u05e4", + "\ufb46": "\u05e6", + "\ufb47": "\u05e7", + "\ufb48": "\u05e8", + "\ufb49": "\u05e9", + "\ufb4a": "\u05ea", + "\ufb4b": "\u05d5", + "\ufb4c": "\u05d1", + "\ufb4d": "\u05db", + "\ufb4e": "\u05e4", + "\ud804\udc9a": "\ud804\udc99", + "\ud804\udc9c": "\ud804\udc9b", + "\ud804\udcab": "\ud804\udca5", + "\ud837\udf09": "t", + "\ud837\udf11": "l", + "\ud837\udf13": "l", + "\ud837\udf16": "r", + "\ud837\udf1a": "i", + "\ud837\udf1b": "o", + "\ud837\udf1d": "c", + "\ud837\udf1e": "s" +} diff --git a/web/src/dinners/names.ts b/web/src/dinners/names.ts index b5afe11..52cc50d 100644 --- a/web/src/dinners/names.ts +++ b/web/src/dinners/names.ts @@ -3,37 +3,17 @@ // API would refuse a name as a duplicate. The server still decides. // internal/ai/harness/testdata/dedupe_cases.json is run against both. +import folds from './letterFolds.json' + // SIMILAR_THRESHOLD is harness.SimilarThreshold. const SIMILAR_THRESHOLD = 0.85 // fillerWords is harness's fillerWords. const fillerWords = new Set(['easy', 'best', 'homemade', 'quick', 'simple', 'the', 'a', 'recipe', 'classic']) -// letterFolds are the letters harness folds that canonical decomposition -// leaves whole: its ligatures and extra letters, and the common stroked -// Latin letters. Letters with accents fold by decomposing and dropping the -// marks, as harness's table does; rarer stroked letters are compared as -// they are. -const letterFolds: Record = { - ß: 'ss', - æ: 'ae', - œ: 'oe', - þ: 'th', - ð: 'd', - ı: 'i', - ʼn: 'n', - ij: 'ij', - ø: 'o', - đ: 'd', - ħ: 'h', - ŀ: 'l', - ł: 'l', - ŧ: 't', - ƀ: 'b', - ƶ: 'z', - ǥ: 'g', - ɨ: 'i', -} +// letterFolds is harness's letterFolds: gen_folds.py writes both tables, +// and harness's TestLetterFoldsMatchPWA fails when they differ. +const letterFolds: Record = folds const combiningMark = /\p{Mn}/u const wordChar = /[\p{L}\p{Nd}]/u @@ -44,12 +24,16 @@ const wordChar = /[\p{L}\p{Nd}]/u // "chicken tikka masala". export function normalizeName(name: string): string { let out = '' - for (const ch of name.toLowerCase().normalize('NFD')) { - if (combiningMark.test(ch) || ch === "'" || ch === '’') continue - const fold = letterFolds[ch] - if (fold !== undefined) out += fold - else if (wordChar.test(ch)) out += ch - else out += ' ' + // Lowercased letter by letter, as Go's strings.ToLower does, so no + // letter is lowered by its context (a final Σ is σ, not ς). + for (const letter of name) { + for (const ch of letter.toLowerCase()) { + if (combiningMark.test(ch) || ch === "'" || ch === '’') continue + const fold = letterFolds[ch] + if (fold !== undefined) out += fold + else if (wordChar.test(ch)) out += ch + else out += ' ' + } } return out .split(' ')