diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index fd4107b..5ebc491 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -164,6 +164,13 @@ automatic** (D-021). last 60 passed Side quests and the plans of the last four ISO weeks. A character budget trims the oldest plans first, then the oldest passes; Regulars and the Quest log are never trimmed. +- **Name dedupe** (`harness.Normalize`, `harness.Similar`, `harness.Deduper`): + a dinner name is normalized by lowercasing, folding accents, stripping + punctuation, dropping filler words (easy, best, homemade, quick, simple, + the, a, recipe, classic) and collapsing whitespace. Two names are the same + dinner when the token-set (Jaccard) similarity of their words is at least + 0.85. A `Deduper` over the Regulars, Quest log, pending suggestions and + passed names rejects a suggestion that repeats any of them. ## Recipe grounding diff --git a/internal/ai/harness/dedupe.go b/internal/ai/harness/dedupe.go new file mode 100644 index 0000000..408d3af --- /dev/null +++ b/internal/ai/harness/dedupe.go @@ -0,0 +1,131 @@ +package harness + +import ( + "strings" + "unicode" +) + +// SimilarThreshold is the token-set similarity at or above which two dinner +// names are the same dinner. +const SimilarThreshold = 0.85 + +// fillerWords carry no meaning in a dinner name and are dropped by +// Normalize. +var fillerWords = map[string]bool{ + "easy": true, "best": true, "homemade": true, "quick": true, "simple": true, + "the": true, "a": true, "recipe": true, "classic": true, +} + +// accentFolds maps each lowercase accented Latin letter to its plain form. +var accentFolds = func() map[rune]string { + m := map[rune]string{'ß': "ss", 'æ': "ae", 'œ': "oe", 'þ': "th", 'ð': "d"} + for plain, accented := range map[string]string{ + "a": "àáâãäåāăą", "c": "çćĉċč", "d": "ďđ", "e": "èéêëēĕėęě", + "g": "ĝğġģ", "h": "ĥħ", "i": "ìíîïĩīĭįı", "j": "ĵ", "k": "ķ", + "l": "ĺļľŀł", "n": "ñńņňʼn", "o": "òóôõöøōŏő", "r": "ŕŗř", + "s": "śŝşšș", "t": "ţťŧț", "u": "ùúûüũūŭůűų", "w": "ŵ", "y": "ýÿŷ", + "z": "źżž", + } { + for _, r := range accented { + m[r] = plain + } + } + return m +}() + +// Normalize returns the comparable form of a dinner name: lowercased, +// accents folded to plain letters, apostrophes removed, other punctuation +// turned into spaces, filler words (see fillerWords) dropped and whitespace +// collapsed to single spaces. "Easy Chicken Tikka-Masala!" becomes +// "chicken tikka masala". +func Normalize(name string) string { + var b strings.Builder + for _, r := range strings.ToLower(name) { + switch { + case unicode.Is(unicode.Mn, r), r == '\'', r == '’': + // Combining accents and apostrophes join their neighbours. + case accentFolds[r] != "": + b.WriteString(accentFolds[r]) + case unicode.IsLetter(r), unicode.IsDigit(r): + b.WriteRune(r) + default: + b.WriteRune(' ') + } + } + words := strings.Fields(b.String()) + kept := words[:0] + for _, w := range words { + if !fillerWords[w] { + kept = append(kept, w) + } + } + return strings.Join(kept, " ") +} + +// Similar reports whether a and b name the same dinner: the token-set +// similarity of their normalized forms is at least SimilarThreshold. A name +// with no words left after normalizing is similar to nothing. +func Similar(a, b string) bool { + return similarity(tokenSet(a), tokenSet(b)) >= SimilarThreshold +} + +// tokenSet returns the distinct words of name's normalized form. +func tokenSet(name string) map[string]bool { + set := map[string]bool{} + for _, w := range strings.Fields(Normalize(name)) { + set[w] = true + } + return set +} + +// similarity is the Jaccard index of a and b: shared words over all words. +// It is 0 when either set is empty. +func similarity(a, b map[string]bool) float64 { + if len(a) == 0 || len(b) == 0 { + return 0 + } + shared := 0 + for w := range a { + if b[w] { + shared++ + } + } + return float64(shared) / float64(len(a)+len(b)-shared) +} + +// DedupeSources are the dinner names a new suggestion must not repeat. +type DedupeSources struct { + Regulars []string + QuestLog []string + // Pending holds the names of suggestions not yet tried or passed. + Pending []string + // Declined holds the names of passed suggestions. + Declined []string +} + +// Deduper answers whether a candidate dinner name repeats a known one. +type Deduper struct { + known []map[string]bool +} + +// NewDeduper returns a Deduper over every name in src. +func NewDeduper(src DedupeSources) *Deduper { + d := &Deduper{} + for _, names := range [][]string{src.Regulars, src.QuestLog, src.Pending, src.Declined} { + for _, name := range names { + d.known = append(d.known, tokenSet(name)) + } + } + return d +} + +// IsDuplicate reports whether name is Similar to any known name. +func (d *Deduper) IsDuplicate(name string) bool { + candidate := tokenSet(name) + for _, k := range d.known { + if similarity(candidate, k) >= SimilarThreshold { + return true + } + } + return false +} diff --git a/internal/ai/harness/dedupe_test.go b/internal/ai/harness/dedupe_test.go new file mode 100644 index 0000000..ba4aaf6 --- /dev/null +++ b/internal/ai/harness/dedupe_test.go @@ -0,0 +1,97 @@ +package harness + +import "testing" + +func TestNormalize(t *testing.T) { + tests := []struct { + name, want string + }{ + {"Easy Chicken Tikka Masala", "chicken tikka masala"}, + {" The BEST homemade\tLasagna ", "lasagna"}, + {"A Quick, Simple Classic Chili Recipe", "chili"}, + {"Tacos (beef)", "tacos beef"}, + {"Mac-and-Cheese!", "mac and cheese"}, + {"Shepherd's Pie", "shepherds pie"}, + {"Shepherd’s Pie", "shepherds pie"}, + {"Crème Brûlée", "creme brulee"}, + {"Jalapeño Poppers", "jalapeno poppers"}, + {"Paella à la Valenciana", "paella la valenciana"}, // "à" folds to the filler "a" + {"Smørrebrød", "smorrebrod"}, + {"Bœuf Bourguignon", "boeuf bourguignon"}, + {"Käsespätzle mit Weißwurst", "kasespatzle mit weisswurst"}, + {"Cre\u0300me bru\u0302le\u0301e", "creme brulee"}, // decomposed accents + {"İskender Kebab", "iskender kebab"}, + {"Pho 24", "pho 24"}, + {"Easy Recipe", ""}, + {"", ""}, + } + for _, tt := range tests { + if got := Normalize(tt.name); got != tt.want { + t.Errorf("Normalize(%q) = %q, want %q", tt.name, got, tt.want) + } + } +} + +func TestSimilar(t *testing.T) { + tests := []struct { + a, b string + want bool + }{ + {"Easy Chicken Tikka Masala", "chicken tikka masala", true}, + {"Beef Tacos", "Tacos (beef)", true}, + {"Chicken tacos", "Fish tacos", false}, + {"Spaghetti Bolognese", "spaghetti bolognese recipe", true}, + {"Crème Brûlée", "creme brulee", true}, + {"Jalapeño Poppers", "JALAPENO POPPERS", true}, + {"Smørrebrød", "Smorrebrod", true}, + {"Chicken Tikka Masala", "Chicken Tikka Masala Curry", false}, // 3/4 + {"Tacos", "Fish tacos", false}, + {"Pad Thai", "Thai Green Curry", false}, + {"Easy Recipe", "The Classic", false}, // nothing left to compare + {"Recipe", "Chili", false}, + } + for _, tt := range tests { + if got := Similar(tt.a, tt.b); got != tt.want { + t.Errorf("Similar(%q, %q) = %v, want %v", tt.a, tt.b, got, tt.want) + } + if got := Similar(tt.b, tt.a); got != tt.want { + t.Errorf("Similar(%q, %q) = %v, want %v", tt.b, tt.a, got, tt.want) + } + } +} + +func TestDeduper(t *testing.T) { + d := NewDeduper(DedupeSources{ + Regulars: []string{"Beef Tacos", "Spaghetti Bolognese"}, + QuestLog: []string{"Chicken Tikka Masala"}, + Pending: []string{"Crème Brûlée"}, + Declined: []string{"Shepherd's Pie"}, + }) + tests := []struct { + name string + want bool + }{ + {"Tacos (beef)", true}, // Regular + {"spaghetti bolognese recipe", true}, // Regular + {"Easy Chicken Tikka Masala", true}, // Quest log + {"creme brulee", true}, // pending suggestion + {"The Best Shepherds Pie", true}, // declined + {"Fish tacos", false}, // shares only "tacos" + {"Chicken Tikka Masala Curry", false}, // below the threshold + {"Jalapeño Poppers", false}, // new + {"Easy Recipe", false}, // nothing left to compare + {"Spaghetti Carbonara", false}, // shares only "spaghetti" + {"Bolognese Spaghetti (Classic)", true}, // word order does not matter + } + for _, tt := range tests { + if got := d.IsDuplicate(tt.name); got != tt.want { + t.Errorf("IsDuplicate(%q) = %v, want %v", tt.name, got, tt.want) + } + } +} + +func TestDeduperEmpty(t *testing.T) { + if NewDeduper(DedupeSources{}).IsDuplicate("Beef Tacos") { + t.Error("an empty Deduper reported a duplicate") + } +}