Name normalization and dedupe

Add harness.Normalize (lowercase, fold accents, strip punctuation, drop
filler words, collapse whitespace), harness.Similar (token-set Jaccard
similarity, threshold 0.85) and a Deduper built from the Regulars, Quest
log, pending suggestions and passed names that answers IsDuplicate.

Maestro task: f046ec6c-dbe2-4b12-bc95-3081c07714eb

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/ai/harness/dedupe.go

@@ -0,0 +1,131 @@
+package harness
+
+import (
+	"strings"
+	"unicode"
+)
+
+// SimilarThreshold is the token-set similarity at or above which two dinner
+// names are the same dinner.
+const SimilarThreshold = 0.85
+
+// fillerWords carry no meaning in a dinner name and are dropped by
+// Normalize.
+var fillerWords = map[string]bool{
+	"easy": true, "best": true, "homemade": true, "quick": true, "simple": true,
+	"the": true, "a": true, "recipe": true, "classic": true,
+}
+
+// accentFolds maps each lowercase accented Latin letter to its plain form.
+var accentFolds = func() map[rune]string {
+	m := map[rune]string{'ß': "ss", 'æ': "ae", 'œ': "oe", 'þ': "th", 'ð': "d"}
+	for plain, accented := range map[string]string{
+		"a": "àáâãäåāăą", "c": "çćĉċč", "d": "ďđ", "e": "èéêëēĕėęě",
+		"g": "ĝğġģ", "h": "ĥħ", "i": "ìíîïĩīĭįı", "j": "ĵ", "k": "ķ",
+		"l": "ĺļľŀł", "n": "ñńņňʼn", "o": "òóôõöøōŏő", "r": "ŕŗř",
+		"s": "śŝşšș", "t": "ţťŧț", "u": "ùúûüũūŭůűų", "w": "ŵ", "y": "ýÿŷ",
+		"z": "źżž",
+	} {
+		for _, r := range accented {
+			m[r] = plain
+		}
+	}
+	return m
+}()
+
+// Normalize returns the comparable form of a dinner name: lowercased,
+// accents folded to plain letters, apostrophes removed, other punctuation
+// turned into spaces, filler words (see fillerWords) dropped and whitespace
+// collapsed to single spaces. "Easy Chicken Tikka-Masala!" becomes
+// "chicken tikka masala".
+func Normalize(name string) string {
+	var b strings.Builder
+	for _, r := range strings.ToLower(name) {
+		switch {
+		case unicode.Is(unicode.Mn, r), r == '\'', r == '’':
+			// Combining accents and apostrophes join their neighbours.
+		case accentFolds[r] != "":
+			b.WriteString(accentFolds[r])
+		case unicode.IsLetter(r), unicode.IsDigit(r):
+			b.WriteRune(r)
+		default:
+			b.WriteRune(' ')
+		}
+	}
+	words := strings.Fields(b.String())
+	kept := words[:0]
+	for _, w := range words {
+		if !fillerWords[w] {
+			kept = append(kept, w)
+		}
+	}
+	return strings.Join(kept, " ")
+}
+
+// Similar reports whether a and b name the same dinner: the token-set
+// similarity of their normalized forms is at least SimilarThreshold. A name
+// with no words left after normalizing is similar to nothing.
+func Similar(a, b string) bool {
+	return similarity(tokenSet(a), tokenSet(b)) >= SimilarThreshold
+}
+
+// tokenSet returns the distinct words of name's normalized form.
+func tokenSet(name string) map[string]bool {
+	set := map[string]bool{}
+	for _, w := range strings.Fields(Normalize(name)) {
+		set[w] = true
+	}
+	return set
+}
+
+// similarity is the Jaccard index of a and b: shared words over all words.
+// It is 0 when either set is empty.
+func similarity(a, b map[string]bool) float64 {
+	if len(a) == 0 || len(b) == 0 {
+		return 0
+	}
+	shared := 0
+	for w := range a {
+		if b[w] {
+			shared++
+		}
+	}
+	return float64(shared) / float64(len(a)+len(b)-shared)
+}
+
+// DedupeSources are the dinner names a new suggestion must not repeat.
+type DedupeSources struct {
+	Regulars []string
+	QuestLog []string
+	// Pending holds the names of suggestions not yet tried or passed.
+	Pending []string
+	// Declined holds the names of passed suggestions.
+	Declined []string
+}
+
+// Deduper answers whether a candidate dinner name repeats a known one.
+type Deduper struct {
+	known []map[string]bool
+}
+
+// NewDeduper returns a Deduper over every name in src.
+func NewDeduper(src DedupeSources) *Deduper {
+	d := &Deduper{}
+	for _, names := range [][]string{src.Regulars, src.QuestLog, src.Pending, src.Declined} {
+		for _, name := range names {
+			d.known = append(d.known, tokenSet(name))
+		}
+	}
+	return d
+}
+
+// IsDuplicate reports whether name is Similar to any known name.
+func (d *Deduper) IsDuplicate(name string) bool {
+	candidate := tokenSet(name)
+	for _, k := range d.known {
+		if similarity(candidate, k) >= SimilarThreshold {
+			return true
+		}
+	}
+	return false
+}

internal/ai/harness/dedupe_test.go

@@ -0,0 +1,97 @@
+package harness
+
+import "testing"
+
+func TestNormalize(t *testing.T) {
+	tests := []struct {
+		name, want string
+	}{
+		{"Easy Chicken Tikka Masala", "chicken tikka masala"},
+		{"  The   BEST homemade\tLasagna  ", "lasagna"},
+		{"A Quick, Simple Classic Chili Recipe", "chili"},
+		{"Tacos (beef)", "tacos beef"},
+		{"Mac-and-Cheese!", "mac and cheese"},
+		{"Shepherd's Pie", "shepherds pie"},
+		{"Shepherd’s Pie", "shepherds pie"},
+		{"Crème Brûlée", "creme brulee"},
+		{"Jalapeño Poppers", "jalapeno poppers"},
+		{"Paella à la Valenciana", "paella la valenciana"}, // "à" folds to the filler "a"
+		{"Smørrebrød", "smorrebrod"},
+		{"Bœuf Bourguignon", "boeuf bourguignon"},
+		{"Käsespätzle mit Weißwurst", "kasespatzle mit weisswurst"},
+		{"Cre\u0300me bru\u0302le\u0301e", "creme brulee"}, // decomposed accents
+		{"İskender Kebab", "iskender kebab"},
+		{"Pho 24", "pho 24"},
+		{"Easy Recipe", ""},
+		{"", ""},
+	}
+	for _, tt := range tests {
+		if got := Normalize(tt.name); got != tt.want {
+			t.Errorf("Normalize(%q) = %q, want %q", tt.name, got, tt.want)
+		}
+	}
+}
+
+func TestSimilar(t *testing.T) {
+	tests := []struct {
+		a, b string
+		want bool
+	}{
+		{"Easy Chicken Tikka Masala", "chicken tikka masala", true},
+		{"Beef Tacos", "Tacos (beef)", true},
+		{"Chicken tacos", "Fish tacos", false},
+		{"Spaghetti Bolognese", "spaghetti bolognese recipe", true},
+		{"Crème Brûlée", "creme brulee", true},
+		{"Jalapeño Poppers", "JALAPENO POPPERS", true},
+		{"Smørrebrød", "Smorrebrod", true},
+		{"Chicken Tikka Masala", "Chicken Tikka Masala Curry", false}, // 3/4
+		{"Tacos", "Fish tacos", false},
+		{"Pad Thai", "Thai Green Curry", false},
+		{"Easy Recipe", "The Classic", false}, // nothing left to compare
+		{"Recipe", "Chili", false},
+	}
+	for _, tt := range tests {
+		if got := Similar(tt.a, tt.b); got != tt.want {
+			t.Errorf("Similar(%q, %q) = %v, want %v", tt.a, tt.b, got, tt.want)
+		}
+		if got := Similar(tt.b, tt.a); got != tt.want {
+			t.Errorf("Similar(%q, %q) = %v, want %v", tt.b, tt.a, got, tt.want)
+		}
+	}
+}
+
+func TestDeduper(t *testing.T) {
+	d := NewDeduper(DedupeSources{
+		Regulars: []string{"Beef Tacos", "Spaghetti Bolognese"},
+		QuestLog: []string{"Chicken Tikka Masala"},
+		Pending:  []string{"Crème Brûlée"},
+		Declined: []string{"Shepherd's Pie"},
+	})
+	tests := []struct {
+		name string
+		want bool
+	}{
+		{"Tacos (beef)", true},                  // Regular
+		{"spaghetti bolognese recipe", true},    // Regular
+		{"Easy Chicken Tikka Masala", true},     // Quest log
+		{"creme brulee", true},                  // pending suggestion
+		{"The Best Shepherds Pie", true},        // declined
+		{"Fish tacos", false},                   // shares only "tacos"
+		{"Chicken Tikka Masala Curry", false},   // below the threshold
+		{"Jalapeño Poppers", false},             // new
+		{"Easy Recipe", false},                  // nothing left to compare
+		{"Spaghetti Carbonara", false},          // shares only "spaghetti"
+		{"Bolognese Spaghetti (Classic)", true}, // word order does not matter
+	}
+	for _, tt := range tests {
+		if got := d.IsDuplicate(tt.name); got != tt.want {
+			t.Errorf("IsDuplicate(%q) = %v, want %v", tt.name, got, tt.want)
+		}
+	}
+}
+
+func TestDeduperEmpty(t *testing.T) {
+	if NewDeduper(DedupeSources{}).IsDuplicate("Beef Tacos") {
+		t.Error("an empty Deduper reported a duplicate")
+	}
+}