Name normalization and dedupe (#59)

Co-authored-by: dev_agent <dev_agent@cubecraftlabs.com>

internal/ai/harness/dedupe.go

@@ -0,0 +1,118 @@
+package harness
+
+import (
+	"strings"
+	"unicode"
+)
+
+// SimilarThreshold is the token-set similarity at or above which two dinner
+// names are the same dinner.
+const SimilarThreshold = 0.85
+
+// fillerWords carry no meaning in a dinner name and are dropped by
+// Normalize.
+var fillerWords = map[string]bool{
+	"easy": true, "best": true, "homemade": true, "quick": true, "simple": true,
+	"the": true, "a": true, "recipe": true, "classic": true,
+}
+
+//go:generate python3 gen_folds.py
+
+// Normalize returns the comparable form of a dinner name: lowercased,
+// combining marks (Unicode category Mn) dropped, precomposed and stroked
+// letters folded to their base letters (see letterFolds) so composed and
+// decomposed spellings are equal, apostrophes removed, other punctuation
+// turned into spaces, filler words (see fillerWords) dropped and whitespace
+// collapsed to single spaces. "Easy Chicken Tikka-Masala!" becomes
+// "chicken tikka masala".
+func Normalize(name string) string {
+	var b strings.Builder
+	for _, r := range strings.ToLower(name) {
+		switch {
+		case unicode.Is(unicode.Mn, r), r == '\'', r == '’':
+			// Combining accents and apostrophes join their neighbours.
+		case letterFolds[r] != "":
+			b.WriteString(letterFolds[r])
+		case unicode.IsLetter(r), unicode.IsDigit(r):
+			b.WriteRune(r)
+		default:
+			b.WriteRune(' ')
+		}
+	}
+	words := strings.Fields(b.String())
+	kept := words[:0]
+	for _, w := range words {
+		if !fillerWords[w] {
+			kept = append(kept, w)
+		}
+	}
+	return strings.Join(kept, " ")
+}
+
+// Similar reports whether a and b name the same dinner: the token-set
+// similarity of their normalized forms is at least SimilarThreshold. A name
+// with no words left after normalizing is similar to nothing.
+func Similar(a, b string) bool {
+	return similarity(tokenSet(a), tokenSet(b)) >= SimilarThreshold
+}
+
+// tokenSet returns the distinct words of name's normalized form.
+func tokenSet(name string) map[string]bool {
+	set := map[string]bool{}
+	for _, w := range strings.Fields(Normalize(name)) {
+		set[w] = true
+	}
+	return set
+}
+
+// similarity is the Jaccard index of a and b: shared words over all words.
+// It is 0 when either set is empty.
+func similarity(a, b map[string]bool) float64 {
+	if len(a) == 0 || len(b) == 0 {
+		return 0
+	}
+	shared := 0
+	for w := range a {
+		if b[w] {
+			shared++
+		}
+	}
+	return float64(shared) / float64(len(a)+len(b)-shared)
+}
+
+// DedupeSources are the dinner names a new suggestion must not repeat.
+type DedupeSources struct {
+	Regulars []string
+	QuestLog []string
+	// Pending holds the names of suggestions not yet tried or passed.
+	Pending []string
+	// Declined holds the names of passed suggestions.
+	Declined []string
+}
+
+// Deduper answers whether a candidate dinner name repeats a known one.
+type Deduper struct {
+	known []map[string]bool
+}
+
+// NewDeduper returns a Deduper over every name in src.
+func NewDeduper(src DedupeSources) *Deduper {
+	d := &Deduper{}
+	for _, names := range [][]string{src.Regulars, src.QuestLog, src.Pending, src.Declined} {
+		for _, name := range names {
+			d.known = append(d.known, tokenSet(name))
+		}
+	}
+	return d
+}
+
+// IsDuplicate reports whether name is Similar to any known name.
+func (d *Deduper) IsDuplicate(name string) bool {
+	candidate := tokenSet(name)
+	for _, k := range d.known {
+		if similarity(candidate, k) >= SimilarThreshold {
+			return true
+		}
+	}
+	return false
+}

internal/ai/harness/dedupe_test.go

@@ -0,0 +1,151 @@
+package harness
+
+import (
+	"testing"
+	"unicode"
+)
+
+func TestNormalize(t *testing.T) {
+	tests := []struct {
+		name, want string
+	}{
+		{"Easy Chicken Tikka Masala", "chicken tikka masala"},
+		{"  The   BEST homemade\tLasagna  ", "lasagna"},
+		{"A Quick, Simple Classic Chili Recipe", "chili"},
+		{"Tacos (beef)", "tacos beef"},
+		{"Mac-and-Cheese!", "mac and cheese"},
+		{"Shepherd's Pie", "shepherds pie"},
+		{"Shepherd’s Pie", "shepherds pie"},
+		{"Crème Brûlée", "creme brulee"},
+		{"Jalapeño Poppers", "jalapeno poppers"},
+		{"Paella à la Valenciana", "paella la valenciana"}, // "à" folds to the filler "a"
+		{"Smørrebrød", "smorrebrod"},
+		{"Bœuf Bourguignon", "boeuf bourguignon"},
+		{"Käsespätzle mit Weißwurst", "kasespatzle mit weisswurst"},
+		{"Cre\u0300me bru\u0302le\u0301e", "creme brulee"}, // decomposed accents
+		{"İskender Kebab", "iskender kebab"},
+		{"Pho 24", "pho 24"},
+		{"Phở Bò", "pho bo"},
+		{"Bún Chả Hà Nội", "bun cha ha noi"},
+		{"Łosoś z Koperkiem", "losos z koperkiem"},
+		{"Ćevapčići", "cevapcici"},
+		{"Easy Recipe", ""},
+		{"", ""},
+		{"!?-- & ...", ""}, // punctuation only
+	}
+	for _, tt := range tests {
+		if got := Normalize(tt.name); got != tt.want {
+			t.Errorf("Normalize(%q) = %q, want %q", tt.name, got, tt.want)
+		}
+	}
+}
+
+// TestNormalizeCanonicalEquivalence checks that a composed spelling and its
+// canonically equivalent decomposed spelling normalize equal and are Similar.
+func TestNormalizeCanonicalEquivalence(t *testing.T) {
+	tests := []struct {
+		composed, decomposed, want string
+	}{
+		{"Phở", "Pho\u031b\u0309", "pho"}, // o + horn + hook above
+		{"Bánh Xèo", "Ba\u0301nh Xe\u0300o", "banh xeo"},
+		{"Cơm Tấm", "Co\u031bm Ta\u0302\u0301m", "com tam"},
+		{"Ẓaʿtar Ȟash", "Z\u0323aʿtar H\u030cash", "zaʿtar hash"},
+		{"Smǫrrebrød", "Smo\u0328rrebrød", "smorrebrod"},
+		{"Smǿrrebrǿd", "Smø\u0301rrebrø\u0301d", "smorrebrod"}, // ǿ folds through ø
+		{"Ǽbleskiver", "Æ\u0301bleskiver", "aebleskiver"},      // ǽ folds through æ
+	}
+	for _, tt := range tests {
+		c, d := Normalize(tt.composed), Normalize(tt.decomposed)
+		if c != tt.want || d != tt.want {
+			t.Errorf("Normalize(%q) = %q, Normalize(%+q) = %q, want both %q",
+				tt.composed, c, tt.decomposed, d, tt.want)
+		}
+		if !Similar(tt.composed, tt.decomposed) {
+			t.Errorf("Similar(%q, %+q) = false, want true", tt.composed, tt.decomposed)
+		}
+		dedupe := NewDeduper(DedupeSources{Regulars: []string{tt.decomposed}})
+		if !dedupe.IsDuplicate(tt.composed) {
+			t.Errorf("IsDuplicate(%q) over %+q = false, want true", tt.composed, tt.decomposed)
+		}
+	}
+}
+
+// TestLetterFoldsFullyReduced checks that every fold result is final:
+// Normalize writes a result without folding it again, so a result that
+// still holds a foldable letter, a combining mark or an upper-case letter
+// would split canonically equivalent spellings (ǿ -> ø instead of o).
+func TestLetterFoldsFullyReduced(t *testing.T) {
+	for r, folded := range letterFolds {
+		for _, c := range folded {
+			if letterFolds[c] != "" || unicode.Is(unicode.Mn, c) || unicode.ToLower(c) != c {
+				t.Errorf("letterFolds[%U] = %q holds %U, which Normalize would change", r, folded, c)
+			}
+		}
+	}
+}
+
+func TestSimilar(t *testing.T) {
+	tests := []struct {
+		a, b string
+		want bool
+	}{
+		{"Easy Chicken Tikka Masala", "chicken tikka masala", true},
+		{"Beef Tacos", "Tacos (beef)", true},
+		{"Chicken tacos", "Fish tacos", false},
+		{"Spaghetti Bolognese", "spaghetti bolognese recipe", true},
+		{"Crème Brûlée", "creme brulee", true},
+		{"Jalapeño Poppers", "JALAPENO POPPERS", true},
+		{"Smørrebrød", "Smorrebrod", true},
+		{"Phở", "Pho\u031b\u0309", true},
+		{"Phở", "Pho", true},
+		{"Chicken Tikka Masala", "Chicken Tikka Masala Curry", false}, // 3/4
+		{"Tacos", "Fish tacos", false},
+		{"Pad Thai", "Thai Green Curry", false},
+		{"Easy Recipe", "The Classic", false}, // nothing left to compare
+		{"Recipe", "Chili", false},
+	}
+	for _, tt := range tests {
+		if got := Similar(tt.a, tt.b); got != tt.want {
+			t.Errorf("Similar(%q, %q) = %v, want %v", tt.a, tt.b, got, tt.want)
+		}
+		if got := Similar(tt.b, tt.a); got != tt.want {
+			t.Errorf("Similar(%q, %q) = %v, want %v", tt.b, tt.a, got, tt.want)
+		}
+	}
+}
+
+func TestDeduper(t *testing.T) {
+	d := NewDeduper(DedupeSources{
+		Regulars: []string{"Beef Tacos", "Spaghetti Bolognese"},
+		QuestLog: []string{"Chicken Tikka Masala"},
+		Pending:  []string{"Crème Brûlée"},
+		Declined: []string{"Shepherd's Pie"},
+	})
+	tests := []struct {
+		name string
+		want bool
+	}{
+		{"Tacos (beef)", true},                  // Regular
+		{"spaghetti bolognese recipe", true},    // Regular
+		{"Easy Chicken Tikka Masala", true},     // Quest log
+		{"creme brulee", true},                  // pending suggestion
+		{"The Best Shepherds Pie", true},        // declined
+		{"Fish tacos", false},                   // shares only "tacos"
+		{"Chicken Tikka Masala Curry", false},   // below the threshold
+		{"Jalapeño Poppers", false},             // new
+		{"Easy Recipe", false},                  // nothing left to compare
+		{"Spaghetti Carbonara", false},          // shares only "spaghetti"
+		{"Bolognese Spaghetti (Classic)", true}, // word order does not matter
+	}
+	for _, tt := range tests {
+		if got := d.IsDuplicate(tt.name); got != tt.want {
+			t.Errorf("IsDuplicate(%q) = %v, want %v", tt.name, got, tt.want)
+		}
+	}
+}
+
+func TestDeduperEmpty(t *testing.T) {
+	if NewDeduper(DedupeSources{}).IsDuplicate("Beef Tacos") {
+		t.Error("an empty Deduper reported a duplicate")
+	}
+}

internal/ai/harness/gen_folds.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+"""Generate folds_table.go, the letter-fold table used by harness.Normalize.
+
+Run from this directory with `go generate ./internal/ai/harness` (or
+`python3 gen_folds.py`; gofmt must be on PATH). It uses only the Python
+standard library's unicodedata, so it works offline; the Unicode version it
+was generated from is recorded in the output.
+
+The table maps a code point to its lowercased base letters:
+
+  - every code point with a canonical decomposition that includes a
+    combining mark (category Mn) maps to its full canonical decomposition
+    with the marks removed, so a precomposed letter folds exactly like its
+    decomposed spelling once Normalize drops the marks (ở -> o, ǖ -> u).
+    Code points whose remaining base is not letters or digits (spacing
+    accents such as U+1FEE) are left out; Normalize treats both spellings
+    of those as punctuation;
+  - Latin letters with no decomposition whose name is "LATIN ... LETTER X
+    WITH ..." map to x (ø -> o, ł -> l, đ -> d);
+  - the Latin ligatures and letters listed in EXTRA map as given.
+
+Each result is then folded again, letter by letter, until nothing changes
+(ǿ -> ø -> o, ǽ -> æ -> ae). Normalize writes a result without folding it
+again, so the table must already be fully reduced; the generator fails if a
+result does not settle.
+
+Hangul syllables are left out: their decomposition is into conjoining jamo,
+not combining marks.
+"""
+
+import re
+import subprocess
+import sys
+import unicodedata
+
+EXTRA = {
+    "ß": "ss", "ẞ": "ss", "æ": "ae", "Æ": "ae", "œ": "oe", "Œ": "oe",
+    "þ": "th", "Þ": "th", "ð": "d", "Ð": "d", "ı": "i", "ʼn": "n",
+    "ij": "ij", "IJ": "ij",
+}
+
+WITH = re.compile(r"^LATIN (?:SMALL|CAPITAL) LETTER ([A-Z]) WITH ")
+
+
+def is_word(ch):
+    """Reports whether Normalize keeps ch: a letter or a decimal digit."""
+    cat = unicodedata.category(ch)
+    return cat.startswith("L") or cat == "Nd"
+
+
+def full_decomposition(ch):
+    return unicodedata.normalize("NFD", ch)
+
+
+def fold(cp):
+    ch = chr(cp)
+    if ch in EXTRA:
+        return EXTRA[ch]
+    if 0xAC00 <= cp <= 0xD7A3:
+        return None
+    nfd = full_decomposition(ch)
+    if nfd != ch:
+        if not any(unicodedata.category(c) == "Mn" for c in nfd):
+            return None
+        base = "".join(c for c in nfd if unicodedata.category(c) != "Mn")
+        if not base or not all(is_word(c) for c in base):
+            return None
+        return base.lower()
+    m = WITH.match(unicodedata.name(ch, ""))
+    if m and unicodedata.category(ch) in ("Ll", "Lu", "Lt"):
+        return m.group(1).lower()
+    return None
+
+
+def reduce(folds, s):
+    """Folds every letter of s until nothing changes."""
+    for _ in range(len(folds)):
+        nxt = "".join(folds.get(c, c) for c in s)
+        if nxt == s:
+            return s
+        s = nxt
+    sys.exit("gen_folds.py: fold of %r does not settle" % s)
+
+
+def main():
+    folds = {}
+    for cp in range(sys.maxunicode + 1):
+        folded = fold(cp)
+        if folded is not None:
+            folds[chr(cp)] = folded
+    entries = [(ord(ch), reduce(folds, folded)) for ch, folded in folds.items()]
+    out = [
+        "// Code generated by gen_folds.py from Unicode "
+        + unicodedata.unidata_version + "; DO NOT EDIT.",
+        "",
+        "package harness",
+        "",
+        "// letterFolds maps a precomposed or stroked letter to its lowercased",
+        "// base letters. See gen_folds.py for how it is built.",
+        "var letterFolds = map[rune]string{",
+    ]
+    for cp, folded in entries:
+        escaped = "".join(
+            c if 0x20 <= ord(c) < 0x7F and c not in '"\\' else "\\u%04x" % ord(c)
+            if ord(c) <= 0xFFFF else "\\U%08x" % ord(c)
+            for c in folded
+        )
+        out.append('\t0x%04X: "%s", // %s' % (cp, escaped, unicodedata.name(chr(cp))))
+    out.append("}")
+    with open("folds_table.go", "w", encoding="utf-8") as f:
+        f.write("\n".join(out) + "\n")
+    subprocess.run(["gofmt", "-w", "folds_table.go"], check=True)
+
+
+if __name__ == "__main__":
+    main()