docs/ARCHITECTURE.md

@@ -165,8 +165,10 @@ automatic** (D-021).
   character budget trims the oldest plans first, then the oldest passes;
   Regulars and the Quest log are never trimmed.
 - **Name dedupe** (`harness.Normalize`, `harness.Similar`, `harness.Deduper`):
-  a dinner name is normalized by lowercasing, folding accents, stripping
+  a dinner name is normalized by lowercasing, dropping combining marks and
-  punctuation, dropping filler words (easy, best, homemade, quick, simple,
+  folding precomposed letters to their base letters through a table generated
+  from Unicode data (`gen_folds.py`), so composed and decomposed spellings
+  match, stripping punctuation, dropping filler words (easy, best, homemade, quick, simple,
   the, a, recipe, classic) and collapsing whitespace. Two names are the same
   dinner when the token-set (Jaccard) similarity of their words is at least
   0.85. A `Deduper` over the Regulars, Quest log, pending suggestions and

internal/ai/harness/dedupe.go

@@ -16,25 +16,12 @@ var fillerWords = map[string]bool{
 	"the": true, "a": true, "recipe": true, "classic": true,
 }
 
-// accentFolds maps each lowercase accented Latin letter to its plain form.
+//go:generate python3 gen_folds.py
-var accentFolds = func() map[rune]string {
-	m := map[rune]string{'ß': "ss", 'æ': "ae", 'œ': "oe", 'þ': "th", 'ð': "d"}
-	for plain, accented := range map[string]string{
-		"a": "àáâãäåāăą", "c": "çćĉċč", "d": "ďđ", "e": "èéêëēĕėęě",
-		"g": "ĝğġģ", "h": "ĥħ", "i": "ìíîïĩīĭįı", "j": "ĵ", "k": "ķ",
-		"l": "ĺļľŀł", "n": "ñńņňʼn", "o": "òóôõöøōŏő", "r": "ŕŗř",
-		"s": "śŝşšș", "t": "ţťŧț", "u": "ùúûüũūŭůűų", "w": "ŵ", "y": "ýÿŷ",
-		"z": "źżž",
-	} {
-		for _, r := range accented {
-			m[r] = plain
-		}
-	}
-	return m
-}()
 
 // Normalize returns the comparable form of a dinner name: lowercased,
-// accents folded to plain letters, apostrophes removed, other punctuation
+// combining marks (Unicode category Mn) dropped, precomposed and stroked
+// letters folded to their base letters (see letterFolds) so composed and
+// decomposed spellings are equal, apostrophes removed, other punctuation
 // turned into spaces, filler words (see fillerWords) dropped and whitespace
 // collapsed to single spaces. "Easy Chicken Tikka-Masala!" becomes
 // "chicken tikka masala".
@@ -44,8 +31,8 @@ func Normalize(name string) string {
 		switch {
 		case unicode.Is(unicode.Mn, r), r == '\'', r == '’':
 			// Combining accents and apostrophes join their neighbours.
-		case accentFolds[r] != "":
+		case letterFolds[r] != "":
-			b.WriteString(accentFolds[r])
+			b.WriteString(letterFolds[r])
 		case unicode.IsLetter(r), unicode.IsDigit(r):
 			b.WriteRune(r)
 		default:

internal/ai/harness/dedupe_test.go

@@ -22,8 +22,13 @@ func TestNormalize(t *testing.T) {
 		{"Cre\u0300me bru\u0302le\u0301e", "creme brulee"}, // decomposed accents
 		{"İskender Kebab", "iskender kebab"},
 		{"Pho 24", "pho 24"},
+		{"Phở Bò", "pho bo"},
+		{"Bún Chả Hà Nội", "bun cha ha noi"},
+		{"Łosoś z Koperkiem", "losos z koperkiem"},
+		{"Ćevapčići", "cevapcici"},
 		{"Easy Recipe", ""},
 		{"", ""},
+		{"!?-- & ...", ""}, // punctuation only
 	}
 	for _, tt := range tests {
 		if got := Normalize(tt.name); got != tt.want {
@@ -32,6 +37,30 @@ func TestNormalize(t *testing.T) {
 	}
 }
 
+// TestNormalizeCanonicalEquivalence checks that a composed spelling and its
+// canonically equivalent decomposed spelling normalize equal and are Similar.
+func TestNormalizeCanonicalEquivalence(t *testing.T) {
+	tests := []struct {
+		composed, decomposed, want string
+	}{
+		{"Phở", "Pho\u031b\u0309", "pho"}, // o + horn + hook above
+		{"Bánh Xèo", "Ba\u0301nh Xe\u0300o", "banh xeo"},
+		{"Cơm Tấm", "Co\u031bm Ta\u0302\u0301m", "com tam"},
+		{"Ẓaʿtar Ȟash", "Z\u0323aʿtar H\u030cash", "zaʿtar hash"},
+		{"Smǫrrebrød", "Smo\u0328rrebrød", "smorrebrod"},
+	}
+	for _, tt := range tests {
+		c, d := Normalize(tt.composed), Normalize(tt.decomposed)
+		if c != tt.want || d != tt.want {
+			t.Errorf("Normalize(%q) = %q, Normalize(%+q) = %q, want both %q",
+				tt.composed, c, tt.decomposed, d, tt.want)
+		}
+		if !Similar(tt.composed, tt.decomposed) {
+			t.Errorf("Similar(%q, %+q) = false, want true", tt.composed, tt.decomposed)
+		}
+	}
+}
+
 func TestSimilar(t *testing.T) {
 	tests := []struct {
 		a, b string
@@ -44,6 +73,8 @@ func TestSimilar(t *testing.T) {
 		{"Crème Brûlée", "creme brulee", true},
 		{"Jalapeño Poppers", "JALAPENO POPPERS", true},
 		{"Smørrebrød", "Smorrebrod", true},
+		{"Phở", "Pho\u031b\u0309", true},
+		{"Phở", "Pho", true},
 		{"Chicken Tikka Masala", "Chicken Tikka Masala Curry", false}, // 3/4
 		{"Tacos", "Fish tacos", false},
 		{"Pad Thai", "Thai Green Curry", false},

internal/ai/harness/gen_folds.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+"""Generate folds_table.go, the letter-fold table used by harness.Normalize.
+
+Run from this directory with `go generate ./internal/ai/harness` (or
+`python3 gen_folds.py`; gofmt must be on PATH). It uses only the Python
+standard library's unicodedata, so it works offline; the Unicode version it
+was generated from is recorded in the output.
+
+The table maps a code point to its lowercased base letters:
+
+  - every code point with a canonical decomposition that includes a
+    combining mark (category Mn) maps to its full canonical decomposition
+    with the marks removed, so a precomposed letter folds exactly like its
+    decomposed spelling once Normalize drops the marks (ở -> o, ǖ -> u).
+    Code points whose remaining base is not letters or digits (spacing
+    accents such as U+1FEE) are left out; Normalize treats both spellings
+    of those as punctuation;
+  - Latin letters with no decomposition whose name is "LATIN ... LETTER X
+    WITH ..." map to x (ø -> o, ł -> l, đ -> d);
+  - the Latin ligatures and letters listed in EXTRA map as given.
+
+Hangul syllables are left out: their decomposition is into conjoining jamo,
+not combining marks.
+"""
+
+import re
+import subprocess
+import sys
+import unicodedata
+
+EXTRA = {
+    "ß": "ss", "ẞ": "ss", "æ": "ae", "Æ": "ae", "œ": "oe", "Œ": "oe",
+    "þ": "th", "Þ": "th", "ð": "d", "Ð": "d", "ı": "i", "ʼn": "n",
+    "ij": "ij", "IJ": "ij",
+}
+
+WITH = re.compile(r"^LATIN (?:SMALL|CAPITAL) LETTER ([A-Z]) WITH ")
+
+
+def is_word(ch):
+    """Reports whether Normalize keeps ch: a letter or a decimal digit."""
+    cat = unicodedata.category(ch)
+    return cat.startswith("L") or cat == "Nd"
+
+
+def full_decomposition(ch):
+    return unicodedata.normalize("NFD", ch)
+
+
+def fold(cp):
+    ch = chr(cp)
+    if ch in EXTRA:
+        return EXTRA[ch]
+    if 0xAC00 <= cp <= 0xD7A3:
+        return None
+    nfd = full_decomposition(ch)
+    if nfd != ch:
+        if not any(unicodedata.category(c) == "Mn" for c in nfd):
+            return None
+        base = "".join(c for c in nfd if unicodedata.category(c) != "Mn")
+        if not base or not all(is_word(c) for c in base):
+            return None
+        return base.lower()
+    m = WITH.match(unicodedata.name(ch, ""))
+    if m and unicodedata.category(ch) in ("Ll", "Lu", "Lt"):
+        return m.group(1).lower()
+    return None
+
+
+def main():
+    entries = []
+    for cp in range(sys.maxunicode + 1):
+        folded = fold(cp)
+        if folded is not None:
+            entries.append((cp, folded))
+    out = [
+        "// Code generated by gen_folds.py from Unicode "
+        + unicodedata.unidata_version + "; DO NOT EDIT.",
+        "",
+        "package harness",
+        "",
+        "// letterFolds maps a precomposed or stroked letter to its lowercased",
+        "// base letters. See gen_folds.py for how it is built.",
+        "var letterFolds = map[rune]string{",
+    ]
+    for cp, folded in entries:
+        escaped = "".join(
+            c if 0x20 <= ord(c) < 0x7F and c not in '"\\' else "\\u%04x" % ord(c)
+            if ord(c) <= 0xFFFF else "\\U%08x" % ord(c)
+            for c in folded
+        )
+        out.append('\t0x%04X: "%s", // %s' % (cp, escaped, unicodedata.name(chr(cp))))
+    out.append("}")
+    with open("folds_table.go", "w", encoding="utf-8") as f:
+        f.write("\n".join(out) + "\n")
+    subprocess.run(["gofmt", "-w", "folds_table.go"], check=True)
+
+
+if __name__ == "__main__":
+    main()