Fold names by Unicode decomposition in Normalize

Replace the hand-written accent map with letterFolds, a table generated
offline by gen_folds.py from Python's unicodedata: every code point whose
canonical decomposition contains combining marks maps to its base letters,
plus stroked Latin letters and ligatures. Combining marks are still dropped,
so composed and decomposed spellings ("Phở" and "Phở") now
normalize equal. No new dependency.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

docs/ARCHITECTURE.md

@@ -165,8 +165,10 @@ automatic** (D-021).
   character budget trims the oldest plans first, then the oldest passes;
   Regulars and the Quest log are never trimmed.
 - **Name dedupe** (`harness.Normalize`, `harness.Similar`, `harness.Deduper`):
-  a dinner name is normalized by lowercasing, folding accents, stripping
-  punctuation, dropping filler words (easy, best, homemade, quick, simple,
+  a dinner name is normalized by lowercasing, dropping combining marks and
+  folding precomposed letters to their base letters through a table generated
+  from Unicode data (`gen_folds.py`), so composed and decomposed spellings
+  match, stripping punctuation, dropping filler words (easy, best, homemade, quick, simple,
   the, a, recipe, classic) and collapsing whitespace. Two names are the same
   dinner when the token-set (Jaccard) similarity of their words is at least
   0.85. A `Deduper` over the Regulars, Quest log, pending suggestions and

internal/ai/harness/dedupe.go

@@ -16,25 +16,12 @@ var fillerWords = map[string]bool{
 	"the": true, "a": true, "recipe": true, "classic": true,
 }
 
-// accentFolds maps each lowercase accented Latin letter to its plain form.
-var accentFolds = func() map[rune]string {
-	m := map[rune]string{'ß': "ss", 'æ': "ae", 'œ': "oe", 'þ': "th", 'ð': "d"}
-	for plain, accented := range map[string]string{
-		"a": "àáâãäåāăą", "c": "çćĉċč", "d": "ďđ", "e": "èéêëēĕėęě",
-		"g": "ĝğġģ", "h": "ĥħ", "i": "ìíîïĩīĭįı", "j": "ĵ", "k": "ķ",
-		"l": "ĺļľŀł", "n": "ñńņňʼn", "o": "òóôõöøōŏő", "r": "ŕŗř",
-		"s": "śŝşšș", "t": "ţťŧț", "u": "ùúûüũūŭůűų", "w": "ŵ", "y": "ýÿŷ",
-		"z": "źżž",
-	} {
-		for _, r := range accented {
-			m[r] = plain
-		}
-	}
-	return m
-}()
+//go:generate python3 gen_folds.py
 
 // Normalize returns the comparable form of a dinner name: lowercased,
-// accents folded to plain letters, apostrophes removed, other punctuation
+// combining marks (Unicode category Mn) dropped, precomposed and stroked
+// letters folded to their base letters (see letterFolds) so composed and
+// decomposed spellings are equal, apostrophes removed, other punctuation
 // turned into spaces, filler words (see fillerWords) dropped and whitespace
 // collapsed to single spaces. "Easy Chicken Tikka-Masala!" becomes
 // "chicken tikka masala".
@@ -44,8 +31,8 @@ func Normalize(name string) string {
 		switch {
 		case unicode.Is(unicode.Mn, r), r == '\'', r == '’':
 			// Combining accents and apostrophes join their neighbours.
-		case accentFolds[r] != "":
-			b.WriteString(accentFolds[r])
+		case letterFolds[r] != "":
+			b.WriteString(letterFolds[r])
 		case unicode.IsLetter(r), unicode.IsDigit(r):
 			b.WriteRune(r)
 		default:

internal/ai/harness/dedupe_test.go

@@ -22,8 +22,13 @@ func TestNormalize(t *testing.T) {
 		{"Cre\u0300me bru\u0302le\u0301e", "creme brulee"}, // decomposed accents
 		{"İskender Kebab", "iskender kebab"},
 		{"Pho 24", "pho 24"},
+		{"Phở Bò", "pho bo"},
+		{"Bún Chả Hà Nội", "bun cha ha noi"},
+		{"Łosoś z Koperkiem", "losos z koperkiem"},
+		{"Ćevapčići", "cevapcici"},
 		{"Easy Recipe", ""},
 		{"", ""},
+		{"!?-- & ...", ""}, // punctuation only
 	}
 	for _, tt := range tests {
 		if got := Normalize(tt.name); got != tt.want {
@@ -32,6 +37,30 @@ func TestNormalize(t *testing.T) {
 	}
 }
 
+// TestNormalizeCanonicalEquivalence checks that a composed spelling and its
+// canonically equivalent decomposed spelling normalize equal and are Similar.
+func TestNormalizeCanonicalEquivalence(t *testing.T) {
+	tests := []struct {
+		composed, decomposed, want string
+	}{
+		{"Phở", "Pho\u031b\u0309", "pho"}, // o + horn + hook above
+		{"Bánh Xèo", "Ba\u0301nh Xe\u0300o", "banh xeo"},
+		{"Cơm Tấm", "Co\u031bm Ta\u0302\u0301m", "com tam"},
+		{"Ẓaʿtar Ȟash", "Z\u0323aʿtar H\u030cash", "zaʿtar hash"},
+		{"Smǫrrebrød", "Smo\u0328rrebrød", "smorrebrod"},
+	}
+	for _, tt := range tests {
+		c, d := Normalize(tt.composed), Normalize(tt.decomposed)
+		if c != tt.want || d != tt.want {
+			t.Errorf("Normalize(%q) = %q, Normalize(%+q) = %q, want both %q",
+				tt.composed, c, tt.decomposed, d, tt.want)
+		}
+		if !Similar(tt.composed, tt.decomposed) {
+			t.Errorf("Similar(%q, %+q) = false, want true", tt.composed, tt.decomposed)
+		}
+	}
+}
+
 func TestSimilar(t *testing.T) {
 	tests := []struct {
 		a, b string
@@ -44,6 +73,8 @@ func TestSimilar(t *testing.T) {
 		{"Crème Brûlée", "creme brulee", true},
 		{"Jalapeño Poppers", "JALAPENO POPPERS", true},
 		{"Smørrebrød", "Smorrebrod", true},
+		{"Phở", "Pho\u031b\u0309", true},
+		{"Phở", "Pho", true},
 		{"Chicken Tikka Masala", "Chicken Tikka Masala Curry", false}, // 3/4
 		{"Tacos", "Fish tacos", false},
 		{"Pad Thai", "Thai Green Curry", false},

internal/ai/harness/gen_folds.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+"""Generate folds_table.go, the letter-fold table used by harness.Normalize.
+
+Run from this directory with `go generate ./internal/ai/harness` (or
+`python3 gen_folds.py`; gofmt must be on PATH). It uses only the Python
+standard library's unicodedata, so it works offline; the Unicode version it
+was generated from is recorded in the output.
+
+The table maps a code point to its lowercased base letters:
+
+  - every code point with a canonical decomposition that includes a
+    combining mark (category Mn) maps to its full canonical decomposition
+    with the marks removed, so a precomposed letter folds exactly like its
+    decomposed spelling once Normalize drops the marks (ở -> o, ǖ -> u).
+    Code points whose remaining base is not letters or digits (spacing
+    accents such as U+1FEE) are left out; Normalize treats both spellings
+    of those as punctuation;
+  - Latin letters with no decomposition whose name is "LATIN ... LETTER X
+    WITH ..." map to x (ø -> o, ł -> l, đ -> d);
+  - the Latin ligatures and letters listed in EXTRA map as given.
+
+Hangul syllables are left out: their decomposition is into conjoining jamo,
+not combining marks.
+"""
+
+import re
+import subprocess
+import sys
+import unicodedata
+
+EXTRA = {
+    "ß": "ss", "ẞ": "ss", "æ": "ae", "Æ": "ae", "œ": "oe", "Œ": "oe",
+    "þ": "th", "Þ": "th", "ð": "d", "Ð": "d", "ı": "i", "ʼn": "n",
+    "ij": "ij", "IJ": "ij",
+}
+
+WITH = re.compile(r"^LATIN (?:SMALL|CAPITAL) LETTER ([A-Z]) WITH ")
+
+
+def is_word(ch):
+    """Reports whether Normalize keeps ch: a letter or a decimal digit."""
+    cat = unicodedata.category(ch)
+    return cat.startswith("L") or cat == "Nd"
+
+
+def full_decomposition(ch):
+    return unicodedata.normalize("NFD", ch)
+
+
+def fold(cp):
+    ch = chr(cp)
+    if ch in EXTRA:
+        return EXTRA[ch]
+    if 0xAC00 <= cp <= 0xD7A3:
+        return None
+    nfd = full_decomposition(ch)
+    if nfd != ch:
+        if not any(unicodedata.category(c) == "Mn" for c in nfd):
+            return None
+        base = "".join(c for c in nfd if unicodedata.category(c) != "Mn")
+        if not base or not all(is_word(c) for c in base):
+            return None
+        return base.lower()
+    m = WITH.match(unicodedata.name(ch, ""))
+    if m and unicodedata.category(ch) in ("Ll", "Lu", "Lt"):
+        return m.group(1).lower()
+    return None
+
+
+def main():
+    entries = []
+    for cp in range(sys.maxunicode + 1):
+        folded = fold(cp)
+        if folded is not None:
+            entries.append((cp, folded))
+    out = [
+        "// Code generated by gen_folds.py from Unicode "
+        + unicodedata.unidata_version + "; DO NOT EDIT.",
+        "",
+        "package harness",
+        "",
+        "// letterFolds maps a precomposed or stroked letter to its lowercased",
+        "// base letters. See gen_folds.py for how it is built.",
+        "var letterFolds = map[rune]string{",
+    ]
+    for cp, folded in entries:
+        escaped = "".join(
+            c if 0x20 <= ord(c) < 0x7F and c not in '"\\' else "\\u%04x" % ord(c)
+            if ord(c) <= 0xFFFF else "\\U%08x" % ord(c)
+            for c in folded
+        )
+        out.append('\t0x%04X: "%s", // %s' % (cp, escaped, unicodedata.name(chr(cp))))
+    out.append("}")
+    with open("folds_table.go", "w", encoding="utf-8") as f:
+        f.write("\n".join(out) + "\n")
+    subprocess.run(["gofmt", "-w", "folds_table.go"], check=True)
+
+
+if __name__ == "__main__":
+    main()