internal/ai/harness/dedupe_test.go

@@ -1,6 +1,9 @@
 package harness
 
-import "testing"
+import (
+	"testing"
+	"unicode"
+)
 
 func TestNormalize(t *testing.T) {
 	tests := []struct {
@@ -48,6 +51,8 @@ func TestNormalizeCanonicalEquivalence(t *testing.T) {
 		{"Cơm Tấm", "Co\u031bm Ta\u0302\u0301m", "com tam"},
 		{"Ẓaʿtar Ȟash", "Z\u0323aʿtar H\u030cash", "zaʿtar hash"},
 		{"Smǫrrebrød", "Smo\u0328rrebrød", "smorrebrod"},
+		{"Smǿrrebrǿd", "Smø\u0301rrebrø\u0301d", "smorrebrod"}, // ǿ folds through ø
+		{"Ǽbleskiver", "Æ\u0301bleskiver", "aebleskiver"},      // ǽ folds through æ
 	}
 	for _, tt := range tests {
 		c, d := Normalize(tt.composed), Normalize(tt.decomposed)
@@ -58,6 +63,24 @@ func TestNormalizeCanonicalEquivalence(t *testing.T) {
 		if !Similar(tt.composed, tt.decomposed) {
 			t.Errorf("Similar(%q, %+q) = false, want true", tt.composed, tt.decomposed)
 		}
+		dedupe := NewDeduper(DedupeSources{Regulars: []string{tt.decomposed}})
+		if !dedupe.IsDuplicate(tt.composed) {
+			t.Errorf("IsDuplicate(%q) over %+q = false, want true", tt.composed, tt.decomposed)
+		}
+	}
+}
+
+// TestLetterFoldsFullyReduced checks that every fold result is final:
+// Normalize writes a result without folding it again, so a result that
+// still holds a foldable letter, a combining mark or an upper-case letter
+// would split canonically equivalent spellings (ǿ -> ø instead of o).
+func TestLetterFoldsFullyReduced(t *testing.T) {
+	for r, folded := range letterFolds {
+		for _, c := range folded {
+			if letterFolds[c] != "" || unicode.Is(unicode.Mn, c) || unicode.ToLower(c) != c {
+				t.Errorf("letterFolds[%U] = %q holds %U, which Normalize would change", r, folded, c)
+			}
+		}
 	}
 }
 

internal/ai/harness/folds_table.go

@@ -248,8 +248,8 @@ var letterFolds = map[rune]string{
 	0x01DF:  "a",          // LATIN SMALL LETTER A WITH DIAERESIS AND MACRON
 	0x01E0:  "a",          // LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON
 	0x01E1:  "a",          // LATIN SMALL LETTER A WITH DOT ABOVE AND MACRON
-	0x01E2:  "\u00e6",     // LATIN CAPITAL LETTER AE WITH MACRON
-	0x01E3:  "\u00e6",     // LATIN SMALL LETTER AE WITH MACRON
+	0x01E2:  "ae",         // LATIN CAPITAL LETTER AE WITH MACRON
+	0x01E3:  "ae",         // LATIN SMALL LETTER AE WITH MACRON
 	0x01E4:  "g",          // LATIN CAPITAL LETTER G WITH STROKE
 	0x01E5:  "g",          // LATIN SMALL LETTER G WITH STROKE
 	0x01E6:  "g",          // LATIN CAPITAL LETTER G WITH CARON
@@ -270,10 +270,10 @@ var letterFolds = map[rune]string{
 	0x01F9:  "n",          // LATIN SMALL LETTER N WITH GRAVE
 	0x01FA:  "a",          // LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE
 	0x01FB:  "a",          // LATIN SMALL LETTER A WITH RING ABOVE AND ACUTE
-	0x01FC:  "\u00e6",     // LATIN CAPITAL LETTER AE WITH ACUTE
-	0x01FD:  "\u00e6",     // LATIN SMALL LETTER AE WITH ACUTE
-	0x01FE:  "\u00f8",     // LATIN CAPITAL LETTER O WITH STROKE AND ACUTE
-	0x01FF:  "\u00f8",     // LATIN SMALL LETTER O WITH STROKE AND ACUTE
+	0x01FC:  "ae",         // LATIN CAPITAL LETTER AE WITH ACUTE
+	0x01FD:  "ae",         // LATIN SMALL LETTER AE WITH ACUTE
+	0x01FE:  "o",          // LATIN CAPITAL LETTER O WITH STROKE AND ACUTE
+	0x01FF:  "o",          // LATIN SMALL LETTER O WITH STROKE AND ACUTE
 	0x0200:  "a",          // LATIN CAPITAL LETTER A WITH DOUBLE GRAVE
 	0x0201:  "a",          // LATIN SMALL LETTER A WITH DOUBLE GRAVE
 	0x0202:  "a",          // LATIN CAPITAL LETTER A WITH INVERTED BREVE

internal/ai/harness/gen_folds.py

@@ -19,6 +19,11 @@ The table maps a code point to its lowercased base letters:
     WITH ..." map to x (ø -> o, ł -> l, đ -> d);
   - the Latin ligatures and letters listed in EXTRA map as given.
 
+Each result is then folded again, letter by letter, until nothing changes
+(ǿ -> ø -> o, ǽ -> æ -> ae). Normalize writes a result without folding it
+again, so the table must already be fully reduced; the generator fails if a
+result does not settle.
+
 Hangul syllables are left out: their decomposition is into conjoining jamo,
 not combining marks.
 """
@@ -67,12 +72,23 @@ def fold(cp):
     return None
 
 
+def reduce(folds, s):
+    """Folds every letter of s until nothing changes."""
+    for _ in range(len(folds)):
+        nxt = "".join(folds.get(c, c) for c in s)
+        if nxt == s:
+            return s
+        s = nxt
+    sys.exit("gen_folds.py: fold of %r does not settle" % s)
+
+
 def main():
-    entries = []
+    folds = {}
     for cp in range(sys.maxunicode + 1):
         folded = fold(cp)
         if folded is not None:
-            entries.append((cp, folded))
+            folds[chr(cp)] = folded
+    entries = [(ord(ch), reduce(folds, folded)) for ch, folded in folds.items()]
     out = [
         "// Code generated by gen_folds.py from Unicode "
         + unicodedata.unidata_version + "; DO NOT EDIT.",