Reduce letter folds to a fixed point in gen_folds.py

A fold result could itself be foldable: ǿ folded to ø and ǽ to æ, while
the canonically equivalent ø+U+0301 and æ+U+0301 reached o and ae.
Normalize writes a result without folding it again, so the generator now
folds every result letter by letter until it settles (and fails if it does
not), making the committed table fully reduced. TestLetterFoldsFullyReduced
pins that, and the canonical-equivalence tests now cover ǿ and ǽ through
Normalize, Similar and Deduper.IsDuplicate.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/ai/harness/dedupe_test.go

@@ -1,6 +1,9 @@
 package harness
 
-import "testing"
+import (
+	"testing"
+	"unicode"
+)
 
 func TestNormalize(t *testing.T) {
 	tests := []struct {
@@ -48,6 +51,8 @@ func TestNormalizeCanonicalEquivalence(t *testing.T) {
 		{"Cơm Tấm", "Co\u031bm Ta\u0302\u0301m", "com tam"},
 		{"Ẓaʿtar Ȟash", "Z\u0323aʿtar H\u030cash", "zaʿtar hash"},
 		{"Smǫrrebrød", "Smo\u0328rrebrød", "smorrebrod"},
+		{"Smǿrrebrǿd", "Smø\u0301rrebrø\u0301d", "smorrebrod"}, // ǿ folds through ø
+		{"Ǽbleskiver", "Æ\u0301bleskiver", "aebleskiver"},      // ǽ folds through æ
 	}
 	for _, tt := range tests {
 		c, d := Normalize(tt.composed), Normalize(tt.decomposed)
@@ -58,6 +63,24 @@ func TestNormalizeCanonicalEquivalence(t *testing.T) {
 		if !Similar(tt.composed, tt.decomposed) {
 			t.Errorf("Similar(%q, %+q) = false, want true", tt.composed, tt.decomposed)
 		}
+		dedupe := NewDeduper(DedupeSources{Regulars: []string{tt.decomposed}})
+		if !dedupe.IsDuplicate(tt.composed) {
+			t.Errorf("IsDuplicate(%q) over %+q = false, want true", tt.composed, tt.decomposed)
+		}
+	}
+}
+
+// TestLetterFoldsFullyReduced checks that every fold result is final:
+// Normalize writes a result without folding it again, so a result that
+// still holds a foldable letter, a combining mark or an upper-case letter
+// would split canonically equivalent spellings (ǿ -> ø instead of o).
+func TestLetterFoldsFullyReduced(t *testing.T) {
+	for r, folded := range letterFolds {
+		for _, c := range folded {
+			if letterFolds[c] != "" || unicode.Is(unicode.Mn, c) || unicode.ToLower(c) != c {
+				t.Errorf("letterFolds[%U] = %q holds %U, which Normalize would change", r, folded, c)
+			}
+		}
 	}
 }
 

internal/ai/harness/folds_table.go

@@ -248,8 +248,8 @@ var letterFolds = map[rune]string{
 	0x01DF:  "a",          // LATIN SMALL LETTER A WITH DIAERESIS AND MACRON
 	0x01E0:  "a",          // LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON
 	0x01E1:  "a",          // LATIN SMALL LETTER A WITH DOT ABOVE AND MACRON
-	0x01E2:  "\u00e6",     // LATIN CAPITAL LETTER AE WITH MACRON
-	0x01E3:  "\u00e6",     // LATIN SMALL LETTER AE WITH MACRON
+	0x01E2:  "ae",         // LATIN CAPITAL LETTER AE WITH MACRON
+	0x01E3:  "ae",         // LATIN SMALL LETTER AE WITH MACRON
 	0x01E4:  "g",          // LATIN CAPITAL LETTER G WITH STROKE
 	0x01E5:  "g",          // LATIN SMALL LETTER G WITH STROKE
 	0x01E6:  "g",          // LATIN CAPITAL LETTER G WITH CARON
@@ -270,10 +270,10 @@ var letterFolds = map[rune]string{
 	0x01F9:  "n",          // LATIN SMALL LETTER N WITH GRAVE
 	0x01FA:  "a",          // LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE
 	0x01FB:  "a",          // LATIN SMALL LETTER A WITH RING ABOVE AND ACUTE
-	0x01FC:  "\u00e6",     // LATIN CAPITAL LETTER AE WITH ACUTE
-	0x01FD:  "\u00e6",     // LATIN SMALL LETTER AE WITH ACUTE
-	0x01FE:  "\u00f8",     // LATIN CAPITAL LETTER O WITH STROKE AND ACUTE
-	0x01FF:  "\u00f8",     // LATIN SMALL LETTER O WITH STROKE AND ACUTE
+	0x01FC:  "ae",         // LATIN CAPITAL LETTER AE WITH ACUTE
+	0x01FD:  "ae",         // LATIN SMALL LETTER AE WITH ACUTE
+	0x01FE:  "o",          // LATIN CAPITAL LETTER O WITH STROKE AND ACUTE
+	0x01FF:  "o",          // LATIN SMALL LETTER O WITH STROKE AND ACUTE
 	0x0200:  "a",          // LATIN CAPITAL LETTER A WITH DOUBLE GRAVE
 	0x0201:  "a",          // LATIN SMALL LETTER A WITH DOUBLE GRAVE
 	0x0202:  "a",          // LATIN CAPITAL LETTER A WITH INVERTED BREVE

internal/ai/harness/gen_folds.py

@@ -19,6 +19,11 @@ The table maps a code point to its lowercased base letters:
     WITH ..." map to x (ø -> o, ł -> l, đ -> d);
   - the Latin ligatures and letters listed in EXTRA map as given.
 
+Each result is then folded again, letter by letter, until nothing changes
+(ǿ -> ø -> o, ǽ -> æ -> ae). Normalize writes a result without folding it
+again, so the table must already be fully reduced; the generator fails if a
+result does not settle.
+
 Hangul syllables are left out: their decomposition is into conjoining jamo,
 not combining marks.
 """
@@ -67,12 +72,23 @@ def fold(cp):
     return None
 
 
+def reduce(folds, s):
+    """Folds every letter of s until nothing changes."""
+    for _ in range(len(folds)):
+        nxt = "".join(folds.get(c, c) for c in s)
+        if nxt == s:
+            return s
+        s = nxt
+    sys.exit("gen_folds.py: fold of %r does not settle" % s)
+
+
 def main():
-    entries = []
+    folds = {}
     for cp in range(sys.maxunicode + 1):
         folded = fold(cp)
         if folded is not None:
-            entries.append((cp, folded))
+            folds[chr(cp)] = folded
+    entries = [(ord(ch), reduce(folds, folded)) for ch, folded in folds.items()]
     out = [
         "// Code generated by gen_folds.py from Unicode "
         + unicodedata.unidata_version + "; DO NOT EDIT.",