From d2d5e9c74289ffe9ee77ddfdaf97823c11c3fbe2 Mon Sep 17 00:00:00 2001 From: dev_agent Date: Mon, 5 Oct 2026 09:07:02 -0400 Subject: [PATCH] Reduce letter folds to a fixed point in gen_folds.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A fold result could itself be foldable: ǿ folded to ø and ǽ to æ, while the canonically equivalent ø+U+0301 and æ+U+0301 reached o and ae. Normalize writes a result without folding it again, so the generator now folds every result letter by letter until it settles (and fails if it does not), making the committed table fully reduced. TestLetterFoldsFullyReduced pins that, and the canonical-equivalence tests now cover ǿ and ǽ through Normalize, Similar and Deduper.IsDuplicate. Co-Authored-By: Claude Opus 5.5 --- internal/ai/harness/dedupe_test.go | 25 ++++++++++++++++++++++++- internal/ai/harness/folds_table.go | 12 ++++++------ internal/ai/harness/gen_folds.py | 20 ++++++++++++++++++-- 3 files changed, 48 insertions(+), 9 deletions(-) diff --git a/internal/ai/harness/dedupe_test.go b/internal/ai/harness/dedupe_test.go index b94bdb0..5779695 100644 --- a/internal/ai/harness/dedupe_test.go +++ b/internal/ai/harness/dedupe_test.go @@ -1,6 +1,9 @@ package harness -import "testing" +import ( + "testing" + "unicode" +) func TestNormalize(t *testing.T) { tests := []struct { @@ -48,6 +51,8 @@ func TestNormalizeCanonicalEquivalence(t *testing.T) { {"Cơm Tấm", "Co\u031bm Ta\u0302\u0301m", "com tam"}, {"Ẓaʿtar Ȟash", "Z\u0323aʿtar H\u030cash", "zaʿtar hash"}, {"Smǫrrebrød", "Smo\u0328rrebrød", "smorrebrod"}, + {"Smǿrrebrǿd", "Smø\u0301rrebrø\u0301d", "smorrebrod"}, // ǿ folds through ø + {"Ǽbleskiver", "Æ\u0301bleskiver", "aebleskiver"}, // ǽ folds through æ } for _, tt := range tests { c, d := Normalize(tt.composed), Normalize(tt.decomposed) @@ -58,6 +63,24 @@ func TestNormalizeCanonicalEquivalence(t *testing.T) { if !Similar(tt.composed, tt.decomposed) { t.Errorf("Similar(%q, %+q) = false, want true", tt.composed, tt.decomposed) } + dedupe := NewDeduper(DedupeSources{Regulars: []string{tt.decomposed}}) + if !dedupe.IsDuplicate(tt.composed) { + t.Errorf("IsDuplicate(%q) over %+q = false, want true", tt.composed, tt.decomposed) + } + } +} + +// TestLetterFoldsFullyReduced checks that every fold result is final: +// Normalize writes a result without folding it again, so a result that +// still holds a foldable letter, a combining mark or an upper-case letter +// would split canonically equivalent spellings (ǿ -> ø instead of o). +func TestLetterFoldsFullyReduced(t *testing.T) { + for r, folded := range letterFolds { + for _, c := range folded { + if letterFolds[c] != "" || unicode.Is(unicode.Mn, c) || unicode.ToLower(c) != c { + t.Errorf("letterFolds[%U] = %q holds %U, which Normalize would change", r, folded, c) + } + } } } diff --git a/internal/ai/harness/folds_table.go b/internal/ai/harness/folds_table.go index 65dd391..c814339 100644 --- a/internal/ai/harness/folds_table.go +++ b/internal/ai/harness/folds_table.go @@ -248,8 +248,8 @@ var letterFolds = map[rune]string{ 0x01DF: "a", // LATIN SMALL LETTER A WITH DIAERESIS AND MACRON 0x01E0: "a", // LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON 0x01E1: "a", // LATIN SMALL LETTER A WITH DOT ABOVE AND MACRON - 0x01E2: "\u00e6", // LATIN CAPITAL LETTER AE WITH MACRON - 0x01E3: "\u00e6", // LATIN SMALL LETTER AE WITH MACRON + 0x01E2: "ae", // LATIN CAPITAL LETTER AE WITH MACRON + 0x01E3: "ae", // LATIN SMALL LETTER AE WITH MACRON 0x01E4: "g", // LATIN CAPITAL LETTER G WITH STROKE 0x01E5: "g", // LATIN SMALL LETTER G WITH STROKE 0x01E6: "g", // LATIN CAPITAL LETTER G WITH CARON @@ -270,10 +270,10 @@ var letterFolds = map[rune]string{ 0x01F9: "n", // LATIN SMALL LETTER N WITH GRAVE 0x01FA: "a", // LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE 0x01FB: "a", // LATIN SMALL LETTER A WITH RING ABOVE AND ACUTE - 0x01FC: "\u00e6", // LATIN CAPITAL LETTER AE WITH ACUTE - 0x01FD: "\u00e6", // LATIN SMALL LETTER AE WITH ACUTE - 0x01FE: "\u00f8", // LATIN CAPITAL LETTER O WITH STROKE AND ACUTE - 0x01FF: "\u00f8", // LATIN SMALL LETTER O WITH STROKE AND ACUTE + 0x01FC: "ae", // LATIN CAPITAL LETTER AE WITH ACUTE + 0x01FD: "ae", // LATIN SMALL LETTER AE WITH ACUTE + 0x01FE: "o", // LATIN CAPITAL LETTER O WITH STROKE AND ACUTE + 0x01FF: "o", // LATIN SMALL LETTER O WITH STROKE AND ACUTE 0x0200: "a", // LATIN CAPITAL LETTER A WITH DOUBLE GRAVE 0x0201: "a", // LATIN SMALL LETTER A WITH DOUBLE GRAVE 0x0202: "a", // LATIN CAPITAL LETTER A WITH INVERTED BREVE diff --git a/internal/ai/harness/gen_folds.py b/internal/ai/harness/gen_folds.py index 1df444f..6e907c4 100644 --- a/internal/ai/harness/gen_folds.py +++ b/internal/ai/harness/gen_folds.py @@ -19,6 +19,11 @@ The table maps a code point to its lowercased base letters: WITH ..." map to x (ø -> o, ł -> l, đ -> d); - the Latin ligatures and letters listed in EXTRA map as given. +Each result is then folded again, letter by letter, until nothing changes +(ǿ -> ø -> o, ǽ -> æ -> ae). Normalize writes a result without folding it +again, so the table must already be fully reduced; the generator fails if a +result does not settle. + Hangul syllables are left out: their decomposition is into conjoining jamo, not combining marks. """ @@ -67,12 +72,23 @@ def fold(cp): return None +def reduce(folds, s): + """Folds every letter of s until nothing changes.""" + for _ in range(len(folds)): + nxt = "".join(folds.get(c, c) for c in s) + if nxt == s: + return s + s = nxt + sys.exit("gen_folds.py: fold of %r does not settle" % s) + + def main(): - entries = [] + folds = {} for cp in range(sys.maxunicode + 1): folded = fold(cp) if folded is not None: - entries.append((cp, folded)) + folds[chr(cp)] = folded + entries = [(ord(ch), reduce(folds, folded)) for ch, folded in folds.items()] out = [ "// Code generated by gen_folds.py from Unicode " + unicodedata.unidata_version + "; DO NOT EDIT.",