diff --git a/internal/ai/harness/dedupe_test.go b/internal/ai/harness/dedupe_test.go index b94bdb0..5779695 100644 --- a/internal/ai/harness/dedupe_test.go +++ b/internal/ai/harness/dedupe_test.go @@ -1,6 +1,9 @@ package harness -import "testing" +import ( + "testing" + "unicode" +) func TestNormalize(t *testing.T) { tests := []struct { @@ -48,6 +51,8 @@ func TestNormalizeCanonicalEquivalence(t *testing.T) { {"Cơm Tấm", "Co\u031bm Ta\u0302\u0301m", "com tam"}, {"Ẓaʿtar Ȟash", "Z\u0323aʿtar H\u030cash", "zaʿtar hash"}, {"Smǫrrebrød", "Smo\u0328rrebrød", "smorrebrod"}, + {"Smǿrrebrǿd", "Smø\u0301rrebrø\u0301d", "smorrebrod"}, // ǿ folds through ø + {"Ǽbleskiver", "Æ\u0301bleskiver", "aebleskiver"}, // ǽ folds through æ } for _, tt := range tests { c, d := Normalize(tt.composed), Normalize(tt.decomposed) @@ -58,6 +63,24 @@ func TestNormalizeCanonicalEquivalence(t *testing.T) { if !Similar(tt.composed, tt.decomposed) { t.Errorf("Similar(%q, %+q) = false, want true", tt.composed, tt.decomposed) } + dedupe := NewDeduper(DedupeSources{Regulars: []string{tt.decomposed}}) + if !dedupe.IsDuplicate(tt.composed) { + t.Errorf("IsDuplicate(%q) over %+q = false, want true", tt.composed, tt.decomposed) + } + } +} + +// TestLetterFoldsFullyReduced checks that every fold result is final: +// Normalize writes a result without folding it again, so a result that +// still holds a foldable letter, a combining mark or an upper-case letter +// would split canonically equivalent spellings (ǿ -> ø instead of o). +func TestLetterFoldsFullyReduced(t *testing.T) { + for r, folded := range letterFolds { + for _, c := range folded { + if letterFolds[c] != "" || unicode.Is(unicode.Mn, c) || unicode.ToLower(c) != c { + t.Errorf("letterFolds[%U] = %q holds %U, which Normalize would change", r, folded, c) + } + } } } diff --git a/internal/ai/harness/folds_table.go b/internal/ai/harness/folds_table.go index 65dd391..c814339 100644 --- a/internal/ai/harness/folds_table.go +++ b/internal/ai/harness/folds_table.go @@ -248,8 +248,8 @@ var letterFolds = map[rune]string{ 0x01DF: "a", // LATIN SMALL LETTER A WITH DIAERESIS AND MACRON 0x01E0: "a", // LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON 0x01E1: "a", // LATIN SMALL LETTER A WITH DOT ABOVE AND MACRON - 0x01E2: "\u00e6", // LATIN CAPITAL LETTER AE WITH MACRON - 0x01E3: "\u00e6", // LATIN SMALL LETTER AE WITH MACRON + 0x01E2: "ae", // LATIN CAPITAL LETTER AE WITH MACRON + 0x01E3: "ae", // LATIN SMALL LETTER AE WITH MACRON 0x01E4: "g", // LATIN CAPITAL LETTER G WITH STROKE 0x01E5: "g", // LATIN SMALL LETTER G WITH STROKE 0x01E6: "g", // LATIN CAPITAL LETTER G WITH CARON @@ -270,10 +270,10 @@ var letterFolds = map[rune]string{ 0x01F9: "n", // LATIN SMALL LETTER N WITH GRAVE 0x01FA: "a", // LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE 0x01FB: "a", // LATIN SMALL LETTER A WITH RING ABOVE AND ACUTE - 0x01FC: "\u00e6", // LATIN CAPITAL LETTER AE WITH ACUTE - 0x01FD: "\u00e6", // LATIN SMALL LETTER AE WITH ACUTE - 0x01FE: "\u00f8", // LATIN CAPITAL LETTER O WITH STROKE AND ACUTE - 0x01FF: "\u00f8", // LATIN SMALL LETTER O WITH STROKE AND ACUTE + 0x01FC: "ae", // LATIN CAPITAL LETTER AE WITH ACUTE + 0x01FD: "ae", // LATIN SMALL LETTER AE WITH ACUTE + 0x01FE: "o", // LATIN CAPITAL LETTER O WITH STROKE AND ACUTE + 0x01FF: "o", // LATIN SMALL LETTER O WITH STROKE AND ACUTE 0x0200: "a", // LATIN CAPITAL LETTER A WITH DOUBLE GRAVE 0x0201: "a", // LATIN SMALL LETTER A WITH DOUBLE GRAVE 0x0202: "a", // LATIN CAPITAL LETTER A WITH INVERTED BREVE diff --git a/internal/ai/harness/gen_folds.py b/internal/ai/harness/gen_folds.py index 1df444f..6e907c4 100644 --- a/internal/ai/harness/gen_folds.py +++ b/internal/ai/harness/gen_folds.py @@ -19,6 +19,11 @@ The table maps a code point to its lowercased base letters: WITH ..." map to x (ø -> o, ł -> l, đ -> d); - the Latin ligatures and letters listed in EXTRA map as given. +Each result is then folded again, letter by letter, until nothing changes +(ǿ -> ø -> o, ǽ -> æ -> ae). Normalize writes a result without folding it +again, so the table must already be fully reduced; the generator fails if a +result does not settle. + Hangul syllables are left out: their decomposition is into conjoining jamo, not combining marks. """ @@ -67,12 +72,23 @@ def fold(cp): return None +def reduce(folds, s): + """Folds every letter of s until nothing changes.""" + for _ in range(len(folds)): + nxt = "".join(folds.get(c, c) for c in s) + if nxt == s: + return s + s = nxt + sys.exit("gen_folds.py: fold of %r does not settle" % s) + + def main(): - entries = [] + folds = {} for cp in range(sys.maxunicode + 1): folded = fold(cp) if folded is not None: - entries.append((cp, folded)) + folds[chr(cp)] = folded + entries = [(ord(ch), reduce(folds, folded)) for ch, folded in folds.items()] out = [ "// Code generated by gen_folds.py from Unicode " + unicodedata.unidata_version + "; DO NOT EDIT.",