From 5d5404b3cb91702763ed18ae34c9233ae32f90cf Mon Sep 17 00:00:00 2001 From: dev_agent Date: Mon, 5 Oct 2026 09:16:16 -0400 Subject: [PATCH] Name normalization and dedupe (#59) Co-authored-by: dev_agent --- docs/ARCHITECTURE.md | 9 + internal/ai/harness/dedupe.go | 118 +++ internal/ai/harness/dedupe_test.go | 151 ++++ internal/ai/harness/folds_table.go | 1173 ++++++++++++++++++++++++++++ internal/ai/harness/gen_folds.py | 116 +++ 5 files changed, 1567 insertions(+) create mode 100644 internal/ai/harness/dedupe.go create mode 100644 internal/ai/harness/dedupe_test.go create mode 100644 internal/ai/harness/folds_table.go create mode 100644 internal/ai/harness/gen_folds.py diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index fd4107b..9fd67a4 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -164,6 +164,15 @@ automatic** (D-021). last 60 passed Side quests and the plans of the last four ISO weeks. A character budget trims the oldest plans first, then the oldest passes; Regulars and the Quest log are never trimmed. +- **Name dedupe** (`harness.Normalize`, `harness.Similar`, `harness.Deduper`): + a dinner name is normalized by lowercasing, dropping combining marks and + folding precomposed letters to their base letters through a table generated + from Unicode data (`gen_folds.py`), so composed and decomposed spellings + match, stripping punctuation, dropping filler words (easy, best, homemade, quick, simple, + the, a, recipe, classic) and collapsing whitespace. Two names are the same + dinner when the token-set (Jaccard) similarity of their words is at least + 0.85. A `Deduper` over the Regulars, Quest log, pending suggestions and + passed names rejects a suggestion that repeats any of them. ## Recipe grounding diff --git a/internal/ai/harness/dedupe.go b/internal/ai/harness/dedupe.go new file mode 100644 index 0000000..ce6c446 --- /dev/null +++ b/internal/ai/harness/dedupe.go @@ -0,0 +1,118 @@ +package harness + +import ( + "strings" + "unicode" +) + +// SimilarThreshold is the token-set similarity at or above which two dinner +// names are the same dinner. +const SimilarThreshold = 0.85 + +// fillerWords carry no meaning in a dinner name and are dropped by +// Normalize. +var fillerWords = map[string]bool{ + "easy": true, "best": true, "homemade": true, "quick": true, "simple": true, + "the": true, "a": true, "recipe": true, "classic": true, +} + +//go:generate python3 gen_folds.py + +// Normalize returns the comparable form of a dinner name: lowercased, +// combining marks (Unicode category Mn) dropped, precomposed and stroked +// letters folded to their base letters (see letterFolds) so composed and +// decomposed spellings are equal, apostrophes removed, other punctuation +// turned into spaces, filler words (see fillerWords) dropped and whitespace +// collapsed to single spaces. "Easy Chicken Tikka-Masala!" becomes +// "chicken tikka masala". +func Normalize(name string) string { + var b strings.Builder + for _, r := range strings.ToLower(name) { + switch { + case unicode.Is(unicode.Mn, r), r == '\'', r == '’': + // Combining accents and apostrophes join their neighbours. + case letterFolds[r] != "": + b.WriteString(letterFolds[r]) + case unicode.IsLetter(r), unicode.IsDigit(r): + b.WriteRune(r) + default: + b.WriteRune(' ') + } + } + words := strings.Fields(b.String()) + kept := words[:0] + for _, w := range words { + if !fillerWords[w] { + kept = append(kept, w) + } + } + return strings.Join(kept, " ") +} + +// Similar reports whether a and b name the same dinner: the token-set +// similarity of their normalized forms is at least SimilarThreshold. A name +// with no words left after normalizing is similar to nothing. +func Similar(a, b string) bool { + return similarity(tokenSet(a), tokenSet(b)) >= SimilarThreshold +} + +// tokenSet returns the distinct words of name's normalized form. +func tokenSet(name string) map[string]bool { + set := map[string]bool{} + for _, w := range strings.Fields(Normalize(name)) { + set[w] = true + } + return set +} + +// similarity is the Jaccard index of a and b: shared words over all words. +// It is 0 when either set is empty. +func similarity(a, b map[string]bool) float64 { + if len(a) == 0 || len(b) == 0 { + return 0 + } + shared := 0 + for w := range a { + if b[w] { + shared++ + } + } + return float64(shared) / float64(len(a)+len(b)-shared) +} + +// DedupeSources are the dinner names a new suggestion must not repeat. +type DedupeSources struct { + Regulars []string + QuestLog []string + // Pending holds the names of suggestions not yet tried or passed. + Pending []string + // Declined holds the names of passed suggestions. + Declined []string +} + +// Deduper answers whether a candidate dinner name repeats a known one. +type Deduper struct { + known []map[string]bool +} + +// NewDeduper returns a Deduper over every name in src. +func NewDeduper(src DedupeSources) *Deduper { + d := &Deduper{} + for _, names := range [][]string{src.Regulars, src.QuestLog, src.Pending, src.Declined} { + for _, name := range names { + d.known = append(d.known, tokenSet(name)) + } + } + return d +} + +// IsDuplicate reports whether name is Similar to any known name. +func (d *Deduper) IsDuplicate(name string) bool { + candidate := tokenSet(name) + for _, k := range d.known { + if similarity(candidate, k) >= SimilarThreshold { + return true + } + } + return false +} diff --git a/internal/ai/harness/dedupe_test.go b/internal/ai/harness/dedupe_test.go new file mode 100644 index 0000000..5779695 --- /dev/null +++ b/internal/ai/harness/dedupe_test.go @@ -0,0 +1,151 @@ +package harness + +import ( + "testing" + "unicode" +) + +func TestNormalize(t *testing.T) { + tests := []struct { + name, want string + }{ + {"Easy Chicken Tikka Masala", "chicken tikka masala"}, + {" The BEST homemade\tLasagna ", "lasagna"}, + {"A Quick, Simple Classic Chili Recipe", "chili"}, + {"Tacos (beef)", "tacos beef"}, + {"Mac-and-Cheese!", "mac and cheese"}, + {"Shepherd's Pie", "shepherds pie"}, + {"Shepherd’s Pie", "shepherds pie"}, + {"Crème Brûlée", "creme brulee"}, + {"Jalapeño Poppers", "jalapeno poppers"}, + {"Paella à la Valenciana", "paella la valenciana"}, // "à" folds to the filler "a" + {"Smørrebrød", "smorrebrod"}, + {"Bœuf Bourguignon", "boeuf bourguignon"}, + {"Käsespätzle mit Weißwurst", "kasespatzle mit weisswurst"}, + {"Cre\u0300me bru\u0302le\u0301e", "creme brulee"}, // decomposed accents + {"İskender Kebab", "iskender kebab"}, + {"Pho 24", "pho 24"}, + {"Phở Bò", "pho bo"}, + {"Bún Chả Hà Nội", "bun cha ha noi"}, + {"Łosoś z Koperkiem", "losos z koperkiem"}, + {"Ćevapčići", "cevapcici"}, + {"Easy Recipe", ""}, + {"", ""}, + {"!?-- & ...", ""}, // punctuation only + } + for _, tt := range tests { + if got := Normalize(tt.name); got != tt.want { + t.Errorf("Normalize(%q) = %q, want %q", tt.name, got, tt.want) + } + } +} + +// TestNormalizeCanonicalEquivalence checks that a composed spelling and its +// canonically equivalent decomposed spelling normalize equal and are Similar. +func TestNormalizeCanonicalEquivalence(t *testing.T) { + tests := []struct { + composed, decomposed, want string + }{ + {"Phở", "Pho\u031b\u0309", "pho"}, // o + horn + hook above + {"Bánh Xèo", "Ba\u0301nh Xe\u0300o", "banh xeo"}, + {"Cơm Tấm", "Co\u031bm Ta\u0302\u0301m", "com tam"}, + {"Ẓaʿtar Ȟash", "Z\u0323aʿtar H\u030cash", "zaʿtar hash"}, + {"Smǫrrebrød", "Smo\u0328rrebrød", "smorrebrod"}, + {"Smǿrrebrǿd", "Smø\u0301rrebrø\u0301d", "smorrebrod"}, // ǿ folds through ø + {"Ǽbleskiver", "Æ\u0301bleskiver", "aebleskiver"}, // ǽ folds through æ + } + for _, tt := range tests { + c, d := Normalize(tt.composed), Normalize(tt.decomposed) + if c != tt.want || d != tt.want { + t.Errorf("Normalize(%q) = %q, Normalize(%+q) = %q, want both %q", + tt.composed, c, tt.decomposed, d, tt.want) + } + if !Similar(tt.composed, tt.decomposed) { + t.Errorf("Similar(%q, %+q) = false, want true", tt.composed, tt.decomposed) + } + dedupe := NewDeduper(DedupeSources{Regulars: []string{tt.decomposed}}) + if !dedupe.IsDuplicate(tt.composed) { + t.Errorf("IsDuplicate(%q) over %+q = false, want true", tt.composed, tt.decomposed) + } + } +} + +// TestLetterFoldsFullyReduced checks that every fold result is final: +// Normalize writes a result without folding it again, so a result that +// still holds a foldable letter, a combining mark or an upper-case letter +// would split canonically equivalent spellings (ǿ -> ø instead of o). +func TestLetterFoldsFullyReduced(t *testing.T) { + for r, folded := range letterFolds { + for _, c := range folded { + if letterFolds[c] != "" || unicode.Is(unicode.Mn, c) || unicode.ToLower(c) != c { + t.Errorf("letterFolds[%U] = %q holds %U, which Normalize would change", r, folded, c) + } + } + } +} + +func TestSimilar(t *testing.T) { + tests := []struct { + a, b string + want bool + }{ + {"Easy Chicken Tikka Masala", "chicken tikka masala", true}, + {"Beef Tacos", "Tacos (beef)", true}, + {"Chicken tacos", "Fish tacos", false}, + {"Spaghetti Bolognese", "spaghetti bolognese recipe", true}, + {"Crème Brûlée", "creme brulee", true}, + {"Jalapeño Poppers", "JALAPENO POPPERS", true}, + {"Smørrebrød", "Smorrebrod", true}, + {"Phở", "Pho\u031b\u0309", true}, + {"Phở", "Pho", true}, + {"Chicken Tikka Masala", "Chicken Tikka Masala Curry", false}, // 3/4 + {"Tacos", "Fish tacos", false}, + {"Pad Thai", "Thai Green Curry", false}, + {"Easy Recipe", "The Classic", false}, // nothing left to compare + {"Recipe", "Chili", false}, + } + for _, tt := range tests { + if got := Similar(tt.a, tt.b); got != tt.want { + t.Errorf("Similar(%q, %q) = %v, want %v", tt.a, tt.b, got, tt.want) + } + if got := Similar(tt.b, tt.a); got != tt.want { + t.Errorf("Similar(%q, %q) = %v, want %v", tt.b, tt.a, got, tt.want) + } + } +} + +func TestDeduper(t *testing.T) { + d := NewDeduper(DedupeSources{ + Regulars: []string{"Beef Tacos", "Spaghetti Bolognese"}, + QuestLog: []string{"Chicken Tikka Masala"}, + Pending: []string{"Crème Brûlée"}, + Declined: []string{"Shepherd's Pie"}, + }) + tests := []struct { + name string + want bool + }{ + {"Tacos (beef)", true}, // Regular + {"spaghetti bolognese recipe", true}, // Regular + {"Easy Chicken Tikka Masala", true}, // Quest log + {"creme brulee", true}, // pending suggestion + {"The Best Shepherds Pie", true}, // declined + {"Fish tacos", false}, // shares only "tacos" + {"Chicken Tikka Masala Curry", false}, // below the threshold + {"Jalapeño Poppers", false}, // new + {"Easy Recipe", false}, // nothing left to compare + {"Spaghetti Carbonara", false}, // shares only "spaghetti" + {"Bolognese Spaghetti (Classic)", true}, // word order does not matter + } + for _, tt := range tests { + if got := d.IsDuplicate(tt.name); got != tt.want { + t.Errorf("IsDuplicate(%q) = %v, want %v", tt.name, got, tt.want) + } + } +} + +func TestDeduperEmpty(t *testing.T) { + if NewDeduper(DedupeSources{}).IsDuplicate("Beef Tacos") { + t.Error("an empty Deduper reported a duplicate") + } +} diff --git a/internal/ai/harness/folds_table.go b/internal/ai/harness/folds_table.go new file mode 100644 index 0000000..c814339 --- /dev/null +++ b/internal/ai/harness/folds_table.go @@ -0,0 +1,1173 @@ +// Code generated by gen_folds.py from Unicode 14.0.0; DO NOT EDIT. + +package harness + +// letterFolds maps a precomposed or stroked letter to its lowercased +// base letters. See gen_folds.py for how it is built. +var letterFolds = map[rune]string{ + 0x00C0: "a", // LATIN CAPITAL LETTER A WITH GRAVE + 0x00C1: "a", // LATIN CAPITAL LETTER A WITH ACUTE + 0x00C2: "a", // LATIN CAPITAL LETTER A WITH CIRCUMFLEX + 0x00C3: "a", // LATIN CAPITAL LETTER A WITH TILDE + 0x00C4: "a", // LATIN CAPITAL LETTER A WITH DIAERESIS + 0x00C5: "a", // LATIN CAPITAL LETTER A WITH RING ABOVE + 0x00C6: "ae", // LATIN CAPITAL LETTER AE + 0x00C7: "c", // LATIN CAPITAL LETTER C WITH CEDILLA + 0x00C8: "e", // LATIN CAPITAL LETTER E WITH GRAVE + 0x00C9: "e", // LATIN CAPITAL LETTER E WITH ACUTE + 0x00CA: "e", // LATIN CAPITAL LETTER E WITH CIRCUMFLEX + 0x00CB: "e", // LATIN CAPITAL LETTER E WITH DIAERESIS + 0x00CC: "i", // LATIN CAPITAL LETTER I WITH GRAVE + 0x00CD: "i", // LATIN CAPITAL LETTER I WITH ACUTE + 0x00CE: "i", // LATIN CAPITAL LETTER I WITH CIRCUMFLEX + 0x00CF: "i", // LATIN CAPITAL LETTER I WITH DIAERESIS + 0x00D0: "d", // LATIN CAPITAL LETTER ETH + 0x00D1: "n", // LATIN CAPITAL LETTER N WITH TILDE + 0x00D2: "o", // LATIN CAPITAL LETTER O WITH GRAVE + 0x00D3: "o", // LATIN CAPITAL LETTER O WITH ACUTE + 0x00D4: "o", // LATIN CAPITAL LETTER O WITH CIRCUMFLEX + 0x00D5: "o", // LATIN CAPITAL LETTER O WITH TILDE + 0x00D6: "o", // LATIN CAPITAL LETTER O WITH DIAERESIS + 0x00D8: "o", // LATIN CAPITAL LETTER O WITH STROKE + 0x00D9: "u", // LATIN CAPITAL LETTER U WITH GRAVE + 0x00DA: "u", // LATIN CAPITAL LETTER U WITH ACUTE + 0x00DB: "u", // LATIN CAPITAL LETTER U WITH CIRCUMFLEX + 0x00DC: "u", // LATIN CAPITAL LETTER U WITH DIAERESIS + 0x00DD: "y", // LATIN CAPITAL LETTER Y WITH ACUTE + 0x00DE: "th", // LATIN CAPITAL LETTER THORN + 0x00DF: "ss", // LATIN SMALL LETTER SHARP S + 0x00E0: "a", // LATIN SMALL LETTER A WITH GRAVE + 0x00E1: "a", // LATIN SMALL LETTER A WITH ACUTE + 0x00E2: "a", // LATIN SMALL LETTER A WITH CIRCUMFLEX + 0x00E3: "a", // LATIN SMALL LETTER A WITH TILDE + 0x00E4: "a", // LATIN SMALL LETTER A WITH DIAERESIS + 0x00E5: "a", // LATIN SMALL LETTER A WITH RING ABOVE + 0x00E6: "ae", // LATIN SMALL LETTER AE + 0x00E7: "c", // LATIN SMALL LETTER C WITH CEDILLA + 0x00E8: "e", // LATIN SMALL LETTER E WITH GRAVE + 0x00E9: "e", // LATIN SMALL LETTER E WITH ACUTE + 0x00EA: "e", // LATIN SMALL LETTER E WITH CIRCUMFLEX + 0x00EB: "e", // LATIN SMALL LETTER E WITH DIAERESIS + 0x00EC: "i", // LATIN SMALL LETTER I WITH GRAVE + 0x00ED: "i", // LATIN SMALL LETTER I WITH ACUTE + 0x00EE: "i", // LATIN SMALL LETTER I WITH CIRCUMFLEX + 0x00EF: "i", // LATIN SMALL LETTER I WITH DIAERESIS + 0x00F0: "d", // LATIN SMALL LETTER ETH + 0x00F1: "n", // LATIN SMALL LETTER N WITH TILDE + 0x00F2: "o", // LATIN SMALL LETTER O WITH GRAVE + 0x00F3: "o", // LATIN SMALL LETTER O WITH ACUTE + 0x00F4: "o", // LATIN SMALL LETTER O WITH CIRCUMFLEX + 0x00F5: "o", // LATIN SMALL LETTER O WITH TILDE + 0x00F6: "o", // LATIN SMALL LETTER O WITH DIAERESIS + 0x00F8: "o", // LATIN SMALL LETTER O WITH STROKE + 0x00F9: "u", // LATIN SMALL LETTER U WITH GRAVE + 0x00FA: "u", // LATIN SMALL LETTER U WITH ACUTE + 0x00FB: "u", // LATIN SMALL LETTER U WITH CIRCUMFLEX + 0x00FC: "u", // LATIN SMALL LETTER U WITH DIAERESIS + 0x00FD: "y", // LATIN SMALL LETTER Y WITH ACUTE + 0x00FE: "th", // LATIN SMALL LETTER THORN + 0x00FF: "y", // LATIN SMALL LETTER Y WITH DIAERESIS + 0x0100: "a", // LATIN CAPITAL LETTER A WITH MACRON + 0x0101: "a", // LATIN SMALL LETTER A WITH MACRON + 0x0102: "a", // LATIN CAPITAL LETTER A WITH BREVE + 0x0103: "a", // LATIN SMALL LETTER A WITH BREVE + 0x0104: "a", // LATIN CAPITAL LETTER A WITH OGONEK + 0x0105: "a", // LATIN SMALL LETTER A WITH OGONEK + 0x0106: "c", // LATIN CAPITAL LETTER C WITH ACUTE + 0x0107: "c", // LATIN SMALL LETTER C WITH ACUTE + 0x0108: "c", // LATIN CAPITAL LETTER C WITH CIRCUMFLEX + 0x0109: "c", // LATIN SMALL LETTER C WITH CIRCUMFLEX + 0x010A: "c", // LATIN CAPITAL LETTER C WITH DOT ABOVE + 0x010B: "c", // LATIN SMALL LETTER C WITH DOT ABOVE + 0x010C: "c", // LATIN CAPITAL LETTER C WITH CARON + 0x010D: "c", // LATIN SMALL LETTER C WITH CARON + 0x010E: "d", // LATIN CAPITAL LETTER D WITH CARON + 0x010F: "d", // LATIN SMALL LETTER D WITH CARON + 0x0110: "d", // LATIN CAPITAL LETTER D WITH STROKE + 0x0111: "d", // LATIN SMALL LETTER D WITH STROKE + 0x0112: "e", // LATIN CAPITAL LETTER E WITH MACRON + 0x0113: "e", // LATIN SMALL LETTER E WITH MACRON + 0x0114: "e", // LATIN CAPITAL LETTER E WITH BREVE + 0x0115: "e", // LATIN SMALL LETTER E WITH BREVE + 0x0116: "e", // LATIN CAPITAL LETTER E WITH DOT ABOVE + 0x0117: "e", // LATIN SMALL LETTER E WITH DOT ABOVE + 0x0118: "e", // LATIN CAPITAL LETTER E WITH OGONEK + 0x0119: "e", // LATIN SMALL LETTER E WITH OGONEK + 0x011A: "e", // LATIN CAPITAL LETTER E WITH CARON + 0x011B: "e", // LATIN SMALL LETTER E WITH CARON + 0x011C: "g", // LATIN CAPITAL LETTER G WITH CIRCUMFLEX + 0x011D: "g", // LATIN SMALL LETTER G WITH CIRCUMFLEX + 0x011E: "g", // LATIN CAPITAL LETTER G WITH BREVE + 0x011F: "g", // LATIN SMALL LETTER G WITH BREVE + 0x0120: "g", // LATIN CAPITAL LETTER G WITH DOT ABOVE + 0x0121: "g", // LATIN SMALL LETTER G WITH DOT ABOVE + 0x0122: "g", // LATIN CAPITAL LETTER G WITH CEDILLA + 0x0123: "g", // LATIN SMALL LETTER G WITH CEDILLA + 0x0124: "h", // LATIN CAPITAL LETTER H WITH CIRCUMFLEX + 0x0125: "h", // LATIN SMALL LETTER H WITH CIRCUMFLEX + 0x0126: "h", // LATIN CAPITAL LETTER H WITH STROKE + 0x0127: "h", // LATIN SMALL LETTER H WITH STROKE + 0x0128: "i", // LATIN CAPITAL LETTER I WITH TILDE + 0x0129: "i", // LATIN SMALL LETTER I WITH TILDE + 0x012A: "i", // LATIN CAPITAL LETTER I WITH MACRON + 0x012B: "i", // LATIN SMALL LETTER I WITH MACRON + 0x012C: "i", // LATIN CAPITAL LETTER I WITH BREVE + 0x012D: "i", // LATIN SMALL LETTER I WITH BREVE + 0x012E: "i", // LATIN CAPITAL LETTER I WITH OGONEK + 0x012F: "i", // LATIN SMALL LETTER I WITH OGONEK + 0x0130: "i", // LATIN CAPITAL LETTER I WITH DOT ABOVE + 0x0131: "i", // LATIN SMALL LETTER DOTLESS I + 0x0132: "ij", // LATIN CAPITAL LIGATURE IJ + 0x0133: "ij", // LATIN SMALL LIGATURE IJ + 0x0134: "j", // LATIN CAPITAL LETTER J WITH CIRCUMFLEX + 0x0135: "j", // LATIN SMALL LETTER J WITH CIRCUMFLEX + 0x0136: "k", // LATIN CAPITAL LETTER K WITH CEDILLA + 0x0137: "k", // LATIN SMALL LETTER K WITH CEDILLA + 0x0139: "l", // LATIN CAPITAL LETTER L WITH ACUTE + 0x013A: "l", // LATIN SMALL LETTER L WITH ACUTE + 0x013B: "l", // LATIN CAPITAL LETTER L WITH CEDILLA + 0x013C: "l", // LATIN SMALL LETTER L WITH CEDILLA + 0x013D: "l", // LATIN CAPITAL LETTER L WITH CARON + 0x013E: "l", // LATIN SMALL LETTER L WITH CARON + 0x013F: "l", // LATIN CAPITAL LETTER L WITH MIDDLE DOT + 0x0140: "l", // LATIN SMALL LETTER L WITH MIDDLE DOT + 0x0141: "l", // LATIN CAPITAL LETTER L WITH STROKE + 0x0142: "l", // LATIN SMALL LETTER L WITH STROKE + 0x0143: "n", // LATIN CAPITAL LETTER N WITH ACUTE + 0x0144: "n", // LATIN SMALL LETTER N WITH ACUTE + 0x0145: "n", // LATIN CAPITAL LETTER N WITH CEDILLA + 0x0146: "n", // LATIN SMALL LETTER N WITH CEDILLA + 0x0147: "n", // LATIN CAPITAL LETTER N WITH CARON + 0x0148: "n", // LATIN SMALL LETTER N WITH CARON + 0x0149: "n", // LATIN SMALL LETTER N PRECEDED BY APOSTROPHE + 0x014C: "o", // LATIN CAPITAL LETTER O WITH MACRON + 0x014D: "o", // LATIN SMALL LETTER O WITH MACRON + 0x014E: "o", // LATIN CAPITAL LETTER O WITH BREVE + 0x014F: "o", // LATIN SMALL LETTER O WITH BREVE + 0x0150: "o", // LATIN CAPITAL LETTER O WITH DOUBLE ACUTE + 0x0151: "o", // LATIN SMALL LETTER O WITH DOUBLE ACUTE + 0x0152: "oe", // LATIN CAPITAL LIGATURE OE + 0x0153: "oe", // LATIN SMALL LIGATURE OE + 0x0154: "r", // LATIN CAPITAL LETTER R WITH ACUTE + 0x0155: "r", // LATIN SMALL LETTER R WITH ACUTE + 0x0156: "r", // LATIN CAPITAL LETTER R WITH CEDILLA + 0x0157: "r", // LATIN SMALL LETTER R WITH CEDILLA + 0x0158: "r", // LATIN CAPITAL LETTER R WITH CARON + 0x0159: "r", // LATIN SMALL LETTER R WITH CARON + 0x015A: "s", // LATIN CAPITAL LETTER S WITH ACUTE + 0x015B: "s", // LATIN SMALL LETTER S WITH ACUTE + 0x015C: "s", // LATIN CAPITAL LETTER S WITH CIRCUMFLEX + 0x015D: "s", // LATIN SMALL LETTER S WITH CIRCUMFLEX + 0x015E: "s", // LATIN CAPITAL LETTER S WITH CEDILLA + 0x015F: "s", // LATIN SMALL LETTER S WITH CEDILLA + 0x0160: "s", // LATIN CAPITAL LETTER S WITH CARON + 0x0161: "s", // LATIN SMALL LETTER S WITH CARON + 0x0162: "t", // LATIN CAPITAL LETTER T WITH CEDILLA + 0x0163: "t", // LATIN SMALL LETTER T WITH CEDILLA + 0x0164: "t", // LATIN CAPITAL LETTER T WITH CARON + 0x0165: "t", // LATIN SMALL LETTER T WITH CARON + 0x0166: "t", // LATIN CAPITAL LETTER T WITH STROKE + 0x0167: "t", // LATIN SMALL LETTER T WITH STROKE + 0x0168: "u", // LATIN CAPITAL LETTER U WITH TILDE + 0x0169: "u", // LATIN SMALL LETTER U WITH TILDE + 0x016A: "u", // LATIN CAPITAL LETTER U WITH MACRON + 0x016B: "u", // LATIN SMALL LETTER U WITH MACRON + 0x016C: "u", // LATIN CAPITAL LETTER U WITH BREVE + 0x016D: "u", // LATIN SMALL LETTER U WITH BREVE + 0x016E: "u", // LATIN CAPITAL LETTER U WITH RING ABOVE + 0x016F: "u", // LATIN SMALL LETTER U WITH RING ABOVE + 0x0170: "u", // LATIN CAPITAL LETTER U WITH DOUBLE ACUTE + 0x0171: "u", // LATIN SMALL LETTER U WITH DOUBLE ACUTE + 0x0172: "u", // LATIN CAPITAL LETTER U WITH OGONEK + 0x0173: "u", // LATIN SMALL LETTER U WITH OGONEK + 0x0174: "w", // LATIN CAPITAL LETTER W WITH CIRCUMFLEX + 0x0175: "w", // LATIN SMALL LETTER W WITH CIRCUMFLEX + 0x0176: "y", // LATIN CAPITAL LETTER Y WITH CIRCUMFLEX + 0x0177: "y", // LATIN SMALL LETTER Y WITH CIRCUMFLEX + 0x0178: "y", // LATIN CAPITAL LETTER Y WITH DIAERESIS + 0x0179: "z", // LATIN CAPITAL LETTER Z WITH ACUTE + 0x017A: "z", // LATIN SMALL LETTER Z WITH ACUTE + 0x017B: "z", // LATIN CAPITAL LETTER Z WITH DOT ABOVE + 0x017C: "z", // LATIN SMALL LETTER Z WITH DOT ABOVE + 0x017D: "z", // LATIN CAPITAL LETTER Z WITH CARON + 0x017E: "z", // LATIN SMALL LETTER Z WITH CARON + 0x0180: "b", // LATIN SMALL LETTER B WITH STROKE + 0x0181: "b", // LATIN CAPITAL LETTER B WITH HOOK + 0x0182: "b", // LATIN CAPITAL LETTER B WITH TOPBAR + 0x0183: "b", // LATIN SMALL LETTER B WITH TOPBAR + 0x0187: "c", // LATIN CAPITAL LETTER C WITH HOOK + 0x0188: "c", // LATIN SMALL LETTER C WITH HOOK + 0x018A: "d", // LATIN CAPITAL LETTER D WITH HOOK + 0x018B: "d", // LATIN CAPITAL LETTER D WITH TOPBAR + 0x018C: "d", // LATIN SMALL LETTER D WITH TOPBAR + 0x0191: "f", // LATIN CAPITAL LETTER F WITH HOOK + 0x0192: "f", // LATIN SMALL LETTER F WITH HOOK + 0x0193: "g", // LATIN CAPITAL LETTER G WITH HOOK + 0x0197: "i", // LATIN CAPITAL LETTER I WITH STROKE + 0x0198: "k", // LATIN CAPITAL LETTER K WITH HOOK + 0x0199: "k", // LATIN SMALL LETTER K WITH HOOK + 0x019A: "l", // LATIN SMALL LETTER L WITH BAR + 0x019D: "n", // LATIN CAPITAL LETTER N WITH LEFT HOOK + 0x019E: "n", // LATIN SMALL LETTER N WITH LONG RIGHT LEG + 0x019F: "o", // LATIN CAPITAL LETTER O WITH MIDDLE TILDE + 0x01A0: "o", // LATIN CAPITAL LETTER O WITH HORN + 0x01A1: "o", // LATIN SMALL LETTER O WITH HORN + 0x01A4: "p", // LATIN CAPITAL LETTER P WITH HOOK + 0x01A5: "p", // LATIN SMALL LETTER P WITH HOOK + 0x01AB: "t", // LATIN SMALL LETTER T WITH PALATAL HOOK + 0x01AC: "t", // LATIN CAPITAL LETTER T WITH HOOK + 0x01AD: "t", // LATIN SMALL LETTER T WITH HOOK + 0x01AE: "t", // LATIN CAPITAL LETTER T WITH RETROFLEX HOOK + 0x01AF: "u", // LATIN CAPITAL LETTER U WITH HORN + 0x01B0: "u", // LATIN SMALL LETTER U WITH HORN + 0x01B2: "v", // LATIN CAPITAL LETTER V WITH HOOK + 0x01B3: "y", // LATIN CAPITAL LETTER Y WITH HOOK + 0x01B4: "y", // LATIN SMALL LETTER Y WITH HOOK + 0x01B5: "z", // LATIN CAPITAL LETTER Z WITH STROKE + 0x01B6: "z", // LATIN SMALL LETTER Z WITH STROKE + 0x01C5: "d", // LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON + 0x01C8: "l", // LATIN CAPITAL LETTER L WITH SMALL LETTER J + 0x01CB: "n", // LATIN CAPITAL LETTER N WITH SMALL LETTER J + 0x01CD: "a", // LATIN CAPITAL LETTER A WITH CARON + 0x01CE: "a", // LATIN SMALL LETTER A WITH CARON + 0x01CF: "i", // LATIN CAPITAL LETTER I WITH CARON + 0x01D0: "i", // LATIN SMALL LETTER I WITH CARON + 0x01D1: "o", // LATIN CAPITAL LETTER O WITH CARON + 0x01D2: "o", // LATIN SMALL LETTER O WITH CARON + 0x01D3: "u", // LATIN CAPITAL LETTER U WITH CARON + 0x01D4: "u", // LATIN SMALL LETTER U WITH CARON + 0x01D5: "u", // LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON + 0x01D6: "u", // LATIN SMALL LETTER U WITH DIAERESIS AND MACRON + 0x01D7: "u", // LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE + 0x01D8: "u", // LATIN SMALL LETTER U WITH DIAERESIS AND ACUTE + 0x01D9: "u", // LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON + 0x01DA: "u", // LATIN SMALL LETTER U WITH DIAERESIS AND CARON + 0x01DB: "u", // LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE + 0x01DC: "u", // LATIN SMALL LETTER U WITH DIAERESIS AND GRAVE + 0x01DE: "a", // LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON + 0x01DF: "a", // LATIN SMALL LETTER A WITH DIAERESIS AND MACRON + 0x01E0: "a", // LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON + 0x01E1: "a", // LATIN SMALL LETTER A WITH DOT ABOVE AND MACRON + 0x01E2: "ae", // LATIN CAPITAL LETTER AE WITH MACRON + 0x01E3: "ae", // LATIN SMALL LETTER AE WITH MACRON + 0x01E4: "g", // LATIN CAPITAL LETTER G WITH STROKE + 0x01E5: "g", // LATIN SMALL LETTER G WITH STROKE + 0x01E6: "g", // LATIN CAPITAL LETTER G WITH CARON + 0x01E7: "g", // LATIN SMALL LETTER G WITH CARON + 0x01E8: "k", // LATIN CAPITAL LETTER K WITH CARON + 0x01E9: "k", // LATIN SMALL LETTER K WITH CARON + 0x01EA: "o", // LATIN CAPITAL LETTER O WITH OGONEK + 0x01EB: "o", // LATIN SMALL LETTER O WITH OGONEK + 0x01EC: "o", // LATIN CAPITAL LETTER O WITH OGONEK AND MACRON + 0x01ED: "o", // LATIN SMALL LETTER O WITH OGONEK AND MACRON + 0x01EE: "\u0292", // LATIN CAPITAL LETTER EZH WITH CARON + 0x01EF: "\u0292", // LATIN SMALL LETTER EZH WITH CARON + 0x01F0: "j", // LATIN SMALL LETTER J WITH CARON + 0x01F2: "d", // LATIN CAPITAL LETTER D WITH SMALL LETTER Z + 0x01F4: "g", // LATIN CAPITAL LETTER G WITH ACUTE + 0x01F5: "g", // LATIN SMALL LETTER G WITH ACUTE + 0x01F8: "n", // LATIN CAPITAL LETTER N WITH GRAVE + 0x01F9: "n", // LATIN SMALL LETTER N WITH GRAVE + 0x01FA: "a", // LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE + 0x01FB: "a", // LATIN SMALL LETTER A WITH RING ABOVE AND ACUTE + 0x01FC: "ae", // LATIN CAPITAL LETTER AE WITH ACUTE + 0x01FD: "ae", // LATIN SMALL LETTER AE WITH ACUTE + 0x01FE: "o", // LATIN CAPITAL LETTER O WITH STROKE AND ACUTE + 0x01FF: "o", // LATIN SMALL LETTER O WITH STROKE AND ACUTE + 0x0200: "a", // LATIN CAPITAL LETTER A WITH DOUBLE GRAVE + 0x0201: "a", // LATIN SMALL LETTER A WITH DOUBLE GRAVE + 0x0202: "a", // LATIN CAPITAL LETTER A WITH INVERTED BREVE + 0x0203: "a", // LATIN SMALL LETTER A WITH INVERTED BREVE + 0x0204: "e", // LATIN CAPITAL LETTER E WITH DOUBLE GRAVE + 0x0205: "e", // LATIN SMALL LETTER E WITH DOUBLE GRAVE + 0x0206: "e", // LATIN CAPITAL LETTER E WITH INVERTED BREVE + 0x0207: "e", // LATIN SMALL LETTER E WITH INVERTED BREVE + 0x0208: "i", // LATIN CAPITAL LETTER I WITH DOUBLE GRAVE + 0x0209: "i", // LATIN SMALL LETTER I WITH DOUBLE GRAVE + 0x020A: "i", // LATIN CAPITAL LETTER I WITH INVERTED BREVE + 0x020B: "i", // LATIN SMALL LETTER I WITH INVERTED BREVE + 0x020C: "o", // LATIN CAPITAL LETTER O WITH DOUBLE GRAVE + 0x020D: "o", // LATIN SMALL LETTER O WITH DOUBLE GRAVE + 0x020E: "o", // LATIN CAPITAL LETTER O WITH INVERTED BREVE + 0x020F: "o", // LATIN SMALL LETTER O WITH INVERTED BREVE + 0x0210: "r", // LATIN CAPITAL LETTER R WITH DOUBLE GRAVE + 0x0211: "r", // LATIN SMALL LETTER R WITH DOUBLE GRAVE + 0x0212: "r", // LATIN CAPITAL LETTER R WITH INVERTED BREVE + 0x0213: "r", // LATIN SMALL LETTER R WITH INVERTED BREVE + 0x0214: "u", // LATIN CAPITAL LETTER U WITH DOUBLE GRAVE + 0x0215: "u", // LATIN SMALL LETTER U WITH DOUBLE GRAVE + 0x0216: "u", // LATIN CAPITAL LETTER U WITH INVERTED BREVE + 0x0217: "u", // LATIN SMALL LETTER U WITH INVERTED BREVE + 0x0218: "s", // LATIN CAPITAL LETTER S WITH COMMA BELOW + 0x0219: "s", // LATIN SMALL LETTER S WITH COMMA BELOW + 0x021A: "t", // LATIN CAPITAL LETTER T WITH COMMA BELOW + 0x021B: "t", // LATIN SMALL LETTER T WITH COMMA BELOW + 0x021E: "h", // LATIN CAPITAL LETTER H WITH CARON + 0x021F: "h", // LATIN SMALL LETTER H WITH CARON + 0x0220: "n", // LATIN CAPITAL LETTER N WITH LONG RIGHT LEG + 0x0221: "d", // LATIN SMALL LETTER D WITH CURL + 0x0224: "z", // LATIN CAPITAL LETTER Z WITH HOOK + 0x0225: "z", // LATIN SMALL LETTER Z WITH HOOK + 0x0226: "a", // LATIN CAPITAL LETTER A WITH DOT ABOVE + 0x0227: "a", // LATIN SMALL LETTER A WITH DOT ABOVE + 0x0228: "e", // LATIN CAPITAL LETTER E WITH CEDILLA + 0x0229: "e", // LATIN SMALL LETTER E WITH CEDILLA + 0x022A: "o", // LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON + 0x022B: "o", // LATIN SMALL LETTER O WITH DIAERESIS AND MACRON + 0x022C: "o", // LATIN CAPITAL LETTER O WITH TILDE AND MACRON + 0x022D: "o", // LATIN SMALL LETTER O WITH TILDE AND MACRON + 0x022E: "o", // LATIN CAPITAL LETTER O WITH DOT ABOVE + 0x022F: "o", // LATIN SMALL LETTER O WITH DOT ABOVE + 0x0230: "o", // LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON + 0x0231: "o", // LATIN SMALL LETTER O WITH DOT ABOVE AND MACRON + 0x0232: "y", // LATIN CAPITAL LETTER Y WITH MACRON + 0x0233: "y", // LATIN SMALL LETTER Y WITH MACRON + 0x0234: "l", // LATIN SMALL LETTER L WITH CURL + 0x0235: "n", // LATIN SMALL LETTER N WITH CURL + 0x0236: "t", // LATIN SMALL LETTER T WITH CURL + 0x023A: "a", // LATIN CAPITAL LETTER A WITH STROKE + 0x023B: "c", // LATIN CAPITAL LETTER C WITH STROKE + 0x023C: "c", // LATIN SMALL LETTER C WITH STROKE + 0x023D: "l", // LATIN CAPITAL LETTER L WITH BAR + 0x023E: "t", // LATIN CAPITAL LETTER T WITH DIAGONAL STROKE + 0x023F: "s", // LATIN SMALL LETTER S WITH SWASH TAIL + 0x0240: "z", // LATIN SMALL LETTER Z WITH SWASH TAIL + 0x0243: "b", // LATIN CAPITAL LETTER B WITH STROKE + 0x0246: "e", // LATIN CAPITAL LETTER E WITH STROKE + 0x0247: "e", // LATIN SMALL LETTER E WITH STROKE + 0x0248: "j", // LATIN CAPITAL LETTER J WITH STROKE + 0x0249: "j", // LATIN SMALL LETTER J WITH STROKE + 0x024B: "q", // LATIN SMALL LETTER Q WITH HOOK TAIL + 0x024C: "r", // LATIN CAPITAL LETTER R WITH STROKE + 0x024D: "r", // LATIN SMALL LETTER R WITH STROKE + 0x024E: "y", // LATIN CAPITAL LETTER Y WITH STROKE + 0x024F: "y", // LATIN SMALL LETTER Y WITH STROKE + 0x0253: "b", // LATIN SMALL LETTER B WITH HOOK + 0x0255: "c", // LATIN SMALL LETTER C WITH CURL + 0x0256: "d", // LATIN SMALL LETTER D WITH TAIL + 0x0257: "d", // LATIN SMALL LETTER D WITH HOOK + 0x0260: "g", // LATIN SMALL LETTER G WITH HOOK + 0x0266: "h", // LATIN SMALL LETTER H WITH HOOK + 0x0268: "i", // LATIN SMALL LETTER I WITH STROKE + 0x026B: "l", // LATIN SMALL LETTER L WITH MIDDLE TILDE + 0x026C: "l", // LATIN SMALL LETTER L WITH BELT + 0x026D: "l", // LATIN SMALL LETTER L WITH RETROFLEX HOOK + 0x0271: "m", // LATIN SMALL LETTER M WITH HOOK + 0x0272: "n", // LATIN SMALL LETTER N WITH LEFT HOOK + 0x0273: "n", // LATIN SMALL LETTER N WITH RETROFLEX HOOK + 0x027C: "r", // LATIN SMALL LETTER R WITH LONG LEG + 0x027D: "r", // LATIN SMALL LETTER R WITH TAIL + 0x027E: "r", // LATIN SMALL LETTER R WITH FISHHOOK + 0x0282: "s", // LATIN SMALL LETTER S WITH HOOK + 0x0288: "t", // LATIN SMALL LETTER T WITH RETROFLEX HOOK + 0x028B: "v", // LATIN SMALL LETTER V WITH HOOK + 0x0290: "z", // LATIN SMALL LETTER Z WITH RETROFLEX HOOK + 0x0291: "z", // LATIN SMALL LETTER Z WITH CURL + 0x029D: "j", // LATIN SMALL LETTER J WITH CROSSED-TAIL + 0x02A0: "q", // LATIN SMALL LETTER Q WITH HOOK + 0x0386: "\u03b1", // GREEK CAPITAL LETTER ALPHA WITH TONOS + 0x0388: "\u03b5", // GREEK CAPITAL LETTER EPSILON WITH TONOS + 0x0389: "\u03b7", // GREEK CAPITAL LETTER ETA WITH TONOS + 0x038A: "\u03b9", // GREEK CAPITAL LETTER IOTA WITH TONOS + 0x038C: "\u03bf", // GREEK CAPITAL LETTER OMICRON WITH TONOS + 0x038E: "\u03c5", // GREEK CAPITAL LETTER UPSILON WITH TONOS + 0x038F: "\u03c9", // GREEK CAPITAL LETTER OMEGA WITH TONOS + 0x0390: "\u03b9", // GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS + 0x03AA: "\u03b9", // GREEK CAPITAL LETTER IOTA WITH DIALYTIKA + 0x03AB: "\u03c5", // GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA + 0x03AC: "\u03b1", // GREEK SMALL LETTER ALPHA WITH TONOS + 0x03AD: "\u03b5", // GREEK SMALL LETTER EPSILON WITH TONOS + 0x03AE: "\u03b7", // GREEK SMALL LETTER ETA WITH TONOS + 0x03AF: "\u03b9", // GREEK SMALL LETTER IOTA WITH TONOS + 0x03B0: "\u03c5", // GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS + 0x03CA: "\u03b9", // GREEK SMALL LETTER IOTA WITH DIALYTIKA + 0x03CB: "\u03c5", // GREEK SMALL LETTER UPSILON WITH DIALYTIKA + 0x03CC: "\u03bf", // GREEK SMALL LETTER OMICRON WITH TONOS + 0x03CD: "\u03c5", // GREEK SMALL LETTER UPSILON WITH TONOS + 0x03CE: "\u03c9", // GREEK SMALL LETTER OMEGA WITH TONOS + 0x03D3: "\u03d2", // GREEK UPSILON WITH ACUTE AND HOOK SYMBOL + 0x03D4: "\u03d2", // GREEK UPSILON WITH DIAERESIS AND HOOK SYMBOL + 0x0400: "\u0435", // CYRILLIC CAPITAL LETTER IE WITH GRAVE + 0x0401: "\u0435", // CYRILLIC CAPITAL LETTER IO + 0x0403: "\u0433", // CYRILLIC CAPITAL LETTER GJE + 0x0407: "\u0456", // CYRILLIC CAPITAL LETTER YI + 0x040C: "\u043a", // CYRILLIC CAPITAL LETTER KJE + 0x040D: "\u0438", // CYRILLIC CAPITAL LETTER I WITH GRAVE + 0x040E: "\u0443", // CYRILLIC CAPITAL LETTER SHORT U + 0x0419: "\u0438", // CYRILLIC CAPITAL LETTER SHORT I + 0x0439: "\u0438", // CYRILLIC SMALL LETTER SHORT I + 0x0450: "\u0435", // CYRILLIC SMALL LETTER IE WITH GRAVE + 0x0451: "\u0435", // CYRILLIC SMALL LETTER IO + 0x0453: "\u0433", // CYRILLIC SMALL LETTER GJE + 0x0457: "\u0456", // CYRILLIC SMALL LETTER YI + 0x045C: "\u043a", // CYRILLIC SMALL LETTER KJE + 0x045D: "\u0438", // CYRILLIC SMALL LETTER I WITH GRAVE + 0x045E: "\u0443", // CYRILLIC SMALL LETTER SHORT U + 0x0476: "\u0475", // CYRILLIC CAPITAL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT + 0x0477: "\u0475", // CYRILLIC SMALL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT + 0x04C1: "\u0436", // CYRILLIC CAPITAL LETTER ZHE WITH BREVE + 0x04C2: "\u0436", // CYRILLIC SMALL LETTER ZHE WITH BREVE + 0x04D0: "\u0430", // CYRILLIC CAPITAL LETTER A WITH BREVE + 0x04D1: "\u0430", // CYRILLIC SMALL LETTER A WITH BREVE + 0x04D2: "\u0430", // CYRILLIC CAPITAL LETTER A WITH DIAERESIS + 0x04D3: "\u0430", // CYRILLIC SMALL LETTER A WITH DIAERESIS + 0x04D6: "\u0435", // CYRILLIC CAPITAL LETTER IE WITH BREVE + 0x04D7: "\u0435", // CYRILLIC SMALL LETTER IE WITH BREVE + 0x04DA: "\u04d9", // CYRILLIC CAPITAL LETTER SCHWA WITH DIAERESIS + 0x04DB: "\u04d9", // CYRILLIC SMALL LETTER SCHWA WITH DIAERESIS + 0x04DC: "\u0436", // CYRILLIC CAPITAL LETTER ZHE WITH DIAERESIS + 0x04DD: "\u0436", // CYRILLIC SMALL LETTER ZHE WITH DIAERESIS + 0x04DE: "\u0437", // CYRILLIC CAPITAL LETTER ZE WITH DIAERESIS + 0x04DF: "\u0437", // CYRILLIC SMALL LETTER ZE WITH DIAERESIS + 0x04E2: "\u0438", // CYRILLIC CAPITAL LETTER I WITH MACRON + 0x04E3: "\u0438", // CYRILLIC SMALL LETTER I WITH MACRON + 0x04E4: "\u0438", // CYRILLIC CAPITAL LETTER I WITH DIAERESIS + 0x04E5: "\u0438", // CYRILLIC SMALL LETTER I WITH DIAERESIS + 0x04E6: "\u043e", // CYRILLIC CAPITAL LETTER O WITH DIAERESIS + 0x04E7: "\u043e", // CYRILLIC SMALL LETTER O WITH DIAERESIS + 0x04EA: "\u04e9", // CYRILLIC CAPITAL LETTER BARRED O WITH DIAERESIS + 0x04EB: "\u04e9", // CYRILLIC SMALL LETTER BARRED O WITH DIAERESIS + 0x04EC: "\u044d", // CYRILLIC CAPITAL LETTER E WITH DIAERESIS + 0x04ED: "\u044d", // CYRILLIC SMALL LETTER E WITH DIAERESIS + 0x04EE: "\u0443", // CYRILLIC CAPITAL LETTER U WITH MACRON + 0x04EF: "\u0443", // CYRILLIC SMALL LETTER U WITH MACRON + 0x04F0: "\u0443", // CYRILLIC CAPITAL LETTER U WITH DIAERESIS + 0x04F1: "\u0443", // CYRILLIC SMALL LETTER U WITH DIAERESIS + 0x04F2: "\u0443", // CYRILLIC CAPITAL LETTER U WITH DOUBLE ACUTE + 0x04F3: "\u0443", // CYRILLIC SMALL LETTER U WITH DOUBLE ACUTE + 0x04F4: "\u0447", // CYRILLIC CAPITAL LETTER CHE WITH DIAERESIS + 0x04F5: "\u0447", // CYRILLIC SMALL LETTER CHE WITH DIAERESIS + 0x04F8: "\u044b", // CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS + 0x04F9: "\u044b", // CYRILLIC SMALL LETTER YERU WITH DIAERESIS + 0x0622: "\u0627", // ARABIC LETTER ALEF WITH MADDA ABOVE + 0x0623: "\u0627", // ARABIC LETTER ALEF WITH HAMZA ABOVE + 0x0624: "\u0648", // ARABIC LETTER WAW WITH HAMZA ABOVE + 0x0625: "\u0627", // ARABIC LETTER ALEF WITH HAMZA BELOW + 0x0626: "\u064a", // ARABIC LETTER YEH WITH HAMZA ABOVE + 0x06C0: "\u06d5", // ARABIC LETTER HEH WITH YEH ABOVE + 0x06C2: "\u06c1", // ARABIC LETTER HEH GOAL WITH HAMZA ABOVE + 0x06D3: "\u06d2", // ARABIC LETTER YEH BARREE WITH HAMZA ABOVE + 0x0929: "\u0928", // DEVANAGARI LETTER NNNA + 0x0931: "\u0930", // DEVANAGARI LETTER RRA + 0x0934: "\u0933", // DEVANAGARI LETTER LLLA + 0x0958: "\u0915", // DEVANAGARI LETTER QA + 0x0959: "\u0916", // DEVANAGARI LETTER KHHA + 0x095A: "\u0917", // DEVANAGARI LETTER GHHA + 0x095B: "\u091c", // DEVANAGARI LETTER ZA + 0x095C: "\u0921", // DEVANAGARI LETTER DDDHA + 0x095D: "\u0922", // DEVANAGARI LETTER RHA + 0x095E: "\u092b", // DEVANAGARI LETTER FA + 0x095F: "\u092f", // DEVANAGARI LETTER YYA + 0x09DC: "\u09a1", // BENGALI LETTER RRA + 0x09DD: "\u09a2", // BENGALI LETTER RHA + 0x09DF: "\u09af", // BENGALI LETTER YYA + 0x0A33: "\u0a32", // GURMUKHI LETTER LLA + 0x0A36: "\u0a38", // GURMUKHI LETTER SHA + 0x0A59: "\u0a16", // GURMUKHI LETTER KHHA + 0x0A5A: "\u0a17", // GURMUKHI LETTER GHHA + 0x0A5B: "\u0a1c", // GURMUKHI LETTER ZA + 0x0A5E: "\u0a2b", // GURMUKHI LETTER FA + 0x0B5C: "\u0b21", // ORIYA LETTER RRA + 0x0B5D: "\u0b22", // ORIYA LETTER RHA + 0x0F43: "\u0f42", // TIBETAN LETTER GHA + 0x0F4D: "\u0f4c", // TIBETAN LETTER DDHA + 0x0F52: "\u0f51", // TIBETAN LETTER DHA + 0x0F57: "\u0f56", // TIBETAN LETTER BHA + 0x0F5C: "\u0f5b", // TIBETAN LETTER DZHA + 0x0F69: "\u0f40", // TIBETAN LETTER KSSA + 0x1026: "\u1025", // MYANMAR LETTER UU + 0x1D6C: "b", // LATIN SMALL LETTER B WITH MIDDLE TILDE + 0x1D6D: "d", // LATIN SMALL LETTER D WITH MIDDLE TILDE + 0x1D6E: "f", // LATIN SMALL LETTER F WITH MIDDLE TILDE + 0x1D6F: "m", // LATIN SMALL LETTER M WITH MIDDLE TILDE + 0x1D70: "n", // LATIN SMALL LETTER N WITH MIDDLE TILDE + 0x1D71: "p", // LATIN SMALL LETTER P WITH MIDDLE TILDE + 0x1D72: "r", // LATIN SMALL LETTER R WITH MIDDLE TILDE + 0x1D73: "r", // LATIN SMALL LETTER R WITH FISHHOOK AND MIDDLE TILDE + 0x1D74: "s", // LATIN SMALL LETTER S WITH MIDDLE TILDE + 0x1D75: "t", // LATIN SMALL LETTER T WITH MIDDLE TILDE + 0x1D76: "z", // LATIN SMALL LETTER Z WITH MIDDLE TILDE + 0x1D7D: "p", // LATIN SMALL LETTER P WITH STROKE + 0x1D80: "b", // LATIN SMALL LETTER B WITH PALATAL HOOK + 0x1D81: "d", // LATIN SMALL LETTER D WITH PALATAL HOOK + 0x1D82: "f", // LATIN SMALL LETTER F WITH PALATAL HOOK + 0x1D83: "g", // LATIN SMALL LETTER G WITH PALATAL HOOK + 0x1D84: "k", // LATIN SMALL LETTER K WITH PALATAL HOOK + 0x1D85: "l", // LATIN SMALL LETTER L WITH PALATAL HOOK + 0x1D86: "m", // LATIN SMALL LETTER M WITH PALATAL HOOK + 0x1D87: "n", // LATIN SMALL LETTER N WITH PALATAL HOOK + 0x1D88: "p", // LATIN SMALL LETTER P WITH PALATAL HOOK + 0x1D89: "r", // LATIN SMALL LETTER R WITH PALATAL HOOK + 0x1D8A: "s", // LATIN SMALL LETTER S WITH PALATAL HOOK + 0x1D8C: "v", // LATIN SMALL LETTER V WITH PALATAL HOOK + 0x1D8D: "x", // LATIN SMALL LETTER X WITH PALATAL HOOK + 0x1D8E: "z", // LATIN SMALL LETTER Z WITH PALATAL HOOK + 0x1D8F: "a", // LATIN SMALL LETTER A WITH RETROFLEX HOOK + 0x1D91: "d", // LATIN SMALL LETTER D WITH HOOK AND TAIL + 0x1D92: "e", // LATIN SMALL LETTER E WITH RETROFLEX HOOK + 0x1D96: "i", // LATIN SMALL LETTER I WITH RETROFLEX HOOK + 0x1D99: "u", // LATIN SMALL LETTER U WITH RETROFLEX HOOK + 0x1E00: "a", // LATIN CAPITAL LETTER A WITH RING BELOW + 0x1E01: "a", // LATIN SMALL LETTER A WITH RING BELOW + 0x1E02: "b", // LATIN CAPITAL LETTER B WITH DOT ABOVE + 0x1E03: "b", // LATIN SMALL LETTER B WITH DOT ABOVE + 0x1E04: "b", // LATIN CAPITAL LETTER B WITH DOT BELOW + 0x1E05: "b", // LATIN SMALL LETTER B WITH DOT BELOW + 0x1E06: "b", // LATIN CAPITAL LETTER B WITH LINE BELOW + 0x1E07: "b", // LATIN SMALL LETTER B WITH LINE BELOW + 0x1E08: "c", // LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE + 0x1E09: "c", // LATIN SMALL LETTER C WITH CEDILLA AND ACUTE + 0x1E0A: "d", // LATIN CAPITAL LETTER D WITH DOT ABOVE + 0x1E0B: "d", // LATIN SMALL LETTER D WITH DOT ABOVE + 0x1E0C: "d", // LATIN CAPITAL LETTER D WITH DOT BELOW + 0x1E0D: "d", // LATIN SMALL LETTER D WITH DOT BELOW + 0x1E0E: "d", // LATIN CAPITAL LETTER D WITH LINE BELOW + 0x1E0F: "d", // LATIN SMALL LETTER D WITH LINE BELOW + 0x1E10: "d", // LATIN CAPITAL LETTER D WITH CEDILLA + 0x1E11: "d", // LATIN SMALL LETTER D WITH CEDILLA + 0x1E12: "d", // LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW + 0x1E13: "d", // LATIN SMALL LETTER D WITH CIRCUMFLEX BELOW + 0x1E14: "e", // LATIN CAPITAL LETTER E WITH MACRON AND GRAVE + 0x1E15: "e", // LATIN SMALL LETTER E WITH MACRON AND GRAVE + 0x1E16: "e", // LATIN CAPITAL LETTER E WITH MACRON AND ACUTE + 0x1E17: "e", // LATIN SMALL LETTER E WITH MACRON AND ACUTE + 0x1E18: "e", // LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW + 0x1E19: "e", // LATIN SMALL LETTER E WITH CIRCUMFLEX BELOW + 0x1E1A: "e", // LATIN CAPITAL LETTER E WITH TILDE BELOW + 0x1E1B: "e", // LATIN SMALL LETTER E WITH TILDE BELOW + 0x1E1C: "e", // LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE + 0x1E1D: "e", // LATIN SMALL LETTER E WITH CEDILLA AND BREVE + 0x1E1E: "f", // LATIN CAPITAL LETTER F WITH DOT ABOVE + 0x1E1F: "f", // LATIN SMALL LETTER F WITH DOT ABOVE + 0x1E20: "g", // LATIN CAPITAL LETTER G WITH MACRON + 0x1E21: "g", // LATIN SMALL LETTER G WITH MACRON + 0x1E22: "h", // LATIN CAPITAL LETTER H WITH DOT ABOVE + 0x1E23: "h", // LATIN SMALL LETTER H WITH DOT ABOVE + 0x1E24: "h", // LATIN CAPITAL LETTER H WITH DOT BELOW + 0x1E25: "h", // LATIN SMALL LETTER H WITH DOT BELOW + 0x1E26: "h", // LATIN CAPITAL LETTER H WITH DIAERESIS + 0x1E27: "h", // LATIN SMALL LETTER H WITH DIAERESIS + 0x1E28: "h", // LATIN CAPITAL LETTER H WITH CEDILLA + 0x1E29: "h", // LATIN SMALL LETTER H WITH CEDILLA + 0x1E2A: "h", // LATIN CAPITAL LETTER H WITH BREVE BELOW + 0x1E2B: "h", // LATIN SMALL LETTER H WITH BREVE BELOW + 0x1E2C: "i", // LATIN CAPITAL LETTER I WITH TILDE BELOW + 0x1E2D: "i", // LATIN SMALL LETTER I WITH TILDE BELOW + 0x1E2E: "i", // LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE + 0x1E2F: "i", // LATIN SMALL LETTER I WITH DIAERESIS AND ACUTE + 0x1E30: "k", // LATIN CAPITAL LETTER K WITH ACUTE + 0x1E31: "k", // LATIN SMALL LETTER K WITH ACUTE + 0x1E32: "k", // LATIN CAPITAL LETTER K WITH DOT BELOW + 0x1E33: "k", // LATIN SMALL LETTER K WITH DOT BELOW + 0x1E34: "k", // LATIN CAPITAL LETTER K WITH LINE BELOW + 0x1E35: "k", // LATIN SMALL LETTER K WITH LINE BELOW + 0x1E36: "l", // LATIN CAPITAL LETTER L WITH DOT BELOW + 0x1E37: "l", // LATIN SMALL LETTER L WITH DOT BELOW + 0x1E38: "l", // LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON + 0x1E39: "l", // LATIN SMALL LETTER L WITH DOT BELOW AND MACRON + 0x1E3A: "l", // LATIN CAPITAL LETTER L WITH LINE BELOW + 0x1E3B: "l", // LATIN SMALL LETTER L WITH LINE BELOW + 0x1E3C: "l", // LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW + 0x1E3D: "l", // LATIN SMALL LETTER L WITH CIRCUMFLEX BELOW + 0x1E3E: "m", // LATIN CAPITAL LETTER M WITH ACUTE + 0x1E3F: "m", // LATIN SMALL LETTER M WITH ACUTE + 0x1E40: "m", // LATIN CAPITAL LETTER M WITH DOT ABOVE + 0x1E41: "m", // LATIN SMALL LETTER M WITH DOT ABOVE + 0x1E42: "m", // LATIN CAPITAL LETTER M WITH DOT BELOW + 0x1E43: "m", // LATIN SMALL LETTER M WITH DOT BELOW + 0x1E44: "n", // LATIN CAPITAL LETTER N WITH DOT ABOVE + 0x1E45: "n", // LATIN SMALL LETTER N WITH DOT ABOVE + 0x1E46: "n", // LATIN CAPITAL LETTER N WITH DOT BELOW + 0x1E47: "n", // LATIN SMALL LETTER N WITH DOT BELOW + 0x1E48: "n", // LATIN CAPITAL LETTER N WITH LINE BELOW + 0x1E49: "n", // LATIN SMALL LETTER N WITH LINE BELOW + 0x1E4A: "n", // LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW + 0x1E4B: "n", // LATIN SMALL LETTER N WITH CIRCUMFLEX BELOW + 0x1E4C: "o", // LATIN CAPITAL LETTER O WITH TILDE AND ACUTE + 0x1E4D: "o", // LATIN SMALL LETTER O WITH TILDE AND ACUTE + 0x1E4E: "o", // LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS + 0x1E4F: "o", // LATIN SMALL LETTER O WITH TILDE AND DIAERESIS + 0x1E50: "o", // LATIN CAPITAL LETTER O WITH MACRON AND GRAVE + 0x1E51: "o", // LATIN SMALL LETTER O WITH MACRON AND GRAVE + 0x1E52: "o", // LATIN CAPITAL LETTER O WITH MACRON AND ACUTE + 0x1E53: "o", // LATIN SMALL LETTER O WITH MACRON AND ACUTE + 0x1E54: "p", // LATIN CAPITAL LETTER P WITH ACUTE + 0x1E55: "p", // LATIN SMALL LETTER P WITH ACUTE + 0x1E56: "p", // LATIN CAPITAL LETTER P WITH DOT ABOVE + 0x1E57: "p", // LATIN SMALL LETTER P WITH DOT ABOVE + 0x1E58: "r", // LATIN CAPITAL LETTER R WITH DOT ABOVE + 0x1E59: "r", // LATIN SMALL LETTER R WITH DOT ABOVE + 0x1E5A: "r", // LATIN CAPITAL LETTER R WITH DOT BELOW + 0x1E5B: "r", // LATIN SMALL LETTER R WITH DOT BELOW + 0x1E5C: "r", // LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON + 0x1E5D: "r", // LATIN SMALL LETTER R WITH DOT BELOW AND MACRON + 0x1E5E: "r", // LATIN CAPITAL LETTER R WITH LINE BELOW + 0x1E5F: "r", // LATIN SMALL LETTER R WITH LINE BELOW + 0x1E60: "s", // LATIN CAPITAL LETTER S WITH DOT ABOVE + 0x1E61: "s", // LATIN SMALL LETTER S WITH DOT ABOVE + 0x1E62: "s", // LATIN CAPITAL LETTER S WITH DOT BELOW + 0x1E63: "s", // LATIN SMALL LETTER S WITH DOT BELOW + 0x1E64: "s", // LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE + 0x1E65: "s", // LATIN SMALL LETTER S WITH ACUTE AND DOT ABOVE + 0x1E66: "s", // LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE + 0x1E67: "s", // LATIN SMALL LETTER S WITH CARON AND DOT ABOVE + 0x1E68: "s", // LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE + 0x1E69: "s", // LATIN SMALL LETTER S WITH DOT BELOW AND DOT ABOVE + 0x1E6A: "t", // LATIN CAPITAL LETTER T WITH DOT ABOVE + 0x1E6B: "t", // LATIN SMALL LETTER T WITH DOT ABOVE + 0x1E6C: "t", // LATIN CAPITAL LETTER T WITH DOT BELOW + 0x1E6D: "t", // LATIN SMALL LETTER T WITH DOT BELOW + 0x1E6E: "t", // LATIN CAPITAL LETTER T WITH LINE BELOW + 0x1E6F: "t", // LATIN SMALL LETTER T WITH LINE BELOW + 0x1E70: "t", // LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW + 0x1E71: "t", // LATIN SMALL LETTER T WITH CIRCUMFLEX BELOW + 0x1E72: "u", // LATIN CAPITAL LETTER U WITH DIAERESIS BELOW + 0x1E73: "u", // LATIN SMALL LETTER U WITH DIAERESIS BELOW + 0x1E74: "u", // LATIN CAPITAL LETTER U WITH TILDE BELOW + 0x1E75: "u", // LATIN SMALL LETTER U WITH TILDE BELOW + 0x1E76: "u", // LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW + 0x1E77: "u", // LATIN SMALL LETTER U WITH CIRCUMFLEX BELOW + 0x1E78: "u", // LATIN CAPITAL LETTER U WITH TILDE AND ACUTE + 0x1E79: "u", // LATIN SMALL LETTER U WITH TILDE AND ACUTE + 0x1E7A: "u", // LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS + 0x1E7B: "u", // LATIN SMALL LETTER U WITH MACRON AND DIAERESIS + 0x1E7C: "v", // LATIN CAPITAL LETTER V WITH TILDE + 0x1E7D: "v", // LATIN SMALL LETTER V WITH TILDE + 0x1E7E: "v", // LATIN CAPITAL LETTER V WITH DOT BELOW + 0x1E7F: "v", // LATIN SMALL LETTER V WITH DOT BELOW + 0x1E80: "w", // LATIN CAPITAL LETTER W WITH GRAVE + 0x1E81: "w", // LATIN SMALL LETTER W WITH GRAVE + 0x1E82: "w", // LATIN CAPITAL LETTER W WITH ACUTE + 0x1E83: "w", // LATIN SMALL LETTER W WITH ACUTE + 0x1E84: "w", // LATIN CAPITAL LETTER W WITH DIAERESIS + 0x1E85: "w", // LATIN SMALL LETTER W WITH DIAERESIS + 0x1E86: "w", // LATIN CAPITAL LETTER W WITH DOT ABOVE + 0x1E87: "w", // LATIN SMALL LETTER W WITH DOT ABOVE + 0x1E88: "w", // LATIN CAPITAL LETTER W WITH DOT BELOW + 0x1E89: "w", // LATIN SMALL LETTER W WITH DOT BELOW + 0x1E8A: "x", // LATIN CAPITAL LETTER X WITH DOT ABOVE + 0x1E8B: "x", // LATIN SMALL LETTER X WITH DOT ABOVE + 0x1E8C: "x", // LATIN CAPITAL LETTER X WITH DIAERESIS + 0x1E8D: "x", // LATIN SMALL LETTER X WITH DIAERESIS + 0x1E8E: "y", // LATIN CAPITAL LETTER Y WITH DOT ABOVE + 0x1E8F: "y", // LATIN SMALL LETTER Y WITH DOT ABOVE + 0x1E90: "z", // LATIN CAPITAL LETTER Z WITH CIRCUMFLEX + 0x1E91: "z", // LATIN SMALL LETTER Z WITH CIRCUMFLEX + 0x1E92: "z", // LATIN CAPITAL LETTER Z WITH DOT BELOW + 0x1E93: "z", // LATIN SMALL LETTER Z WITH DOT BELOW + 0x1E94: "z", // LATIN CAPITAL LETTER Z WITH LINE BELOW + 0x1E95: "z", // LATIN SMALL LETTER Z WITH LINE BELOW + 0x1E96: "h", // LATIN SMALL LETTER H WITH LINE BELOW + 0x1E97: "t", // LATIN SMALL LETTER T WITH DIAERESIS + 0x1E98: "w", // LATIN SMALL LETTER W WITH RING ABOVE + 0x1E99: "y", // LATIN SMALL LETTER Y WITH RING ABOVE + 0x1E9A: "a", // LATIN SMALL LETTER A WITH RIGHT HALF RING + 0x1E9B: "\u017f", // LATIN SMALL LETTER LONG S WITH DOT ABOVE + 0x1E9E: "ss", // LATIN CAPITAL LETTER SHARP S + 0x1EA0: "a", // LATIN CAPITAL LETTER A WITH DOT BELOW + 0x1EA1: "a", // LATIN SMALL LETTER A WITH DOT BELOW + 0x1EA2: "a", // LATIN CAPITAL LETTER A WITH HOOK ABOVE + 0x1EA3: "a", // LATIN SMALL LETTER A WITH HOOK ABOVE + 0x1EA4: "a", // LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE + 0x1EA5: "a", // LATIN SMALL LETTER A WITH CIRCUMFLEX AND ACUTE + 0x1EA6: "a", // LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE + 0x1EA7: "a", // LATIN SMALL LETTER A WITH CIRCUMFLEX AND GRAVE + 0x1EA8: "a", // LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE + 0x1EA9: "a", // LATIN SMALL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE + 0x1EAA: "a", // LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE + 0x1EAB: "a", // LATIN SMALL LETTER A WITH CIRCUMFLEX AND TILDE + 0x1EAC: "a", // LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW + 0x1EAD: "a", // LATIN SMALL LETTER A WITH CIRCUMFLEX AND DOT BELOW + 0x1EAE: "a", // LATIN CAPITAL LETTER A WITH BREVE AND ACUTE + 0x1EAF: "a", // LATIN SMALL LETTER A WITH BREVE AND ACUTE + 0x1EB0: "a", // LATIN CAPITAL LETTER A WITH BREVE AND GRAVE + 0x1EB1: "a", // LATIN SMALL LETTER A WITH BREVE AND GRAVE + 0x1EB2: "a", // LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE + 0x1EB3: "a", // LATIN SMALL LETTER A WITH BREVE AND HOOK ABOVE + 0x1EB4: "a", // LATIN CAPITAL LETTER A WITH BREVE AND TILDE + 0x1EB5: "a", // LATIN SMALL LETTER A WITH BREVE AND TILDE + 0x1EB6: "a", // LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW + 0x1EB7: "a", // LATIN SMALL LETTER A WITH BREVE AND DOT BELOW + 0x1EB8: "e", // LATIN CAPITAL LETTER E WITH DOT BELOW + 0x1EB9: "e", // LATIN SMALL LETTER E WITH DOT BELOW + 0x1EBA: "e", // LATIN CAPITAL LETTER E WITH HOOK ABOVE + 0x1EBB: "e", // LATIN SMALL LETTER E WITH HOOK ABOVE + 0x1EBC: "e", // LATIN CAPITAL LETTER E WITH TILDE + 0x1EBD: "e", // LATIN SMALL LETTER E WITH TILDE + 0x1EBE: "e", // LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE + 0x1EBF: "e", // LATIN SMALL LETTER E WITH CIRCUMFLEX AND ACUTE + 0x1EC0: "e", // LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE + 0x1EC1: "e", // LATIN SMALL LETTER E WITH CIRCUMFLEX AND GRAVE + 0x1EC2: "e", // LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE + 0x1EC3: "e", // LATIN SMALL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE + 0x1EC4: "e", // LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE + 0x1EC5: "e", // LATIN SMALL LETTER E WITH CIRCUMFLEX AND TILDE + 0x1EC6: "e", // LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW + 0x1EC7: "e", // LATIN SMALL LETTER E WITH CIRCUMFLEX AND DOT BELOW + 0x1EC8: "i", // LATIN CAPITAL LETTER I WITH HOOK ABOVE + 0x1EC9: "i", // LATIN SMALL LETTER I WITH HOOK ABOVE + 0x1ECA: "i", // LATIN CAPITAL LETTER I WITH DOT BELOW + 0x1ECB: "i", // LATIN SMALL LETTER I WITH DOT BELOW + 0x1ECC: "o", // LATIN CAPITAL LETTER O WITH DOT BELOW + 0x1ECD: "o", // LATIN SMALL LETTER O WITH DOT BELOW + 0x1ECE: "o", // LATIN CAPITAL LETTER O WITH HOOK ABOVE + 0x1ECF: "o", // LATIN SMALL LETTER O WITH HOOK ABOVE + 0x1ED0: "o", // LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE + 0x1ED1: "o", // LATIN SMALL LETTER O WITH CIRCUMFLEX AND ACUTE + 0x1ED2: "o", // LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE + 0x1ED3: "o", // LATIN SMALL LETTER O WITH CIRCUMFLEX AND GRAVE + 0x1ED4: "o", // LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE + 0x1ED5: "o", // LATIN SMALL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE + 0x1ED6: "o", // LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE + 0x1ED7: "o", // LATIN SMALL LETTER O WITH CIRCUMFLEX AND TILDE + 0x1ED8: "o", // LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW + 0x1ED9: "o", // LATIN SMALL LETTER O WITH CIRCUMFLEX AND DOT BELOW + 0x1EDA: "o", // LATIN CAPITAL LETTER O WITH HORN AND ACUTE + 0x1EDB: "o", // LATIN SMALL LETTER O WITH HORN AND ACUTE + 0x1EDC: "o", // LATIN CAPITAL LETTER O WITH HORN AND GRAVE + 0x1EDD: "o", // LATIN SMALL LETTER O WITH HORN AND GRAVE + 0x1EDE: "o", // LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE + 0x1EDF: "o", // LATIN SMALL LETTER O WITH HORN AND HOOK ABOVE + 0x1EE0: "o", // LATIN CAPITAL LETTER O WITH HORN AND TILDE + 0x1EE1: "o", // LATIN SMALL LETTER O WITH HORN AND TILDE + 0x1EE2: "o", // LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW + 0x1EE3: "o", // LATIN SMALL LETTER O WITH HORN AND DOT BELOW + 0x1EE4: "u", // LATIN CAPITAL LETTER U WITH DOT BELOW + 0x1EE5: "u", // LATIN SMALL LETTER U WITH DOT BELOW + 0x1EE6: "u", // LATIN CAPITAL LETTER U WITH HOOK ABOVE + 0x1EE7: "u", // LATIN SMALL LETTER U WITH HOOK ABOVE + 0x1EE8: "u", // LATIN CAPITAL LETTER U WITH HORN AND ACUTE + 0x1EE9: "u", // LATIN SMALL LETTER U WITH HORN AND ACUTE + 0x1EEA: "u", // LATIN CAPITAL LETTER U WITH HORN AND GRAVE + 0x1EEB: "u", // LATIN SMALL LETTER U WITH HORN AND GRAVE + 0x1EEC: "u", // LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE + 0x1EED: "u", // LATIN SMALL LETTER U WITH HORN AND HOOK ABOVE + 0x1EEE: "u", // LATIN CAPITAL LETTER U WITH HORN AND TILDE + 0x1EEF: "u", // LATIN SMALL LETTER U WITH HORN AND TILDE + 0x1EF0: "u", // LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW + 0x1EF1: "u", // LATIN SMALL LETTER U WITH HORN AND DOT BELOW + 0x1EF2: "y", // LATIN CAPITAL LETTER Y WITH GRAVE + 0x1EF3: "y", // LATIN SMALL LETTER Y WITH GRAVE + 0x1EF4: "y", // LATIN CAPITAL LETTER Y WITH DOT BELOW + 0x1EF5: "y", // LATIN SMALL LETTER Y WITH DOT BELOW + 0x1EF6: "y", // LATIN CAPITAL LETTER Y WITH HOOK ABOVE + 0x1EF7: "y", // LATIN SMALL LETTER Y WITH HOOK ABOVE + 0x1EF8: "y", // LATIN CAPITAL LETTER Y WITH TILDE + 0x1EF9: "y", // LATIN SMALL LETTER Y WITH TILDE + 0x1EFE: "y", // LATIN CAPITAL LETTER Y WITH LOOP + 0x1EFF: "y", // LATIN SMALL LETTER Y WITH LOOP + 0x1F00: "\u03b1", // GREEK SMALL LETTER ALPHA WITH PSILI + 0x1F01: "\u03b1", // GREEK SMALL LETTER ALPHA WITH DASIA + 0x1F02: "\u03b1", // GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA + 0x1F03: "\u03b1", // GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA + 0x1F04: "\u03b1", // GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA + 0x1F05: "\u03b1", // GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA + 0x1F06: "\u03b1", // GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI + 0x1F07: "\u03b1", // GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI + 0x1F08: "\u03b1", // GREEK CAPITAL LETTER ALPHA WITH PSILI + 0x1F09: "\u03b1", // GREEK CAPITAL LETTER ALPHA WITH DASIA + 0x1F0A: "\u03b1", // GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA + 0x1F0B: "\u03b1", // GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA + 0x1F0C: "\u03b1", // GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA + 0x1F0D: "\u03b1", // GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA + 0x1F0E: "\u03b1", // GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI + 0x1F0F: "\u03b1", // GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI + 0x1F10: "\u03b5", // GREEK SMALL LETTER EPSILON WITH PSILI + 0x1F11: "\u03b5", // GREEK SMALL LETTER EPSILON WITH DASIA + 0x1F12: "\u03b5", // GREEK SMALL LETTER EPSILON WITH PSILI AND VARIA + 0x1F13: "\u03b5", // GREEK SMALL LETTER EPSILON WITH DASIA AND VARIA + 0x1F14: "\u03b5", // GREEK SMALL LETTER EPSILON WITH PSILI AND OXIA + 0x1F15: "\u03b5", // GREEK SMALL LETTER EPSILON WITH DASIA AND OXIA + 0x1F18: "\u03b5", // GREEK CAPITAL LETTER EPSILON WITH PSILI + 0x1F19: "\u03b5", // GREEK CAPITAL LETTER EPSILON WITH DASIA + 0x1F1A: "\u03b5", // GREEK CAPITAL LETTER EPSILON WITH PSILI AND VARIA + 0x1F1B: "\u03b5", // GREEK CAPITAL LETTER EPSILON WITH DASIA AND VARIA + 0x1F1C: "\u03b5", // GREEK CAPITAL LETTER EPSILON WITH PSILI AND OXIA + 0x1F1D: "\u03b5", // GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA + 0x1F20: "\u03b7", // GREEK SMALL LETTER ETA WITH PSILI + 0x1F21: "\u03b7", // GREEK SMALL LETTER ETA WITH DASIA + 0x1F22: "\u03b7", // GREEK SMALL LETTER ETA WITH PSILI AND VARIA + 0x1F23: "\u03b7", // GREEK SMALL LETTER ETA WITH DASIA AND VARIA + 0x1F24: "\u03b7", // GREEK SMALL LETTER ETA WITH PSILI AND OXIA + 0x1F25: "\u03b7", // GREEK SMALL LETTER ETA WITH DASIA AND OXIA + 0x1F26: "\u03b7", // GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI + 0x1F27: "\u03b7", // GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI + 0x1F28: "\u03b7", // GREEK CAPITAL LETTER ETA WITH PSILI + 0x1F29: "\u03b7", // GREEK CAPITAL LETTER ETA WITH DASIA + 0x1F2A: "\u03b7", // GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA + 0x1F2B: "\u03b7", // GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA + 0x1F2C: "\u03b7", // GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA + 0x1F2D: "\u03b7", // GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA + 0x1F2E: "\u03b7", // GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI + 0x1F2F: "\u03b7", // GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI + 0x1F30: "\u03b9", // GREEK SMALL LETTER IOTA WITH PSILI + 0x1F31: "\u03b9", // GREEK SMALL LETTER IOTA WITH DASIA + 0x1F32: "\u03b9", // GREEK SMALL LETTER IOTA WITH PSILI AND VARIA + 0x1F33: "\u03b9", // GREEK SMALL LETTER IOTA WITH DASIA AND VARIA + 0x1F34: "\u03b9", // GREEK SMALL LETTER IOTA WITH PSILI AND OXIA + 0x1F35: "\u03b9", // GREEK SMALL LETTER IOTA WITH DASIA AND OXIA + 0x1F36: "\u03b9", // GREEK SMALL LETTER IOTA WITH PSILI AND PERISPOMENI + 0x1F37: "\u03b9", // GREEK SMALL LETTER IOTA WITH DASIA AND PERISPOMENI + 0x1F38: "\u03b9", // GREEK CAPITAL LETTER IOTA WITH PSILI + 0x1F39: "\u03b9", // GREEK CAPITAL LETTER IOTA WITH DASIA + 0x1F3A: "\u03b9", // GREEK CAPITAL LETTER IOTA WITH PSILI AND VARIA + 0x1F3B: "\u03b9", // GREEK CAPITAL LETTER IOTA WITH DASIA AND VARIA + 0x1F3C: "\u03b9", // GREEK CAPITAL LETTER IOTA WITH PSILI AND OXIA + 0x1F3D: "\u03b9", // GREEK CAPITAL LETTER IOTA WITH DASIA AND OXIA + 0x1F3E: "\u03b9", // GREEK CAPITAL LETTER IOTA WITH PSILI AND PERISPOMENI + 0x1F3F: "\u03b9", // GREEK CAPITAL LETTER IOTA WITH DASIA AND PERISPOMENI + 0x1F40: "\u03bf", // GREEK SMALL LETTER OMICRON WITH PSILI + 0x1F41: "\u03bf", // GREEK SMALL LETTER OMICRON WITH DASIA + 0x1F42: "\u03bf", // GREEK SMALL LETTER OMICRON WITH PSILI AND VARIA + 0x1F43: "\u03bf", // GREEK SMALL LETTER OMICRON WITH DASIA AND VARIA + 0x1F44: "\u03bf", // GREEK SMALL LETTER OMICRON WITH PSILI AND OXIA + 0x1F45: "\u03bf", // GREEK SMALL LETTER OMICRON WITH DASIA AND OXIA + 0x1F48: "\u03bf", // GREEK CAPITAL LETTER OMICRON WITH PSILI + 0x1F49: "\u03bf", // GREEK CAPITAL LETTER OMICRON WITH DASIA + 0x1F4A: "\u03bf", // GREEK CAPITAL LETTER OMICRON WITH PSILI AND VARIA + 0x1F4B: "\u03bf", // GREEK CAPITAL LETTER OMICRON WITH DASIA AND VARIA + 0x1F4C: "\u03bf", // GREEK CAPITAL LETTER OMICRON WITH PSILI AND OXIA + 0x1F4D: "\u03bf", // GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA + 0x1F50: "\u03c5", // GREEK SMALL LETTER UPSILON WITH PSILI + 0x1F51: "\u03c5", // GREEK SMALL LETTER UPSILON WITH DASIA + 0x1F52: "\u03c5", // GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA + 0x1F53: "\u03c5", // GREEK SMALL LETTER UPSILON WITH DASIA AND VARIA + 0x1F54: "\u03c5", // GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA + 0x1F55: "\u03c5", // GREEK SMALL LETTER UPSILON WITH DASIA AND OXIA + 0x1F56: "\u03c5", // GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI + 0x1F57: "\u03c5", // GREEK SMALL LETTER UPSILON WITH DASIA AND PERISPOMENI + 0x1F59: "\u03c5", // GREEK CAPITAL LETTER UPSILON WITH DASIA + 0x1F5B: "\u03c5", // GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA + 0x1F5D: "\u03c5", // GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA + 0x1F5F: "\u03c5", // GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI + 0x1F60: "\u03c9", // GREEK SMALL LETTER OMEGA WITH PSILI + 0x1F61: "\u03c9", // GREEK SMALL LETTER OMEGA WITH DASIA + 0x1F62: "\u03c9", // GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA + 0x1F63: "\u03c9", // GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA + 0x1F64: "\u03c9", // GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA + 0x1F65: "\u03c9", // GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA + 0x1F66: "\u03c9", // GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI + 0x1F67: "\u03c9", // GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI + 0x1F68: "\u03c9", // GREEK CAPITAL LETTER OMEGA WITH PSILI + 0x1F69: "\u03c9", // GREEK CAPITAL LETTER OMEGA WITH DASIA + 0x1F6A: "\u03c9", // GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA + 0x1F6B: "\u03c9", // GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA + 0x1F6C: "\u03c9", // GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA + 0x1F6D: "\u03c9", // GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA + 0x1F6E: "\u03c9", // GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI + 0x1F6F: "\u03c9", // GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI + 0x1F70: "\u03b1", // GREEK SMALL LETTER ALPHA WITH VARIA + 0x1F71: "\u03b1", // GREEK SMALL LETTER ALPHA WITH OXIA + 0x1F72: "\u03b5", // GREEK SMALL LETTER EPSILON WITH VARIA + 0x1F73: "\u03b5", // GREEK SMALL LETTER EPSILON WITH OXIA + 0x1F74: "\u03b7", // GREEK SMALL LETTER ETA WITH VARIA + 0x1F75: "\u03b7", // GREEK SMALL LETTER ETA WITH OXIA + 0x1F76: "\u03b9", // GREEK SMALL LETTER IOTA WITH VARIA + 0x1F77: "\u03b9", // GREEK SMALL LETTER IOTA WITH OXIA + 0x1F78: "\u03bf", // GREEK SMALL LETTER OMICRON WITH VARIA + 0x1F79: "\u03bf", // GREEK SMALL LETTER OMICRON WITH OXIA + 0x1F7A: "\u03c5", // GREEK SMALL LETTER UPSILON WITH VARIA + 0x1F7B: "\u03c5", // GREEK SMALL LETTER UPSILON WITH OXIA + 0x1F7C: "\u03c9", // GREEK SMALL LETTER OMEGA WITH VARIA + 0x1F7D: "\u03c9", // GREEK SMALL LETTER OMEGA WITH OXIA + 0x1F80: "\u03b1", // GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI + 0x1F81: "\u03b1", // GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI + 0x1F82: "\u03b1", // GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI + 0x1F83: "\u03b1", // GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI + 0x1F84: "\u03b1", // GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI + 0x1F85: "\u03b1", // GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI + 0x1F86: "\u03b1", // GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + 0x1F87: "\u03b1", // GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + 0x1F88: "\u03b1", // GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI + 0x1F89: "\u03b1", // GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI + 0x1F8A: "\u03b1", // GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI + 0x1F8B: "\u03b1", // GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI + 0x1F8C: "\u03b1", // GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI + 0x1F8D: "\u03b1", // GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI + 0x1F8E: "\u03b1", // GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + 0x1F8F: "\u03b1", // GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + 0x1F90: "\u03b7", // GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI + 0x1F91: "\u03b7", // GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI + 0x1F92: "\u03b7", // GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI + 0x1F93: "\u03b7", // GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI + 0x1F94: "\u03b7", // GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI + 0x1F95: "\u03b7", // GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI + 0x1F96: "\u03b7", // GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + 0x1F97: "\u03b7", // GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + 0x1F98: "\u03b7", // GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI + 0x1F99: "\u03b7", // GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI + 0x1F9A: "\u03b7", // GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI + 0x1F9B: "\u03b7", // GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI + 0x1F9C: "\u03b7", // GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI + 0x1F9D: "\u03b7", // GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI + 0x1F9E: "\u03b7", // GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + 0x1F9F: "\u03b7", // GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + 0x1FA0: "\u03c9", // GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI + 0x1FA1: "\u03c9", // GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI + 0x1FA2: "\u03c9", // GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI + 0x1FA3: "\u03c9", // GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI + 0x1FA4: "\u03c9", // GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI + 0x1FA5: "\u03c9", // GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI + 0x1FA6: "\u03c9", // GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI + 0x1FA7: "\u03c9", // GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI + 0x1FA8: "\u03c9", // GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI + 0x1FA9: "\u03c9", // GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI + 0x1FAA: "\u03c9", // GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI + 0x1FAB: "\u03c9", // GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI + 0x1FAC: "\u03c9", // GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI + 0x1FAD: "\u03c9", // GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI + 0x1FAE: "\u03c9", // GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI + 0x1FAF: "\u03c9", // GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI + 0x1FB0: "\u03b1", // GREEK SMALL LETTER ALPHA WITH VRACHY + 0x1FB1: "\u03b1", // GREEK SMALL LETTER ALPHA WITH MACRON + 0x1FB2: "\u03b1", // GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI + 0x1FB3: "\u03b1", // GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI + 0x1FB4: "\u03b1", // GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI + 0x1FB6: "\u03b1", // GREEK SMALL LETTER ALPHA WITH PERISPOMENI + 0x1FB7: "\u03b1", // GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI + 0x1FB8: "\u03b1", // GREEK CAPITAL LETTER ALPHA WITH VRACHY + 0x1FB9: "\u03b1", // GREEK CAPITAL LETTER ALPHA WITH MACRON + 0x1FBA: "\u03b1", // GREEK CAPITAL LETTER ALPHA WITH VARIA + 0x1FBB: "\u03b1", // GREEK CAPITAL LETTER ALPHA WITH OXIA + 0x1FBC: "\u03b1", // GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI + 0x1FC2: "\u03b7", // GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI + 0x1FC3: "\u03b7", // GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI + 0x1FC4: "\u03b7", // GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI + 0x1FC6: "\u03b7", // GREEK SMALL LETTER ETA WITH PERISPOMENI + 0x1FC7: "\u03b7", // GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI + 0x1FC8: "\u03b5", // GREEK CAPITAL LETTER EPSILON WITH VARIA + 0x1FC9: "\u03b5", // GREEK CAPITAL LETTER EPSILON WITH OXIA + 0x1FCA: "\u03b7", // GREEK CAPITAL LETTER ETA WITH VARIA + 0x1FCB: "\u03b7", // GREEK CAPITAL LETTER ETA WITH OXIA + 0x1FCC: "\u03b7", // GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI + 0x1FD0: "\u03b9", // GREEK SMALL LETTER IOTA WITH VRACHY + 0x1FD1: "\u03b9", // GREEK SMALL LETTER IOTA WITH MACRON + 0x1FD2: "\u03b9", // GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA + 0x1FD3: "\u03b9", // GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA + 0x1FD6: "\u03b9", // GREEK SMALL LETTER IOTA WITH PERISPOMENI + 0x1FD7: "\u03b9", // GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI + 0x1FD8: "\u03b9", // GREEK CAPITAL LETTER IOTA WITH VRACHY + 0x1FD9: "\u03b9", // GREEK CAPITAL LETTER IOTA WITH MACRON + 0x1FDA: "\u03b9", // GREEK CAPITAL LETTER IOTA WITH VARIA + 0x1FDB: "\u03b9", // GREEK CAPITAL LETTER IOTA WITH OXIA + 0x1FE0: "\u03c5", // GREEK SMALL LETTER UPSILON WITH VRACHY + 0x1FE1: "\u03c5", // GREEK SMALL LETTER UPSILON WITH MACRON + 0x1FE2: "\u03c5", // GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA + 0x1FE3: "\u03c5", // GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA + 0x1FE4: "\u03c1", // GREEK SMALL LETTER RHO WITH PSILI + 0x1FE5: "\u03c1", // GREEK SMALL LETTER RHO WITH DASIA + 0x1FE6: "\u03c5", // GREEK SMALL LETTER UPSILON WITH PERISPOMENI + 0x1FE7: "\u03c5", // GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI + 0x1FE8: "\u03c5", // GREEK CAPITAL LETTER UPSILON WITH VRACHY + 0x1FE9: "\u03c5", // GREEK CAPITAL LETTER UPSILON WITH MACRON + 0x1FEA: "\u03c5", // GREEK CAPITAL LETTER UPSILON WITH VARIA + 0x1FEB: "\u03c5", // GREEK CAPITAL LETTER UPSILON WITH OXIA + 0x1FEC: "\u03c1", // GREEK CAPITAL LETTER RHO WITH DASIA + 0x1FF2: "\u03c9", // GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI + 0x1FF3: "\u03c9", // GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI + 0x1FF4: "\u03c9", // GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI + 0x1FF6: "\u03c9", // GREEK SMALL LETTER OMEGA WITH PERISPOMENI + 0x1FF7: "\u03c9", // GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI + 0x1FF8: "\u03bf", // GREEK CAPITAL LETTER OMICRON WITH VARIA + 0x1FF9: "\u03bf", // GREEK CAPITAL LETTER OMICRON WITH OXIA + 0x1FFA: "\u03c9", // GREEK CAPITAL LETTER OMEGA WITH VARIA + 0x1FFB: "\u03c9", // GREEK CAPITAL LETTER OMEGA WITH OXIA + 0x1FFC: "\u03c9", // GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI + 0x212B: "a", // ANGSTROM SIGN + 0x2C60: "l", // LATIN CAPITAL LETTER L WITH DOUBLE BAR + 0x2C61: "l", // LATIN SMALL LETTER L WITH DOUBLE BAR + 0x2C62: "l", // LATIN CAPITAL LETTER L WITH MIDDLE TILDE + 0x2C63: "p", // LATIN CAPITAL LETTER P WITH STROKE + 0x2C64: "r", // LATIN CAPITAL LETTER R WITH TAIL + 0x2C65: "a", // LATIN SMALL LETTER A WITH STROKE + 0x2C66: "t", // LATIN SMALL LETTER T WITH DIAGONAL STROKE + 0x2C67: "h", // LATIN CAPITAL LETTER H WITH DESCENDER + 0x2C68: "h", // LATIN SMALL LETTER H WITH DESCENDER + 0x2C69: "k", // LATIN CAPITAL LETTER K WITH DESCENDER + 0x2C6A: "k", // LATIN SMALL LETTER K WITH DESCENDER + 0x2C6B: "z", // LATIN CAPITAL LETTER Z WITH DESCENDER + 0x2C6C: "z", // LATIN SMALL LETTER Z WITH DESCENDER + 0x2C6E: "m", // LATIN CAPITAL LETTER M WITH HOOK + 0x2C71: "v", // LATIN SMALL LETTER V WITH RIGHT HOOK + 0x2C72: "w", // LATIN CAPITAL LETTER W WITH HOOK + 0x2C73: "w", // LATIN SMALL LETTER W WITH HOOK + 0x2C74: "v", // LATIN SMALL LETTER V WITH CURL + 0x2C78: "e", // LATIN SMALL LETTER E WITH NOTCH + 0x2C7A: "o", // LATIN SMALL LETTER O WITH LOW RING INSIDE + 0x2C7E: "s", // LATIN CAPITAL LETTER S WITH SWASH TAIL + 0x2C7F: "z", // LATIN CAPITAL LETTER Z WITH SWASH TAIL + 0x304C: "\u304b", // HIRAGANA LETTER GA + 0x304E: "\u304d", // HIRAGANA LETTER GI + 0x3050: "\u304f", // HIRAGANA LETTER GU + 0x3052: "\u3051", // HIRAGANA LETTER GE + 0x3054: "\u3053", // HIRAGANA LETTER GO + 0x3056: "\u3055", // HIRAGANA LETTER ZA + 0x3058: "\u3057", // HIRAGANA LETTER ZI + 0x305A: "\u3059", // HIRAGANA LETTER ZU + 0x305C: "\u305b", // HIRAGANA LETTER ZE + 0x305E: "\u305d", // HIRAGANA LETTER ZO + 0x3060: "\u305f", // HIRAGANA LETTER DA + 0x3062: "\u3061", // HIRAGANA LETTER DI + 0x3065: "\u3064", // HIRAGANA LETTER DU + 0x3067: "\u3066", // HIRAGANA LETTER DE + 0x3069: "\u3068", // HIRAGANA LETTER DO + 0x3070: "\u306f", // HIRAGANA LETTER BA + 0x3071: "\u306f", // HIRAGANA LETTER PA + 0x3073: "\u3072", // HIRAGANA LETTER BI + 0x3074: "\u3072", // HIRAGANA LETTER PI + 0x3076: "\u3075", // HIRAGANA LETTER BU + 0x3077: "\u3075", // HIRAGANA LETTER PU + 0x3079: "\u3078", // HIRAGANA LETTER BE + 0x307A: "\u3078", // HIRAGANA LETTER PE + 0x307C: "\u307b", // HIRAGANA LETTER BO + 0x307D: "\u307b", // HIRAGANA LETTER PO + 0x3094: "\u3046", // HIRAGANA LETTER VU + 0x309E: "\u309d", // HIRAGANA VOICED ITERATION MARK + 0x30AC: "\u30ab", // KATAKANA LETTER GA + 0x30AE: "\u30ad", // KATAKANA LETTER GI + 0x30B0: "\u30af", // KATAKANA LETTER GU + 0x30B2: "\u30b1", // KATAKANA LETTER GE + 0x30B4: "\u30b3", // KATAKANA LETTER GO + 0x30B6: "\u30b5", // KATAKANA LETTER ZA + 0x30B8: "\u30b7", // KATAKANA LETTER ZI + 0x30BA: "\u30b9", // KATAKANA LETTER ZU + 0x30BC: "\u30bb", // KATAKANA LETTER ZE + 0x30BE: "\u30bd", // KATAKANA LETTER ZO + 0x30C0: "\u30bf", // KATAKANA LETTER DA + 0x30C2: "\u30c1", // KATAKANA LETTER DI + 0x30C5: "\u30c4", // KATAKANA LETTER DU + 0x30C7: "\u30c6", // KATAKANA LETTER DE + 0x30C9: "\u30c8", // KATAKANA LETTER DO + 0x30D0: "\u30cf", // KATAKANA LETTER BA + 0x30D1: "\u30cf", // KATAKANA LETTER PA + 0x30D3: "\u30d2", // KATAKANA LETTER BI + 0x30D4: "\u30d2", // KATAKANA LETTER PI + 0x30D6: "\u30d5", // KATAKANA LETTER BU + 0x30D7: "\u30d5", // KATAKANA LETTER PU + 0x30D9: "\u30d8", // KATAKANA LETTER BE + 0x30DA: "\u30d8", // KATAKANA LETTER PE + 0x30DC: "\u30db", // KATAKANA LETTER BO + 0x30DD: "\u30db", // KATAKANA LETTER PO + 0x30F4: "\u30a6", // KATAKANA LETTER VU + 0x30F7: "\u30ef", // KATAKANA LETTER VA + 0x30F8: "\u30f0", // KATAKANA LETTER VI + 0x30F9: "\u30f1", // KATAKANA LETTER VE + 0x30FA: "\u30f2", // KATAKANA LETTER VO + 0x30FE: "\u30fd", // KATAKANA VOICED ITERATION MARK + 0xA740: "k", // LATIN CAPITAL LETTER K WITH STROKE + 0xA741: "k", // LATIN SMALL LETTER K WITH STROKE + 0xA742: "k", // LATIN CAPITAL LETTER K WITH DIAGONAL STROKE + 0xA743: "k", // LATIN SMALL LETTER K WITH DIAGONAL STROKE + 0xA744: "k", // LATIN CAPITAL LETTER K WITH STROKE AND DIAGONAL STROKE + 0xA745: "k", // LATIN SMALL LETTER K WITH STROKE AND DIAGONAL STROKE + 0xA748: "l", // LATIN CAPITAL LETTER L WITH HIGH STROKE + 0xA749: "l", // LATIN SMALL LETTER L WITH HIGH STROKE + 0xA74A: "o", // LATIN CAPITAL LETTER O WITH LONG STROKE OVERLAY + 0xA74B: "o", // LATIN SMALL LETTER O WITH LONG STROKE OVERLAY + 0xA74C: "o", // LATIN CAPITAL LETTER O WITH LOOP + 0xA74D: "o", // LATIN SMALL LETTER O WITH LOOP + 0xA750: "p", // LATIN CAPITAL LETTER P WITH STROKE THROUGH DESCENDER + 0xA751: "p", // LATIN SMALL LETTER P WITH STROKE THROUGH DESCENDER + 0xA752: "p", // LATIN CAPITAL LETTER P WITH FLOURISH + 0xA753: "p", // LATIN SMALL LETTER P WITH FLOURISH + 0xA754: "p", // LATIN CAPITAL LETTER P WITH SQUIRREL TAIL + 0xA755: "p", // LATIN SMALL LETTER P WITH SQUIRREL TAIL + 0xA756: "q", // LATIN CAPITAL LETTER Q WITH STROKE THROUGH DESCENDER + 0xA757: "q", // LATIN SMALL LETTER Q WITH STROKE THROUGH DESCENDER + 0xA758: "q", // LATIN CAPITAL LETTER Q WITH DIAGONAL STROKE + 0xA759: "q", // LATIN SMALL LETTER Q WITH DIAGONAL STROKE + 0xA75E: "v", // LATIN CAPITAL LETTER V WITH DIAGONAL STROKE + 0xA75F: "v", // LATIN SMALL LETTER V WITH DIAGONAL STROKE + 0xA78E: "l", // LATIN SMALL LETTER L WITH RETROFLEX HOOK AND BELT + 0xA790: "n", // LATIN CAPITAL LETTER N WITH DESCENDER + 0xA791: "n", // LATIN SMALL LETTER N WITH DESCENDER + 0xA792: "c", // LATIN CAPITAL LETTER C WITH BAR + 0xA793: "c", // LATIN SMALL LETTER C WITH BAR + 0xA794: "c", // LATIN SMALL LETTER C WITH PALATAL HOOK + 0xA795: "h", // LATIN SMALL LETTER H WITH PALATAL HOOK + 0xA796: "b", // LATIN CAPITAL LETTER B WITH FLOURISH + 0xA797: "b", // LATIN SMALL LETTER B WITH FLOURISH + 0xA798: "f", // LATIN CAPITAL LETTER F WITH STROKE + 0xA799: "f", // LATIN SMALL LETTER F WITH STROKE + 0xA7A0: "g", // LATIN CAPITAL LETTER G WITH OBLIQUE STROKE + 0xA7A1: "g", // LATIN SMALL LETTER G WITH OBLIQUE STROKE + 0xA7A2: "k", // LATIN CAPITAL LETTER K WITH OBLIQUE STROKE + 0xA7A3: "k", // LATIN SMALL LETTER K WITH OBLIQUE STROKE + 0xA7A4: "n", // LATIN CAPITAL LETTER N WITH OBLIQUE STROKE + 0xA7A5: "n", // LATIN SMALL LETTER N WITH OBLIQUE STROKE + 0xA7A6: "r", // LATIN CAPITAL LETTER R WITH OBLIQUE STROKE + 0xA7A7: "r", // LATIN SMALL LETTER R WITH OBLIQUE STROKE + 0xA7A8: "s", // LATIN CAPITAL LETTER S WITH OBLIQUE STROKE + 0xA7A9: "s", // LATIN SMALL LETTER S WITH OBLIQUE STROKE + 0xA7AA: "h", // LATIN CAPITAL LETTER H WITH HOOK + 0xA7AD: "l", // LATIN CAPITAL LETTER L WITH BELT + 0xA7B2: "j", // LATIN CAPITAL LETTER J WITH CROSSED-TAIL + 0xA7B8: "u", // LATIN CAPITAL LETTER U WITH STROKE + 0xA7B9: "u", // LATIN SMALL LETTER U WITH STROKE + 0xA7C4: "c", // LATIN CAPITAL LETTER C WITH PALATAL HOOK + 0xA7C5: "s", // LATIN CAPITAL LETTER S WITH HOOK + 0xA7C6: "z", // LATIN CAPITAL LETTER Z WITH PALATAL HOOK + 0xA7C7: "d", // LATIN CAPITAL LETTER D WITH SHORT STROKE OVERLAY + 0xA7C8: "d", // LATIN SMALL LETTER D WITH SHORT STROKE OVERLAY + 0xA7C9: "s", // LATIN CAPITAL LETTER S WITH SHORT STROKE OVERLAY + 0xA7CA: "s", // LATIN SMALL LETTER S WITH SHORT STROKE OVERLAY + 0xAB34: "e", // LATIN SMALL LETTER E WITH FLOURISH + 0xAB37: "l", // LATIN SMALL LETTER L WITH INVERTED LAZY S + 0xAB38: "l", // LATIN SMALL LETTER L WITH DOUBLE MIDDLE TILDE + 0xAB39: "l", // LATIN SMALL LETTER L WITH MIDDLE RING + 0xAB3A: "m", // LATIN SMALL LETTER M WITH CROSSED-TAIL + 0xAB3B: "n", // LATIN SMALL LETTER N WITH CROSSED-TAIL + 0xAB49: "r", // LATIN SMALL LETTER R WITH CROSSED-TAIL + 0xAB4E: "u", // LATIN SMALL LETTER U WITH SHORT RIGHT LEG + 0xAB52: "u", // LATIN SMALL LETTER U WITH LEFT HOOK + 0xAB56: "x", // LATIN SMALL LETTER X WITH LOW RIGHT RING + 0xAB57: "x", // LATIN SMALL LETTER X WITH LONG LEFT LEG + 0xAB58: "x", // LATIN SMALL LETTER X WITH LONG LEFT LEG AND LOW RIGHT RING + 0xAB59: "x", // LATIN SMALL LETTER X WITH LONG LEFT LEG WITH SERIF + 0xAB5A: "y", // LATIN SMALL LETTER Y WITH SHORT RIGHT LEG + 0xFB1D: "\u05d9", // HEBREW LETTER YOD WITH HIRIQ + 0xFB1F: "\u05f2", // HEBREW LIGATURE YIDDISH YOD YOD PATAH + 0xFB2A: "\u05e9", // HEBREW LETTER SHIN WITH SHIN DOT + 0xFB2B: "\u05e9", // HEBREW LETTER SHIN WITH SIN DOT + 0xFB2C: "\u05e9", // HEBREW LETTER SHIN WITH DAGESH AND SHIN DOT + 0xFB2D: "\u05e9", // HEBREW LETTER SHIN WITH DAGESH AND SIN DOT + 0xFB2E: "\u05d0", // HEBREW LETTER ALEF WITH PATAH + 0xFB2F: "\u05d0", // HEBREW LETTER ALEF WITH QAMATS + 0xFB30: "\u05d0", // HEBREW LETTER ALEF WITH MAPIQ + 0xFB31: "\u05d1", // HEBREW LETTER BET WITH DAGESH + 0xFB32: "\u05d2", // HEBREW LETTER GIMEL WITH DAGESH + 0xFB33: "\u05d3", // HEBREW LETTER DALET WITH DAGESH + 0xFB34: "\u05d4", // HEBREW LETTER HE WITH MAPIQ + 0xFB35: "\u05d5", // HEBREW LETTER VAV WITH DAGESH + 0xFB36: "\u05d6", // HEBREW LETTER ZAYIN WITH DAGESH + 0xFB38: "\u05d8", // HEBREW LETTER TET WITH DAGESH + 0xFB39: "\u05d9", // HEBREW LETTER YOD WITH DAGESH + 0xFB3A: "\u05da", // HEBREW LETTER FINAL KAF WITH DAGESH + 0xFB3B: "\u05db", // HEBREW LETTER KAF WITH DAGESH + 0xFB3C: "\u05dc", // HEBREW LETTER LAMED WITH DAGESH + 0xFB3E: "\u05de", // HEBREW LETTER MEM WITH DAGESH + 0xFB40: "\u05e0", // HEBREW LETTER NUN WITH DAGESH + 0xFB41: "\u05e1", // HEBREW LETTER SAMEKH WITH DAGESH + 0xFB43: "\u05e3", // HEBREW LETTER FINAL PE WITH DAGESH + 0xFB44: "\u05e4", // HEBREW LETTER PE WITH DAGESH + 0xFB46: "\u05e6", // HEBREW LETTER TSADI WITH DAGESH + 0xFB47: "\u05e7", // HEBREW LETTER QOF WITH DAGESH + 0xFB48: "\u05e8", // HEBREW LETTER RESH WITH DAGESH + 0xFB49: "\u05e9", // HEBREW LETTER SHIN WITH DAGESH + 0xFB4A: "\u05ea", // HEBREW LETTER TAV WITH DAGESH + 0xFB4B: "\u05d5", // HEBREW LETTER VAV WITH HOLAM + 0xFB4C: "\u05d1", // HEBREW LETTER BET WITH RAFE + 0xFB4D: "\u05db", // HEBREW LETTER KAF WITH RAFE + 0xFB4E: "\u05e4", // HEBREW LETTER PE WITH RAFE + 0x1109A: "\U00011099", // KAITHI LETTER DDDHA + 0x1109C: "\U0001109b", // KAITHI LETTER RHA + 0x110AB: "\U000110a5", // KAITHI LETTER VA + 0x1DF09: "t", // LATIN SMALL LETTER T WITH HOOK AND RETROFLEX HOOK + 0x1DF11: "l", // LATIN SMALL LETTER L WITH FISHHOOK + 0x1DF13: "l", // LATIN SMALL LETTER L WITH BELT AND PALATAL HOOK + 0x1DF16: "r", // LATIN SMALL LETTER R WITH FISHHOOK AND PALATAL HOOK + 0x1DF1A: "i", // LATIN SMALL LETTER I WITH STROKE AND RETROFLEX HOOK + 0x1DF1B: "o", // LATIN SMALL LETTER O WITH RETROFLEX HOOK + 0x1DF1D: "c", // LATIN SMALL LETTER C WITH RETROFLEX HOOK + 0x1DF1E: "s", // LATIN SMALL LETTER S WITH CURL +} diff --git a/internal/ai/harness/gen_folds.py b/internal/ai/harness/gen_folds.py new file mode 100644 index 0000000..6e907c4 --- /dev/null +++ b/internal/ai/harness/gen_folds.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Generate folds_table.go, the letter-fold table used by harness.Normalize. + +Run from this directory with `go generate ./internal/ai/harness` (or +`python3 gen_folds.py`; gofmt must be on PATH). It uses only the Python +standard library's unicodedata, so it works offline; the Unicode version it +was generated from is recorded in the output. + +The table maps a code point to its lowercased base letters: + + - every code point with a canonical decomposition that includes a + combining mark (category Mn) maps to its full canonical decomposition + with the marks removed, so a precomposed letter folds exactly like its + decomposed spelling once Normalize drops the marks (ở -> o, ǖ -> u). + Code points whose remaining base is not letters or digits (spacing + accents such as U+1FEE) are left out; Normalize treats both spellings + of those as punctuation; + - Latin letters with no decomposition whose name is "LATIN ... LETTER X + WITH ..." map to x (ø -> o, ł -> l, đ -> d); + - the Latin ligatures and letters listed in EXTRA map as given. + +Each result is then folded again, letter by letter, until nothing changes +(ǿ -> ø -> o, ǽ -> æ -> ae). Normalize writes a result without folding it +again, so the table must already be fully reduced; the generator fails if a +result does not settle. + +Hangul syllables are left out: their decomposition is into conjoining jamo, +not combining marks. +""" + +import re +import subprocess +import sys +import unicodedata + +EXTRA = { + "ß": "ss", "ẞ": "ss", "æ": "ae", "Æ": "ae", "œ": "oe", "Œ": "oe", + "þ": "th", "Þ": "th", "ð": "d", "Ð": "d", "ı": "i", "ʼn": "n", + "ij": "ij", "IJ": "ij", +} + +WITH = re.compile(r"^LATIN (?:SMALL|CAPITAL) LETTER ([A-Z]) WITH ") + + +def is_word(ch): + """Reports whether Normalize keeps ch: a letter or a decimal digit.""" + cat = unicodedata.category(ch) + return cat.startswith("L") or cat == "Nd" + + +def full_decomposition(ch): + return unicodedata.normalize("NFD", ch) + + +def fold(cp): + ch = chr(cp) + if ch in EXTRA: + return EXTRA[ch] + if 0xAC00 <= cp <= 0xD7A3: + return None + nfd = full_decomposition(ch) + if nfd != ch: + if not any(unicodedata.category(c) == "Mn" for c in nfd): + return None + base = "".join(c for c in nfd if unicodedata.category(c) != "Mn") + if not base or not all(is_word(c) for c in base): + return None + return base.lower() + m = WITH.match(unicodedata.name(ch, "")) + if m and unicodedata.category(ch) in ("Ll", "Lu", "Lt"): + return m.group(1).lower() + return None + + +def reduce(folds, s): + """Folds every letter of s until nothing changes.""" + for _ in range(len(folds)): + nxt = "".join(folds.get(c, c) for c in s) + if nxt == s: + return s + s = nxt + sys.exit("gen_folds.py: fold of %r does not settle" % s) + + +def main(): + folds = {} + for cp in range(sys.maxunicode + 1): + folded = fold(cp) + if folded is not None: + folds[chr(cp)] = folded + entries = [(ord(ch), reduce(folds, folded)) for ch, folded in folds.items()] + out = [ + "// Code generated by gen_folds.py from Unicode " + + unicodedata.unidata_version + "; DO NOT EDIT.", + "", + "package harness", + "", + "// letterFolds maps a precomposed or stroked letter to its lowercased", + "// base letters. See gen_folds.py for how it is built.", + "var letterFolds = map[rune]string{", + ] + for cp, folded in entries: + escaped = "".join( + c if 0x20 <= ord(c) < 0x7F and c not in '"\\' else "\\u%04x" % ord(c) + if ord(c) <= 0xFFFF else "\\U%08x" % ord(c) + for c in folded + ) + out.append('\t0x%04X: "%s", // %s' % (cp, escaped, unicodedata.name(chr(cp)))) + out.append("}") + with open("folds_table.go", "w", encoding="utf-8") as f: + f.write("\n".join(out) + "\n") + subprocess.run(["gofmt", "-w", "folds_table.go"], check=True) + + +if __name__ == "__main__": + main()