101 lines
3.4 KiB
Python
101 lines
3.4 KiB
Python
 
|
#!/usr/bin/env python3
|
|||
|
|
"""Generate folds_table.go, the letter-fold table used by harness.Normalize.
|
|||
|
|
|
|||
|
|
Run from this directory with `go generate ./internal/ai/harness` (or
|
|||
|
|
`python3 gen_folds.py`; gofmt must be on PATH). It uses only the Python
|
|||
|
|
standard library's unicodedata, so it works offline; the Unicode version it
|
|||
|
|
was generated from is recorded in the output.
|
|||
|
|
|
|||
|
|
The table maps a code point to its lowercased base letters:
|
|||
|
|
|
|||
|
|
- every code point with a canonical decomposition that includes a
|
|||
|
|
combining mark (category Mn) maps to its full canonical decomposition
|
|||
|
|
with the marks removed, so a precomposed letter folds exactly like its
|
|||
|
|
decomposed spelling once Normalize drops the marks (ở -> o, ǖ -> u).
|
|||
|
|
Code points whose remaining base is not letters or digits (spacing
|
|||
|
|
accents such as U+1FEE) are left out; Normalize treats both spellings
|
|||
|
|
of those as punctuation;
|
|||
|
|
- Latin letters with no decomposition whose name is "LATIN ... LETTER X
|
|||
|
|
WITH ..." map to x (ø -> o, ł -> l, đ -> d);
|
|||
|
|
- the Latin ligatures and letters listed in EXTRA map as given.
|
|||
|
|
|
|||
|
|
Hangul syllables are left out: their decomposition is into conjoining jamo,
|
|||
|
|
not combining marks.
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
import re
|
|||
|
|
import subprocess
|
|||
|
|
import sys
|
|||
|
|
import unicodedata
|
|||
|
|
|
|||
|
|
EXTRA = {
|
|||
|
|
"ß": "ss", "ẞ": "ss", "æ": "ae", "Æ": "ae", "œ": "oe", "Œ": "oe",
|
|||
|
|
"þ": "th", "Þ": "th", "ð": "d", "Ð": "d", "ı": "i", "ʼn": "n",
|
|||
|
|
"ij": "ij", "IJ": "ij",
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
WITH = re.compile(r"^LATIN (?:SMALL|CAPITAL) LETTER ([A-Z]) WITH ")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def is_word(ch):
|
|||
|
|
"""Reports whether Normalize keeps ch: a letter or a decimal digit."""
|
|||
|
|
cat = unicodedata.category(ch)
|
|||
|
|
return cat.startswith("L") or cat == "Nd"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def full_decomposition(ch):
|
|||
|
|
return unicodedata.normalize("NFD", ch)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def fold(cp):
|
|||
|
|
ch = chr(cp)
|
|||
|
|
if ch in EXTRA:
|
|||
|
|
return EXTRA[ch]
|
|||
|
|
if 0xAC00 <= cp <= 0xD7A3:
|
|||
|
|
return None
|
|||
|
|
nfd = full_decomposition(ch)
|
|||
|
|
if nfd != ch:
|
|||
|
|
if not any(unicodedata.category(c) == "Mn" for c in nfd):
|
|||
|
|
return None
|
|||
|
|
base = "".join(c for c in nfd if unicodedata.category(c) != "Mn")
|
|||
|
|
if not base or not all(is_word(c) for c in base):
|
|||
|
|
return None
|
|||
|
|
return base.lower()
|
|||
|
|
m = WITH.match(unicodedata.name(ch, ""))
|
|||
|
|
if m and unicodedata.category(ch) in ("Ll", "Lu", "Lt"):
|
|||
|
|
return m.group(1).lower()
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
|
|||
|
|
def main():
|
|||
|
|
entries = []
|
|||
|
|
for cp in range(sys.maxunicode + 1):
|
|||
|
|
folded = fold(cp)
|
|||
|
|
if folded is not None:
|
|||
|
|
entries.append((cp, folded))
|
|||
|
|
out = [
|
|||
|
|
"// Code generated by gen_folds.py from Unicode "
|
|||
|
|
+ unicodedata.unidata_version + "; DO NOT EDIT.",
|
|||
|
|
"",
|
|||
|
|
"package harness",
|
|||
|
|
"",
|
|||
|
|
"// letterFolds maps a precomposed or stroked letter to its lowercased",
|
|||
|
|
"// base letters. See gen_folds.py for how it is built.",
|
|||
|
|
"var letterFolds = map[rune]string{",
|
|||
|
|
]
|
|||
|
|
for cp, folded in entries:
|
|||
|
|
escaped = "".join(
|
|||
|
|
c if 0x20 <= ord(c) < 0x7F and c not in '"\\' else "\\u%04x" % ord(c)
|
|||
|
|
if ord(c) <= 0xFFFF else "\\U%08x" % ord(c)
|
|||
|
|
for c in folded
|
|||
|
|
)
|
|||
|
|
out.append('\t0x%04X: "%s", // %s' % (cp, escaped, unicodedata.name(chr(cp))))
|
|||
|
|
out.append("}")
|
|||
|
|
with open("folds_table.go", "w", encoding="utf-8") as f:
|
|||
|
|
f.write("\n".join(out) + "\n")
|
|||
|
|
subprocess.run(["gofmt", "-w", "folds_table.go"], check=True)
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
main()
|