dev_agentandClaude Opus 5.5
7729488cde
Replace the hand-written accent map with letterFolds, a table generated
offline by gen_folds.py from Python's unicodedata: every code point whose
canonical decomposition contains combining marks maps to its base letters,
plus stroked Latin letters and ligatures. Combining marks are still dropped,
so composed and decomposed spellings ("Phở" and "Phở") now
normalize equal. No new dependency.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
101 lines
3.4 KiB
Python
101 lines
3.4 KiB
Python
#!/usr/bin/env python3
|
||
"""Generate folds_table.go, the letter-fold table used by harness.Normalize.
|
||
|
||
Run from this directory with `go generate ./internal/ai/harness` (or
|
||
`python3 gen_folds.py`; gofmt must be on PATH). It uses only the Python
|
||
standard library's unicodedata, so it works offline; the Unicode version it
|
||
was generated from is recorded in the output.
|
||
|
||
The table maps a code point to its lowercased base letters:
|
||
|
||
- every code point with a canonical decomposition that includes a
|
||
combining mark (category Mn) maps to its full canonical decomposition
|
||
with the marks removed, so a precomposed letter folds exactly like its
|
||
decomposed spelling once Normalize drops the marks (ở -> o, ǖ -> u).
|
||
Code points whose remaining base is not letters or digits (spacing
|
||
accents such as U+1FEE) are left out; Normalize treats both spellings
|
||
of those as punctuation;
|
||
- Latin letters with no decomposition whose name is "LATIN ... LETTER X
|
||
WITH ..." map to x (ø -> o, ł -> l, đ -> d);
|
||
- the Latin ligatures and letters listed in EXTRA map as given.
|
||
|
||
Hangul syllables are left out: their decomposition is into conjoining jamo,
|
||
not combining marks.
|
||
"""
|
||
|
||
import re
|
||
import subprocess
|
||
import sys
|
||
import unicodedata
|
||
|
||
EXTRA = {
|
||
"ß": "ss", "ẞ": "ss", "æ": "ae", "Æ": "ae", "œ": "oe", "Œ": "oe",
|
||
"þ": "th", "Þ": "th", "ð": "d", "Ð": "d", "ı": "i", "ʼn": "n",
|
||
"ij": "ij", "IJ": "ij",
|
||
}
|
||
|
||
WITH = re.compile(r"^LATIN (?:SMALL|CAPITAL) LETTER ([A-Z]) WITH ")
|
||
|
||
|
||
def is_word(ch):
|
||
"""Reports whether Normalize keeps ch: a letter or a decimal digit."""
|
||
cat = unicodedata.category(ch)
|
||
return cat.startswith("L") or cat == "Nd"
|
||
|
||
|
||
def full_decomposition(ch):
|
||
return unicodedata.normalize("NFD", ch)
|
||
|
||
|
||
def fold(cp):
|
||
ch = chr(cp)
|
||
if ch in EXTRA:
|
||
return EXTRA[ch]
|
||
if 0xAC00 <= cp <= 0xD7A3:
|
||
return None
|
||
nfd = full_decomposition(ch)
|
||
if nfd != ch:
|
||
if not any(unicodedata.category(c) == "Mn" for c in nfd):
|
||
return None
|
||
base = "".join(c for c in nfd if unicodedata.category(c) != "Mn")
|
||
if not base or not all(is_word(c) for c in base):
|
||
return None
|
||
return base.lower()
|
||
m = WITH.match(unicodedata.name(ch, ""))
|
||
if m and unicodedata.category(ch) in ("Ll", "Lu", "Lt"):
|
||
return m.group(1).lower()
|
||
return None
|
||
|
||
|
||
def main():
|
||
entries = []
|
||
for cp in range(sys.maxunicode + 1):
|
||
folded = fold(cp)
|
||
if folded is not None:
|
||
entries.append((cp, folded))
|
||
out = [
|
||
"// Code generated by gen_folds.py from Unicode "
|
||
+ unicodedata.unidata_version + "; DO NOT EDIT.",
|
||
"",
|
||
"package harness",
|
||
"",
|
||
"// letterFolds maps a precomposed or stroked letter to its lowercased",
|
||
"// base letters. See gen_folds.py for how it is built.",
|
||
"var letterFolds = map[rune]string{",
|
||
]
|
||
for cp, folded in entries:
|
||
escaped = "".join(
|
||
c if 0x20 <= ord(c) < 0x7F and c not in '"\\' else "\\u%04x" % ord(c)
|
||
if ord(c) <= 0xFFFF else "\\U%08x" % ord(c)
|
||
for c in folded
|
||
)
|
||
out.append('\t0x%04X: "%s", // %s' % (cp, escaped, unicodedata.name(chr(cp))))
|
||
out.append("}")
|
||
with open("folds_table.go", "w", encoding="utf-8") as f:
|
||
f.write("\n".join(out) + "\n")
|
||
subprocess.run(["gofmt", "-w", "folds_table.go"], check=True)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|