MealQuest/internal/ai/harness/gen_folds.py

#!/usr/bin/env python3
"""Generate folds_table.go, the letter-fold table used by harness.Normalize.

Run from this directory with `go generate ./internal/ai/harness` (or
`python3 gen_folds.py`; gofmt must be on PATH). It uses only the Python
standard library's unicodedata, so it works offline; the Unicode version it
was generated from is recorded in the output.

The table maps a code point to its lowercased base letters:

  - every code point with a canonical decomposition that includes a
    combining mark (category Mn) maps to its full canonical decomposition
    with the marks removed, so a precomposed letter folds exactly like its
    decomposed spelling once Normalize drops the marks (ở -> o, ǖ -> u).
    Code points whose remaining base is not letters or digits (spacing
    accents such as U+1FEE) are left out; Normalize treats both spellings
    of those as punctuation;
  - Latin letters with no decomposition whose name is "LATIN ... LETTER X
    WITH ..." map to x (ø -> o, ł -> l, đ -> d);
  - the Latin ligatures and letters listed in EXTRA map as given.

Each result is then folded again, letter by letter, until nothing changes
(ǿ -> ø -> o, ǽ -> æ -> ae). Normalize writes a result without folding it
again, so the table must already be fully reduced; the generator fails if a
result does not settle.

Hangul syllables are left out: their decomposition is into conjoining jamo,
not combining marks.
"""

import re
import subprocess
import sys
import unicodedata

EXTRA = {
    "ß": "ss", "ẞ": "ss", "æ": "ae", "Æ": "ae", "œ": "oe", "Œ": "oe",
    "þ": "th", "Þ": "th", "ð": "d", "Ð": "d", "ı": "i", "ʼn": "n",
    "ij": "ij", "IJ": "ij",
}

WITH = re.compile(r"^LATIN (?:SMALL|CAPITAL) LETTER ([A-Z]) WITH ")


def is_word(ch):
    """Reports whether Normalize keeps ch: a letter or a decimal digit."""
    cat = unicodedata.category(ch)
    return cat.startswith("L") or cat == "Nd"


def full_decomposition(ch):
    return unicodedata.normalize("NFD", ch)


def fold(cp):
    ch = chr(cp)
    if ch in EXTRA:
        return EXTRA[ch]
    if 0xAC00 <= cp <= 0xD7A3:
        return None
    nfd = full_decomposition(ch)
    if nfd != ch:
        if not any(unicodedata.category(c) == "Mn" for c in nfd):
            return None
        base = "".join(c for c in nfd if unicodedata.category(c) != "Mn")
        if not base or not all(is_word(c) for c in base):
            return None
        return base.lower()
    m = WITH.match(unicodedata.name(ch, ""))
    if m and unicodedata.category(ch) in ("Ll", "Lu", "Lt"):
        return m.group(1).lower()
    return None


def reduce(folds, s):
    """Folds every letter of s until nothing changes."""
    for _ in range(len(folds)):
        nxt = "".join(folds.get(c, c) for c in s)
        if nxt == s:
            return s
        s = nxt
    sys.exit("gen_folds.py: fold of %r does not settle" % s)


def main():
    folds = {}
    for cp in range(sys.maxunicode + 1):
        folded = fold(cp)
        if folded is not None:
            folds[chr(cp)] = folded
    entries = [(ord(ch), reduce(folds, folded)) for ch, folded in folds.items()]
    out = [
        "// Code generated by gen_folds.py from Unicode "
        + unicodedata.unidata_version + "; DO NOT EDIT.",
        "",
        "package harness",
        "",
        "// letterFolds maps a precomposed or stroked letter to its lowercased",
        "// base letters. See gen_folds.py for how it is built.",
        "var letterFolds = map[rune]string{",
    ]
    for cp, folded in entries:
        escaped = "".join(
            c if 0x20 <= ord(c) < 0x7F and c not in '"\\' else "\\u%04x" % ord(c)
            if ord(c) <= 0xFFFF else "\\U%08x" % ord(c)
            for c in folded
        )
        out.append('\t0x%04X: "%s", // %s' % (cp, escaped, unicodedata.name(chr(cp))))
    out.append("}")
    with open("folds_table.go", "w", encoding="utf-8") as f:
        f.write("\n".join(out) + "\n")
    subprocess.run(["gofmt", "-w", "folds_table.go"], check=True)


if __name__ == "__main__":
    main()