dev_agentandClaude Opus 5.5
7729488cde
Replace the hand-written accent map with letterFolds, a table generated
offline by gen_folds.py from Python's unicodedata: every code point whose
canonical decomposition contains combining marks maps to its base letters,
plus stroked Latin letters and ligatures. Combining marks are still dropped,
so composed and decomposed spellings ("Phở" and "Phở") now
normalize equal. No new dependency.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
119 lines
3.3 KiB
Go
119 lines
3.3 KiB
Go
package harness
|
||
|
||
import (
|
||
"strings"
|
||
"unicode"
|
||
)
|
||
|
||
// SimilarThreshold is the token-set similarity at or above which two dinner
|
||
// names are the same dinner.
|
||
const SimilarThreshold = 0.85
|
||
|
||
// fillerWords carry no meaning in a dinner name and are dropped by
|
||
// Normalize.
|
||
var fillerWords = map[string]bool{
|
||
"easy": true, "best": true, "homemade": true, "quick": true, "simple": true,
|
||
"the": true, "a": true, "recipe": true, "classic": true,
|
||
}
|
||
|
||
//go:generate python3 gen_folds.py
|
||
|
||
// Normalize returns the comparable form of a dinner name: lowercased,
|
||
// combining marks (Unicode category Mn) dropped, precomposed and stroked
|
||
// letters folded to their base letters (see letterFolds) so composed and
|
||
// decomposed spellings are equal, apostrophes removed, other punctuation
|
||
// turned into spaces, filler words (see fillerWords) dropped and whitespace
|
||
// collapsed to single spaces. "Easy Chicken Tikka-Masala!" becomes
|
||
// "chicken tikka masala".
|
||
func Normalize(name string) string {
|
||
var b strings.Builder
|
||
for _, r := range strings.ToLower(name) {
|
||
switch {
|
||
case unicode.Is(unicode.Mn, r), r == '\'', r == '’':
|
||
// Combining accents and apostrophes join their neighbours.
|
||
case letterFolds[r] != "":
|
||
b.WriteString(letterFolds[r])
|
||
case unicode.IsLetter(r), unicode.IsDigit(r):
|
||
b.WriteRune(r)
|
||
default:
|
||
b.WriteRune(' ')
|
||
}
|
||
}
|
||
words := strings.Fields(b.String())
|
||
kept := words[:0]
|
||
for _, w := range words {
|
||
if !fillerWords[w] {
|
||
kept = append(kept, w)
|
||
}
|
||
}
|
||
return strings.Join(kept, " ")
|
||
}
|
||
|
||
// Similar reports whether a and b name the same dinner: the token-set
|
||
// similarity of their normalized forms is at least SimilarThreshold. A name
|
||
// with no words left after normalizing is similar to nothing.
|
||
func Similar(a, b string) bool {
|
||
return similarity(tokenSet(a), tokenSet(b)) >= SimilarThreshold
|
||
}
|
||
|
||
// tokenSet returns the distinct words of name's normalized form.
|
||
func tokenSet(name string) map[string]bool {
|
||
set := map[string]bool{}
|
||
for _, w := range strings.Fields(Normalize(name)) {
|
||
set[w] = true
|
||
}
|
||
return set
|
||
}
|
||
|
||
// similarity is the Jaccard index of a and b: shared words over all words.
|
||
// It is 0 when either set is empty.
|
||
func similarity(a, b map[string]bool) float64 {
|
||
if len(a) == 0 || len(b) == 0 {
|
||
return 0
|
||
}
|
||
shared := 0
|
||
for w := range a {
|
||
if b[w] {
|
||
shared++
|
||
}
|
||
}
|
||
return float64(shared) / float64(len(a)+len(b)-shared)
|
||
}
|
||
|
||
// DedupeSources are the dinner names a new suggestion must not repeat.
|
||
type DedupeSources struct {
|
||
Regulars []string
|
||
QuestLog []string
|
||
// Pending holds the names of suggestions not yet tried or passed.
|
||
Pending []string
|
||
// Declined holds the names of passed suggestions.
|
||
Declined []string
|
||
}
|
||
|
||
// Deduper answers whether a candidate dinner name repeats a known one.
|
||
type Deduper struct {
|
||
known []map[string]bool
|
||
}
|
||
|
||
// NewDeduper returns a Deduper over every name in src.
|
||
func NewDeduper(src DedupeSources) *Deduper {
|
||
d := &Deduper{}
|
||
for _, names := range [][]string{src.Regulars, src.QuestLog, src.Pending, src.Declined} {
|
||
for _, name := range names {
|
||
d.known = append(d.known, tokenSet(name))
|
||
}
|
||
}
|
||
return d
|
||
}
|
||
|
||
// IsDuplicate reports whether name is Similar to any known name.
|
||
func (d *Deduper) IsDuplicate(name string) bool {
|
||
candidate := tokenSet(name)
|
||
for _, k := range d.known {
|
||
if similarity(candidate, k) >= SimilarThreshold {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|