dev_agent
f25b28b39b
CI / ci (push) Successful in 2m0s
Co-authored-by: dev_agent <dev_agent@cubecraftlabs.com>
126 lines
3.6 KiB
Go
126 lines
3.6 KiB
Go
package harness
|
||
|
||
import (
|
||
"strings"
|
||
"unicode"
|
||
)
|
||
|
||
// SimilarThreshold is the token-set similarity at or above which two dinner
|
||
// names are the same dinner.
|
||
const SimilarThreshold = 0.85
|
||
|
||
// fillerWords carry no meaning in a dinner name and are dropped by
|
||
// Normalize.
|
||
var fillerWords = map[string]bool{
|
||
"easy": true, "best": true, "homemade": true, "quick": true, "simple": true,
|
||
"the": true, "a": true, "recipe": true, "classic": true,
|
||
}
|
||
|
||
//go:generate python3 gen_folds.py
|
||
|
||
// Normalize returns the comparable form of a dinner name: lowercased,
|
||
// combining marks (Unicode category Mn) dropped, precomposed and stroked
|
||
// letters folded to their base letters (see letterFolds) so composed and
|
||
// decomposed spellings are equal, apostrophes removed, other punctuation
|
||
// turned into spaces, filler words (see fillerWords) dropped and whitespace
|
||
// collapsed to single spaces. "Easy Chicken Tikka-Masala!" becomes
|
||
// "chicken tikka masala".
|
||
func Normalize(name string) string {
|
||
var b strings.Builder
|
||
for _, r := range strings.ToLower(name) {
|
||
switch {
|
||
case unicode.Is(unicode.Mn, r), r == '\'', r == '’':
|
||
// Combining accents and apostrophes join their neighbours.
|
||
case letterFolds[r] != "":
|
||
b.WriteString(letterFolds[r])
|
||
case unicode.IsLetter(r), unicode.IsDigit(r):
|
||
b.WriteRune(r)
|
||
default:
|
||
b.WriteRune(' ')
|
||
}
|
||
}
|
||
words := strings.Fields(b.String())
|
||
kept := words[:0]
|
||
for _, w := range words {
|
||
if !fillerWords[w] {
|
||
kept = append(kept, w)
|
||
}
|
||
}
|
||
return strings.Join(kept, " ")
|
||
}
|
||
|
||
// Similar reports whether a and b name the same dinner: the token-set
|
||
// similarity of their normalized forms is at least SimilarThreshold. A name
|
||
// with no words left after normalizing is similar to nothing.
|
||
func Similar(a, b string) bool {
|
||
return Similarity(a, b) >= SimilarThreshold
|
||
}
|
||
|
||
// Similarity is the token-set (Jaccard) similarity of a's and b's normalized
|
||
// forms, from 0 (no shared word) to 1 (the same words). A name with no words
|
||
// left after normalizing scores 0 against everything.
|
||
func Similarity(a, b string) float64 {
|
||
return similarity(tokenSet(a), tokenSet(b))
|
||
}
|
||
|
||
// tokenSet returns the distinct words of name's normalized form.
|
||
func tokenSet(name string) map[string]bool {
|
||
set := map[string]bool{}
|
||
for _, w := range strings.Fields(Normalize(name)) {
|
||
set[w] = true
|
||
}
|
||
return set
|
||
}
|
||
|
||
// similarity is the Jaccard index of a and b: shared words over all words.
|
||
// It is 0 when either set is empty.
|
||
func similarity(a, b map[string]bool) float64 {
|
||
if len(a) == 0 || len(b) == 0 {
|
||
return 0
|
||
}
|
||
shared := 0
|
||
for w := range a {
|
||
if b[w] {
|
||
shared++
|
||
}
|
||
}
|
||
return float64(shared) / float64(len(a)+len(b)-shared)
|
||
}
|
||
|
||
// DedupeSources are the dinner names a new suggestion must not repeat.
|
||
type DedupeSources struct {
|
||
Regulars []string
|
||
QuestLog []string
|
||
// Pending holds the names of suggestions not yet tried or passed.
|
||
Pending []string
|
||
// Declined holds the names of passed suggestions.
|
||
Declined []string
|
||
}
|
||
|
||
// Deduper answers whether a candidate dinner name repeats a known one.
|
||
type Deduper struct {
|
||
known []map[string]bool
|
||
}
|
||
|
||
// NewDeduper returns a Deduper over every name in src.
|
||
func NewDeduper(src DedupeSources) *Deduper {
|
||
d := &Deduper{}
|
||
for _, names := range [][]string{src.Regulars, src.QuestLog, src.Pending, src.Declined} {
|
||
for _, name := range names {
|
||
d.known = append(d.known, tokenSet(name))
|
||
}
|
||
}
|
||
return d
|
||
}
|
||
|
||
// IsDuplicate reports whether name is Similar to any known name.
|
||
func (d *Deduper) IsDuplicate(name string) bool {
|
||
candidate := tokenSet(name)
|
||
for _, k := range d.known {
|
||
if similarity(candidate, k) >= SimilarThreshold {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|