MealQuest/internal/ai/harness/dedupe.go

package harness

import (
	"strings"
	"unicode"
)

// SimilarThreshold is the token-set similarity at or above which two dinner
// names are the same dinner.
const SimilarThreshold = 0.85

// fillerWords carry no meaning in a dinner name and are dropped by
// Normalize.
var fillerWords = map[string]bool{
	"easy": true, "best": true, "homemade": true, "quick": true, "simple": true,
	"the": true, "a": true, "recipe": true, "classic": true,
}

//go:generate python3 gen_folds.py

// Normalize returns the comparable form of a dinner name: lowercased,
// combining marks (Unicode category Mn) dropped, precomposed and stroked
// letters folded to their base letters (see letterFolds) so composed and
// decomposed spellings are equal, apostrophes removed, other punctuation
// turned into spaces, filler words (see fillerWords) dropped and whitespace
// collapsed to single spaces. "Easy Chicken Tikka-Masala!" becomes
// "chicken tikka masala".
func Normalize(name string) string {
	var b strings.Builder
	for _, r := range strings.ToLower(name) {
		switch {
		case unicode.Is(unicode.Mn, r), r == '\'', r == '’':
			// Combining accents and apostrophes join their neighbours.
		case letterFolds[r] != "":
			b.WriteString(letterFolds[r])
		case unicode.IsLetter(r), unicode.IsDigit(r):
			b.WriteRune(r)
		default:
			b.WriteRune(' ')
		}
	}
	words := strings.Fields(b.String())
	kept := words[:0]
	for _, w := range words {
		if !fillerWords[w] {
			kept = append(kept, w)
		}
	}
	return strings.Join(kept, " ")
}

// Similar reports whether a and b name the same dinner: the token-set
// similarity of their normalized forms is at least SimilarThreshold. A name
// with no words left after normalizing is similar to nothing.
func Similar(a, b string) bool {
	return Similarity(a, b) >= SimilarThreshold
}

// Similarity is the token-set (Jaccard) similarity of a's and b's normalized
// forms, from 0 (no shared word) to 1 (the same words). A name with no words
// left after normalizing scores 0 against everything.
func Similarity(a, b string) float64 {
	return similarity(tokenSet(a), tokenSet(b))
}

// tokenSet returns the distinct words of name's normalized form.
func tokenSet(name string) map[string]bool {
	set := map[string]bool{}
	for _, w := range strings.Fields(Normalize(name)) {
		set[w] = true
	}
	return set
}

// similarity is the Jaccard index of a and b: shared words over all words.
// It is 0 when either set is empty.
func similarity(a, b map[string]bool) float64 {
	if len(a) == 0 || len(b) == 0 {
		return 0
	}
	shared := 0
	for w := range a {
		if b[w] {
			shared++
		}
	}
	return float64(shared) / float64(len(a)+len(b)-shared)
}

// DedupeSources are the dinner names a new suggestion must not repeat.
type DedupeSources struct {
	Regulars []string
	QuestLog []string
	// Pending holds the names of suggestions not yet tried or passed.
	Pending []string
	// Declined holds the names of passed suggestions.
	Declined []string
}

// Deduper answers whether a candidate dinner name repeats a known one.
type Deduper struct {
	known []map[string]bool
}

// NewDeduper returns a Deduper over every name in src.
func NewDeduper(src DedupeSources) *Deduper {
	d := &Deduper{}
	for _, names := range [][]string{src.Regulars, src.QuestLog, src.Pending, src.Declined} {
		for _, name := range names {
			d.known = append(d.known, tokenSet(name))
		}
	}
	return d
}

// IsDuplicate reports whether name is Similar to any known name.
func (d *Deduper) IsDuplicate(name string) bool {
	candidate := tokenSet(name)
	for _, k := range d.known {
		if similarity(candidate, k) >= SimilarThreshold {
			return true
		}
	}
	return false
}