MealQuest/internal/ai/harness/dedupe.go

package harness

import (
	"strings"
	"unicode"
)

// SimilarThreshold is the token-set similarity at or above which two dinner
// names are the same dinner.
const SimilarThreshold = 0.85

// fillerWords carry no meaning in a dinner name and are dropped by
// Normalize.
var fillerWords = map[string]bool{
	"easy": true, "best": true, "homemade": true, "quick": true, "simple": true,
	"the": true, "a": true, "recipe": true, "classic": true,
}

// accentFolds maps each lowercase accented Latin letter to its plain form.
var accentFolds = func() map[rune]string {
	m := map[rune]string{'ß': "ss", 'æ': "ae", 'œ': "oe", 'þ': "th", 'ð': "d"}
	for plain, accented := range map[string]string{
		"a": "àáâãäåāăą", "c": "çćĉċč", "d": "ďđ", "e": "èéêëēĕėęě",
		"g": "ĝğġģ", "h": "ĥħ", "i": "ìíîïĩīĭįı", "j": "ĵ", "k": "ķ",
		"l": "ĺļľŀł", "n": "ñńņňʼn", "o": "òóôõöøōŏő", "r": "ŕŗř",
		"s": "śŝşšș", "t": "ţťŧț", "u": "ùúûüũūŭůűų", "w": "ŵ", "y": "ýÿŷ",
		"z": "źżž",
	} {
		for _, r := range accented {
			m[r] = plain
		}
	}
	return m
}()

// Normalize returns the comparable form of a dinner name: lowercased,
// accents folded to plain letters, apostrophes removed, other punctuation
// turned into spaces, filler words (see fillerWords) dropped and whitespace
// collapsed to single spaces. "Easy Chicken Tikka-Masala!" becomes
// "chicken tikka masala".
func Normalize(name string) string {
	var b strings.Builder
	for _, r := range strings.ToLower(name) {
		switch {
		case unicode.Is(unicode.Mn, r), r == '\'', r == '’':
			// Combining accents and apostrophes join their neighbours.
		case accentFolds[r] != "":
			b.WriteString(accentFolds[r])
		case unicode.IsLetter(r), unicode.IsDigit(r):
			b.WriteRune(r)
		default:
			b.WriteRune(' ')
		}
	}
	words := strings.Fields(b.String())
	kept := words[:0]
	for _, w := range words {
		if !fillerWords[w] {
			kept = append(kept, w)
		}
	}
	return strings.Join(kept, " ")
}

// Similar reports whether a and b name the same dinner: the token-set
// similarity of their normalized forms is at least SimilarThreshold. A name
// with no words left after normalizing is similar to nothing.
func Similar(a, b string) bool {
	return similarity(tokenSet(a), tokenSet(b)) >= SimilarThreshold
}

// tokenSet returns the distinct words of name's normalized form.
func tokenSet(name string) map[string]bool {
	set := map[string]bool{}
	for _, w := range strings.Fields(Normalize(name)) {
		set[w] = true
	}
	return set
}

// similarity is the Jaccard index of a and b: shared words over all words.
// It is 0 when either set is empty.
func similarity(a, b map[string]bool) float64 {
	if len(a) == 0 || len(b) == 0 {
		return 0
	}
	shared := 0
	for w := range a {
		if b[w] {
			shared++
		}
	}
	return float64(shared) / float64(len(a)+len(b)-shared)
}

// DedupeSources are the dinner names a new suggestion must not repeat.
type DedupeSources struct {
	Regulars []string
	QuestLog []string
	// Pending holds the names of suggestions not yet tried or passed.
	Pending []string
	// Declined holds the names of passed suggestions.
	Declined []string
}

// Deduper answers whether a candidate dinner name repeats a known one.
type Deduper struct {
	known []map[string]bool
}

// NewDeduper returns a Deduper over every name in src.
func NewDeduper(src DedupeSources) *Deduper {
	d := &Deduper{}
	for _, names := range [][]string{src.Regulars, src.QuestLog, src.Pending, src.Declined} {
		for _, name := range names {
			d.known = append(d.known, tokenSet(name))
		}
	}
	return d
}

// IsDuplicate reports whether name is Similar to any known name.
func (d *Deduper) IsDuplicate(name string) bool {
	candidate := tokenSet(name)
	for _, k := range d.known {
		if similarity(candidate, k) >= SimilarThreshold {
			return true
		}
	}
	return false
}