Feature 09: AI harness and skills framework (#62)

Co-authored-by: dev_agent <dev_agent@cubecraftlabs.com>

internal/ai/harness/dedupe.go

@@ -0,0 +1,118 @@
+package harness
+
+import (
+	"strings"
+	"unicode"
+)
+
+// SimilarThreshold is the token-set similarity at or above which two dinner
+// names are the same dinner.
+const SimilarThreshold = 0.85
+
+// fillerWords carry no meaning in a dinner name and are dropped by
+// Normalize.
+var fillerWords = map[string]bool{
+	"easy": true, "best": true, "homemade": true, "quick": true, "simple": true,
+	"the": true, "a": true, "recipe": true, "classic": true,
+}
+
+//go:generate python3 gen_folds.py
+
+// Normalize returns the comparable form of a dinner name: lowercased,
+// combining marks (Unicode category Mn) dropped, precomposed and stroked
+// letters folded to their base letters (see letterFolds) so composed and
+// decomposed spellings are equal, apostrophes removed, other punctuation
+// turned into spaces, filler words (see fillerWords) dropped and whitespace
+// collapsed to single spaces. "Easy Chicken Tikka-Masala!" becomes
+// "chicken tikka masala".
+func Normalize(name string) string {
+	var b strings.Builder
+	for _, r := range strings.ToLower(name) {
+		switch {
+		case unicode.Is(unicode.Mn, r), r == '\'', r == '’':
+			// Combining accents and apostrophes join their neighbours.
+		case letterFolds[r] != "":
+			b.WriteString(letterFolds[r])
+		case unicode.IsLetter(r), unicode.IsDigit(r):
+			b.WriteRune(r)
+		default:
+			b.WriteRune(' ')
+		}
+	}
+	words := strings.Fields(b.String())
+	kept := words[:0]
+	for _, w := range words {
+		if !fillerWords[w] {
+			kept = append(kept, w)
+		}
+	}
+	return strings.Join(kept, " ")
+}
+
+// Similar reports whether a and b name the same dinner: the token-set
+// similarity of their normalized forms is at least SimilarThreshold. A name
+// with no words left after normalizing is similar to nothing.
+func Similar(a, b string) bool {
+	return similarity(tokenSet(a), tokenSet(b)) >= SimilarThreshold
+}
+
+// tokenSet returns the distinct words of name's normalized form.
+func tokenSet(name string) map[string]bool {
+	set := map[string]bool{}
+	for _, w := range strings.Fields(Normalize(name)) {
+		set[w] = true
+	}
+	return set
+}
+
+// similarity is the Jaccard index of a and b: shared words over all words.
+// It is 0 when either set is empty.
+func similarity(a, b map[string]bool) float64 {
+	if len(a) == 0 || len(b) == 0 {
+		return 0
+	}
+	shared := 0
+	for w := range a {
+		if b[w] {
+			shared++
+		}
+	}
+	return float64(shared) / float64(len(a)+len(b)-shared)
+}
+
+// DedupeSources are the dinner names a new suggestion must not repeat.
+type DedupeSources struct {
+	Regulars []string
+	QuestLog []string
+	// Pending holds the names of suggestions not yet tried or passed.
+	Pending []string
+	// Declined holds the names of passed suggestions.
+	Declined []string
+}
+
+// Deduper answers whether a candidate dinner name repeats a known one.
+type Deduper struct {
+	known []map[string]bool
+}
+
+// NewDeduper returns a Deduper over every name in src.
+func NewDeduper(src DedupeSources) *Deduper {
+	d := &Deduper{}
+	for _, names := range [][]string{src.Regulars, src.QuestLog, src.Pending, src.Declined} {
+		for _, name := range names {
+			d.known = append(d.known, tokenSet(name))
+		}
+	}
+	return d
+}
+
+// IsDuplicate reports whether name is Similar to any known name.
+func (d *Deduper) IsDuplicate(name string) bool {
+	candidate := tokenSet(name)
+	for _, k := range d.known {
+		if similarity(candidate, k) >= SimilarThreshold {
+			return true
+		}
+	}
+	return false
+}