Feature 12: Regular dinners and onboarding (#94)

Co-authored-by: dev_agent <dev_agent@cubecraftlabs.com>

internal/recipes/ground.go

@@ -25,6 +25,13 @@ var (
 	// ErrTotalTimeNotPositive rejects a recipe whose page gives a total
 	// time of zero minutes, which normalize_recipe refuses.
 	ErrTotalTimeNotPositive = errors.New("recipes: recipe total time is not positive")
+	// ErrSearchFailed wraps every failure to search: the search provider
+	// could not be built (ErrSearchNotConfigured among them) or its search
+	// failed.
+	ErrSearchFailed = errors.New("recipes: recipe search failed")
+	// ErrNormalizeFailed wraps a normalize_recipe failure that is not the
+	// recipe's: the model provider could not be built or called.
+	ErrNormalizeFailed = errors.New("recipes: normalize_recipe failed")
 )
 
 // ErrUngroundable is returned when no recipe page could ground a dish or a
@@ -118,43 +125,16 @@ func (g *Grounder) Ground(ctx context.Context, dishName, query string) (Grounded
 	if strings.TrimSpace(dishName) == "" {
 		return Grounded{}, errors.New("recipes: ground: dish name is empty")
 	}
-	provider, err := g.search.Provider(ctx)
+	found, err := g.searchRanked(ctx, dishName, query)
 	if err != nil {
 		return Grounded{}, fmt.Errorf("recipes: ground %q: %w", dishName, err)
 	}
-	cands, err := provider.Search(ctx, query, RankDepth)
-	if err != nil {
-		return Grounded{}, fmt.Errorf("recipes: ground %q: search: %w", dishName, err)
-	}
-	if len(cands) == 0 {
-		return Grounded{}, &ErrUngroundable{Target: dishName, Reasons: []error{ErrNoCandidates}}
-	}
-	cands = cands[:min(len(cands), RankDepth)]
-	stored := make(map[string]storedRecipe)
-	for _, c := range cands {
-		sr, ok, err := g.storedRecipe(ctx, c.URL)
-		if err != nil {
-			return Grounded{}, fmt.Errorf("recipes: ground %q: %w", dishName, err)
-		}
-		if ok {
-			stored[c.URL] = sr
-		}
-	}
-	ranked, rankErr := rank(ctx, dishName, cands, func(ctx context.Context, c Candidate) (Recipe, error) {
-		if sr, ok := stored[c.URL]; ok {
-			return sr.recipe, nil
-		}
-		return fetchRecipe(ctx, g.pages, c.URL)
-	})
-	if err := ctx.Err(); err != nil {
-		return Grounded{}, err
-	}
 	var reasons []error
-	if rankErr != nil {
-		reasons = append(reasons, rankErr)
+	if found.rankErr != nil {
+		reasons = append(reasons, found.rankErr)
 	}
-	for _, r := range ranked {
-		if sr, ok := stored[r.Candidate.URL]; ok {
+	for _, r := range found.ranked {
+		if sr, ok := found.stored[r.Candidate.URL]; ok {
 			return sr.grounded, nil
 		}
 		gr, reason, err := g.ground(ctx, r.Candidate.URL, r.Recipe)
@@ -169,6 +149,96 @@ func (g *Grounder) Ground(ctx context.Context, dishName, query string) (Grounded
 	return Grounded{}, &ErrUngroundable{Target: dishName, Reasons: reasons}
 }
 
+// rankedSearch is a search's candidates ranked: the ranked ones best first,
+// those already stored by URL, and why any were not ranked.
+type rankedSearch struct {
+	ranked []Ranked
+	stored map[string]storedRecipe
+	// rankErr is Rank's error, or ErrNoCandidates when the search found
+	// nothing.
+	rankErr error
+}
+
+// searchRanked searches for query and ranks the top RankDepth candidates
+// for dishName as Ground describes. A search, database or context failure
+// is its error; candidates that do not rank are only rankErr's.
+func (g *Grounder) searchRanked(ctx context.Context, dishName, query string) (rankedSearch, error) {
+	provider, err := g.search.Provider(ctx)
+	if err != nil {
+		return rankedSearch{}, fmt.Errorf("%w: %w", ErrSearchFailed, err)
+	}
+	cands, err := provider.Search(ctx, query, RankDepth)
+	if err != nil {
+		return rankedSearch{}, fmt.Errorf("%w: %w", ErrSearchFailed, err)
+	}
+	if len(cands) == 0 {
+		return rankedSearch{rankErr: ErrNoCandidates}, nil
+	}
+	cands = cands[:min(len(cands), RankDepth)]
+	stored := make(map[string]storedRecipe)
+	for _, c := range cands {
+		sr, ok, err := g.storedRecipe(ctx, c.URL)
+		if err != nil {
+			return rankedSearch{}, err
+		}
+		if ok {
+			stored[c.URL] = sr
+		}
+	}
+	ranked, rankErr := rank(ctx, dishName, cands, func(ctx context.Context, c Candidate) (Recipe, error) {
+		if sr, ok := stored[c.URL]; ok {
+			return sr.recipe, nil
+		}
+		return fetchRecipe(ctx, g.pages, c.URL)
+	})
+	if err := ctx.Err(); err != nil {
+		return rankedSearch{}, err
+	}
+	return rankedSearch{ranked: ranked, stored: stored, rankErr: rankErr}, nil
+}
+
+// Offer is a ranked candidate offered for the household to ground a dish
+// in, with GroundURL on its URL.
+type Offer struct {
+	URL      string
+	Title    string
+	SiteName string
+	// PhotoID is the cached photo of a candidate already stored, "" for
+	// one that is not: a candidate's photo is cached only when it grounds.
+	PhotoID string
+	// Minutes is a stored candidate's normalized minutes_total, else its
+	// page's total time; nil when unknown.
+	Minutes *int
+}
+
+// Candidates searches for query and returns up to n of the candidates
+// Ground would try for dishName, best first. It fetches and extracts pages
+// but downloads no photo and runs no model. Candidates that hold no usable
+// recipe are not offered, so the list may be empty; a search that fails is
+// an error wrapping ErrSearchFailed.
+func (g *Grounder) Candidates(ctx context.Context, dishName, query string, n int) ([]Offer, error) {
+	if strings.TrimSpace(dishName) == "" {
+		return nil, errors.New("recipes: candidates: dish name is empty")
+	}
+	if n < 1 {
+		return nil, fmt.Errorf("recipes: candidates: n is %d, want at least 1", n)
+	}
+	found, err := g.searchRanked(ctx, dishName, query)
+	if err != nil {
+		return nil, fmt.Errorf("recipes: candidates for %q: %w", dishName, err)
+	}
+	offers := make([]Offer, 0, min(n, len(found.ranked)))
+	for _, r := range found.ranked[:min(n, len(found.ranked))] {
+		o := Offer{URL: r.Candidate.URL, Title: r.Recipe.Title, SiteName: r.Recipe.SiteName, Minutes: r.Recipe.TotalMinutes}
+		if sr, ok := found.stored[r.Candidate.URL]; ok {
+			o.PhotoID = sr.grounded.PhotoID
+			o.Minutes = sr.grounded.MinutesTotal
+		}
+		offers = append(offers, o)
+	}
+	return offers, nil
+}
+
 // GroundURL grounds the recipe on the page at rawURL, a pasted URL, as
 // Ground does a candidate. An already stored rawURL is returned without
 // fetching it. A page that cannot be fetched, holds no usable recipe, or
@@ -196,7 +266,7 @@ func (g *Grounder) GroundURL(ctx context.Context, rawURL string) (Grounded, erro
 	}
 	gr, reason, err := g.ground(ctx, rawURL, r)
 	if err != nil {
-		return Grounded{}, fmt.Errorf("recipes: ground %s: %w", rawURL, err)
+		return Grounded{}, fmt.Errorf("recipes: ground pasted link: %w", err)
 	}
 	if reason != nil {
 		return ungroundable(reason)
@@ -344,7 +414,7 @@ func (g *Grounder) ground(ctx context.Context, sourceURL string, r Recipe) (gr G
 		return Grounded{}, fmt.Errorf("normalize: %w", err), nil
 	}
 	if err != nil {
-		return Grounded{}, nil, fmt.Errorf("normalize: %w", err)
+		return Grounded{}, nil, fmt.Errorf("%w: %w", ErrNormalizeFailed, err)
 	}
 	if len(norm.Ingredients) != len(r.Ingredients) {
 		return Grounded{}, nil, fmt.Errorf("normalize: %d ingredients for %d lines", len(norm.Ingredients), len(r.Ingredients))

internal/recipes/ground_test.go

@@ -516,3 +516,38 @@ func TestGroundManyLimitsConcurrency(t *testing.T) {
 		t.Errorf("max in flight %d of %d calls, want %d of 5", search.maxActive, search.callCount(), GroundConcurrency)
 	}
 }
+
+func TestCandidatesOffersBestFirstUpToN(t *testing.T) {
+	t.Parallel()
+	var cands []Candidate
+	pages := map[string][]byte{}
+	for i, title := range []string{"Fish Tacos", "Fish Taco Bowls", "Baja Fish Tacos", "Tacos"} {
+		u := fmt.Sprintf("https://site%d.example/tacos", i)
+		cands = append(cands, Candidate{URL: u})
+		pages[u] = groundPage(title, "/img/tacos.jpg")
+	}
+	search := &fakeSearch{results: map[string][]Candidate{"fish tacos": cands}}
+	env := newGroundEnv(t, pages, nil, search)
+
+	got, err := env.g.Candidates(t.Context(), "Fish Tacos", "fish tacos", 3)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(got) != 3 || got[0].URL != cands[0].URL || got[0].Title != "Fish Tacos" || got[0].SiteName != "Example Kitchen" ||
+		got[0].PhotoID != "" || got[0].Minutes == nil || *got[0].Minutes != 40 {
+		t.Fatalf("Candidates = %+v, want 3 with Fish Tacos first", got)
+	}
+	if env.images.requests.Load() != 0 || len(env.llm.Requests()) != 0 {
+		t.Errorf("Candidates downloaded a photo or ran the model")
+	}
+}
+
+func TestCandidatesSearchFailureWrapsErrSearchFailed(t *testing.T) {
+	t.Parallel()
+	search := &fakeSearch{errs: map[string]error{"stew": ErrSearchRateLimited}}
+	env := newGroundEnv(t, nil, nil, search)
+	_, err := env.g.Candidates(t.Context(), "Stew", "stew", 3)
+	if !errors.Is(err, ErrSearchFailed) || !errors.Is(err, ErrSearchRateLimited) {
+		t.Errorf("err = %v, want ErrSearchFailed wrapping ErrSearchRateLimited", err)
+	}
+}

internal/recipes/photo.go

@@ -65,22 +65,28 @@ var photoDecoders = map[string]func([]byte) (image.Image, error){
 	"webp": func(b []byte) (image.Image, error) { return webp.Decode(bytes.NewReader(b)) },
 }
 
-// PhotoCache downloads photos through a Fetcher and stores each at every
+// ImageFetcher fetches an image; *Fetcher satisfies it.
+type ImageFetcher interface {
+	FetchImage(ctx context.Context, rawURL string) (Page, error)
+}
+
+// PhotoCache downloads photos through an ImageFetcher and stores each at every
 // PhotoSizes length as {id}-{size}.jpg under PhotoDir, with a photos row.
 // Photos are deduplicated by the SHA-256 of the downloaded bytes. A
 // PhotoCache is safe for concurrent use.
 type PhotoCache struct {
-	fetcher *Fetcher
-	store   *store.Store
+	images ImageFetcher
+	store  *store.Store
 	// dir is the absolute photo directory.
 	dir string
 }
 
-// NewPhotoCache returns a PhotoCache storing photos under dataDir, creating
-// the photo directory if it is absent.
-func NewPhotoCache(fetcher *Fetcher, st *store.Store, dataDir string) (*PhotoCache, error) {
-	if fetcher == nil {
-		return nil, errors.New("recipes: nil fetcher")
+// NewPhotoCache returns a PhotoCache downloading images with images and
+// storing photos under dataDir, creating the photo directory if it is
+// absent.
+func NewPhotoCache(images ImageFetcher, st *store.Store, dataDir string) (*PhotoCache, error) {
+	if images == nil {
+		return nil, errors.New("recipes: nil image fetcher")
 	}
 	if st == nil {
 		return nil, errors.New("recipes: nil store")
@@ -92,7 +98,7 @@ func NewPhotoCache(fetcher *Fetcher, st *store.Store, dataDir string) (*PhotoCac
 	if err := os.MkdirAll(dir, 0o750); err != nil {
 		return nil, fmt.Errorf("recipes: create photo directory: %w", err)
 	}
-	return &PhotoCache{fetcher: fetcher, store: st, dir: dir}, nil
+	return &PhotoCache{images: images, store: st, dir: dir}, nil
 }
 
 // DownloadedPhoto is an image DownloadPhoto has downloaded and decoded,
@@ -108,7 +114,7 @@ type DownloadedPhoto struct {
 // whose shorter side is under PhotoMinShortSide. It records nothing, so a
 // caller can try another image before opening a transaction.
 func (c *PhotoCache) DownloadPhoto(ctx context.Context, rawURL string) (DownloadedPhoto, error) {
-	page, err := c.fetcher.FetchImage(ctx, rawURL)
+	page, err := c.images.FetchImage(ctx, rawURL)
 	if err != nil {
 		return DownloadedPhoto{}, err
 	}

internal/recipes/recipestest/recipestest.go

@@ -0,0 +1,183 @@
+// Package recipestest provides fakes for tests that ground dinners outside
+// package recipes: a search, page and image fetchers serving fixtures, and
+// a Grounder over them that normalizes through the real harness and
+// normalize_recipe skill with a scripted model.
+package recipestest
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"image"
+	"image/color"
+	"image/jpeg"
+	"sync"
+	"testing"
+
+	"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/harness"
+	"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm"
+	"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/skills"
+	"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/recipes"
+	"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/store"
+)
+
+// Search is a recipes.SearchSource and recipes.SearchProvider answering
+// each query from Results. Err, when set, fails every search. Gate, when
+// set, makes every search wait for it to close.
+type Search struct {
+	Results map[string][]recipes.Candidate
+	Err     error
+	Gate    chan struct{}
+
+	mu      sync.Mutex
+	queries []string
+}
+
+func (s *Search) Provider(context.Context) (recipes.SearchProvider, error) { return s, nil }
+
+func (s *Search) Search(ctx context.Context, query string, n int) ([]recipes.Candidate, error) {
+	s.mu.Lock()
+	s.queries = append(s.queries, query)
+	s.mu.Unlock()
+	if s.Gate != nil {
+		select {
+		case <-s.Gate:
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		}
+	}
+	if s.Err != nil {
+		return nil, s.Err
+	}
+	c := s.Results[query]
+	return c[:min(n, len(c))], nil
+}
+
+// Queries returns every query searched, in order.
+func (s *Search) Queries() []string {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return append([]string(nil), s.queries...)
+}
+
+// Pages is a recipes.PageFetcher serving Bodies by URL; any other URL is a
+// 404. Gate, when set, makes every fetch wait for it to close.
+type Pages struct {
+	Bodies map[string][]byte
+	Gate   chan struct{}
+
+	mu      sync.Mutex
+	fetched []string
+}
+
+func (p *Pages) Fetch(ctx context.Context, rawURL string) (recipes.Page, error) {
+	p.mu.Lock()
+	p.fetched = append(p.fetched, rawURL)
+	p.mu.Unlock()
+	if p.Gate != nil {
+		select {
+		case <-p.Gate:
+		case <-ctx.Done():
+			return recipes.Page{}, ctx.Err()
+		}
+	}
+	body, ok := p.Bodies[rawURL]
+	if !ok {
+		return recipes.Page{}, &recipes.StatusError{URL: rawURL, Status: 404}
+	}
+	return recipes.Page{URL: rawURL, ContentType: "text/html", Body: body}, nil
+}
+
+// Fetched returns every URL fetched, in order.
+func (p *Pages) Fetched() []string {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	return append([]string(nil), p.fetched...)
+}
+
+// Images is a recipes.ImageFetcher serving JPEGs by URL; any other URL is
+// a 404.
+type Images struct {
+	JPEGs map[string][]byte
+}
+
+func (i *Images) FetchImage(_ context.Context, rawURL string) (recipes.Page, error) {
+	body, ok := i.JPEGs[rawURL]
+	if !ok {
+		return recipes.Page{}, &recipes.StatusError{URL: rawURL, Status: 404}
+	}
+	return recipes.Page{URL: rawURL, ContentType: "image/jpeg", Body: body}, nil
+}
+
+// JPEG is a w×h JPEG whose pixels depend on seed, so different seeds give
+// different photos.
+func JPEG(t testing.TB, w, h int, seed uint8) []byte {
+	t.Helper()
+	img := image.NewRGBA(image.Rect(0, 0, w, h))
+	for y := range h {
+		for x := range w {
+			img.Set(x, y, color.RGBA{uint8(x), uint8(y), seed, 255})
+		}
+	}
+	var buf bytes.Buffer
+	if err := jpeg.Encode(&buf, img, nil); err != nil {
+		t.Fatal(err)
+	}
+	return buf.Bytes()
+}
+
+// RecipePage is a page whose schema.org Recipe is title, from site, with
+// two ingredients, two steps, a total time of minutes and the image at
+// imageURL.
+func RecipePage(title, site, imageURL string, minutes int) []byte {
+	return fmt.Appendf(nil, `<!DOCTYPE html><html><head>
+<meta property="og:site_name" content=%q>
+<script type="application/ld+json">{
+	"@context": "https://schema.org",
+	"@type": "Recipe",
+	"name": %q,
+	"image": %q,
+	"recipeIngredient": ["1 onion", "2 cloves garlic, minced"],
+	"recipeInstructions": ["Chop the onion.", "Fry with the garlic."],
+	"totalTime": "PT%dM"
+}</script></head><body></body></html>`, site, title, imageURL, minutes)
+}
+
+// NormalizedReply is a usable normalize_recipe reply for a RecipePage with
+// method and minutes.
+func NormalizedReply(method string, minutes int) llm.GenerateResult {
+	js := fmt.Sprintf(`{"ingredients":[
+		{"name":"onion","qty":"1","unit":"","note":""},
+		{"name":"garlic","qty":"2","unit":"cloves","note":"minced"}
+	],"method":%q,"difficulty":null,"minutes_total":%d}`, method, minutes)
+	return llm.GenerateResult{Response: llm.Response{Text: js, JSON: json.RawMessage(js)}}
+}
+
+// NewGrounder returns a Grounder storing in st, with photos cached under
+// dataDir, normalizing through the real harness with model.
+func NewGrounder(t testing.TB, st *store.Store, dataDir string, search recipes.SearchSource, pages recipes.PageFetcher, images recipes.ImageFetcher, model llm.Provider) *recipes.Grounder {
+	t.Helper()
+	photos, err := recipes.NewPhotoCache(images, st, dataDir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	guard, err := harness.NewGuard(harness.DefaultMaxCalls)
+	if err != nil {
+		t.Fatal(err)
+	}
+	cfg := harness.Config{
+		Settings:    llm.Settings{Provider: "anthropic", Model: "claude-test"},
+		NewProvider: func(llm.Settings, string) (llm.Provider, error) { return model, nil },
+		Store:       st,
+		Guard:       guard,
+	}
+	normalize := func(ctx context.Context, scope string, in skills.RecipeText) (skills.NormalizedRecipe, error) {
+		return skills.NormalizeRecipe(ctx, cfg, scope, in)
+	}
+	g, err := recipes.NewGrounder(st, search, pages, photos, normalize)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return g
+}