package recipes import ( "context" "database/sql" "encoding/json" "errors" "fmt" "strings" "sync" "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm" "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/skills" "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/db" "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/store" ) // GroundConcurrency is how many items GroundMany grounds at once. const GroundConcurrency = 3 var ( // ErrNoCandidates is a reason a dish is ungroundable: the search found // no pages at all. ErrNoCandidates = errors.New("recipes: search found no candidates") // ErrTotalTimeNotPositive rejects a recipe whose page gives a total // time of zero minutes, which normalize_recipe refuses. ErrTotalTimeNotPositive = errors.New("recipes: recipe total time is not positive") // ErrSearchFailed wraps every failure to search: the search provider // could not be built (ErrSearchNotConfigured among them) or its search // failed. ErrSearchFailed = errors.New("recipes: recipe search failed") // ErrNormalizeFailed wraps a normalize_recipe failure that is not the // recipe's: the model provider could not be built or called. ErrNormalizeFailed = errors.New("recipes: normalize_recipe failed") ) // ErrUngroundable is returned when no recipe page could ground a dish or a // pasted URL. Reasons says why each page tried was rejected, or why there // were none; errors.Is matches any of them. type ErrUngroundable struct { // Target is the dish name or the URL. Target string Reasons []error } func (e *ErrUngroundable) Error() string { parts := make([]string, len(e.Reasons)) for i, r := range e.Reasons { parts[i] = r.Error() } return fmt.Sprintf("recipes: cannot ground %q: %s", e.Target, strings.Join(parts, "; ")) } func (e *ErrUngroundable) Unwrap() []error { return e.Reasons } // Grounded is a recipe stored for a dish: the recipe and its cached photo, // with normalize_recipe's method, minutes and difficulty. type Grounded struct { RecipeID string `json:"recipe_id"` PhotoID string `json:"photo_id"` // Method is stovetop, oven, crockpot, grill or other. Method string `json:"method"` // MinutesTotal is prep to table, nil when unknown. MinutesTotal *int `json:"minutes_total"` // Difficulty is easy, medium or hard. Difficulty string `json:"difficulty"` SourceURL string `json:"source_url"` SiteName string `json:"site_name"` } // SearchSource returns the household's current search provider; // *SearchSettingsService satisfies it. type SearchSource interface { Provider(ctx context.Context) (SearchProvider, error) } // Normalizer runs normalize_recipe on a recipe; scope is its source URL. In // production it is skills.NormalizeRecipe with the household's // harness.Config. type Normalizer func(ctx context.Context, scope string, in skills.RecipeText) (skills.NormalizedRecipe, error) // Grounder grounds dishes in recipe pages: it searches, fetches, extracts // and ranks candidates, caches the best one's photo, normalizes its // ingredients and stores it all. It is safe for concurrent use. type Grounder struct { st *store.Store search SearchSource pages PageFetcher photos *PhotoCache normalize Normalizer } // NewGrounder returns a Grounder storing recipes in st, searching with // search, fetching pages with pages, caching photos with photos and // normalizing with normalize. func NewGrounder(st *store.Store, search SearchSource, pages PageFetcher, photos *PhotoCache, normalize Normalizer) (*Grounder, error) { switch { case st == nil: return nil, errors.New("recipes: nil store") case search == nil: return nil, errors.New("recipes: nil search source") case pages == nil: return nil, errors.New("recipes: nil page fetcher") case photos == nil: return nil, errors.New("recipes: nil photo cache") case normalize == nil: return nil, errors.New("recipes: nil normalizer") } return &Grounder{st: st, search: search, pages: pages, photos: photos, normalize: normalize}, nil } // Ground searches for query and grounds dishName in the best recipe found. // The candidates are ranked as Rank does, except that a candidate already // stored is scored on its stored recipe and never fetched. They are tried // best first: a stored candidate is returned as it is, and otherwise the // first whose photo can be cached and whose ingredients normalize is stored // — the recipe, its ingredients, steps and photo in one transaction — and // returned. A stored recipe is reused only when it ranks above every // candidate that grounds. // // When no candidate grounds the dish the error is an *ErrUngroundable with // each reason. A failure that is not the candidates' — the search, the // model provider, the database — is returned as it is. func (g *Grounder) Ground(ctx context.Context, dishName, query string) (Grounded, error) { if strings.TrimSpace(dishName) == "" { return Grounded{}, errors.New("recipes: ground: dish name is empty") } found, err := g.searchRanked(ctx, dishName, query) if err != nil { return Grounded{}, fmt.Errorf("recipes: ground %q: %w", dishName, err) } var reasons []error if found.rankErr != nil { reasons = append(reasons, found.rankErr) } for _, r := range found.ranked { if sr, ok := found.stored[r.Candidate.URL]; ok { return sr.grounded, nil } gr, reason, err := g.ground(ctx, r.Candidate.URL, r.Recipe) if err != nil { return Grounded{}, fmt.Errorf("recipes: ground %q: %w", dishName, err) } if reason == nil { return gr, nil } reasons = append(reasons, fmt.Errorf("%s: %w", r.Candidate.URL, reason)) } return Grounded{}, &ErrUngroundable{Target: dishName, Reasons: reasons} } // rankedSearch is a search's candidates ranked: the ranked ones best first, // those already stored by URL, and why any were not ranked. type rankedSearch struct { ranked []Ranked stored map[string]storedRecipe // rankErr is Rank's error, or ErrNoCandidates when the search found // nothing. rankErr error } // searchRanked searches for query and ranks the top RankDepth candidates // for dishName as Ground describes. A search, database or context failure // is its error; candidates that do not rank are only rankErr's. func (g *Grounder) searchRanked(ctx context.Context, dishName, query string) (rankedSearch, error) { provider, err := g.search.Provider(ctx) if err != nil { return rankedSearch{}, fmt.Errorf("%w: %w", ErrSearchFailed, err) } cands, err := provider.Search(ctx, query, RankDepth) if err != nil { return rankedSearch{}, fmt.Errorf("%w: %w", ErrSearchFailed, err) } if len(cands) == 0 { return rankedSearch{rankErr: ErrNoCandidates}, nil } cands = cands[:min(len(cands), RankDepth)] stored := make(map[string]storedRecipe) for _, c := range cands { sr, ok, err := g.storedRecipe(ctx, c.URL) if err != nil { return rankedSearch{}, err } if ok { stored[c.URL] = sr } } ranked, rankErr := rank(ctx, dishName, cands, func(ctx context.Context, c Candidate) (Recipe, error) { if sr, ok := stored[c.URL]; ok { return sr.recipe, nil } return fetchRecipe(ctx, g.pages, c.URL) }) if err := ctx.Err(); err != nil { return rankedSearch{}, err } return rankedSearch{ranked: ranked, stored: stored, rankErr: rankErr}, nil } // Offer is a ranked candidate offered for the household to ground a dish // in, with GroundURL on its URL. type Offer struct { URL string Title string SiteName string // PhotoID is the cached photo of a candidate already stored, "" for // one that is not: a candidate's photo is cached only when it grounds. PhotoID string // Minutes is a stored candidate's normalized minutes_total, else its // page's total time; nil when unknown. Minutes *int } // Candidates searches for query and returns up to n of the candidates // Ground would try for dishName, best first. It fetches and extracts pages // but downloads no photo and runs no model. Candidates that hold no usable // recipe are not offered, so the list may be empty; a search that fails is // an error wrapping ErrSearchFailed. func (g *Grounder) Candidates(ctx context.Context, dishName, query string, n int) ([]Offer, error) { if strings.TrimSpace(dishName) == "" { return nil, errors.New("recipes: candidates: dish name is empty") } if n < 1 { return nil, fmt.Errorf("recipes: candidates: n is %d, want at least 1", n) } found, err := g.searchRanked(ctx, dishName, query) if err != nil { return nil, fmt.Errorf("recipes: candidates for %q: %w", dishName, err) } offers := make([]Offer, 0, min(n, len(found.ranked))) for _, r := range found.ranked[:min(n, len(found.ranked))] { o := Offer{URL: r.Candidate.URL, Title: r.Recipe.Title, SiteName: r.Recipe.SiteName, Minutes: r.Recipe.TotalMinutes} if sr, ok := found.stored[r.Candidate.URL]; ok { o.PhotoID = sr.grounded.PhotoID o.Minutes = sr.grounded.MinutesTotal } offers = append(offers, o) } return offers, nil } // GroundURL grounds the recipe on the page at rawURL, a pasted URL, as // Ground does a candidate. An already stored rawURL is returned without // fetching it. A page that cannot be fetched, holds no usable recipe, or // whose photo or ingredients fail is an *ErrUngroundable. func (g *Grounder) GroundURL(ctx context.Context, rawURL string) (Grounded, error) { if gr, ok, err := g.stored(ctx, rawURL); err != nil || ok { return gr, err } ungroundable := func(err error) (Grounded, error) { if ctxErr := ctx.Err(); ctxErr != nil { return Grounded{}, ctxErr } return Grounded{}, &ErrUngroundable{Target: rawURL, Reasons: []error{err}} } page, err := g.pages.Fetch(ctx, rawURL) if err != nil { return ungroundable(err) } r, err := Extract(page.Body, page.URL) if err != nil { return ungroundable(err) } if err := usable(r); err != nil { return ungroundable(err) } gr, reason, err := g.ground(ctx, rawURL, r) if err != nil { return Grounded{}, fmt.Errorf("recipes: ground pasted link: %w", err) } if reason != nil { return ungroundable(reason) } return gr, nil } // GroundRequest is one dish for GroundMany. type GroundRequest struct { DishName string Query string } // GroundResult is one GroundMany outcome: Grounded, or Err when the item // failed. type GroundResult struct { Grounded Grounded Err error } // GroundMany grounds each request with Ground, GroundConcurrency at a time, // and returns their results in request order. One item's failure does not // affect the others. func (g *Grounder) GroundMany(ctx context.Context, reqs []GroundRequest) []GroundResult { results := make([]GroundResult, len(reqs)) g.GroundEach(ctx, reqs, func(i int, r GroundResult) { results[i] = r }) return results } // GroundEach grounds each request as GroundMany does, but calls done with // each item's index and result as soon as that item finishes, one call at // a time. It returns once every item has finished and every call has // returned. func (g *Grounder) GroundEach(ctx context.Context, reqs []GroundRequest, done func(i int, r GroundResult)) { slots := make(chan struct{}, GroundConcurrency) var mu sync.Mutex report := func(i int, r GroundResult) { mu.Lock() defer mu.Unlock() done(i, r) } var wg sync.WaitGroup for i, req := range reqs { wg.Add(1) go func() { defer wg.Done() select { case slots <- struct{}{}: case <-ctx.Done(): report(i, GroundResult{Err: ctx.Err()}) return } gr, err := g.Ground(ctx, req.DishName, req.Query) <-slots report(i, GroundResult{Grounded: gr, Err: err}) }() } wg.Wait() } // stored returns the grounding of the recipe stored for sourceURL, and // false when there is none. func (g *Grounder) stored(ctx context.Context, sourceURL string) (Grounded, bool, error) { row, ok, err := g.storedRow(ctx, sourceURL) if err != nil || !ok { return Grounded{}, false, err } return groundedFrom(row), true, nil } // storedRow returns the recipe row stored for sourceURL, and false when // there is none. func (g *Grounder) storedRow(ctx context.Context, sourceURL string) (db.Recipe, bool, error) { row, err := db.New(g.st.Reader).GetRecipeBySourceURL(ctx, sourceURL) if errors.Is(err, sql.ErrNoRows) { return db.Recipe{}, false, nil } if err != nil { return db.Recipe{}, false, fmt.Errorf("recipes: look up recipe %s: %w", sourceURL, err) } if !row.PhotoID.Valid || !row.Method.Valid || !row.Difficulty.Valid { return db.Recipe{}, false, fmt.Errorf("recipes: stored recipe %s for %s has no photo, method or difficulty", row.ID, sourceURL) } return row, true, nil } // storedRecipe is a recipe already stored: its grounding, and the recipe // rebuilt from what was stored so it can be ranked without its page. type storedRecipe struct { grounded Grounded recipe Recipe } // storedRecipe returns the recipe stored for sourceURL with its photo, // ingredients and steps, and false when there is none. func (g *Grounder) storedRecipe(ctx context.Context, sourceURL string) (storedRecipe, bool, error) { row, ok, err := g.storedRow(ctx, sourceURL) if err != nil || !ok { return storedRecipe{}, false, err } gr := groundedFrom(row) q := db.New(g.st.Reader) photo, err := q.GetPhoto(ctx, gr.PhotoID) if err != nil { return storedRecipe{}, false, fmt.Errorf("recipes: load photo %s of recipe %s: %w", gr.PhotoID, gr.RecipeID, err) } ings, err := q.ListRecipeIngredients(ctx, gr.RecipeID) if err != nil { return storedRecipe{}, false, fmt.Errorf("recipes: load ingredients of recipe %s: %w", gr.RecipeID, err) } steps, err := q.ListRecipeSteps(ctx, gr.RecipeID) if err != nil { return storedRecipe{}, false, fmt.Errorf("recipes: load steps of recipe %s: %w", gr.RecipeID, err) } r := Recipe{ Title: row.Title, SiteName: row.SiteName.String, ImageURL: photo.SourceUrl, PrepMinutes: intFrom(row.PrepMinutes), CookMinutes: intFrom(row.CookMinutes), TotalMinutes: intFrom(row.TotalMinutes), Yield: row.Yield.String, JSONLD: json.RawMessage(row.RawJsonld), } for _, ing := range ings { r.Ingredients = append(r.Ingredients, ing.RawText) } for _, st := range steps { r.Steps = append(r.Steps, st.Text) } return storedRecipe{grounded: gr, recipe: r}, true, nil } // ground downloads r's photo, normalizes it and stores it as sourceURL's // recipe. A problem with this recipe is returned as reason, so the caller // can try another; err is a failure that is not the recipe's. // // The order is fixed: the photo is downloaded and validated first, so a // recipe without a usable photo never costs a normalize_recipe call; then // normalize_recipe runs; then save records the photo (CachePhoto) and the // recipe in one transaction. func (g *Grounder) ground(ctx context.Context, sourceURL string, r Recipe) (gr Grounded, reason, err error) { photo, err := g.photos.DownloadPhoto(ctx, r.ImageURL) if err != nil { if ctxErr := ctx.Err(); ctxErr != nil { return Grounded{}, nil, ctxErr } return Grounded{}, fmt.Errorf("photo: %w", err), nil } norm, err := g.normalize(ctx, sourceURL, skills.RecipeText{ Title: r.Title, Ingredients: r.Ingredients, Steps: r.Steps, PrepMinutes: r.PrepMinutes, CookMinutes: r.CookMinutes, TotalMinutes: r.TotalMinutes, }) if errors.Is(err, llm.ErrBadOutput) { return Grounded{}, fmt.Errorf("normalize: %w", err), nil } if err != nil { return Grounded{}, nil, fmt.Errorf("%w: %w", ErrNormalizeFailed, err) } if len(norm.Ingredients) != len(r.Ingredients) { return Grounded{}, nil, fmt.Errorf("normalize: %d ingredients for %d lines", len(norm.Ingredients), len(r.Ingredients)) } gr, err = g.save(ctx, sourceURL, r, photo, norm) return gr, nil, err } // save stores the recipe, its ingredients and steps and its photo in one // transaction. A photo's files written for a transaction that does not // commit are removed. func (g *Grounder) save(ctx context.Context, sourceURL string, r Recipe, p DownloadedPhoto, norm skills.NormalizedRecipe) (Grounded, error) { var ( row db.Recipe photo db.Photo created bool ) err := g.st.WithTx(ctx, func(tx *sql.Tx) error { var err error photo, created, err = g.photos.CachePhoto(ctx, tx, p) if err != nil { return err } q := db.New(tx) now := store.Now().String() row, err = q.UpsertRecipe(ctx, db.UpsertRecipeParams{ ID: store.NewID(), SourceUrl: sourceURL, SiteName: sql.NullString{String: r.SiteName, Valid: r.SiteName != ""}, Title: r.Title, TotalMinutes: nullInt(r.TotalMinutes), PrepMinutes: nullInt(r.PrepMinutes), CookMinutes: nullInt(r.CookMinutes), Yield: sql.NullString{String: r.Yield, Valid: r.Yield != ""}, RawJsonld: string(r.JSONLD), PhotoID: sql.NullString{String: photo.ID, Valid: true}, Method: sql.NullString{String: norm.Method, Valid: true}, Difficulty: sql.NullString{String: norm.Difficulty, Valid: true}, MinutesTotal: nullInt(norm.MinutesTotal), FetchedAt: now, CreatedAt: now, UpdatedAt: now, }) if err != nil { return fmt.Errorf("recipes: store recipe: %w", err) } if err := q.DeleteRecipeIngredients(ctx, row.ID); err != nil { return fmt.Errorf("recipes: replace ingredients: %w", err) } if err := q.DeleteRecipeSteps(ctx, row.ID); err != nil { return fmt.Errorf("recipes: replace steps: %w", err) } for i, line := range r.Ingredients { ing := norm.Ingredients[i] err := q.CreateRecipeIngredient(ctx, db.CreateRecipeIngredientParams{ ID: store.NewID(), RecipeID: row.ID, Position: int64(i), RawText: line, Name: sql.NullString{String: ing.Name, Valid: true}, Qty: sql.NullString{String: ing.Qty, Valid: ing.Qty != ""}, Unit: sql.NullString{String: ing.Unit, Valid: ing.Unit != ""}, Note: sql.NullString{String: ing.Note, Valid: ing.Note != ""}, CreatedAt: now, UpdatedAt: now, }) if err != nil { return fmt.Errorf("recipes: store ingredient %d: %w", i+1, err) } } for i, text := range r.Steps { err := q.CreateRecipeStep(ctx, db.CreateRecipeStepParams{ ID: store.NewID(), RecipeID: row.ID, Position: int64(i), Text: text, CreatedAt: now, UpdatedAt: now, }) if err != nil { return fmt.Errorf("recipes: store step %d: %w", i+1, err) } } return nil }) if err != nil { if created { err = errors.Join(err, g.photos.RemovePhotoFiles(photo.ID)) } return Grounded{}, err } return groundedFrom(row), nil } func groundedFrom(row db.Recipe) Grounded { gr := Grounded{ RecipeID: row.ID, PhotoID: row.PhotoID.String, Method: row.Method.String, Difficulty: row.Difficulty.String, SourceURL: row.SourceUrl, SiteName: row.SiteName.String, } if row.MinutesTotal.Valid { m := int(row.MinutesTotal.Int64) gr.MinutesTotal = &m } return gr } func intFrom(v sql.NullInt64) *int { if !v.Valid { return nil } n := int(v.Int64) return &n } func nullInt(v *int) sql.NullInt64 { if v == nil { return sql.NullInt64{} } return sql.NullInt64{Int64: int64(*v), Valid: true} }