Add photo backfill for failed dinners without a photo

Task 36c40130-a69a-46ed-a047-2cdc5ee2d2e0 (Feature 23, task 1 of 2).

A backfill_dinner_photos job grounds, by name with the existing
recipes.Grounder.Ground, every unarchived dinner whose grounding failed
and that has no photo and no pasted recipe_url. Ground already tries the
ranked candidates best first and keeps the first whose photo caches and
whose ingredients normalize, reusing stored recipes and photos. Success
records the recipe and photo as grounded; a failure leaves the dinner
failed with the new grounding_error for the fix-recipe flow, without
stopping its siblings. Writes are guarded on status, photo and
grounding_revision, so grounded, manual, pending, renamed and archived
dinners are never touched.

POST /api/v1/settings/photo-backfill starts a full pass and answers 202
with the job id; the kind is service-only.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/dinners/backfill.go

@@ -0,0 +1,145 @@
+package dinners
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"fmt"
+
+	"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/db"
+	"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/store"
+)
+
+// KindBackfillPhotos is the job that grounds, by name, the dinners left
+// without a photo: those whose grounding failed and that have no pasted
+// recipe_url (ListPhotoBackfillDinners).
+const KindBackfillPhotos = "backfill_dinner_photos"
+
+// Photo backfill outcomes for one dinner.
+const (
+	BackfillGrounded = "grounded"
+	BackfillFailed   = "failed"
+	// BackfillSkipped: the dinner was grounded by hand, renamed or archived
+	// while the backfill grounded it, so the backfill left it as it is.
+	BackfillSkipped = "skipped"
+)
+
+// BackfillJob is the KindBackfillPhotos payload. DinnerID narrows the
+// backfill to that dinner; empty backfills every dinner without a photo.
+type BackfillJob struct {
+	DinnerID string `json:"dinner_id,omitempty"`
+}
+
+// BackfillOutcome is what a photo backfill did for one dinner. Reason is
+// set only for BackfillFailed: the Reason recorded as its grounding_error.
+type BackfillOutcome struct {
+	DinnerID string  `json:"dinner_id"`
+	Outcome  string  `json:"outcome"`
+	Reason   *string `json:"reason"`
+}
+
+// backfillResult is the KindBackfillPhotos result: the outcome of each
+// dinner the backfill tried, in the order it tried them.
+type backfillResult struct {
+	Dinners []BackfillOutcome `json:"dinners"`
+}
+
+func (r backfillResult) PublicJSON() ([]byte, error) { return json.Marshal(r) }
+
+// StartPhotoBackfill enqueues a KindBackfillPhotos job over every dinner
+// without a photo and returns its id, without waiting for it.
+func (s *Service) StartPhotoBackfill(ctx context.Context) (string, error) {
+	id, err := s.backfillJob.Enqueue(ctx, BackfillJob{})
+	if err != nil {
+		return "", fmt.Errorf("dinners: enqueue photo backfill: %w", err)
+	}
+	return id, nil
+}
+
+// runBackfillJob grounds each dinner ListPhotoBackfillDinners lists for p,
+// one at a time, with recipes.Grounder.Ground on its name: the first ranked
+// candidate whose photo caches and whose ingredients normalize grounds it,
+// with the recipe's source URL and site kept as any grounding keeps them.
+// Success records the recipe and photo with grounding_status grounded;
+// failure leaves the dinner failed with the new Reason as its
+// grounding_error, for the household to fix its recipe by hand. One
+// dinner's failure, its search's, model's or database write's, never stops
+// the others. Only a failure to list the dinners, or the job's context
+// ending, fails the job.
+func (s *Service) runBackfillJob(ctx context.Context, p BackfillJob, _ func(backfillResult) error) (backfillResult, error) {
+	q := db.New(s.st.Reader)
+	rows, err := q.ListPhotoBackfillDinners(ctx, sql.NullString{String: p.DinnerID, Valid: p.DinnerID != ""})
+	if err != nil {
+		return backfillResult{}, fmt.Errorf("dinners: photo backfill: list: %w", err)
+	}
+	res := backfillResult{Dinners: make([]BackfillOutcome, 0, len(rows))}
+	grounded := false
+	for _, row := range rows {
+		out, err := s.backfillDinner(ctx, row)
+		if err != nil {
+			return backfillResult{}, err
+		}
+		grounded = grounded || out.Outcome == BackfillGrounded
+		res.Dinners = append(res.Dinners, out)
+	}
+	if grounded {
+		s.wake()
+	}
+	return res, nil
+}
+
+// backfillDinner grounds row, a dinner ListPhotoBackfillDinners listed, as
+// runBackfillJob describes. Its error is only the job's context ending.
+func (s *Service) backfillDinner(ctx context.Context, row db.Dinner) (BackfillOutcome, error) {
+	out := BackfillOutcome{DinnerID: row.ID}
+	w := db.New(s.st.Writer)
+	gr, err := s.grounder.Ground(ctx, row.Name, row.Name)
+	if ctxErr := ctx.Err(); ctxErr != nil {
+		return BackfillOutcome{}, ctxErr
+	}
+	if err != nil {
+		reason := reasonFor(err, false)
+		// Only the reason is logged: the error may quote a dependency's
+		// response, which jobs never store, return or log.
+		s.logger.InfoContext(ctx, "dinner photo backfill failed", "dinner_id", row.ID, "reason", reason)
+		n, err := w.RefailDinnerGrounding(ctx, db.RefailDinnerGroundingParams{
+			GroundingError:    sql.NullString{String: reason, Valid: true},
+			UpdatedAt:         store.Now().String(),
+			ID:                row.ID,
+			GroundingRevision: row.GroundingRevision,
+		})
+		return s.backfillRecorded(ctx, out, BackfillFailed, reason, n, err), nil
+	}
+	n, err := w.GroundFailedDinner(ctx, db.GroundFailedDinnerParams{
+		RecipeID:          sql.NullString{String: gr.RecipeID, Valid: true},
+		PhotoID:           sql.NullString{String: gr.PhotoID, Valid: true},
+		Method:            sql.NullString{String: gr.Method, Valid: true},
+		MinutesTotal:      nullInt(gr.MinutesTotal),
+		Difficulty:        sql.NullString{String: gr.Difficulty, Valid: true},
+		UpdatedAt:         store.Now().String(),
+		ID:                row.ID,
+		GroundingRevision: row.GroundingRevision,
+	})
+	return s.backfillRecorded(ctx, out, BackfillGrounded, "", n, err), nil
+}
+
+// backfillRecorded is out once the backfill recorded outcome (with reason,
+// for BackfillFailed) on the dinner, writing n rows with err: a dinner that
+// changed meanwhile (no row) is BackfillSkipped, and one whose write failed
+// is BackfillFailed with ReasonGroundingError, the dinner left as it was.
+func (s *Service) backfillRecorded(ctx context.Context, out BackfillOutcome, outcome, reason string, n int64, err error) BackfillOutcome {
+	switch {
+	case err != nil:
+		// The error is a database error, never a dependency's text.
+		s.logger.ErrorContext(ctx, "dinner photo backfill not recorded", "dinner_id", out.DinnerID, "error", err)
+		out.Outcome, reason = BackfillFailed, ReasonGroundingError
+	case n == 0:
+		out.Outcome, reason = BackfillSkipped, ""
+	default:
+		out.Outcome = outcome
+	}
+	if reason != "" {
+		out.Reason = &reason
+	}
+	return out
+}

internal/dinners/backfill_test.go

@@ -0,0 +1,241 @@
+package dinners_test
+
+import (
+	"database/sql"
+	"encoding/json"
+	"errors"
+	"slices"
+	"testing"
+	"time"
+
+	"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/harness"
+	"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/db"
+	"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/dinners"
+	"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/recipes"
+	"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/recipes/recipestest"
+	"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/store"
+)
+
+// failedDinner stores a Regular named name whose grounding failed with
+// no_recipe_found, without a photo or recipe_url, and returns its id.
+func (e *env) failedDinner(t *testing.T, name string) string {
+	t.Helper()
+	q := db.New(e.st.Writer)
+	now := store.Now().String()
+	row, err := q.CreateDinner(t.Context(), db.CreateDinnerParams{
+		ID:              store.NewID(),
+		Name:            name,
+		NormalizedName:  harness.Normalize(name),
+		Status:          dinners.StatusRegular,
+		Origin:          "regular",
+		GroundingStatus: dinners.GroundingPending,
+		UpdatedBy:       sql.NullString{String: member, Valid: true},
+		CreatedAt:       now,
+		UpdatedAt:       now,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := q.FailPendingDinnerGrounding(t.Context(), db.FailPendingDinnerGroundingParams{
+		GroundingError:    sql.NullString{String: dinners.ReasonNoRecipe, Valid: true},
+		UpdatedAt:         now,
+		ID:                row.ID,
+		GroundingRevision: row.GroundingRevision,
+	}); err != nil {
+		t.Fatal(err)
+	}
+	return row.ID
+}
+
+func outcome(id, out string, reason *string) dinners.BackfillOutcome {
+	return dinners.BackfillOutcome{DinnerID: id, Outcome: out, Reason: reason}
+}
+
+func checkOutcomes(t *testing.T, got, want []dinners.BackfillOutcome) {
+	t.Helper()
+	gj, _ := json.Marshal(got)
+	wj, _ := json.Marshal(want)
+	if string(gj) != string(wj) {
+		t.Fatalf("outcomes = %s, want %s", gj, wj)
+	}
+}
+
+func TestBackfillGroundsFirstCandidateWithAPhoto(t *testing.T) {
+	t.Parallel()
+	e := newUnstartedEnv(t)
+	// curryA ranks first but its photo is gone; curryB's photo caches.
+	delete(e.images.JPEGs, "https://a.example/curry.jpg")
+	e.model.QueueGenerate(recipestest.NormalizedReply("stovetop", 25))
+	id := e.failedDinner(t, "Chicken curry")
+
+	got, err := e.svc.RunBackfillJob(t.Context(), dinners.BackfillJob{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	checkOutcomes(t, got, []dinners.BackfillOutcome{outcome(id, dinners.BackfillGrounded, nil)})
+
+	row := e.dinner(t, id)
+	if row.GroundingStatus != dinners.GroundingGrounded || !row.PhotoID.Valid || !row.RecipeID.Valid || row.GroundingError.Valid {
+		t.Fatalf("dinner = %+v, want grounded with a photo and no error", row)
+	}
+	rec, err := db.New(e.st.Reader).GetRecipe(t.Context(), row.RecipeID.String)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if rec.SourceUrl != curryB || rec.SiteName.String != "B Kitchen" || rec.PhotoID != row.PhotoID {
+		t.Errorf("recipe = %s from %q photo %s, want %s from B Kitchen photo %s", rec.SourceUrl, rec.SiteName.String, rec.PhotoID.String, curryB, row.PhotoID.String)
+	}
+	if got := e.search.Queries(); !slices.Equal(got, []string{"Chicken curry"}) {
+		t.Errorf("searched %q, want the dinner's name", got)
+	}
+}
+
+func TestBackfillReusesCachedRecipeAndPhoto(t *testing.T) {
+	t.Parallel()
+	e := newUnstartedEnv(t)
+	e.search.Results["Curry chicken"] = []recipes.Candidate{{URL: curryA}}
+	// One reply: the second dinner must reuse the recipe the first stored.
+	e.model.QueueGenerate(recipestest.NormalizedReply("stovetop", 40))
+	first := e.failedDinner(t, "Chicken curry")
+	second := e.failedDinner(t, "Curry chicken")
+
+	got, err := e.svc.RunBackfillJob(t.Context(), dinners.BackfillJob{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	checkOutcomes(t, got, []dinners.BackfillOutcome{
+		outcome(first, dinners.BackfillGrounded, nil),
+		outcome(second, dinners.BackfillGrounded, nil),
+	})
+	a, b := e.dinner(t, first), e.dinner(t, second)
+	if a.RecipeID != b.RecipeID || a.PhotoID != b.PhotoID || !a.PhotoID.Valid {
+		t.Errorf("dinners grounded in recipe %s photo %s and recipe %s photo %s, want the same cached ones",
+			a.RecipeID.String, a.PhotoID.String, b.RecipeID.String, b.PhotoID.String)
+	}
+	if n := len(e.model.Requests()); n != 1 {
+		t.Errorf("normalize_recipe ran %d times, want 1", n)
+	}
+	if n := slices.Index(e.pages.Fetched(), curryA); n < 0 || slices.Contains(e.pages.Fetched()[n+1:], curryA) {
+		t.Errorf("fetched %q, want %s fetched once", e.pages.Fetched(), curryA)
+	}
+}
+
+func TestBackfillIsolatesEachDinnersFailure(t *testing.T) {
+	t.Parallel()
+	e := newUnstartedEnv(t)
+	e.search.Fail = map[string]error{"Broken search": errors.New("search provider down")}
+	e.model.QueueGenerate(recipestest.NormalizedReply("stovetop", 40))
+	// Listed by normalized name: both failures come before the curry.
+	broken := e.failedDinner(t, "Broken search")
+	curry := e.failedDinner(t, "Chicken curry")
+	absent := e.failedDinner(t, "Abalone surprise")
+
+	got, err := e.svc.RunBackfillJob(t.Context(), dinners.BackfillJob{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	checkOutcomes(t, got, []dinners.BackfillOutcome{
+		outcome(absent, dinners.BackfillFailed, ptr(dinners.ReasonNoRecipe)),
+		outcome(broken, dinners.BackfillFailed, ptr(dinners.ReasonSearchFailed)),
+		outcome(curry, dinners.BackfillGrounded, nil),
+	})
+	for id, reason := range map[string]string{absent: dinners.ReasonNoRecipe, broken: dinners.ReasonSearchFailed} {
+		row := e.dinner(t, id)
+		if row.GroundingStatus != dinners.GroundingFailed || row.GroundingError.String != reason || row.PhotoID.Valid {
+			t.Errorf("dinner %s = %+v, want failed with %s and no photo", row.Name, row, reason)
+		}
+	}
+	if row := e.dinner(t, curry); row.GroundingStatus != dinners.GroundingGrounded || !row.PhotoID.Valid {
+		t.Errorf("curry = %+v, want grounded with a photo", row)
+	}
+}
+
+func TestBackfillSkipsDinnersNotMissingAPhoto(t *testing.T) {
+	t.Parallel()
+	e := newEnv(t)
+	e.model.QueueGenerate(recipestest.NormalizedReply("stovetop", 40))
+	c, err := e.svc.CreateRegular(t.Context(), dinners.NewRegular{Name: ptr("Chicken curry")}, member)
+	if err != nil {
+		t.Fatal(err)
+	}
+	e.waitJob(t, *c.GroundingJobID)
+	if row := e.dinner(t, c.ID); row.GroundingStatus != dinners.GroundingGrounded {
+		t.Fatalf("dinner = %+v, want grounded", row)
+	}
+	// A failed dinner with a pasted link keeps that link for a fix by hand.
+	linked, err := e.svc.CreateRegular(t.Context(), dinners.NewRegular{Name: ptr("Mystery pie"), RecipeURL: ptr("https://d.example/pie")}, member)
+	if err != nil {
+		t.Fatal(err)
+	}
+	e.waitJob(t, *linked.GroundingJobID)
+	if row := e.dinner(t, linked.ID); row.GroundingStatus != dinners.GroundingFailed {
+		t.Fatalf("linked dinner = %+v, want failed", row)
+	}
+	archived := e.failedDinner(t, "Old stew")
+	if _, err := e.svc.Archive(t.Context(), archived, member); err != nil {
+		t.Fatal(err)
+	}
+	searched := len(e.search.Queries())
+
+	id, err := e.svc.StartPhotoBackfill(t.Context())
+	if err != nil {
+		t.Fatal(err)
+	}
+	job := e.waitJob(t, id)
+	if job.Kind != dinners.KindBackfillPhotos || string(job.Result) != `{"dinners":[]}` {
+		t.Fatalf("job = %+v (result %s), want a backfill with no dinners", job, job.Result)
+	}
+	if got := e.search.Queries()[searched:]; len(got) != 0 {
+		t.Errorf("backfill searched %q, want nothing", got)
+	}
+}
+
+func TestBackfillOfOneDinnerGroundsOnlyIt(t *testing.T) {
+	t.Parallel()
+	e := newUnstartedEnv(t)
+	e.model.QueueGenerate(recipestest.NormalizedReply("stovetop", 40))
+	curry := e.failedDinner(t, "Chicken curry")
+	other := e.failedDinner(t, "Abalone surprise")
+
+	got, err := e.svc.RunBackfillJob(t.Context(), dinners.BackfillJob{DinnerID: curry})
+	if err != nil {
+		t.Fatal(err)
+	}
+	checkOutcomes(t, got, []dinners.BackfillOutcome{outcome(curry, dinners.BackfillGrounded, nil)})
+	if row := e.dinner(t, other); row.GroundingStatus != dinners.GroundingFailed || row.PhotoID.Valid {
+		t.Errorf("other dinner = %+v, want left failed", row)
+	}
+	if got := e.search.Queries(); !slices.Equal(got, []string{"Chicken curry"}) {
+		t.Errorf("searched %q, want only the dinner's name", got)
+	}
+}
+
+func TestBackfillLeavesDinnerChangedWhileGrounding(t *testing.T) {
+	t.Parallel()
+	e := newUnstartedEnv(t)
+	id := e.failedDinner(t, "Chicken curry")
+	gate := make(chan struct{})
+	e.search.Gate = gate
+	e.model.QueueGenerate(recipestest.NormalizedReply("stovetop", 40))
+
+	done := make(chan []dinners.BackfillOutcome)
+	go func() {
+		got, err := e.svc.RunBackfillJob(t.Context(), dinners.BackfillJob{DinnerID: id})
+		if err != nil {
+			t.Error(err)
+		}
+		done <- got
+	}()
+	// The backfill listed the dinner and waits at its search.
+	for len(e.search.Queries()) == 0 {
+		time.Sleep(time.Millisecond)
+	}
+	if _, err := e.svc.Update(t.Context(), id, dinners.Patch{Name: ptr("Chicken korma")}, member); err != nil {
+		t.Fatal(err)
+	}
+	close(gate)
+	checkOutcomes(t, <-done, []dinners.BackfillOutcome{outcome(id, dinners.BackfillSkipped, nil)})
+	if row := e.dinner(t, id); row.GroundingStatus != dinners.GroundingFailed || row.PhotoID.Valid {
+		t.Errorf("dinner = %+v, want left failed without a photo", row)
+	}
+}

internal/dinners/dinners.go

@@ -138,6 +138,8 @@ type Service struct {
 	grounder  *recipes.Grounder
 	logger    *slog.Logger
 	groundJob *jobs.Kind[GroundJob]
+	// backfillJob enqueues KindBackfillPhotos jobs.
+	backfillJob *jobs.Kind[BackfillJob]
 	// wake is called after every committed change to a dinner's method or
 	// minutes, which can move its crock-pot start reminder.
 	wake func()
@@ -162,8 +164,8 @@ type Service struct {
 
 // NewService returns a service over st that grounds dinners with grounder,
 // calls wake (the scheduler's Wake) after every committed change to a
-// dinner's method or minutes, and registers the KindGroundDinner job on
-// runner, which must not have started yet.
+// dinner's method or minutes, and registers the KindGroundDinner and
+// KindBackfillPhotos jobs on runner, which must not have started yet.
 func NewService(st *store.Store, grounder *recipes.Grounder, runner *jobs.Runner, wake func(), logger *slog.Logger) (*Service, error) {
 	switch {
 	case st == nil:
@@ -184,6 +186,14 @@ func NewService(st *store.Store, grounder *recipes.Grounder, runner *jobs.Runner
 		return nil, err
 	}
 	s.groundJob = kind
+	backfill, err := jobs.Register(runner, KindBackfillPhotos, s.runBackfillJob)
+	if err != nil {
+		return nil, err
+	}
+	if err := backfill.ServiceOnly(); err != nil {
+		return nil, err
+	}
+	s.backfillJob = backfill
 	return s, nil
 }
 

internal/dinners/export_test.go

@@ -74,3 +74,10 @@ func (s *Service) RunGroundJob(ctx context.Context, p GroundJob) (string, error)
 
 // SafeSourceURL is safeSourceURL.
 var SafeSourceURL = safeSourceURL
+
+// RunBackfillJob runs the KindBackfillPhotos handler for p in the caller's
+// goroutine, as the job runner would, and returns its outcomes.
+func (s *Service) RunBackfillJob(ctx context.Context, p BackfillJob) ([]BackfillOutcome, error) {
+	r, err := s.runBackfillJob(ctx, p, nil)
+	return r.Dinners, err
+}

internal/dinners/grounding_test.go

@@ -39,6 +39,7 @@ type env struct {
 	runner *jobs.Runner
 	search *recipestest.Search
 	pages  *recipestest.Pages
+	images *recipestest.Images
 	model  *llm.Fake
 }
 
@@ -108,7 +109,7 @@ func newUnstartedEnvOn(t *testing.T, st *store.Store, dataDir string) *env {
 	if err != nil {
 		t.Fatal(err)
 	}
-	return &env{svc: svc, st: st, runner: runner, search: search, pages: pages, model: model}
+	return &env{svc: svc, st: st, runner: runner, search: search, pages: pages, images: images, model: model}
 }
 
 // start starts the env's job runner, stopping it when the test ends.