Fail interrupted groundings at boot and bound hand grounding

At boot, after the job runner fails the jobs a previous process left
running, FailInterruptedGroundings sets every dinner still pending with
no ground_dinner job queued or running to failed (interrupted). A crash
mid-job, or between saving a dinner and enqueuing its job, no longer
leaves the dinner pending forever.

A search failure on recipe-candidates or ground now logs only the reason
code search_failed, never the wrapped error, which may quote the
provider; the response stays redacted.

When the grounding job cannot be enqueued, POST /dinners answers 201
with the saved dinner failed (grounding_error) and grounding_job_id
null, instead of a 500 that hid the dinner it had created.

Grounding a candidate or link by hand runs under a 90-second deadline
(fetch, photo, normalize_recipe); past it the request answers 504
ground_timeout and the dinner is unchanged.

Maestro task: 0cbaa5bf-8be4-4cbd-aab2-891e7f1bc2b5

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/dinners/dinners.go

@@ -155,10 +155,12 @@ func NewService(st *store.Store, grounder *recipes.Grounder, runner *jobs.Runner
 	return s, nil
 }
 
-// Created is a new Regular with the id of the job grounding it.
+// Created is a new Regular with the id of the job grounding it, which is
+// nil when the job could not be enqueued and the dinner's grounding is
+// failed instead.
 type Created struct {
 	Dinner
-	GroundingJobID string `json:"grounding_job_id"`
+	GroundingJobID *string `json:"grounding_job_id"`
 }
 
 // CreateRegular adds a Regular named by r.Name, or by the last path segment
@@ -166,24 +168,35 @@ type Created struct {
 // origin regular and grounding_status pending, recording updatedBy, and
 // enqueues the KindGroundDinner job that grounds it. It returns without
 // waiting for the job. A name repeating an unarchived dinner is a
-// *DuplicateError.
+// *DuplicateError. When the job cannot be enqueued, the stored dinner is
+// returned with grounding_status failed (ReasonGroundingError) and no job
+// id, so the household can fix its recipe by hand.
 func (s *Service) CreateRegular(ctx context.Context, r NewRegular, updatedBy string) (Created, error) {
 	d, err := s.createRegular(ctx, r, updatedBy)
 	if err != nil {
 		return Created{}, err
 	}
 	jobID, err := s.groundJob.Enqueue(ctx, GroundJob{DinnerID: d.ID})
-	if err != nil {
-		// The dinner is stored; record that it will not be grounded so it
-		// is not left pending with no job, and can be fixed by hand.
-		_, failErr := db.New(s.st.Writer).FailPendingDinnerGrounding(context.WithoutCancel(ctx), db.FailPendingDinnerGroundingParams{
-			GroundingError: sql.NullString{String: ReasonGroundingError, Valid: true},
-			UpdatedAt:      store.Now().String(),
-			ID:             d.ID,
-		})
-		return Created{}, errors.Join(fmt.Errorf("dinners: enqueue grounding of %s: %w", d.ID, err), failErr)
+	if err == nil {
+		return Created{Dinner: d, GroundingJobID: &jobID}, nil
 	}
-	return Created{Dinner: d, GroundingJobID: jobID}, nil
+	// The dinner is stored; record that it will not be grounded so it is
+	// not left pending with no job. The error is a database error, never a
+	// dependency's text.
+	s.logger.ErrorContext(ctx, "dinner grounding not enqueued", "dinner_id", d.ID, "error", err)
+	write := context.WithoutCancel(ctx)
+	if _, err := db.New(s.st.Writer).FailPendingDinnerGrounding(write, db.FailPendingDinnerGroundingParams{
+		GroundingError: sql.NullString{String: ReasonGroundingError, Valid: true},
+		UpdatedAt:      store.Now().String(),
+		ID:             d.ID,
+	}); err != nil {
+		return Created{}, fmt.Errorf("dinners: record failed grounding of %s: %w", d.ID, err)
+	}
+	row, err := db.New(s.st.Reader).GetDinner(write, d.ID)
+	if err != nil {
+		return Created{}, fmt.Errorf("dinners: read %s: %w", d.ID, err)
+	}
+	return Created{Dinner: fromRow(row)}, nil
 }
 
 func (s *Service) createRegular(ctx context.Context, r NewRegular, updatedBy string) (Dinner, error) {

internal/dinners/export_test.go

@@ -0,0 +1,14 @@
+package dinners
+
+import (
+	"testing"
+	"time"
+)
+
+// SetGroundTimeout sets the deadline of grounding by hand until t ends. A
+// test calling it must not run in parallel.
+func SetGroundTimeout(t *testing.T, d time.Duration) {
+	old := groundTimeout
+	groundTimeout = d
+	t.Cleanup(func() { groundTimeout = old })
+}

internal/dinners/grounding.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"slices"
 	"strings"
+	"time"
 	"unicode/utf8"
 
 	"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/db"
@@ -63,6 +64,18 @@ const (
 	manualSourcePrefix = "mealquest:manual:"
 )
 
+// GroundTimeout bounds grounding a candidate or link by hand, which runs in
+// the request: the page fetch (recipes.Timeout), its photo download and
+// normalize_recipe (harness.DefaultTimeout), with some slack.
+const GroundTimeout = 90 * time.Second
+
+// groundTimeout is GroundTimeout; tests shorten it.
+var groundTimeout = GroundTimeout
+
+// ErrGroundTimeout is returned by Ground when grounding a candidate or link
+// runs past GroundTimeout; the dinner is left as it was.
+var ErrGroundTimeout = errors.New("dinners: grounding the recipe took too long")
+
 // GroundJob is the KindGroundDinner payload.
 type GroundJob struct {
 	DinnerID string `json:"dinner_id"`
@@ -143,6 +156,27 @@ func (s *Service) runGroundJob(ctx context.Context, p GroundJob, _ func(groundRe
 	return groundResult{DinnerID: row.ID, GroundingStatus: GroundingGrounded}, nil
 }
 
+// FailInterruptedGroundings fails, with ReasonInterrupted, every dinner
+// still pending whose KindGroundDinner job is no longer queued or running:
+// a previous process stopped before enqueuing the job or before its
+// handler recorded a result. Call it once at boot, after the job runner's
+// Start has failed the jobs that process left running, so no dinner stays
+// pending with nothing left to ground it. It returns how many it failed.
+func (s *Service) FailInterruptedGroundings(ctx context.Context) (int64, error) {
+	n, err := db.New(s.st.Writer).FailOrphanedPendingDinnerGroundings(ctx, db.FailOrphanedPendingDinnerGroundingsParams{
+		GroundingError: sql.NullString{String: ReasonInterrupted, Valid: true},
+		UpdatedAt:      store.Now().String(),
+		Kind:           KindGroundDinner,
+	})
+	if err != nil {
+		return 0, fmt.Errorf("dinners: fail interrupted groundings: %w", err)
+	}
+	if n > 0 {
+		s.logger.WarnContext(ctx, "failed dinner groundings interrupted by restart", "count", n)
+	}
+	return n, nil
+}
+
 // reasonFor is the Reason for a grounding error; fromURL says whether the
 // dinner's recipe_url was grounded rather than its name.
 func reasonFor(err error, fromURL bool) string {
@@ -236,8 +270,13 @@ func (s *Service) Ground(ctx context.Context, id string, f Fix, updatedBy string
 	if _, err := s.get(ctx, id); err != nil {
 		return Dinner{}, err
 	}
-	gr, err := s.grounder.GroundURL(ctx, link)
+	groundCtx, cancel := context.WithTimeout(ctx, groundTimeout)
+	defer cancel()
+	gr, err := s.grounder.GroundURL(groundCtx, link)
 	if err != nil {
+		if ctx.Err() == nil && errors.Is(groundCtx.Err(), context.DeadlineExceeded) {
+			return Dinner{}, ErrGroundTimeout
+		}
 		return Dinner{}, fmt.Errorf("dinners: ground %s: %w", id, err)
 	}
 	row, err := db.New(s.st.Writer).GroundDinner(ctx, db.GroundDinnerParams{

internal/dinners/grounding_test.go

@@ -33,6 +33,7 @@ type env struct {
 	st     *store.Store
 	runner *jobs.Runner
 	search *recipestest.Search
+	pages  *recipestest.Pages
 	model  *llm.Fake
 }
 
@@ -40,6 +41,15 @@ type env struct {
 // with fake search, pages, images and model. Searching "Chicken curry"
 // finds curryA, curryB and a missing page.
 func newEnv(t *testing.T) *env {
+	t.Helper()
+	e := newUnstartedEnv(t)
+	e.start(t)
+	return e
+}
+
+// newUnstartedEnv is newEnv with its job runner not yet started, so jobs
+// enqueued stay queued until start.
+func newUnstartedEnv(t *testing.T) *env {
 	t.Helper()
 	dataDir := t.TempDir()
 	st, err := store.Open(t.Context(), dataDir)
@@ -79,16 +89,21 @@ func newEnv(t *testing.T) *env {
 	if err != nil {
 		t.Fatal(err)
 	}
+	return &env{svc: svc, st: st, runner: runner, search: search, pages: pages, model: model}
+}
+
+// start starts the env's job runner, stopping it when the test ends.
+func (e *env) start(t *testing.T) {
+	t.Helper()
 	ctx, cancel := context.WithCancel(context.Background())
-	if err := runner.Start(ctx); err != nil {
+	if err := e.runner.Start(ctx); err != nil {
 		cancel()
 		t.Fatal(err)
 	}
 	t.Cleanup(func() {
 		cancel()
-		runner.Wait()
+		e.runner.Wait()
 	})
-	return &env{svc: svc, st: st, runner: runner, search: search, model: model}
 }
 
 func ptr[T any](v T) *T { return &v }
@@ -130,10 +145,10 @@ func TestCreateGroundsInBackground(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if c.GroundingStatus != dinners.GroundingPending || c.GroundingJobID == "" {
+	if c.GroundingStatus != dinners.GroundingPending || c.GroundingJobID == nil {
 		t.Fatalf("created = %+v, want pending with a job id", c)
 	}
-	job := e.waitJob(t, c.GroundingJobID)
+	job := e.waitJob(t, *c.GroundingJobID)
 	if job.Kind != dinners.KindGroundDinner || job.Status != jobs.StatusSucceeded ||
 		string(job.Result) != `{"dinner_id":"`+c.ID+`","grounding_status":"grounded"}` {
 		t.Fatalf("job = %+v (result %s)", job, job.Result)
@@ -165,7 +180,7 @@ func TestCreateByRecipeURLGroundsThatURL(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if job := e.waitJob(t, c.GroundingJobID); job.Status != jobs.StatusSucceeded {
+	if job := e.waitJob(t, *c.GroundingJobID); job.Status != jobs.StatusSucceeded {
 		t.Fatalf("job = %+v", job)
 	}
 	row := e.dinner(t, c.ID)
@@ -208,7 +223,7 @@ func TestCreateGroundingFailureRecordsReason(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-			job := e.waitJob(t, c.GroundingJobID)
+			job := e.waitJob(t, *c.GroundingJobID)
 			if job.Status != jobs.StatusFailed || job.Error == nil || *job.Error != tt.reason {
 				t.Errorf("job = %+v, want failed with %s", job, tt.reason)
 			}
@@ -234,11 +249,11 @@ func TestCreateReturnsBeforeGroundingCompletes(t *testing.T) {
 	if row := e.dinner(t, c.ID); row.GroundingStatus != "pending" {
 		t.Fatalf("grounding_status = %s right after create, want pending", row.GroundingStatus)
 	}
-	if job, err := e.runner.Get(t.Context(), c.GroundingJobID); err != nil || (job.Status != jobs.StatusQueued && job.Status != jobs.StatusRunning) {
+	if job, err := e.runner.Get(t.Context(), *c.GroundingJobID); err != nil || (job.Status != jobs.StatusQueued && job.Status != jobs.StatusRunning) {
 		t.Fatalf("job = %+v, %v; want queued or running", job, err)
 	}
 	close(e.search.Gate)
-	if job := e.waitJob(t, c.GroundingJobID); job.Status != jobs.StatusSucceeded {
+	if job := e.waitJob(t, *c.GroundingJobID); job.Status != jobs.StatusSucceeded {
 		t.Fatalf("job = %+v", job)
 	}
 	if row := e.dinner(t, c.ID); row.GroundingStatus != "grounded" {
@@ -262,7 +277,7 @@ func TestManualFixWhilePendingIsKept(t *testing.T) {
 		t.Fatal(err)
 	}
 	close(e.search.Gate)
-	job := e.waitJob(t, c.GroundingJobID)
+	job := e.waitJob(t, *c.GroundingJobID)
 	if job.Status != jobs.StatusSucceeded || string(job.Result) != `{"dinner_id":"`+c.ID+`","grounding_status":"manual"}` {
 		t.Errorf("job = %+v (result %s), want succeeded reporting manual", job, job.Result)
 	}
@@ -278,7 +293,7 @@ func failedDinner(t *testing.T, e *env, name string) string {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if job := e.waitJob(t, c.GroundingJobID); job.Status != jobs.StatusFailed {
+	if job := e.waitJob(t, *c.GroundingJobID); job.Status != jobs.StatusFailed {
 		t.Fatalf("job = %+v, want failed", job)
 	}
 	return c.ID
@@ -293,7 +308,7 @@ func TestCandidates(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if job := e.waitJob(t, c.GroundingJobID); job.Status != jobs.StatusSucceeded {
+	if job := e.waitJob(t, *c.GroundingJobID); job.Status != jobs.StatusSucceeded {
 		t.Fatalf("job = %+v", job)
 	}
 
@@ -476,3 +491,105 @@ func TestFixValidation(t *testing.T) {
 		t.Errorf("unknown dinner, link: err = %v, want ErrNotFound", err)
 	}
 }
+
+// TestRestartFailsInterruptedGrounding simulates a crash: one dinner's job
+// was left running and another dinner was saved before its job was
+// enqueued. At the next boot both end failed as interrupted, while a
+// dinner whose job is still queued stays pending and is grounded.
+func TestRestartFailsInterruptedGrounding(t *testing.T) {
+	t.Parallel()
+	e := newUnstartedEnv(t)
+	e.search.Gate = make(chan struct{})
+	e.model.QueueGenerate(recipestest.NormalizedReply("stovetop", 40))
+
+	running, err := e.svc.CreateRegular(t.Context(), dinners.NewRegular{Name: ptr("Fish pie")}, member)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := e.st.Writer.ExecContext(t.Context(),
+		`UPDATE jobs SET status = 'running', started_at = updated_at WHERE id = ?`, *running.GroundingJobID); err != nil {
+		t.Fatal(err)
+	}
+	unqueued, err := e.svc.CreateRegular(t.Context(), dinners.NewRegular{Name: ptr("Lentil soup")}, member)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := e.st.Writer.ExecContext(t.Context(), `DELETE FROM jobs WHERE id = ?`, *unqueued.GroundingJobID); err != nil {
+		t.Fatal(err)
+	}
+	queued, err := e.svc.CreateRegular(t.Context(), dinners.NewRegular{Name: ptr("Chicken curry")}, member)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Boot: the runner fails the running job, then the service fails the
+	// dinners nothing is left to ground. The search is held, so the queued
+	// dinner's job cannot finish meanwhile.
+	e.start(t)
+	n, err := e.svc.FailInterruptedGroundings(t.Context())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if n != 2 {
+		t.Errorf("failed %d dinners, want 2", n)
+	}
+	if job, err := e.runner.Get(t.Context(), *running.GroundingJobID); err != nil || job.Status != jobs.StatusFailed {
+		t.Errorf("running job = %+v, %v; want failed", job, err)
+	}
+	for _, id := range []string{running.ID, unqueued.ID} {
+		if row := e.dinner(t, id); row.GroundingStatus != dinners.GroundingFailed || row.GroundingError.String != dinners.ReasonInterrupted {
+			t.Errorf("dinner %s = %s (%s), want failed (interrupted)", id, row.GroundingStatus, row.GroundingError.String)
+		}
+	}
+	if row := e.dinner(t, queued.ID); row.GroundingStatus != dinners.GroundingPending {
+		t.Errorf("queued dinner = %s, want pending", row.GroundingStatus)
+	}
+	close(e.search.Gate)
+	if job := e.waitJob(t, *queued.GroundingJobID); job.Status != jobs.StatusSucceeded {
+		t.Fatalf("queued job = %+v", job)
+	}
+	if row := e.dinner(t, queued.ID); row.GroundingStatus != dinners.GroundingGrounded {
+		t.Errorf("queued dinner = %s, want grounded", row.GroundingStatus)
+	}
+}
+
+// TestCreateEnqueueFailureReturnsFailedDinner: a dinner saved when its job
+// cannot be enqueued is returned, with its id, failed and no job id.
+func TestCreateEnqueueFailureReturnsFailedDinner(t *testing.T) {
+	t.Parallel()
+	e := newEnv(t)
+	if _, err := e.st.Writer.ExecContext(t.Context(),
+		`CREATE TRIGGER refuse_jobs BEFORE INSERT ON jobs BEGIN SELECT RAISE(ABORT, 'no jobs'); END`); err != nil {
+		t.Fatal(err)
+	}
+	c, err := e.svc.CreateRegular(t.Context(), dinners.NewRegular{Name: ptr("Chicken curry")}, member)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if c.ID == "" || c.GroundingJobID != nil || c.GroundingStatus != dinners.GroundingFailed ||
+		c.GroundingError == nil || *c.GroundingError != dinners.ReasonGroundingError {
+		t.Fatalf("created = %+v, want its id, failed (grounding_error) and no job", c)
+	}
+	if row := e.dinner(t, c.ID); row.GroundingStatus != dinners.GroundingFailed {
+		t.Errorf("stored grounding_status = %s, want failed", row.GroundingStatus)
+	}
+}
+
+// TestGroundTimesOut: grounding a link by hand that runs past its deadline
+// is ErrGroundTimeout and leaves the dinner as it was. It shortens the
+// package deadline, so it does not run in parallel.
+func TestGroundTimesOut(t *testing.T) {
+	dinners.SetGroundTimeout(t, 50*time.Millisecond)
+	e := newEnv(t)
+	id := failedDinner(t, e, "Beef stew")
+	e.pages.Gate = make(chan struct{})
+	defer close(e.pages.Gate)
+
+	_, err := e.svc.Ground(t.Context(), id, dinners.Fix{RecipeURL: ptr(stew)}, member)
+	if !errors.Is(err, dinners.ErrGroundTimeout) {
+		t.Fatalf("err = %v, want ErrGroundTimeout", err)
+	}
+	if row := e.dinner(t, id); row.GroundingStatus != dinners.GroundingFailed {
+		t.Errorf("grounding_status = %s, want failed as before", row.GroundingStatus)
+	}
+}