Fail interrupted groundings at boot and bound hand grounding
CI / ci (push) Successful in 2m25s
CI / ci (pull_request) Successful in 2m20s

At boot, after the job runner fails the jobs a previous process left
running, FailInterruptedGroundings sets every dinner still pending with
no ground_dinner job queued or running to failed (interrupted). A crash
mid-job, or between saving a dinner and enqueuing its job, no longer
leaves the dinner pending forever.

A search failure on recipe-candidates or ground now logs only the reason
code search_failed, never the wrapped error, which may quote the
provider; the response stays redacted.

When the grounding job cannot be enqueued, POST /dinners answers 201
with the saved dinner failed (grounding_error) and grounding_job_id
null, instead of a 500 that hid the dinner it had created.

Grounding a candidate or link by hand runs under a 90-second deadline
(fetch, photo, normalize_recipe); past it the request answers 504
ground_timeout and the dinner is unchanged.

Maestro task: 0cbaa5bf-8be4-4cbd-aab2-891e7f1bc2b5

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-06 16:30:53 -04:00
co-authored by Claude Opus 5.5
parent d6d82e8368
commit f4bed5d82d
13 changed files with 357 additions and 37 deletions
+40 -1
View File
@@ -8,6 +8,7 @@ import (
"fmt"
"slices"
"strings"
"time"
"unicode/utf8"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/db"
@@ -63,6 +64,18 @@ const (
manualSourcePrefix = "mealquest:manual:"
)
// GroundTimeout bounds grounding a candidate or link by hand, which runs in
// the request: the page fetch (recipes.Timeout), its photo download and
// normalize_recipe (harness.DefaultTimeout), with some slack.
const GroundTimeout = 90 * time.Second
// groundTimeout is GroundTimeout; tests shorten it.
var groundTimeout = GroundTimeout
// ErrGroundTimeout is returned by Ground when grounding a candidate or link
// runs past GroundTimeout; the dinner is left as it was.
var ErrGroundTimeout = errors.New("dinners: grounding the recipe took too long")
// GroundJob is the KindGroundDinner payload.
type GroundJob struct {
DinnerID string `json:"dinner_id"`
@@ -143,6 +156,27 @@ func (s *Service) runGroundJob(ctx context.Context, p GroundJob, _ func(groundRe
return groundResult{DinnerID: row.ID, GroundingStatus: GroundingGrounded}, nil
}
// FailInterruptedGroundings fails, with ReasonInterrupted, every dinner
// still pending whose KindGroundDinner job is no longer queued or running:
// a previous process stopped before enqueuing the job or before its
// handler recorded a result. Call it once at boot, after the job runner's
// Start has failed the jobs that process left running, so no dinner stays
// pending with nothing left to ground it. It returns how many it failed.
func (s *Service) FailInterruptedGroundings(ctx context.Context) (int64, error) {
n, err := db.New(s.st.Writer).FailOrphanedPendingDinnerGroundings(ctx, db.FailOrphanedPendingDinnerGroundingsParams{
GroundingError: sql.NullString{String: ReasonInterrupted, Valid: true},
UpdatedAt: store.Now().String(),
Kind: KindGroundDinner,
})
if err != nil {
return 0, fmt.Errorf("dinners: fail interrupted groundings: %w", err)
}
if n > 0 {
s.logger.WarnContext(ctx, "failed dinner groundings interrupted by restart", "count", n)
}
return n, nil
}
// reasonFor is the Reason for a grounding error; fromURL says whether the
// dinner's recipe_url was grounded rather than its name.
func reasonFor(err error, fromURL bool) string {
@@ -236,8 +270,13 @@ func (s *Service) Ground(ctx context.Context, id string, f Fix, updatedBy string
if _, err := s.get(ctx, id); err != nil {
return Dinner{}, err
}
gr, err := s.grounder.GroundURL(ctx, link)
groundCtx, cancel := context.WithTimeout(ctx, groundTimeout)
defer cancel()
gr, err := s.grounder.GroundURL(groundCtx, link)
if err != nil {
if ctx.Err() == nil && errors.Is(groundCtx.Err(), context.DeadlineExceeded) {
return Dinner{}, ErrGroundTimeout
}
return Dinner{}, fmt.Errorf("dinners: ground %s: %w", id, err)
}
row, err := db.New(s.st.Writer).GroundDinner(ctx, db.GroundDinnerParams{