From 423b898f54adc455c4f90030688a115492965f17 Mon Sep 17 00:00:00 2001 From: dev_agent Date: Tue, 6 Oct 2026 16:39:10 -0400 Subject: [PATCH] Background grounding and recipe fix-up API (#90) Co-authored-by: dev_agent --- cmd/server/main.go | 46 +- docs/ARCHITECTURE.md | 44 +- internal/db/dinners.sql.go | 186 +++++- internal/db/models.go | 1 + internal/db/querier.go | 17 + internal/db/recipes.sql.go | 61 ++ internal/dinners/dinners.go | 90 ++- internal/dinners/export_test.go | 14 + internal/dinners/grounding.go | 497 ++++++++++++++++ internal/dinners/grounding_test.go | 595 ++++++++++++++++++++ internal/recipes/ground.go | 136 +++-- internal/recipes/ground_test.go | 35 ++ internal/recipes/photo.go | 26 +- internal/recipes/recipestest/recipestest.go | 170 ++++++ internal/server/dinners.go | 92 ++- internal/server/dinners_test.go | 87 ++- internal/server/grounding_test.go | 303 ++++++++++ internal/server/server.go | 6 +- internal/server/server_test.go | 18 +- internal/store/migrate_test.go | 8 +- migrations/0009_dinner_grounding_error.sql | 7 + migrations/queries/dinners.sql | 56 ++ migrations/queries/recipes.sql | 12 + 23 files changed, 2424 insertions(+), 83 deletions(-) create mode 100644 internal/dinners/export_test.go create mode 100644 internal/dinners/grounding.go create mode 100644 internal/dinners/grounding_test.go create mode 100644 internal/recipes/recipestest/recipestest.go create mode 100644 internal/server/grounding_test.go create mode 100644 migrations/0009_dinner_grounding_error.sql diff --git a/cmd/server/main.go b/cmd/server/main.go index acda7fb..3935c0a 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -14,6 +14,8 @@ import ( "time" "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/aisettings" + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/harness" + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/skills" "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/auth" "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/config" "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/dinners" @@ -76,7 +78,10 @@ func run(cfg config.Config, logger *slog.Logger) error { return err } - photos, err := recipes.NewPhotoCache(recipes.NewFetcher(), db, cfg.DataDir) + // One fetcher serves pages and photos, so its per-host spacing and + // page cache cover both. + fetcher := recipes.NewFetcher() + photos, err := recipes.NewPhotoCache(fetcher, db, cfg.DataDir) if err != nil { return err } @@ -89,6 +94,18 @@ func run(cfg config.Config, logger *slog.Logger) error { if err != nil { return err } + guard, err := harness.NewGuard(harness.DefaultMaxCalls) + if err != nil { + return err + } + grounder, err := recipes.NewGrounder(db, search, fetcher, photos, normalizer(ai, guard)) + if err != nil { + return err + } + dinnerSvc, err := dinners.NewService(db, grounder, runner, logger) + if err != nil { + return err + } sessions, err := auth.NewSessionManager(db, cfg.SessionIdle, cfg.SessionMax) if err != nil { @@ -106,7 +123,7 @@ func run(cfg config.Config, logger *slog.Logger) error { return err } - router, err := server.NewRouter(logger, version, cfg.TrustedProxy, authFlow, sessions, settings, search, ai, dinners.NewService(db), runner, photos, frontend.Assets()) + router, err := server.NewRouter(logger, version, cfg.TrustedProxy, authFlow, sessions, settings, search, ai, dinnerSvc, runner, photos, frontend.Assets()) if err != nil { return err } @@ -122,6 +139,12 @@ func run(cfg config.Config, logger *slog.Logger) error { if err := runner.Start(jobsCtx); err != nil { return err } + // With those jobs failed, a dinner still pending with no grounding job + // queued or running is failed as interrupted, before any request can + // create one. + if _, err := dinnerSvc.FailInterruptedGroundings(ctx); err != nil { + return err + } srv := &http.Server{ Addr: cfg.HTTPAddr, Handler: router, @@ -203,7 +226,7 @@ func seedDefaults(ctx context.Context, db *store.Store, defaultTZ string) error return household.SeedNotificationPrefs(ctx, db) } -// jobKinds enqueues every job kind the server runs. +// jobKinds enqueues the framework's job kinds. type jobKinds struct { builtins *jobs.Builtins } @@ -222,8 +245,9 @@ func newJobRunner(db *store.Store, logger *slog.Logger) (*jobs.Runner, *jobKinds return runner, kinds, nil } -// registerJobKinds registers every job kind's handler on runner. Each -// feature's job kinds are registered here. +// registerJobKinds registers the framework's job kinds on runner. A feature +// service registers its own kinds when it is built, before the runner +// starts (dinners.NewService registers dinners.KindGroundDinner). func registerJobKinds(runner *jobs.Runner) (*jobKinds, error) { builtins, err := jobs.RegisterBuiltins(runner) if err != nil { @@ -232,6 +256,18 @@ func registerJobKinds(runner *jobs.Runner) (*jobKinds, error) { return &jobKinds{builtins: builtins}, nil } +// normalizer runs normalize_recipe with the AI settings current at each +// run, sharing guard. +func normalizer(ai *aisettings.Service, guard *harness.Guard) recipes.Normalizer { + return func(ctx context.Context, scope string, in skills.RecipeText) (skills.NormalizedRecipe, error) { + cfg, err := ai.HarnessConfig(ctx, guard) + if err != nil { + return skills.NormalizedRecipe{}, err + } + return skills.NormalizeRecipe(ctx, cfg, scope, in) + } +} + // logListening logs the startup line. Only the public URL's origin is logged // so no credential-bearing URL component can reach the log. func logListening(logger *slog.Logger, cfg config.Config) { diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 4e6d7d6..2b64f31 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -96,7 +96,7 @@ Columns ending in `_ct` hold AES-256-GCM ciphertext (via `secretbox`). | `recipe_ingredients` | recipe_id, position, raw_text, name, qty, unit, note. | | `recipe_steps` | recipe_id, position, text. | | `photos` | source_url, local_path, width, height, sha256, fetched_at. | -| `dinners` | The meal record and its lifecycle: name, normalized_name, status (`suggested` / `quest_log` / `tried` / `regular` / `archived`), origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, recipe_url (a pasted recipe link awaiting grounding), photo_id, grounding_status, updated_by. | +| `dinners` | The meal record and its lifecycle: name, normalized_name, status (`suggested` / `quest_log` / `tried` / `regular` / `archived`), origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, recipe_url (a pasted recipe link awaiting grounding), photo_id, grounding_status (`pending` / `grounded` / `failed` / `manual`), grounding_error (why grounding failed), updated_by. | | `suggestion_batches` | trigger, status, ai_run_id. | | `suggestions` | dinner_id, batch_id, reason, cited_regular_ids (JSON), status, decided_at. | | `week_plans` | week_start (unique), status (`draft` / `synced` / `changed`), ai_note, ai_run_id, calendar_synced_at. | @@ -200,6 +200,43 @@ automatic** (D-021). sets status `archived` and keeps the row; `POST /api/v1/dinners/{id}/favorite` toggles `is_favorite`. Every write records the session's subject as `updated_by`; an unknown id is 404. +- Dinner grounding. `POST /api/v1/dinners` also enqueues a `ground_dinner` + job and answers with its `grounding_job_id` without waiting for it + (`GET /api/v1/jobs/{id}` reports it); if the job cannot be enqueued it + still answers 201, with grounding_status `failed` (`grounding_error`) + and `grounding_job_id` null. The job grounds the dinner's + `recipe_url` directly with `GroundURL`, else its name with `Ground` + (see Recipe grounding), and sets grounding_status `grounded` with the + recipe's recipe_id, photo_id, method, minutes_total and difficulty, or + `failed` with `grounding_error` — also the job's failure code — one of + `no_recipe_found`, `recipe_unusable` (the pasted link), `search_not_configured`, + `search_failed`, `ai_failed`, `interrupted` (the server stopped) or + `grounding_error`. It only writes a dinner still `pending`, so a fix-up + made meanwhile wins. At boot, after the job runner fails the jobs a + previous process left running, every dinner still `pending` with no + `ground_dinner` job queued or running is set `failed` with `interrupted`, + so no dinner stays pending with nothing left to ground it. + `GET /api/v1/dinners/{id}/recipe-candidates` + searches for the dinner's name and returns `{candidates: [{candidate_id, + title, site_name, photo_url, minutes}]}`, at most 3, ranked as `Ground` + ranks them; it fetches pages but downloads no photo and runs no model, + so `photo_url` is set only for a page already stored, and `candidate_id` + is the page's URL. `POST /api/v1/dinners/{id}/ground` takes exactly one + of `{candidate_id}`, `{recipe_url}` (both grounded with `GroundURL`, + setting `grounded`) or `{manual: {ingredients[], steps[], minutes, + method}}` (1–100 ingredient lines, up to 100 steps, each non-blank and at + most 500 characters; minutes 1–1440). A manual recipe is stored as typed + — no model runs — as a `recipes` row whose source_url is + `mealquest:manual:{recipe_id}` and whose raw_jsonld is a schema.org + Recipe built from it, and sets `manual` with its method and minutes, no + photo and the dinner's difficulty unchanged. It answers 200 with the + dinner; an invalid body is 400, an unknown dinner 404, a page that cannot + ground it 422 `ungroundable` (the dinner is unchanged), recipe search not + set up 409 `not_configured`, a search or model failure 502 + `search_failed` / `ai_failed` (only the reason code is logged, never the + provider's text), and grounding a candidate or link that runs past 90 + seconds (fetch, photo and `normalize_recipe`, all in the request) 504 + `ground_timeout`, leaving the dinner unchanged. - `/health` — liveness. - Everything else serves the embedded PWA with SPA fallback. @@ -368,7 +405,10 @@ automatic** (D-021). Rank's per-candidate reasons, photo or normalize failures). Failures that are not the candidates' — search, model provider, database — are returned as they are. `GroundURL(ctx, url)` does the same for one pasted - URL. `GroundMany` grounds items 3 at a time and returns one result per + URL. `Candidates(ctx, dishName, query, n)` returns the first n of the + candidates `Ground` would try, without grounding any. Search failures + wrap `recipes.ErrSearchFailed` and model provider failures + `recipes.ErrNormalizeFailed`. `GroundMany` grounds items 3 at a time and returns one result per item, so one failure never affects its siblings. A recipe is usable only with ingredients, an image and, if given, a positive total time. diff --git a/internal/db/dinners.sql.go b/internal/db/dinners.sql.go index b62e9b1..9e867ce 100644 --- a/internal/db/dinners.sql.go +++ b/internal/db/dinners.sql.go @@ -16,7 +16,7 @@ SET status = 'archived', updated_by = ?, updated_at = ? WHERE id = ? -RETURNING id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url +RETURNING id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error ` type ArchiveDinnerParams struct { @@ -47,6 +47,7 @@ func (q *Queries) ArchiveDinner(ctx context.Context, arg ArchiveDinnerParams) (D &i.CreatedAt, &i.UpdatedAt, &i.RecipeUrl, + &i.GroundingError, ) return i, err } @@ -61,7 +62,7 @@ INSERT INTO dinners ( ?, ?, ?, ?, ?, ?, ?, ?, ?, ? ) -RETURNING id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url +RETURNING id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error ` type CreateDinnerParams struct { @@ -123,12 +124,69 @@ func (q *Queries) CreateDinner(ctx context.Context, arg CreateDinnerParams) (Din &i.CreatedAt, &i.UpdatedAt, &i.RecipeUrl, + &i.GroundingError, ) return i, err } +const failOrphanedPendingDinnerGroundings = `-- name: FailOrphanedPendingDinnerGroundings :execrows +UPDATE dinners +SET grounding_status = 'failed', + grounding_error = ?1, + updated_at = ?2 +WHERE grounding_status = 'pending' + AND NOT EXISTS ( + SELECT 1 FROM jobs + WHERE jobs.kind = ?3 + AND jobs.status IN ('queued', 'running') + AND json_extract(jobs.payload, '$.dinner_id') = dinners.id + ) +` + +type FailOrphanedPendingDinnerGroundingsParams struct { + GroundingError sql.NullString + UpdatedAt string + Kind string +} + +// Fails every dinner still pending whose grounding job is no longer queued +// or running: the process stopped before the job, or before its handler +// recorded a result. Run at boot, after the job runner fails the jobs a +// previous process left running. +func (q *Queries) FailOrphanedPendingDinnerGroundings(ctx context.Context, arg FailOrphanedPendingDinnerGroundingsParams) (int64, error) { + result, err := q.db.ExecContext(ctx, failOrphanedPendingDinnerGroundings, arg.GroundingError, arg.UpdatedAt, arg.Kind) + if err != nil { + return 0, err + } + return result.RowsAffected() +} + +const failPendingDinnerGrounding = `-- name: FailPendingDinnerGrounding :execrows +UPDATE dinners +SET grounding_status = 'failed', + grounding_error = ?, + updated_at = ? +WHERE id = ? AND grounding_status = 'pending' +` + +type FailPendingDinnerGroundingParams struct { + GroundingError sql.NullString + UpdatedAt string + ID string +} + +// Records why the background grounding of a dinner still pending failed; a +// dinner grounded some other way meanwhile is left as it is (0 rows). +func (q *Queries) FailPendingDinnerGrounding(ctx context.Context, arg FailPendingDinnerGroundingParams) (int64, error) { + result, err := q.db.ExecContext(ctx, failPendingDinnerGrounding, arg.GroundingError, arg.UpdatedAt, arg.ID) + if err != nil { + return 0, err + } + return result.RowsAffected() +} + const getDinner = `-- name: GetDinner :one -SELECT id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url FROM dinners +SELECT id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error FROM dinners WHERE id = ? ` @@ -153,12 +211,119 @@ func (q *Queries) GetDinner(ctx context.Context, id string) (Dinner, error) { &i.CreatedAt, &i.UpdatedAt, &i.RecipeUrl, + &i.GroundingError, ) return i, err } +const groundDinner = `-- name: GroundDinner :one +UPDATE dinners +SET recipe_id = ?, + photo_id = ?, + method = ?, + minutes_total = ?, + difficulty = ?, + grounding_status = ?, + grounding_error = NULL, + updated_by = ?, + updated_at = ? +WHERE id = ? +RETURNING id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error +` + +type GroundDinnerParams struct { + RecipeID sql.NullString + PhotoID sql.NullString + Method sql.NullString + MinutesTotal sql.NullInt64 + Difficulty sql.NullString + GroundingStatus string + UpdatedBy sql.NullString + UpdatedAt string + ID string +} + +// Records a dinner's grounding (grounded or manual) and clears any +// grounding_error. An unknown id returns sql.ErrNoRows. +func (q *Queries) GroundDinner(ctx context.Context, arg GroundDinnerParams) (Dinner, error) { + row := q.db.QueryRowContext(ctx, groundDinner, + arg.RecipeID, + arg.PhotoID, + arg.Method, + arg.MinutesTotal, + arg.Difficulty, + arg.GroundingStatus, + arg.UpdatedBy, + arg.UpdatedAt, + arg.ID, + ) + var i Dinner + err := row.Scan( + &i.ID, + &i.Name, + &i.NormalizedName, + &i.Status, + &i.Origin, + &i.Method, + &i.MinutesTotal, + &i.Difficulty, + &i.Cuisine, + &i.IsFavorite, + &i.RecipeID, + &i.PhotoID, + &i.GroundingStatus, + &i.UpdatedBy, + &i.CreatedAt, + &i.UpdatedAt, + &i.RecipeUrl, + &i.GroundingError, + ) + return i, err +} + +const groundPendingDinner = `-- name: GroundPendingDinner :execrows +UPDATE dinners +SET recipe_id = ?, + photo_id = ?, + method = ?, + minutes_total = ?, + difficulty = ?, + grounding_status = 'grounded', + grounding_error = NULL, + updated_at = ? +WHERE id = ? AND grounding_status = 'pending' +` + +type GroundPendingDinnerParams struct { + RecipeID sql.NullString + PhotoID sql.NullString + Method sql.NullString + MinutesTotal sql.NullInt64 + Difficulty sql.NullString + UpdatedAt string + ID string +} + +// Records the background grounding of a dinner still pending; a dinner +// grounded some other way meanwhile is left as it is (0 rows). +func (q *Queries) GroundPendingDinner(ctx context.Context, arg GroundPendingDinnerParams) (int64, error) { + result, err := q.db.ExecContext(ctx, groundPendingDinner, + arg.RecipeID, + arg.PhotoID, + arg.Method, + arg.MinutesTotal, + arg.Difficulty, + arg.UpdatedAt, + arg.ID, + ) + if err != nil { + return 0, err + } + return result.RowsAffected() +} + const listDinnersByStatus = `-- name: ListDinnersByStatus :many -SELECT id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url FROM dinners +SELECT id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error FROM dinners WHERE status = ? ORDER BY normalized_name, id ` @@ -190,6 +355,7 @@ func (q *Queries) ListDinnersByStatus(ctx context.Context, status string) ([]Din &i.CreatedAt, &i.UpdatedAt, &i.RecipeUrl, + &i.GroundingError, ); err != nil { return nil, err } @@ -205,7 +371,7 @@ func (q *Queries) ListDinnersByStatus(ctx context.Context, status string) ([]Din } const listUnarchivedDinners = `-- name: ListUnarchivedDinners :many -SELECT id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url FROM dinners +SELECT id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error FROM dinners WHERE status <> 'archived' ORDER BY id ` @@ -238,6 +404,7 @@ func (q *Queries) ListUnarchivedDinners(ctx context.Context) ([]Dinner, error) { &i.CreatedAt, &i.UpdatedAt, &i.RecipeUrl, + &i.GroundingError, ); err != nil { return nil, err } @@ -258,7 +425,7 @@ SET is_favorite = ?, updated_by = ?, updated_at = ? WHERE id = ? -RETURNING id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url +RETURNING id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error ` type SetDinnerFavoriteParams struct { @@ -295,6 +462,7 @@ func (q *Queries) SetDinnerFavorite(ctx context.Context, arg SetDinnerFavoritePa &i.CreatedAt, &i.UpdatedAt, &i.RecipeUrl, + &i.GroundingError, ) return i, err } @@ -305,7 +473,7 @@ SET is_favorite = 1 - is_favorite, updated_by = ?, updated_at = ? WHERE id = ? -RETURNING id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url +RETURNING id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error ` type ToggleDinnerFavoriteParams struct { @@ -336,6 +504,7 @@ func (q *Queries) ToggleDinnerFavorite(ctx context.Context, arg ToggleDinnerFavo &i.CreatedAt, &i.UpdatedAt, &i.RecipeUrl, + &i.GroundingError, ) return i, err } @@ -356,7 +525,7 @@ SET name = ?, updated_by = ?, updated_at = ? WHERE id = ? -RETURNING id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url +RETURNING id, name, normalized_name, status, origin, method, minutes_total, difficulty, cuisine, is_favorite, recipe_id, photo_id, grounding_status, updated_by, created_at, updated_at, recipe_url, grounding_error ` type UpdateDinnerParams struct { @@ -413,6 +582,7 @@ func (q *Queries) UpdateDinner(ctx context.Context, arg UpdateDinnerParams) (Din &i.CreatedAt, &i.UpdatedAt, &i.RecipeUrl, + &i.GroundingError, ) return i, err } diff --git a/internal/db/models.go b/internal/db/models.go index 0166d56..ac5ef62 100644 --- a/internal/db/models.go +++ b/internal/db/models.go @@ -68,6 +68,7 @@ type Dinner struct { CreatedAt string UpdatedAt string RecipeUrl sql.NullString + GroundingError sql.NullString } type HouseholdSetting struct { diff --git a/internal/db/querier.go b/internal/db/querier.go index 90bbb89..3c804d8 100644 --- a/internal/db/querier.go +++ b/internal/db/querier.go @@ -24,6 +24,9 @@ type Querier interface { CreateAIRun(ctx context.Context, arg CreateAIRunParams) error CreateDinner(ctx context.Context, arg CreateDinnerParams) (Dinner, error) CreateJob(ctx context.Context, arg CreateJobParams) error + // A recipe the household typed in. Its source_url is a mealquest:manual: + // URN rather than a page, so it never matches a fetched page. + CreateManualRecipe(ctx context.Context, arg CreateManualRecipeParams) (Recipe, error) CreatePhoto(ctx context.Context, arg CreatePhotoParams) (Photo, error) CreateRecipeIngredient(ctx context.Context, arg CreateRecipeIngredientParams) error CreateRecipeStep(ctx context.Context, arg CreateRecipeStepParams) error @@ -31,6 +34,14 @@ type Querier interface { DeleteRecipeIngredients(ctx context.Context, recipeID string) error DeleteRecipeSteps(ctx context.Context, recipeID string) error FailJob(ctx context.Context, arg FailJobParams) (int64, error) + // Fails every dinner still pending whose grounding job is no longer queued + // or running: the process stopped before the job, or before its handler + // recorded a result. Run at boot, after the job runner fails the jobs a + // previous process left running. + FailOrphanedPendingDinnerGroundings(ctx context.Context, arg FailOrphanedPendingDinnerGroundingsParams) (int64, error) + // Records why the background grounding of a dinner still pending failed; a + // dinner grounded some other way meanwhile is left as it is (0 rows). + FailPendingDinnerGrounding(ctx context.Context, arg FailPendingDinnerGroundingParams) (int64, error) // Fails every job left running, at boot before any worker starts. FailRunningJobs(ctx context.Context, arg FailRunningJobsParams) (int64, error) // Returns sql.ErrNoRows until the row exists. @@ -51,6 +62,12 @@ type Querier interface { GetRecipeBySourceURL(ctx context.Context, sourceUrl string) (Recipe, error) // Returns sql.ErrNoRows until the row exists. GetRecipeSearchSettings(ctx context.Context) (RecipeSearchSetting, error) + // Records a dinner's grounding (grounded or manual) and clears any + // grounding_error. An unknown id returns sql.ErrNoRows. + GroundDinner(ctx context.Context, arg GroundDinnerParams) (Dinner, error) + // Records the background grounding of a dinner still pending; a dinner + // grounded some other way meanwhile is left as it is (0 rows). + GroundPendingDinner(ctx context.Context, arg GroundPendingDinnerParams) (int64, error) ListDinnersByStatus(ctx context.Context, status string) ([]Dinner, error) // The planned dinners dated from from_date up to but not including // to_date, oldest first. Days with no dinner are left out. diff --git a/internal/db/recipes.sql.go b/internal/db/recipes.sql.go index 2098ef8..b43b465 100644 --- a/internal/db/recipes.sql.go +++ b/internal/db/recipes.sql.go @@ -10,6 +10,67 @@ import ( "database/sql" ) +const createManualRecipe = `-- name: CreateManualRecipe :one +INSERT INTO recipes ( + id, source_url, title, total_minutes, raw_jsonld, method, + minutes_total, fetched_at, created_at, updated_at +) VALUES ( + ?, ?, ?, ?, ?, ?, + ?, ?, ?, ? +) +RETURNING id, source_url, site_name, title, total_minutes, prep_minutes, cook_minutes, yield, raw_jsonld, fetched_at, created_at, updated_at, photo_id, method, difficulty, minutes_total +` + +type CreateManualRecipeParams struct { + ID string + SourceUrl string + Title string + TotalMinutes sql.NullInt64 + RawJsonld string + Method sql.NullString + MinutesTotal sql.NullInt64 + FetchedAt string + CreatedAt string + UpdatedAt string +} + +// A recipe the household typed in. Its source_url is a mealquest:manual: +// URN rather than a page, so it never matches a fetched page. +func (q *Queries) CreateManualRecipe(ctx context.Context, arg CreateManualRecipeParams) (Recipe, error) { + row := q.db.QueryRowContext(ctx, createManualRecipe, + arg.ID, + arg.SourceUrl, + arg.Title, + arg.TotalMinutes, + arg.RawJsonld, + arg.Method, + arg.MinutesTotal, + arg.FetchedAt, + arg.CreatedAt, + arg.UpdatedAt, + ) + var i Recipe + err := row.Scan( + &i.ID, + &i.SourceUrl, + &i.SiteName, + &i.Title, + &i.TotalMinutes, + &i.PrepMinutes, + &i.CookMinutes, + &i.Yield, + &i.RawJsonld, + &i.FetchedAt, + &i.CreatedAt, + &i.UpdatedAt, + &i.PhotoID, + &i.Method, + &i.Difficulty, + &i.MinutesTotal, + ) + return i, err +} + const createRecipeIngredient = `-- name: CreateRecipeIngredient :exec INSERT INTO recipe_ingredients ( id, recipe_id, position, raw_text, name, qty, unit, note, diff --git a/internal/dinners/dinners.go b/internal/dinners/dinners.go index cdf0bfe..2af0e2f 100644 --- a/internal/dinners/dinners.go +++ b/internal/dinners/dinners.go @@ -1,6 +1,6 @@ // Package dinners keeps the household's dinners: adding Regulars by name or // recipe link, listing Regulars and the Quest log, editing, archiving and -// favoriting. Names are deduplicated with the harness's dinner-name rules, +// favoriting, and grounding each dinner in a recipe. Names are deduplicated with the harness's dinner-name rules, // so the Regulars the household types and the Side quests the AI suggests // are compared the same way. package dinners @@ -10,6 +10,7 @@ import ( "database/sql" "errors" "fmt" + "log/slog" "net/url" "slices" "strings" @@ -18,6 +19,8 @@ import ( "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/harness" "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/db" + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/jobs" + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/recipes" "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/store" ) @@ -65,9 +68,12 @@ type Dinner struct { RecipeURL *string `json:"recipe_url"` PhotoURL *string `json:"photo_url"` GroundingStatus string `json:"grounding_status"` - UpdatedBy *string `json:"updated_by"` - CreatedAt string `json:"created_at"` - UpdatedAt string `json:"updated_at"` + // GroundingError is why grounding failed, one of the Reason codes; + // null unless GroundingStatus is failed. + GroundingError *string `json:"grounding_error"` + UpdatedBy *string `json:"updated_by"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` } // NewRegular is a Regular to add: exactly one of Name and RecipeURL. @@ -120,19 +126,80 @@ func (e *DuplicateError) Error() string { // Service reads and changes dinners. type Service struct { - st *store.Store + st *store.Store + grounder *recipes.Grounder + logger *slog.Logger + groundJob *jobs.Kind[GroundJob] } -// NewService returns a service over st. -func NewService(st *store.Store) *Service { - return &Service{st: st} +// NewService returns a service over st that grounds dinners with grounder +// and registers the KindGroundDinner job on runner, which must not have +// started yet. +func NewService(st *store.Store, grounder *recipes.Grounder, runner *jobs.Runner, logger *slog.Logger) (*Service, error) { + switch { + case st == nil: + return nil, errors.New("dinners: nil store") + case grounder == nil: + return nil, errors.New("dinners: nil grounder") + case runner == nil: + return nil, errors.New("dinners: nil job runner") + case logger == nil: + return nil, errors.New("dinners: nil logger") + } + s := &Service{st: st, grounder: grounder, logger: logger} + kind, err := jobs.Register(runner, KindGroundDinner, s.runGroundJob, Reasons...) + if err != nil { + return nil, err + } + s.groundJob = kind + return s, nil +} + +// Created is a new Regular with the id of the job grounding it, which is +// nil when the job could not be enqueued and the dinner's grounding is +// failed instead. +type Created struct { + Dinner + GroundingJobID *string `json:"grounding_job_id"` } // CreateRegular adds a Regular named by r.Name, or by the last path segment // of r.RecipeURL (whose recipe grounding later replaces), with status and -// origin regular and grounding_status pending, recording updatedBy. A name -// repeating an unarchived dinner is a *DuplicateError. -func (s *Service) CreateRegular(ctx context.Context, r NewRegular, updatedBy string) (Dinner, error) { +// origin regular and grounding_status pending, recording updatedBy, and +// enqueues the KindGroundDinner job that grounds it. It returns without +// waiting for the job. A name repeating an unarchived dinner is a +// *DuplicateError. When the job cannot be enqueued, the stored dinner is +// returned with grounding_status failed (ReasonGroundingError) and no job +// id, so the household can fix its recipe by hand. +func (s *Service) CreateRegular(ctx context.Context, r NewRegular, updatedBy string) (Created, error) { + d, err := s.createRegular(ctx, r, updatedBy) + if err != nil { + return Created{}, err + } + jobID, err := s.groundJob.Enqueue(ctx, GroundJob{DinnerID: d.ID}) + if err == nil { + return Created{Dinner: d, GroundingJobID: &jobID}, nil + } + // The dinner is stored; record that it will not be grounded so it is + // not left pending with no job. The error is a database error, never a + // dependency's text. + s.logger.ErrorContext(ctx, "dinner grounding not enqueued", "dinner_id", d.ID, "error", err) + write := context.WithoutCancel(ctx) + if _, err := db.New(s.st.Writer).FailPendingDinnerGrounding(write, db.FailPendingDinnerGroundingParams{ + GroundingError: sql.NullString{String: ReasonGroundingError, Valid: true}, + UpdatedAt: store.Now().String(), + ID: d.ID, + }); err != nil { + return Created{}, fmt.Errorf("dinners: record failed grounding of %s: %w", d.ID, err) + } + row, err := db.New(s.st.Reader).GetDinner(write, d.ID) + if err != nil { + return Created{}, fmt.Errorf("dinners: read %s: %w", d.ID, err) + } + return Created{Dinner: fromRow(row)}, nil +} + +func (s *Service) createRegular(ctx context.Context, r NewRegular, updatedBy string) (Dinner, error) { if updatedBy == "" { return Dinner{}, errors.New("dinners: create: no member to record as updated_by") } @@ -445,6 +512,7 @@ func fromRow(r db.Dinner) Dinner { RecipeID: nullString(r.RecipeID), RecipeURL: nullString(r.RecipeUrl), GroundingStatus: r.GroundingStatus, + GroundingError: nullString(r.GroundingError), UpdatedBy: nullString(r.UpdatedBy), CreatedAt: r.CreatedAt, UpdatedAt: r.UpdatedAt, diff --git a/internal/dinners/export_test.go b/internal/dinners/export_test.go new file mode 100644 index 0000000..9eb3511 --- /dev/null +++ b/internal/dinners/export_test.go @@ -0,0 +1,14 @@ +package dinners + +import ( + "testing" + "time" +) + +// SetGroundTimeout sets the deadline of grounding by hand until t ends. A +// test calling it must not run in parallel. +func SetGroundTimeout(t *testing.T, d time.Duration) { + old := groundTimeout + groundTimeout = d + t.Cleanup(func() { groundTimeout = old }) +} diff --git a/internal/dinners/grounding.go b/internal/dinners/grounding.go new file mode 100644 index 0000000..9f8335f --- /dev/null +++ b/internal/dinners/grounding.go @@ -0,0 +1,497 @@ +package dinners + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "slices" + "strings" + "time" + "unicode/utf8" + + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/db" + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/jobs" + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/recipes" + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/store" +) + +// KindGroundDinner is the job that grounds a new Regular in a recipe. +const KindGroundDinner = "ground_dinner" + +// Grounding statuses (the dinners.grounding_status column). +const ( + GroundingPending = "pending" + GroundingGrounded = "grounded" + GroundingFailed = "failed" + GroundingManual = "manual" +) + +// Reasons a dinner's background grounding failed, stored in +// dinners.grounding_error and as the KindGroundDinner job's failure code. +const ( + // ReasonNoRecipe: no page the search found holds a usable recipe. + ReasonNoRecipe = "no_recipe_found" + // ReasonRecipeUnusable: the dinner's recipe_url holds no usable recipe. + ReasonRecipeUnusable = "recipe_unusable" + // ReasonSearchNotConfigured: recipe search is not set up. + ReasonSearchNotConfigured = "search_not_configured" + // ReasonSearchFailed: the search provider failed. + ReasonSearchFailed = "search_failed" + // ReasonAIFailed: normalize_recipe's model provider could not be built + // or called. + ReasonAIFailed = "ai_failed" + // ReasonInterrupted: the server stopped while grounding. + ReasonInterrupted = "interrupted" + // ReasonGroundingError: any other failure, such as the database's. + ReasonGroundingError = "grounding_error" +) + +// Reasons are every grounding failure reason. +var Reasons = []string{ + ReasonNoRecipe, ReasonRecipeUnusable, ReasonSearchNotConfigured, ReasonSearchFailed, + ReasonAIFailed, ReasonInterrupted, ReasonGroundingError, +} + +// Limits on candidates and on a recipe entered by hand. +const ( + MaxCandidates = 3 + MaxManualLines = 100 + MaxManualLineChar = 500 + // manualSourcePrefix starts the source_url of a recipe entered by hand, + // followed by the recipe's id; it is never a fetchable page. + manualSourcePrefix = "mealquest:manual:" +) + +// GroundTimeout bounds grounding a candidate or link by hand, which runs in +// the request: the page fetch (recipes.Timeout), its photo download and +// normalize_recipe (harness.DefaultTimeout), with some slack. +const GroundTimeout = 90 * time.Second + +// groundTimeout is GroundTimeout; tests shorten it. +var groundTimeout = GroundTimeout + +// ErrGroundTimeout is returned by Ground when grounding a candidate or link +// runs past GroundTimeout; the dinner is left as it was. +var ErrGroundTimeout = errors.New("dinners: grounding the recipe took too long") + +// GroundJob is the KindGroundDinner payload. +type GroundJob struct { + DinnerID string `json:"dinner_id"` +} + +// groundResult is the KindGroundDinner result: the dinner's grounding +// status once the job ends. It is not grounded when the dinner was +// grounded by hand before the job finished. +type groundResult struct { + DinnerID string `json:"dinner_id"` + GroundingStatus string `json:"grounding_status"` +} + +func (r groundResult) PublicJSON() ([]byte, error) { return json.Marshal(r) } + +// runGroundJob grounds a pending dinner as its creation asked: its +// recipe_url with GroundURL, else its name with Ground. Success records +// the recipe, photo, method, minutes and difficulty with grounding_status +// grounded; failure records grounding_status failed with its Reason, which +// is also the job's failure code. A dinner no longer pending (grounded by +// hand meanwhile) is left as it is. +func (s *Service) runGroundJob(ctx context.Context, p GroundJob, _ func(groundResult) error) (groundResult, error) { + row, err := db.New(s.st.Reader).GetDinner(ctx, p.DinnerID) + if err != nil { + return groundResult{}, fmt.Errorf("dinners: ground: read %s: %w", p.DinnerID, err) + } + if row.GroundingStatus != GroundingPending { + return groundResult{DinnerID: row.ID, GroundingStatus: row.GroundingStatus}, nil + } + + var gr recipes.Grounded + if row.RecipeUrl.Valid { + gr, err = s.grounder.GroundURL(ctx, row.RecipeUrl.String) + } else { + gr, err = s.grounder.Ground(ctx, row.Name, row.Name) + } + if err != nil { + reason := reasonFor(err, row.RecipeUrl.Valid) + if ctx.Err() != nil { + reason = ReasonInterrupted + } + // Only the reason is logged: the error may quote a dependency's + // response, which jobs never store, return or log. + s.logger.InfoContext(ctx, "dinner grounding failed", "dinner_id", row.ID, "reason", reason) + write := context.WithoutCancel(ctx) + if _, failErr := db.New(s.st.Writer).FailPendingDinnerGrounding(write, db.FailPendingDinnerGroundingParams{ + GroundingError: sql.NullString{String: reason, Valid: true}, + UpdatedAt: store.Now().String(), + ID: row.ID, + }); failErr != nil { + return groundResult{}, fmt.Errorf("dinners: ground: record failure of %s: %w", row.ID, failErr) + } + if ctx.Err() != nil { + return groundResult{}, ctx.Err() + } + return groundResult{}, &jobs.Failure{Code: reason} + } + + n, err := db.New(s.st.Writer).GroundPendingDinner(ctx, db.GroundPendingDinnerParams{ + RecipeID: sql.NullString{String: gr.RecipeID, Valid: true}, + PhotoID: sql.NullString{String: gr.PhotoID, Valid: true}, + Method: sql.NullString{String: gr.Method, Valid: true}, + MinutesTotal: nullInt(gr.MinutesTotal), + Difficulty: sql.NullString{String: gr.Difficulty, Valid: true}, + UpdatedAt: store.Now().String(), + ID: row.ID, + }) + if err != nil { + return groundResult{}, fmt.Errorf("dinners: ground: record %s: %w", row.ID, err) + } + if n == 0 { + cur, err := db.New(s.st.Reader).GetDinner(ctx, row.ID) + if err != nil { + return groundResult{}, fmt.Errorf("dinners: ground: read %s: %w", row.ID, err) + } + return groundResult{DinnerID: cur.ID, GroundingStatus: cur.GroundingStatus}, nil + } + return groundResult{DinnerID: row.ID, GroundingStatus: GroundingGrounded}, nil +} + +// FailInterruptedGroundings fails, with ReasonInterrupted, every dinner +// still pending whose KindGroundDinner job is no longer queued or running: +// a previous process stopped before enqueuing the job or before its +// handler recorded a result. Call it once at boot, after the job runner's +// Start has failed the jobs that process left running, so no dinner stays +// pending with nothing left to ground it. It returns how many it failed. +func (s *Service) FailInterruptedGroundings(ctx context.Context) (int64, error) { + n, err := db.New(s.st.Writer).FailOrphanedPendingDinnerGroundings(ctx, db.FailOrphanedPendingDinnerGroundingsParams{ + GroundingError: sql.NullString{String: ReasonInterrupted, Valid: true}, + UpdatedAt: store.Now().String(), + Kind: KindGroundDinner, + }) + if err != nil { + return 0, fmt.Errorf("dinners: fail interrupted groundings: %w", err) + } + if n > 0 { + s.logger.WarnContext(ctx, "failed dinner groundings interrupted by restart", "count", n) + } + return n, nil +} + +// reasonFor is the Reason for a grounding error; fromURL says whether the +// dinner's recipe_url was grounded rather than its name. +func reasonFor(err error, fromURL bool) string { + var ungroundable *recipes.ErrUngroundable + switch { + case errors.As(err, &ungroundable) && fromURL: + return ReasonRecipeUnusable + case errors.As(err, &ungroundable): + return ReasonNoRecipe + case errors.Is(err, recipes.ErrSearchNotConfigured): + return ReasonSearchNotConfigured + case errors.Is(err, recipes.ErrSearchFailed): + return ReasonSearchFailed + case errors.Is(err, recipes.ErrNormalizeFailed): + return ReasonAIFailed + } + return ReasonGroundingError +} + +// Candidate is a recipe page offered to ground a dinner in. +type Candidate struct { + // CandidateID is the page's URL; Fix.CandidateID grounds it. + CandidateID string `json:"candidate_id"` + Title string `json:"title"` + SiteName string `json:"site_name"` + // PhotoURL is the cached photo of a page already grounded, else null. + PhotoURL *string `json:"photo_url"` + Minutes *int `json:"minutes"` +} + +// Candidates searches for dinner id's name and returns up to MaxCandidates +// recipe pages to ground it in, best first; the list may be empty. A +// search that fails wraps recipes.ErrSearchFailed. +func (s *Service) Candidates(ctx context.Context, id string) ([]Candidate, error) { + row, err := s.get(ctx, id) + if err != nil { + return nil, err + } + offers, err := s.grounder.Candidates(ctx, row.Name, row.Name, MaxCandidates) + if err != nil { + return nil, err + } + out := make([]Candidate, len(offers)) + for i, o := range offers { + out[i] = Candidate{CandidateID: o.URL, Title: o.Title, SiteName: o.SiteName, Minutes: o.Minutes} + if o.PhotoID != "" { + u := PhotoURL(o.PhotoID) + out[i].PhotoURL = &u + } + } + return out, nil +} + +// Fix grounds a dinner by hand: exactly one of a candidate from +// Candidates, a pasted recipe link or a recipe typed in. +type Fix struct { + CandidateID *string + RecipeURL *string + Manual *ManualRecipe +} + +// ManualRecipe is a recipe typed in by the household. +type ManualRecipe struct { + // Ingredients are the ingredient lines, at least one. + Ingredients []string + // Steps are the instructions, in order; there may be none. + Steps []string + Minutes *int + Method *string +} + +// Ground grounds dinner id as f says, recording updatedBy. A candidate or +// link is grounded with recipes.Grounder.GroundURL and sets +// grounding_status grounded, with the recipe's photo, method, minutes and +// difficulty. A manual recipe is stored as typed — no model runs — and +// sets grounding_status manual with its method and minutes; it has no +// photo, and the dinner keeps its difficulty. An invalid f is a +// *ValidationError; a page that cannot ground the dinner is a +// *recipes.ErrUngroundable and changes nothing. +func (s *Service) Ground(ctx context.Context, id string, f Fix, updatedBy string) (Dinner, error) { + if updatedBy == "" { + return Dinner{}, errors.New("dinners: ground: no member to record as updated_by") + } + link, manual, err := f.validate() + if err != nil { + return Dinner{}, err + } + if manual != nil { + return s.groundManual(ctx, id, *manual, updatedBy) + } + if _, err := s.get(ctx, id); err != nil { + return Dinner{}, err + } + groundCtx, cancel := context.WithTimeout(ctx, groundTimeout) + defer cancel() + gr, err := s.grounder.GroundURL(groundCtx, link) + if err != nil { + if ctx.Err() == nil && errors.Is(groundCtx.Err(), context.DeadlineExceeded) { + return Dinner{}, ErrGroundTimeout + } + return Dinner{}, fmt.Errorf("dinners: ground %s: %w", id, err) + } + row, err := db.New(s.st.Writer).GroundDinner(ctx, db.GroundDinnerParams{ + RecipeID: sql.NullString{String: gr.RecipeID, Valid: true}, + PhotoID: sql.NullString{String: gr.PhotoID, Valid: true}, + Method: sql.NullString{String: gr.Method, Valid: true}, + MinutesTotal: nullInt(gr.MinutesTotal), + Difficulty: sql.NullString{String: gr.Difficulty, Valid: true}, + GroundingStatus: GroundingGrounded, + UpdatedBy: sql.NullString{String: updatedBy, Valid: true}, + UpdatedAt: store.Now().String(), + ID: id, + }) + if errors.Is(err, sql.ErrNoRows) { + return Dinner{}, ErrNotFound + } + if err != nil { + return Dinner{}, fmt.Errorf("dinners: ground %s: %w", id, err) + } + return fromRow(row), nil +} + +// groundManual stores m as dinner id's recipe, with its ingredients and +// steps, and records it on the dinner, all in one transaction. +func (s *Service) groundManual(ctx context.Context, id string, m ManualRecipe, updatedBy string) (Dinner, error) { + var row db.Dinner + err := s.st.WithTx(ctx, func(tx *sql.Tx) error { + q := db.New(tx) + cur, err := q.GetDinner(ctx, id) + if errors.Is(err, sql.ErrNoRows) { + return ErrNotFound + } + if err != nil { + return fmt.Errorf("dinners: read %s: %w", id, err) + } + ld, err := json.Marshal(manualJSONLD{ + Context: "https://schema.org", + Type: "Recipe", + Name: cur.Name, + Ingredients: m.Ingredients, + Instructions: m.Steps, + TotalTime: fmt.Sprintf("PT%dM", *m.Minutes), + }) + if err != nil { + return fmt.Errorf("dinners: encode manual recipe: %w", err) + } + now := store.Now().String() + recipeID := store.NewID() + minutes := sql.NullInt64{Int64: int64(*m.Minutes), Valid: true} + method := sql.NullString{String: *m.Method, Valid: true} + rec, err := q.CreateManualRecipe(ctx, db.CreateManualRecipeParams{ + ID: recipeID, + SourceUrl: manualSourcePrefix + recipeID, + Title: cur.Name, + TotalMinutes: minutes, + RawJsonld: string(ld), + Method: method, + MinutesTotal: minutes, + FetchedAt: now, + CreatedAt: now, + UpdatedAt: now, + }) + if err != nil { + return fmt.Errorf("dinners: store manual recipe: %w", err) + } + for i, line := range m.Ingredients { + if err := q.CreateRecipeIngredient(ctx, db.CreateRecipeIngredientParams{ + ID: store.NewID(), + RecipeID: rec.ID, + Position: int64(i), + RawText: line, + CreatedAt: now, + UpdatedAt: now, + }); err != nil { + return fmt.Errorf("dinners: store manual ingredient %d: %w", i+1, err) + } + } + for i, text := range m.Steps { + if err := q.CreateRecipeStep(ctx, db.CreateRecipeStepParams{ + ID: store.NewID(), + RecipeID: rec.ID, + Position: int64(i), + Text: text, + CreatedAt: now, + UpdatedAt: now, + }); err != nil { + return fmt.Errorf("dinners: store manual step %d: %w", i+1, err) + } + } + row, err = q.GroundDinner(ctx, db.GroundDinnerParams{ + RecipeID: sql.NullString{String: rec.ID, Valid: true}, + Method: method, + MinutesTotal: minutes, + Difficulty: cur.Difficulty, + GroundingStatus: GroundingManual, + UpdatedBy: sql.NullString{String: updatedBy, Valid: true}, + UpdatedAt: now, + ID: cur.ID, + }) + if err != nil { + return fmt.Errorf("dinners: ground %s: %w", id, err) + } + return nil + }) + if err != nil { + return Dinner{}, err + } + return fromRow(row), nil +} + +// manualJSONLD is the schema.org Recipe stored as a manual recipe's +// raw_jsonld, so it reads like a fetched page's. +type manualJSONLD struct { + Context string `json:"@context"` + Type string `json:"@type"` + Name string `json:"name"` + Ingredients []string `json:"recipeIngredient"` + Instructions []string `json:"recipeInstructions"` + TotalTime string `json:"totalTime"` +} + +// validate checks f and returns the link to ground, or the manual recipe +// with its lines trimmed. +func (f Fix) validate() (link string, manual *ManualRecipe, err error) { + given := 0 + for _, set := range []bool{f.CandidateID != nil, f.RecipeURL != nil, f.Manual != nil} { + if set { + given++ + } + } + if given != 1 { + return "", nil, invalid("ground", "give exactly one of candidate_id, recipe_url and manual") + } + switch { + case f.CandidateID != nil: + u, err := checkRecipeURL(*f.CandidateID) + if err != nil { + return "", nil, invalid("candidate_id", "is not a candidate from recipe-candidates") + } + return u.String(), nil, nil + case f.RecipeURL != nil: + u, err := checkRecipeURL(*f.RecipeURL) + if err != nil { + return "", nil, err + } + return u.String(), nil, nil + } + m, err := f.Manual.validate() + if err != nil { + return "", nil, err + } + return "", &m, nil +} + +// validate checks every field of m and returns it with its lines trimmed. +func (m ManualRecipe) validate() (ManualRecipe, error) { + var problems []Problem + ings, ingProblem := checkLines(m.Ingredients, 1) + if ingProblem != "" { + problems = append(problems, Problem{"manual.ingredients", ingProblem}) + } + steps, stepProblem := checkLines(m.Steps, 0) + if stepProblem != "" { + problems = append(problems, Problem{"manual.steps", stepProblem}) + } + if m.Minutes == nil || *m.Minutes < 1 || *m.Minutes > MaxMinutes { + problems = append(problems, Problem{"manual.minutes", fmt.Sprintf("must be 1 to %d", MaxMinutes)}) + } + if m.Method == nil || !slices.Contains(Methods, *m.Method) { + problems = append(problems, Problem{"manual.method", "must be one of " + strings.Join(Methods, ", ")}) + } + if problems != nil { + return ManualRecipe{}, &ValidationError{Problems: problems} + } + return ManualRecipe{Ingredients: ings, Steps: steps, Minutes: m.Minutes, Method: m.Method}, nil +} + +// checkLines returns lines trimmed, or why they are refused: fewer than +// least, more than MaxManualLines, or one blank or over MaxManualLineChar. +func checkLines(lines []string, least int) ([]string, string) { + if len(lines) < least { + return nil, fmt.Sprintf("must have at least %d line", least) + } + if len(lines) > MaxManualLines { + return nil, fmt.Sprintf("must have at most %d lines", MaxManualLines) + } + out := make([]string, len(lines)) + for i, line := range lines { + line = strings.TrimSpace(line) + switch { + case line == "": + return nil, fmt.Sprintf("line %d is blank", i+1) + case utf8.RuneCountInString(line) > MaxManualLineChar: + return nil, fmt.Sprintf("line %d is over %d characters", i+1, MaxManualLineChar) + } + out[i] = line + } + return out, "" +} + +// get reads dinner id, or ErrNotFound. +func (s *Service) get(ctx context.Context, id string) (db.Dinner, error) { + row, err := db.New(s.st.Reader).GetDinner(ctx, id) + if errors.Is(err, sql.ErrNoRows) { + return db.Dinner{}, ErrNotFound + } + if err != nil { + return db.Dinner{}, fmt.Errorf("dinners: read %s: %w", id, err) + } + return row, nil +} + +func nullInt(v *int) sql.NullInt64 { + if v == nil { + return sql.NullInt64{} + } + return sql.NullInt64{Int64: int64(*v), Valid: true} +} diff --git a/internal/dinners/grounding_test.go b/internal/dinners/grounding_test.go new file mode 100644 index 0000000..1d7e70e --- /dev/null +++ b/internal/dinners/grounding_test.go @@ -0,0 +1,595 @@ +package dinners_test + +import ( + "context" + "errors" + "io" + "log/slog" + "slices" + "testing" + "time" + + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm" + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/db" + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/dinners" + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/jobs" + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/recipes" + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/recipes/recipestest" + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/store" + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/migrations" +) + +const member = "member-1" + +// Fixture pages: two curry recipes from search and a stew at a pasted link. +const ( + curryA = "https://a.example/chicken-curry" + curryB = "https://b.example/curry" + stew = "https://c.example/recipes/beef-stew" +) + +type env struct { + svc *dinners.Service + st *store.Store + runner *jobs.Runner + search *recipestest.Search + pages *recipestest.Pages + model *llm.Fake +} + +// newEnv returns a dinners service over a started job runner, grounding +// with fake search, pages, images and model. Searching "Chicken curry" +// finds curryA, curryB and a missing page. +func newEnv(t *testing.T) *env { + t.Helper() + e := newUnstartedEnv(t) + e.start(t) + return e +} + +// newUnstartedEnv is newEnv with its job runner not yet started, so jobs +// enqueued stay queued until start. +func newUnstartedEnv(t *testing.T) *env { + t.Helper() + dataDir := t.TempDir() + st, err := store.Open(t.Context(), dataDir) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := st.Close(); err != nil { + t.Error(err) + } + }) + if err := store.Migrate(t.Context(), st.Writer, migrations.FS); err != nil { + t.Fatal(err) + } + search := &recipestest.Search{Results: map[string][]recipes.Candidate{ + "Chicken curry": {{URL: curryA}, {URL: "https://missing.example/x"}, {URL: curryB}}, + }} + pages := &recipestest.Pages{Bodies: map[string][]byte{ + curryA: recipestest.RecipePage("Chicken Curry", "Curry House", "https://a.example/curry.jpg", 40), + curryB: recipestest.RecipePage("Easy Weeknight Curry", "B Kitchen", "https://b.example/curry.jpg", 25), + stew: recipestest.RecipePage("Beef Stew", "Stew Co", "https://c.example/stew.jpg", 150), + }} + images := &recipestest.Images{JPEGs: map[string][]byte{ + "https://a.example/curry.jpg": recipestest.JPEG(t, 400, 320, 1), + "https://b.example/curry.jpg": recipestest.JPEG(t, 400, 320, 2), + "https://c.example/stew.jpg": recipestest.JPEG(t, 400, 320, 3), + }} + model := &llm.Fake{} + g := recipestest.NewGrounder(t, st, dataDir, search, pages, images, model) + + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + runner, err := jobs.NewRunner(st, logger, 1) + if err != nil { + t.Fatal(err) + } + svc, err := dinners.NewService(st, g, runner, logger) + if err != nil { + t.Fatal(err) + } + return &env{svc: svc, st: st, runner: runner, search: search, pages: pages, model: model} +} + +// start starts the env's job runner, stopping it when the test ends. +func (e *env) start(t *testing.T) { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + if err := e.runner.Start(ctx); err != nil { + cancel() + t.Fatal(err) + } + t.Cleanup(func() { + cancel() + e.runner.Wait() + }) +} + +func ptr[T any](v T) *T { return &v } + +// waitJob waits for job id to finish and returns it. +func (e *env) waitJob(t *testing.T, id string) jobs.Job { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for { + job, err := e.runner.Get(t.Context(), id) + if err != nil { + t.Fatal(err) + } + if job.Status == jobs.StatusSucceeded || job.Status == jobs.StatusFailed { + return job + } + if time.Now().After(deadline) { + t.Fatalf("job %s never finished; last %+v", id, job) + } + time.Sleep(5 * time.Millisecond) + } +} + +func (e *env) dinner(t *testing.T, id string) db.Dinner { + t.Helper() + row, err := db.New(e.st.Reader).GetDinner(t.Context(), id) + if err != nil { + t.Fatal(err) + } + return row +} + +func TestCreateGroundsInBackground(t *testing.T) { + t.Parallel() + e := newEnv(t) + e.model.QueueGenerate(recipestest.NormalizedReply("stovetop", 40)) + + c, err := e.svc.CreateRegular(t.Context(), dinners.NewRegular{Name: ptr("Chicken curry")}, member) + if err != nil { + t.Fatal(err) + } + if c.GroundingStatus != dinners.GroundingPending || c.GroundingJobID == nil { + t.Fatalf("created = %+v, want pending with a job id", c) + } + job := e.waitJob(t, *c.GroundingJobID) + if job.Kind != dinners.KindGroundDinner || job.Status != jobs.StatusSucceeded || + string(job.Result) != `{"dinner_id":"`+c.ID+`","grounding_status":"grounded"}` { + t.Fatalf("job = %+v (result %s)", job, job.Result) + } + + row := e.dinner(t, c.ID) + if row.GroundingStatus != "grounded" || !row.RecipeID.Valid || !row.PhotoID.Valid || + row.Method.String != "stovetop" || row.MinutesTotal.Int64 != 40 || row.Difficulty.String != "easy" || row.GroundingError.Valid { + t.Fatalf("dinner = %+v, want grounded with recipe, photo, stovetop, 40 minutes, easy", row) + } + rec, err := db.New(e.st.Reader).GetRecipe(t.Context(), row.RecipeID.String) + if err != nil { + t.Fatal(err) + } + if rec.SourceUrl != curryA || rec.PhotoID != row.PhotoID { + t.Errorf("recipe = %s photo %s, want %s photo %s", rec.SourceUrl, rec.PhotoID.String, curryA, row.PhotoID.String) + } + if got := e.search.Queries(); !slices.Equal(got, []string{"Chicken curry"}) { + t.Errorf("searched %q, want the dinner's name", got) + } +} + +func TestCreateByRecipeURLGroundsThatURL(t *testing.T) { + t.Parallel() + e := newEnv(t) + e.model.QueueGenerate(recipestest.NormalizedReply("oven", 150)) + + c, err := e.svc.CreateRegular(t.Context(), dinners.NewRegular{RecipeURL: ptr(stew)}, member) + if err != nil { + t.Fatal(err) + } + if job := e.waitJob(t, *c.GroundingJobID); job.Status != jobs.StatusSucceeded { + t.Fatalf("job = %+v", job) + } + row := e.dinner(t, c.ID) + rec, err := db.New(e.st.Reader).GetRecipe(t.Context(), row.RecipeID.String) + if err != nil { + t.Fatal(err) + } + if row.GroundingStatus != "grounded" || rec.SourceUrl != stew || row.Method.String != "oven" || row.MinutesTotal.Int64 != 150 { + t.Fatalf("dinner = %+v recipe %s, want grounded in %s", row, rec.SourceUrl, stew) + } + if q := e.search.Queries(); len(q) != 0 { + t.Errorf("searched %q; a recipe_url is grounded directly", q) + } +} + +func TestCreateGroundingFailureRecordsReason(t *testing.T) { + t.Parallel() + tests := []struct { + name string + create dinners.NewRegular + setup func(*env) + reason string + }{ + {"no candidates", dinners.NewRegular{Name: ptr("Unknown dish")}, nil, dinners.ReasonNoRecipe}, + {"unusable link", dinners.NewRegular{RecipeURL: ptr("https://gone.example/recipes/lost-soup")}, nil, dinners.ReasonRecipeUnusable}, + {"search fails", dinners.NewRegular{Name: ptr("Chicken curry")}, func(e *env) { e.search.Err = recipes.ErrSearchRateLimited }, dinners.ReasonSearchFailed}, + {"model fails", dinners.NewRegular{Name: ptr("Chicken curry")}, func(e *env) { + // One provider failure per candidate tried. + e.model.QueueGenerate(llm.GenerateResult{Err: llm.ErrAuth}) + }, dinners.ReasonAIFailed}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + e := newEnv(t) + if tt.setup != nil { + tt.setup(e) + } + c, err := e.svc.CreateRegular(t.Context(), tt.create, member) + if err != nil { + t.Fatal(err) + } + job := e.waitJob(t, *c.GroundingJobID) + if job.Status != jobs.StatusFailed || job.Error == nil || *job.Error != tt.reason { + t.Errorf("job = %+v, want failed with %s", job, tt.reason) + } + row := e.dinner(t, c.ID) + if row.GroundingStatus != "failed" || row.GroundingError.String != tt.reason || row.RecipeID.Valid { + t.Errorf("dinner = %+v, want failed with %s", row, tt.reason) + } + }) + } +} + +func TestCreateReturnsBeforeGroundingCompletes(t *testing.T) { + t.Parallel() + e := newEnv(t) + e.search.Gate = make(chan struct{}) + e.model.QueueGenerate(recipestest.NormalizedReply("stovetop", 40)) + + c, err := e.svc.CreateRegular(t.Context(), dinners.NewRegular{Name: ptr("Chicken curry")}, member) + if err != nil { + t.Fatal(err) + } + // The search is held, so the job cannot have finished. + if row := e.dinner(t, c.ID); row.GroundingStatus != "pending" { + t.Fatalf("grounding_status = %s right after create, want pending", row.GroundingStatus) + } + if job, err := e.runner.Get(t.Context(), *c.GroundingJobID); err != nil || (job.Status != jobs.StatusQueued && job.Status != jobs.StatusRunning) { + t.Fatalf("job = %+v, %v; want queued or running", job, err) + } + close(e.search.Gate) + if job := e.waitJob(t, *c.GroundingJobID); job.Status != jobs.StatusSucceeded { + t.Fatalf("job = %+v", job) + } + if row := e.dinner(t, c.ID); row.GroundingStatus != "grounded" { + t.Errorf("grounding_status = %s, want grounded", row.GroundingStatus) + } +} + +func TestManualFixWhilePendingIsKept(t *testing.T) { + t.Parallel() + e := newEnv(t) + e.search.Gate = make(chan struct{}) + e.model.QueueGenerate(recipestest.NormalizedReply("stovetop", 40)) + + c, err := e.svc.CreateRegular(t.Context(), dinners.NewRegular{Name: ptr("Chicken curry")}, member) + if err != nil { + t.Fatal(err) + } + if _, err := e.svc.Ground(t.Context(), c.ID, dinners.Fix{Manual: &dinners.ManualRecipe{ + Ingredients: []string{"1 chicken"}, Minutes: ptr(30), Method: ptr("oven"), + }}, member); err != nil { + t.Fatal(err) + } + close(e.search.Gate) + job := e.waitJob(t, *c.GroundingJobID) + if job.Status != jobs.StatusSucceeded || string(job.Result) != `{"dinner_id":"`+c.ID+`","grounding_status":"manual"}` { + t.Errorf("job = %+v (result %s), want succeeded reporting manual", job, job.Result) + } + if row := e.dinner(t, c.ID); row.GroundingStatus != "manual" || row.Method.String != "oven" { + t.Errorf("dinner = %+v, want the manual grounding kept", row) + } +} + +// failedDinner creates a Regular whose background grounding fails. +func failedDinner(t *testing.T, e *env, name string) string { + t.Helper() + c, err := e.svc.CreateRegular(t.Context(), dinners.NewRegular{Name: ptr(name)}, member) + if err != nil { + t.Fatal(err) + } + if job := e.waitJob(t, *c.GroundingJobID); job.Status != jobs.StatusFailed { + t.Fatalf("job = %+v, want failed", job) + } + return c.ID +} + +func TestCandidates(t *testing.T) { + t.Parallel() + e := newEnv(t) + // The background grounding stores curryA with its cached photo. + e.model.QueueGenerate(recipestest.NormalizedReply("stovetop", 40)) + c, err := e.svc.CreateRegular(t.Context(), dinners.NewRegular{Name: ptr("Chicken curry")}, member) + if err != nil { + t.Fatal(err) + } + if job := e.waitJob(t, *c.GroundingJobID); job.Status != jobs.StatusSucceeded { + t.Fatalf("job = %+v", job) + } + + got, err := e.svc.Candidates(t.Context(), c.ID) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 { + t.Fatalf("candidates = %+v, want the two usable pages", got) + } + first, second := got[0], got[1] + wantPhoto := dinners.PhotoURL(e.dinner(t, c.ID).PhotoID.String) + if first.CandidateID != curryA || first.Title != "Chicken Curry" || first.SiteName != "Curry House" || + first.PhotoURL == nil || *first.PhotoURL != wantPhoto || first.Minutes == nil || *first.Minutes != 40 { + t.Errorf("first = %+v, want the stored curryA with photo %s", first, wantPhoto) + } + if second.CandidateID != curryB || second.Title != "Easy Weeknight Curry" || second.SiteName != "B Kitchen" || + second.PhotoURL != nil || second.Minutes == nil || *second.Minutes != 25 { + t.Errorf("second = %+v, want curryB without a photo", second) + } + + if _, err := e.svc.Candidates(t.Context(), "nope"); !errors.Is(err, dinners.ErrNotFound) { + t.Errorf("unknown dinner: err = %v, want ErrNotFound", err) + } +} + +func TestCandidatesSearchFails(t *testing.T) { + t.Parallel() + e := newEnv(t) + e.search.Err = recipes.ErrSearchRefused + id := failedDinner(t, e, "Chicken curry") + if _, err := e.svc.Candidates(t.Context(), id); !errors.Is(err, recipes.ErrSearchFailed) { + t.Errorf("err = %v, want ErrSearchFailed", err) + } +} + +func TestCandidatesNone(t *testing.T) { + t.Parallel() + e := newEnv(t) + id := failedDinner(t, e, "Fish tacos") + got, err := e.svc.Candidates(t.Context(), id) + if err != nil || len(got) != 0 { + t.Fatalf("no search results: candidates = %+v, %v; want none", got, err) + } +} + +func TestFixViaCandidate(t *testing.T) { + t.Parallel() + e := newEnv(t) + id := failedDinner(t, e, "Mystery curry") + e.model.QueueGenerate(recipestest.NormalizedReply("stovetop", 25)) + + d, err := e.svc.Ground(t.Context(), id, dinners.Fix{CandidateID: ptr(curryB)}, "member-2") + if err != nil { + t.Fatal(err) + } + if d.GroundingStatus != "grounded" || d.GroundingError != nil || d.RecipeID == nil || d.PhotoURL == nil || + *d.Method != "stovetop" || *d.MinutesTotal != 25 || *d.Difficulty != "easy" || *d.UpdatedBy != "member-2" { + t.Fatalf("dinner = %+v", d) + } + rec, err := db.New(e.st.Reader).GetRecipe(t.Context(), *d.RecipeID) + if err != nil { + t.Fatal(err) + } + if rec.SourceUrl != curryB { + t.Errorf("recipe source = %s, want %s", rec.SourceUrl, curryB) + } +} + +func TestFixViaRecipeURL(t *testing.T) { + t.Parallel() + e := newEnv(t) + id := failedDinner(t, e, "Grandma's stew") + e.model.QueueGenerate(recipestest.NormalizedReply("oven", 150)) + + d, err := e.svc.Ground(t.Context(), id, dinners.Fix{RecipeURL: ptr(" " + stew + " ")}, member) + if err != nil { + t.Fatal(err) + } + if d.GroundingStatus != "grounded" || d.PhotoURL == nil || *d.Method != "oven" || *d.MinutesTotal != 150 { + t.Fatalf("dinner = %+v", d) + } + + // A link holding no recipe changes nothing. + before := e.dinner(t, id) + _, err = e.svc.Ground(t.Context(), id, dinners.Fix{RecipeURL: ptr("https://gone.example/x")}, member) + var ungroundable *recipes.ErrUngroundable + if !errors.As(err, &ungroundable) { + t.Fatalf("err = %v, want *recipes.ErrUngroundable", err) + } + if after := e.dinner(t, id); after != before { + t.Errorf("dinner changed by a failed fix: %+v, was %+v", after, before) + } +} + +func TestFixViaManual(t *testing.T) { + t.Parallel() + e := newEnv(t) + id := failedDinner(t, e, "Family chili") + if _, err := e.svc.Update(t.Context(), id, dinners.Patch{Difficulty: ptr("medium")}, member); err != nil { + t.Fatal(err) + } + + d, err := e.svc.Ground(t.Context(), id, dinners.Fix{Manual: &dinners.ManualRecipe{ + Ingredients: []string{" 1 lb beef ", "1 can beans"}, + Steps: []string{"Brown the beef.", "Simmer with the beans."}, + Minutes: ptr(240), + Method: ptr("crockpot"), + }}, member) + if err != nil { + t.Fatal(err) + } + if d.GroundingStatus != "manual" || d.GroundingError != nil || d.RecipeID == nil || d.PhotoURL != nil || + *d.Method != "crockpot" || *d.MinutesTotal != 240 || *d.Difficulty != "medium" { + t.Fatalf("dinner = %+v", d) + } + q := db.New(e.st.Reader) + rec, err := q.GetRecipe(t.Context(), *d.RecipeID) + if err != nil { + t.Fatal(err) + } + if rec.Title != "Family chili" || rec.SourceUrl != "mealquest:manual:"+rec.ID || rec.MinutesTotal.Int64 != 240 || rec.PhotoID.Valid { + t.Errorf("recipe = %+v", rec) + } + ings, err := q.ListRecipeIngredients(t.Context(), rec.ID) + if err != nil { + t.Fatal(err) + } + steps, err := q.ListRecipeSteps(t.Context(), rec.ID) + if err != nil { + t.Fatal(err) + } + if len(ings) != 2 || ings[0].RawText != "1 lb beef" || len(steps) != 2 || steps[1].Text != "Simmer with the beans." { + t.Errorf("ingredients = %+v, steps = %+v", ings, steps) + } +} + +func TestFixValidation(t *testing.T) { + t.Parallel() + e := newEnv(t) + id := failedDinner(t, e, "Pad thai") + good := func() *dinners.ManualRecipe { + return &dinners.ManualRecipe{Ingredients: []string{"noodles"}, Minutes: ptr(20), Method: ptr("stovetop")} + } + tests := map[string]dinners.Fix{ + "nothing": {}, + "two": {CandidateID: ptr(curryA), RecipeURL: ptr(stew)}, + "candidate not URL": {CandidateID: ptr("cand-1")}, + "http link": {RecipeURL: ptr("http://a.example/x")}, + "no ingredients": {Manual: &dinners.ManualRecipe{Minutes: ptr(20), Method: ptr("oven")}}, + "blank ingredient": {Manual: &dinners.ManualRecipe{Ingredients: []string{" "}, Minutes: ptr(20), Method: ptr("oven")}}, + "blank step": {Manual: func() *dinners.ManualRecipe { m := good(); m.Steps = []string{""}; return m }()}, + "no minutes": {Manual: func() *dinners.ManualRecipe { m := good(); m.Minutes = nil; return m }()}, + "zero minutes": {Manual: func() *dinners.ManualRecipe { m := good(); m.Minutes = ptr(0); return m }()}, + "unknown method": {Manual: func() *dinners.ManualRecipe { m := good(); m.Method = ptr("microwave"); return m }()}, + "no method": {Manual: func() *dinners.ManualRecipe { m := good(); m.Method = nil; return m }()}, + "too many lines": {Manual: func() *dinners.ManualRecipe { m := good(); m.Ingredients = make([]string, 101); return m }()}, + "manual and a link": {RecipeURL: ptr(stew), Manual: good()}, + "over-long ing line": {Manual: func() *dinners.ManualRecipe { + m := good() + m.Ingredients = []string{string(make([]byte, 501))} + return m + }()}, + } + before := e.dinner(t, id) + for name, fix := range tests { + _, err := e.svc.Ground(t.Context(), id, fix, member) + var invalid *dinners.ValidationError + if !errors.As(err, &invalid) { + t.Errorf("%s: err = %v, want *ValidationError", name, err) + } + } + if after := e.dinner(t, id); after != before { + t.Errorf("dinner changed by invalid fixes: %+v", after) + } + if _, err := e.svc.Ground(t.Context(), "nope", dinners.Fix{Manual: good()}, member); !errors.Is(err, dinners.ErrNotFound) { + t.Errorf("unknown dinner, manual: err = %v, want ErrNotFound", err) + } + if _, err := e.svc.Ground(t.Context(), "nope", dinners.Fix{RecipeURL: ptr(stew)}, member); !errors.Is(err, dinners.ErrNotFound) { + t.Errorf("unknown dinner, link: err = %v, want ErrNotFound", err) + } +} + +// TestRestartFailsInterruptedGrounding simulates a crash: one dinner's job +// was left running and another dinner was saved before its job was +// enqueued. At the next boot both end failed as interrupted, while a +// dinner whose job is still queued stays pending and is grounded. +func TestRestartFailsInterruptedGrounding(t *testing.T) { + t.Parallel() + e := newUnstartedEnv(t) + e.search.Gate = make(chan struct{}) + e.model.QueueGenerate(recipestest.NormalizedReply("stovetop", 40)) + + running, err := e.svc.CreateRegular(t.Context(), dinners.NewRegular{Name: ptr("Fish pie")}, member) + if err != nil { + t.Fatal(err) + } + if _, err := e.st.Writer.ExecContext(t.Context(), + `UPDATE jobs SET status = 'running', started_at = updated_at WHERE id = ?`, *running.GroundingJobID); err != nil { + t.Fatal(err) + } + unqueued, err := e.svc.CreateRegular(t.Context(), dinners.NewRegular{Name: ptr("Lentil soup")}, member) + if err != nil { + t.Fatal(err) + } + if _, err := e.st.Writer.ExecContext(t.Context(), `DELETE FROM jobs WHERE id = ?`, *unqueued.GroundingJobID); err != nil { + t.Fatal(err) + } + queued, err := e.svc.CreateRegular(t.Context(), dinners.NewRegular{Name: ptr("Chicken curry")}, member) + if err != nil { + t.Fatal(err) + } + + // Boot: the runner fails the running job, then the service fails the + // dinners nothing is left to ground. The search is held, so the queued + // dinner's job cannot finish meanwhile. + e.start(t) + n, err := e.svc.FailInterruptedGroundings(t.Context()) + if err != nil { + t.Fatal(err) + } + if n != 2 { + t.Errorf("failed %d dinners, want 2", n) + } + if job, err := e.runner.Get(t.Context(), *running.GroundingJobID); err != nil || job.Status != jobs.StatusFailed { + t.Errorf("running job = %+v, %v; want failed", job, err) + } + for _, id := range []string{running.ID, unqueued.ID} { + if row := e.dinner(t, id); row.GroundingStatus != dinners.GroundingFailed || row.GroundingError.String != dinners.ReasonInterrupted { + t.Errorf("dinner %s = %s (%s), want failed (interrupted)", id, row.GroundingStatus, row.GroundingError.String) + } + } + if row := e.dinner(t, queued.ID); row.GroundingStatus != dinners.GroundingPending { + t.Errorf("queued dinner = %s, want pending", row.GroundingStatus) + } + close(e.search.Gate) + if job := e.waitJob(t, *queued.GroundingJobID); job.Status != jobs.StatusSucceeded { + t.Fatalf("queued job = %+v", job) + } + if row := e.dinner(t, queued.ID); row.GroundingStatus != dinners.GroundingGrounded { + t.Errorf("queued dinner = %s, want grounded", row.GroundingStatus) + } +} + +// TestCreateEnqueueFailureReturnsFailedDinner: a dinner saved when its job +// cannot be enqueued is returned, with its id, failed and no job id. +func TestCreateEnqueueFailureReturnsFailedDinner(t *testing.T) { + t.Parallel() + e := newEnv(t) + if _, err := e.st.Writer.ExecContext(t.Context(), + `CREATE TRIGGER refuse_jobs BEFORE INSERT ON jobs BEGIN SELECT RAISE(ABORT, 'no jobs'); END`); err != nil { + t.Fatal(err) + } + c, err := e.svc.CreateRegular(t.Context(), dinners.NewRegular{Name: ptr("Chicken curry")}, member) + if err != nil { + t.Fatal(err) + } + if c.ID == "" || c.GroundingJobID != nil || c.GroundingStatus != dinners.GroundingFailed || + c.GroundingError == nil || *c.GroundingError != dinners.ReasonGroundingError { + t.Fatalf("created = %+v, want its id, failed (grounding_error) and no job", c) + } + if row := e.dinner(t, c.ID); row.GroundingStatus != dinners.GroundingFailed { + t.Errorf("stored grounding_status = %s, want failed", row.GroundingStatus) + } +} + +// TestGroundTimesOut: grounding a link by hand that runs past its deadline +// is ErrGroundTimeout and leaves the dinner as it was. It shortens the +// package deadline, so it does not run in parallel. +func TestGroundTimesOut(t *testing.T) { + dinners.SetGroundTimeout(t, 50*time.Millisecond) + e := newEnv(t) + id := failedDinner(t, e, "Beef stew") + e.pages.Gate = make(chan struct{}) + defer close(e.pages.Gate) + + _, err := e.svc.Ground(t.Context(), id, dinners.Fix{RecipeURL: ptr(stew)}, member) + if !errors.Is(err, dinners.ErrGroundTimeout) { + t.Fatalf("err = %v, want ErrGroundTimeout", err) + } + if row := e.dinner(t, id); row.GroundingStatus != dinners.GroundingFailed { + t.Errorf("grounding_status = %s, want failed as before", row.GroundingStatus) + } +} diff --git a/internal/recipes/ground.go b/internal/recipes/ground.go index bb52b71..b5e5b16 100644 --- a/internal/recipes/ground.go +++ b/internal/recipes/ground.go @@ -25,6 +25,13 @@ var ( // ErrTotalTimeNotPositive rejects a recipe whose page gives a total // time of zero minutes, which normalize_recipe refuses. ErrTotalTimeNotPositive = errors.New("recipes: recipe total time is not positive") + // ErrSearchFailed wraps every failure to search: the search provider + // could not be built (ErrSearchNotConfigured among them) or its search + // failed. + ErrSearchFailed = errors.New("recipes: recipe search failed") + // ErrNormalizeFailed wraps a normalize_recipe failure that is not the + // recipe's: the model provider could not be built or called. + ErrNormalizeFailed = errors.New("recipes: normalize_recipe failed") ) // ErrUngroundable is returned when no recipe page could ground a dish or a @@ -118,43 +125,16 @@ func (g *Grounder) Ground(ctx context.Context, dishName, query string) (Grounded if strings.TrimSpace(dishName) == "" { return Grounded{}, errors.New("recipes: ground: dish name is empty") } - provider, err := g.search.Provider(ctx) + found, err := g.searchRanked(ctx, dishName, query) if err != nil { return Grounded{}, fmt.Errorf("recipes: ground %q: %w", dishName, err) } - cands, err := provider.Search(ctx, query, RankDepth) - if err != nil { - return Grounded{}, fmt.Errorf("recipes: ground %q: search: %w", dishName, err) - } - if len(cands) == 0 { - return Grounded{}, &ErrUngroundable{Target: dishName, Reasons: []error{ErrNoCandidates}} - } - cands = cands[:min(len(cands), RankDepth)] - stored := make(map[string]storedRecipe) - for _, c := range cands { - sr, ok, err := g.storedRecipe(ctx, c.URL) - if err != nil { - return Grounded{}, fmt.Errorf("recipes: ground %q: %w", dishName, err) - } - if ok { - stored[c.URL] = sr - } - } - ranked, rankErr := rank(ctx, dishName, cands, func(ctx context.Context, c Candidate) (Recipe, error) { - if sr, ok := stored[c.URL]; ok { - return sr.recipe, nil - } - return fetchRecipe(ctx, g.pages, c.URL) - }) - if err := ctx.Err(); err != nil { - return Grounded{}, err - } var reasons []error - if rankErr != nil { - reasons = append(reasons, rankErr) + if found.rankErr != nil { + reasons = append(reasons, found.rankErr) } - for _, r := range ranked { - if sr, ok := stored[r.Candidate.URL]; ok { + for _, r := range found.ranked { + if sr, ok := found.stored[r.Candidate.URL]; ok { return sr.grounded, nil } gr, reason, err := g.ground(ctx, r.Candidate.URL, r.Recipe) @@ -169,6 +149,96 @@ func (g *Grounder) Ground(ctx context.Context, dishName, query string) (Grounded return Grounded{}, &ErrUngroundable{Target: dishName, Reasons: reasons} } +// rankedSearch is a search's candidates ranked: the ranked ones best first, +// those already stored by URL, and why any were not ranked. +type rankedSearch struct { + ranked []Ranked + stored map[string]storedRecipe + // rankErr is Rank's error, or ErrNoCandidates when the search found + // nothing. + rankErr error +} + +// searchRanked searches for query and ranks the top RankDepth candidates +// for dishName as Ground describes. A search, database or context failure +// is its error; candidates that do not rank are only rankErr's. +func (g *Grounder) searchRanked(ctx context.Context, dishName, query string) (rankedSearch, error) { + provider, err := g.search.Provider(ctx) + if err != nil { + return rankedSearch{}, fmt.Errorf("%w: %w", ErrSearchFailed, err) + } + cands, err := provider.Search(ctx, query, RankDepth) + if err != nil { + return rankedSearch{}, fmt.Errorf("%w: %w", ErrSearchFailed, err) + } + if len(cands) == 0 { + return rankedSearch{rankErr: ErrNoCandidates}, nil + } + cands = cands[:min(len(cands), RankDepth)] + stored := make(map[string]storedRecipe) + for _, c := range cands { + sr, ok, err := g.storedRecipe(ctx, c.URL) + if err != nil { + return rankedSearch{}, err + } + if ok { + stored[c.URL] = sr + } + } + ranked, rankErr := rank(ctx, dishName, cands, func(ctx context.Context, c Candidate) (Recipe, error) { + if sr, ok := stored[c.URL]; ok { + return sr.recipe, nil + } + return fetchRecipe(ctx, g.pages, c.URL) + }) + if err := ctx.Err(); err != nil { + return rankedSearch{}, err + } + return rankedSearch{ranked: ranked, stored: stored, rankErr: rankErr}, nil +} + +// Offer is a ranked candidate offered for the household to ground a dish +// in, with GroundURL on its URL. +type Offer struct { + URL string + Title string + SiteName string + // PhotoID is the cached photo of a candidate already stored, "" for + // one that is not: a candidate's photo is cached only when it grounds. + PhotoID string + // Minutes is a stored candidate's normalized minutes_total, else its + // page's total time; nil when unknown. + Minutes *int +} + +// Candidates searches for query and returns up to n of the candidates +// Ground would try for dishName, best first. It fetches and extracts pages +// but downloads no photo and runs no model. Candidates that hold no usable +// recipe are not offered, so the list may be empty; a search that fails is +// an error wrapping ErrSearchFailed. +func (g *Grounder) Candidates(ctx context.Context, dishName, query string, n int) ([]Offer, error) { + if strings.TrimSpace(dishName) == "" { + return nil, errors.New("recipes: candidates: dish name is empty") + } + if n < 1 { + return nil, fmt.Errorf("recipes: candidates: n is %d, want at least 1", n) + } + found, err := g.searchRanked(ctx, dishName, query) + if err != nil { + return nil, fmt.Errorf("recipes: candidates for %q: %w", dishName, err) + } + offers := make([]Offer, 0, min(n, len(found.ranked))) + for _, r := range found.ranked[:min(n, len(found.ranked))] { + o := Offer{URL: r.Candidate.URL, Title: r.Recipe.Title, SiteName: r.Recipe.SiteName, Minutes: r.Recipe.TotalMinutes} + if sr, ok := found.stored[r.Candidate.URL]; ok { + o.PhotoID = sr.grounded.PhotoID + o.Minutes = sr.grounded.MinutesTotal + } + offers = append(offers, o) + } + return offers, nil +} + // GroundURL grounds the recipe on the page at rawURL, a pasted URL, as // Ground does a candidate. An already stored rawURL is returned without // fetching it. A page that cannot be fetched, holds no usable recipe, or @@ -344,7 +414,7 @@ func (g *Grounder) ground(ctx context.Context, sourceURL string, r Recipe) (gr G return Grounded{}, fmt.Errorf("normalize: %w", err), nil } if err != nil { - return Grounded{}, nil, fmt.Errorf("normalize: %w", err) + return Grounded{}, nil, fmt.Errorf("%w: %w", ErrNormalizeFailed, err) } if len(norm.Ingredients) != len(r.Ingredients) { return Grounded{}, nil, fmt.Errorf("normalize: %d ingredients for %d lines", len(norm.Ingredients), len(r.Ingredients)) diff --git a/internal/recipes/ground_test.go b/internal/recipes/ground_test.go index 8bde81a..3932616 100644 --- a/internal/recipes/ground_test.go +++ b/internal/recipes/ground_test.go @@ -516,3 +516,38 @@ func TestGroundManyLimitsConcurrency(t *testing.T) { t.Errorf("max in flight %d of %d calls, want %d of 5", search.maxActive, search.callCount(), GroundConcurrency) } } + +func TestCandidatesOffersBestFirstUpToN(t *testing.T) { + t.Parallel() + var cands []Candidate + pages := map[string][]byte{} + for i, title := range []string{"Fish Tacos", "Fish Taco Bowls", "Baja Fish Tacos", "Tacos"} { + u := fmt.Sprintf("https://site%d.example/tacos", i) + cands = append(cands, Candidate{URL: u}) + pages[u] = groundPage(title, "/img/tacos.jpg") + } + search := &fakeSearch{results: map[string][]Candidate{"fish tacos": cands}} + env := newGroundEnv(t, pages, nil, search) + + got, err := env.g.Candidates(t.Context(), "Fish Tacos", "fish tacos", 3) + if err != nil { + t.Fatal(err) + } + if len(got) != 3 || got[0].URL != cands[0].URL || got[0].Title != "Fish Tacos" || got[0].SiteName != "Example Kitchen" || + got[0].PhotoID != "" || got[0].Minutes == nil || *got[0].Minutes != 40 { + t.Fatalf("Candidates = %+v, want 3 with Fish Tacos first", got) + } + if env.images.requests.Load() != 0 || len(env.llm.Requests()) != 0 { + t.Errorf("Candidates downloaded a photo or ran the model") + } +} + +func TestCandidatesSearchFailureWrapsErrSearchFailed(t *testing.T) { + t.Parallel() + search := &fakeSearch{errs: map[string]error{"stew": ErrSearchRateLimited}} + env := newGroundEnv(t, nil, nil, search) + _, err := env.g.Candidates(t.Context(), "Stew", "stew", 3) + if !errors.Is(err, ErrSearchFailed) || !errors.Is(err, ErrSearchRateLimited) { + t.Errorf("err = %v, want ErrSearchFailed wrapping ErrSearchRateLimited", err) + } +} diff --git a/internal/recipes/photo.go b/internal/recipes/photo.go index b57d223..f5d22db 100644 --- a/internal/recipes/photo.go +++ b/internal/recipes/photo.go @@ -65,22 +65,28 @@ var photoDecoders = map[string]func([]byte) (image.Image, error){ "webp": func(b []byte) (image.Image, error) { return webp.Decode(bytes.NewReader(b)) }, } -// PhotoCache downloads photos through a Fetcher and stores each at every +// ImageFetcher fetches an image; *Fetcher satisfies it. +type ImageFetcher interface { + FetchImage(ctx context.Context, rawURL string) (Page, error) +} + +// PhotoCache downloads photos through an ImageFetcher and stores each at every // PhotoSizes length as {id}-{size}.jpg under PhotoDir, with a photos row. // Photos are deduplicated by the SHA-256 of the downloaded bytes. A // PhotoCache is safe for concurrent use. type PhotoCache struct { - fetcher *Fetcher - store *store.Store + images ImageFetcher + store *store.Store // dir is the absolute photo directory. dir string } -// NewPhotoCache returns a PhotoCache storing photos under dataDir, creating -// the photo directory if it is absent. -func NewPhotoCache(fetcher *Fetcher, st *store.Store, dataDir string) (*PhotoCache, error) { - if fetcher == nil { - return nil, errors.New("recipes: nil fetcher") +// NewPhotoCache returns a PhotoCache downloading images with images and +// storing photos under dataDir, creating the photo directory if it is +// absent. +func NewPhotoCache(images ImageFetcher, st *store.Store, dataDir string) (*PhotoCache, error) { + if images == nil { + return nil, errors.New("recipes: nil image fetcher") } if st == nil { return nil, errors.New("recipes: nil store") @@ -92,7 +98,7 @@ func NewPhotoCache(fetcher *Fetcher, st *store.Store, dataDir string) (*PhotoCac if err := os.MkdirAll(dir, 0o750); err != nil { return nil, fmt.Errorf("recipes: create photo directory: %w", err) } - return &PhotoCache{fetcher: fetcher, store: st, dir: dir}, nil + return &PhotoCache{images: images, store: st, dir: dir}, nil } // DownloadedPhoto is an image DownloadPhoto has downloaded and decoded, @@ -108,7 +114,7 @@ type DownloadedPhoto struct { // whose shorter side is under PhotoMinShortSide. It records nothing, so a // caller can try another image before opening a transaction. func (c *PhotoCache) DownloadPhoto(ctx context.Context, rawURL string) (DownloadedPhoto, error) { - page, err := c.fetcher.FetchImage(ctx, rawURL) + page, err := c.images.FetchImage(ctx, rawURL) if err != nil { return DownloadedPhoto{}, err } diff --git a/internal/recipes/recipestest/recipestest.go b/internal/recipes/recipestest/recipestest.go new file mode 100644 index 0000000..46aa72a --- /dev/null +++ b/internal/recipes/recipestest/recipestest.go @@ -0,0 +1,170 @@ +// Package recipestest provides fakes for tests that ground dinners outside +// package recipes: a search, page and image fetchers serving fixtures, and +// a Grounder over them that normalizes through the real harness and +// normalize_recipe skill with a scripted model. +package recipestest + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "image" + "image/color" + "image/jpeg" + "sync" + "testing" + + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/harness" + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm" + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/skills" + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/recipes" + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/store" +) + +// Search is a recipes.SearchSource and recipes.SearchProvider answering +// each query from Results. Err, when set, fails every search. Gate, when +// set, makes every search wait for it to close. +type Search struct { + Results map[string][]recipes.Candidate + Err error + Gate chan struct{} + + mu sync.Mutex + queries []string +} + +func (s *Search) Provider(context.Context) (recipes.SearchProvider, error) { return s, nil } + +func (s *Search) Search(ctx context.Context, query string, n int) ([]recipes.Candidate, error) { + s.mu.Lock() + s.queries = append(s.queries, query) + s.mu.Unlock() + if s.Gate != nil { + select { + case <-s.Gate: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + if s.Err != nil { + return nil, s.Err + } + c := s.Results[query] + return c[:min(n, len(c))], nil +} + +// Queries returns every query searched, in order. +func (s *Search) Queries() []string { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string(nil), s.queries...) +} + +// Pages is a recipes.PageFetcher serving Bodies by URL; any other URL is a +// 404. Gate, when set, makes every fetch wait for it to close. +type Pages struct { + Bodies map[string][]byte + Gate chan struct{} +} + +func (p *Pages) Fetch(ctx context.Context, rawURL string) (recipes.Page, error) { + if p.Gate != nil { + select { + case <-p.Gate: + case <-ctx.Done(): + return recipes.Page{}, ctx.Err() + } + } + body, ok := p.Bodies[rawURL] + if !ok { + return recipes.Page{}, &recipes.StatusError{URL: rawURL, Status: 404} + } + return recipes.Page{URL: rawURL, ContentType: "text/html", Body: body}, nil +} + +// Images is a recipes.ImageFetcher serving JPEGs by URL; any other URL is +// a 404. +type Images struct { + JPEGs map[string][]byte +} + +func (i *Images) FetchImage(_ context.Context, rawURL string) (recipes.Page, error) { + body, ok := i.JPEGs[rawURL] + if !ok { + return recipes.Page{}, &recipes.StatusError{URL: rawURL, Status: 404} + } + return recipes.Page{URL: rawURL, ContentType: "image/jpeg", Body: body}, nil +} + +// JPEG is a w×h JPEG whose pixels depend on seed, so different seeds give +// different photos. +func JPEG(t testing.TB, w, h int, seed uint8) []byte { + t.Helper() + img := image.NewRGBA(image.Rect(0, 0, w, h)) + for y := range h { + for x := range w { + img.Set(x, y, color.RGBA{uint8(x), uint8(y), seed, 255}) + } + } + var buf bytes.Buffer + if err := jpeg.Encode(&buf, img, nil); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +// RecipePage is a page whose schema.org Recipe is title, from site, with +// two ingredients, two steps, a total time of minutes and the image at +// imageURL. +func RecipePage(title, site, imageURL string, minutes int) []byte { + return fmt.Appendf(nil, ` + +`, site, title, imageURL, minutes) +} + +// NormalizedReply is a usable normalize_recipe reply for a RecipePage with +// method and minutes. +func NormalizedReply(method string, minutes int) llm.GenerateResult { + js := fmt.Sprintf(`{"ingredients":[ + {"name":"onion","qty":"1","unit":"","note":""}, + {"name":"garlic","qty":"2","unit":"cloves","note":"minced"} + ],"method":%q,"difficulty":null,"minutes_total":%d}`, method, minutes) + return llm.GenerateResult{Response: llm.Response{Text: js, JSON: json.RawMessage(js)}} +} + +// NewGrounder returns a Grounder storing in st, with photos cached under +// dataDir, normalizing through the real harness with model. +func NewGrounder(t testing.TB, st *store.Store, dataDir string, search recipes.SearchSource, pages recipes.PageFetcher, images recipes.ImageFetcher, model llm.Provider) *recipes.Grounder { + t.Helper() + photos, err := recipes.NewPhotoCache(images, st, dataDir) + if err != nil { + t.Fatal(err) + } + guard, err := harness.NewGuard(harness.DefaultMaxCalls) + if err != nil { + t.Fatal(err) + } + cfg := harness.Config{ + Settings: llm.Settings{Provider: "anthropic", Model: "claude-test"}, + NewProvider: func(llm.Settings, string) (llm.Provider, error) { return model, nil }, + Store: st, + Guard: guard, + } + normalize := func(ctx context.Context, scope string, in skills.RecipeText) (skills.NormalizedRecipe, error) { + return skills.NormalizeRecipe(ctx, cfg, scope, in) + } + g, err := recipes.NewGrounder(st, search, pages, photos, normalize) + if err != nil { + t.Fatal(err) + } + return g +} diff --git a/internal/server/dinners.go b/internal/server/dinners.go index b3fbcaa..185b823 100644 --- a/internal/server/dinners.go +++ b/internal/server/dinners.go @@ -9,11 +9,16 @@ import ( "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/auth" "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/dinners" + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/recipes" ) // maxDinnerBody bounds a POST or PATCH /dinners request body. const maxDinnerBody = 16 << 10 +// maxGroundBody bounds a POST /dinners/{id}/ground body, which may carry a +// manual recipe of dinners.MaxManualLines lines of each kind. +const maxGroundBody = 512 << 10 + // dinnerCreate is the POST /dinners body: a name or a recipe_url. type dinnerCreate struct { Name *string `json:"name"` @@ -33,6 +38,25 @@ type dinnerList struct { Dinners []dinners.Dinner `json:"dinners"` } +// dinnerGround is the POST /dinners/{id}/ground body: exactly one of +// candidate_id, recipe_url and manual. +type dinnerGround struct { + CandidateID *string `json:"candidate_id"` + RecipeURL *string `json:"recipe_url"` + Manual *dinnerManualRecipe `json:"manual"` +} + +type dinnerManualRecipe struct { + Ingredients []string `json:"ingredients"` + Steps []string `json:"steps"` + Minutes *int `json:"minutes"` + Method *string `json:"method"` +} + +type candidateList struct { + Candidates []dinners.Candidate `json:"candidates"` +} + // getDinners answers GET /dinners?status=regular|quest_log. func getDinners(logger *slog.Logger, svc *dinners.Service) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { @@ -45,7 +69,10 @@ func getDinners(logger *slog.Logger, svc *dinners.Service) http.HandlerFunc { } } -// postDinner adds a Regular, recording the session's actor as updated_by. +// postDinner adds a Regular, recording the session's actor as updated_by, +// and answers 201 with it and its grounding_job_id without waiting for the +// job. When the job cannot be enqueued it still answers 201, with the dinner +// grounding_status failed and grounding_job_id null. func postDinner(logger *slog.Logger, svc *dinners.Service) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { actor, ok := actorOrFail(w, r, logger) @@ -87,6 +114,46 @@ func patchDinner(logger *slog.Logger, svc *dinners.Service) http.HandlerFunc { } } +// getDinnerCandidates answers GET /dinners/{id}/recipe-candidates with up +// to dinners.MaxCandidates recipe pages for the dinner's name. +func getDinnerCandidates(logger *slog.Logger, svc *dinners.Service) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + list, err := svc.Candidates(r.Context(), chi.URLParam(r, "id")) + if err != nil { + writeDinnerError(w, r, logger, "list recipe candidates", err) + return + } + writeJSON(w, r, logger, http.StatusOK, candidateList{Candidates: list}) + } +} + +// postDinnerGround grounds a dinner by hand in a candidate, a pasted link +// or a recipe typed in, recording the session's actor as updated_by. +func postDinnerGround(logger *slog.Logger, svc *dinners.Service) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + actor, ok := actorOrFail(w, r, logger) + if !ok { + return + } + body, err := decodePatch[dinnerGround](http.MaxBytesReader(w, r.Body, maxGroundBody)) + if err != nil { + writeError(w, r, logger, http.StatusBadRequest, "invalid_request", err.Error()) + return + } + fix := dinners.Fix{CandidateID: body.CandidateID, RecipeURL: body.RecipeURL} + if body.Manual != nil { + m := dinners.ManualRecipe(*body.Manual) + fix.Manual = &m + } + d, err := svc.Ground(r.Context(), chi.URLParam(r, "id"), fix, actor.Subject) + if err != nil { + writeDinnerError(w, r, logger, "ground dinner", err) + return + } + writeJSON(w, r, logger, http.StatusOK, d) + } +} + // deleteDinner archives a dinner; the row is never deleted. func deleteDinner(logger *slog.Logger, svc *dinners.Service) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { @@ -130,11 +197,32 @@ func actorOrFail(w http.ResponseWriter, r *http.Request, logger *slog.Logger) (a return actor, ok } -// writeDinnerError maps a dinners.Service error to its API error. +// writeDinnerError maps a dinners.Service error to its API error. A page +// that cannot ground the dinner is 422 ungroundable; recipe search not set +// up is 409 not_configured; a search or model failure is 502; grounding by +// hand that runs past dinners.GroundTimeout is 504 ground_timeout. func writeDinnerError(w http.ResponseWriter, r *http.Request, logger *slog.Logger, action string, err error) { var invalid *dinners.ValidationError var dup *dinners.DuplicateError + var ungroundable *recipes.ErrUngroundable switch { + case errors.As(err, &ungroundable): + // The reasons quote dependency errors, so they are neither returned + // nor logged. + writeError(w, r, logger, http.StatusUnprocessableEntity, "ungroundable", "no usable recipe was found at that link") + case errors.Is(err, recipes.ErrSearchNotConfigured): + writeError(w, r, logger, http.StatusConflict, "not_configured", "recipe search is not configured") + case errors.Is(err, recipes.ErrSearchFailed): + // The error may quote the search provider's response, so only the + // reason is logged. + logger.WarnContext(r.Context(), action, "reason", "search_failed") + writeError(w, r, logger, http.StatusBadGateway, "search_failed", "recipe search failed") + case errors.Is(err, recipes.ErrNormalizeFailed): + logger.WarnContext(r.Context(), action, "reason", "normalize_recipe failed") + writeError(w, r, logger, http.StatusBadGateway, "ai_failed", "the AI provider failed to read the recipe") + case errors.Is(err, dinners.ErrGroundTimeout): + logger.WarnContext(r.Context(), action, "reason", "ground_timeout") + writeError(w, r, logger, http.StatusGatewayTimeout, "ground_timeout", "grounding the recipe took too long; try again") case errors.As(err, &invalid): writeError(w, r, logger, http.StatusBadRequest, "invalid_dinner", invalid.Error()) case errors.As(err, &dup): diff --git a/internal/server/dinners_test.go b/internal/server/dinners_test.go index 75937a5..402b90d 100644 --- a/internal/server/dinners_test.go +++ b/internal/server/dinners_test.go @@ -1,33 +1,110 @@ package server import ( + "bytes" "context" "io" "log/slog" "net/http" "net/http/httptest" "strings" + "sync" "testing" "testing/fstest" + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm" "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/dinners" + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/jobs" + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/recipes" + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/recipes/recipestest" "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/store" ) const dinnersPath = "/api/v1/dinners" // newDinnersRouter returns a router whose dinners service runs over the -// returned store. +// returned store. Its job runner is not started, so grounding jobs stay +// queued. func newDinnersRouter(t *testing.T) (http.Handler, *store.Store) { + t.Helper() + e := newDinnersEnv(t) + return e.h, e.st +} + +// dinnersEnv is a router whose dinners service, job runner and grounder +// share one store, grounding with fake search, pages, images and model. +type dinnersEnv struct { + h http.Handler + st *store.Store + runner *jobs.Runner + search *recipestest.Search + pages *recipestest.Pages + images *recipestest.Images + model *llm.Fake + // logs holds everything the router and services log. + logs *logBuffer +} + +// logBuffer is a bytes.Buffer safe for concurrent log writes. +type logBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *logBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *logBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + +func newDinnersEnv(t *testing.T) *dinnersEnv { t.Helper() st := newTestStore(t) - logger := slog.New(slog.NewTextHandler(io.Discard, nil)) - r, err := NewRouter(logger, "dev", nil, stubAuth{}, stubSessions{}, newTestSettings(t), newTestSearch(t), newTestAI(t), - dinners.NewService(st), newTestRunner(t), newTestPhotos(t), fstest.MapFS{"index.html": {Data: []byte(fixtureIndex)}}) + e := &dinnersEnv{ + st: st, + search: &recipestest.Search{Results: map[string][]recipes.Candidate{}}, + pages: &recipestest.Pages{Bodies: map[string][]byte{}}, + images: &recipestest.Images{JPEGs: map[string][]byte{}}, + model: &llm.Fake{}, + logs: &logBuffer{}, + } + logger := slog.New(slog.NewTextHandler(e.logs, nil)) + runner, err := jobs.NewRunner(st, logger, 1) if err != nil { t.Fatal(err) } - return r, st + e.runner = runner + g := recipestest.NewGrounder(t, st, t.TempDir(), e.search, e.pages, e.images, e.model) + svc, err := dinners.NewService(st, g, runner, logger) + if err != nil { + t.Fatal(err) + } + e.h, err = NewRouter(logger, "dev", nil, stubAuth{}, stubSessions{}, newTestSettings(t), newTestSearch(t), newTestAI(t), + svc, runner, newTestPhotos(t), fstest.MapFS{"index.html": {Data: []byte(fixtureIndex)}}) + if err != nil { + t.Fatal(err) + } + return e +} + +// start starts the env's job runner, stopping it when the test ends. +func (e *dinnersEnv) start(t *testing.T) { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + if err := e.runner.Start(ctx); err != nil { + cancel() + t.Fatal(err) + } + t.Cleanup(func() { + cancel() + e.runner.Wait() + }) } // callDinners sends a signed-in request with body (none when empty). diff --git a/internal/server/grounding_test.go b/internal/server/grounding_test.go new file mode 100644 index 0000000..1365f9e --- /dev/null +++ b/internal/server/grounding_test.go @@ -0,0 +1,303 @@ +package server + +import ( + "fmt" + "net/http" + "strings" + "testing" + "time" + + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/recipes" + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/recipes/recipestest" +) + +// Fixture pages: two curries found by searching "Chicken curry" and a stew +// at a pasted link. +const ( + curryPage = "https://a.example/chicken-curry" + curry2Page = "https://b.example/curry" + stewPage = "https://c.example/recipes/beef-stew" +) + +// withRecipes adds the fixture pages, their photos and the search for +// "Chicken curry" to e. +func (e *dinnersEnv) withRecipes(t *testing.T) *dinnersEnv { + t.Helper() + e.search.Results["Chicken curry"] = []recipes.Candidate{{URL: curryPage}, {URL: curry2Page}} + e.pages.Bodies[curryPage] = recipestest.RecipePage("Chicken Curry", "Curry House", "https://a.example/curry.jpg", 40) + e.pages.Bodies[curry2Page] = recipestest.RecipePage("Weeknight Curry", "B Kitchen", "https://b.example/curry.jpg", 25) + e.pages.Bodies[stewPage] = recipestest.RecipePage("Beef Stew", "Stew Co", "https://c.example/stew.jpg", 150) + e.images.JPEGs["https://a.example/curry.jpg"] = recipestest.JPEG(t, 400, 320, 1) + e.images.JPEGs["https://b.example/curry.jpg"] = recipestest.JPEG(t, 400, 320, 2) + e.images.JPEGs["https://c.example/stew.jpg"] = recipestest.JPEG(t, 400, 320, 3) + return e +} + +// waitJob polls GET /jobs/{id} until the job ends and returns its body. +func waitJob(t *testing.T, h http.Handler, id string) map[string]any { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for { + rec, body := callDinners(t, h, http.MethodGet, "/api/v1/jobs/"+id, "") + if rec.Code != http.StatusOK { + t.Fatalf("GET job = %d: %s", rec.Code, rec.Body) + } + if s := body["status"]; s == "succeeded" || s == "failed" { + return body + } + if time.Now().After(deadline) { + t.Fatalf("job %s never finished; last %v", id, body) + } + time.Sleep(5 * time.Millisecond) + } +} + +// listedDinner returns dinner id as GET /dinners?status=regular lists it. +func listedDinner(t *testing.T, h http.Handler, id string) map[string]any { + t.Helper() + rec, body := callDinners(t, h, http.MethodGet, dinnersPath+"?status=regular", "") + if rec.Code != http.StatusOK { + t.Fatalf("GET dinners = %d: %s", rec.Code, rec.Body) + } + for _, d := range body["dinners"].([]any) { + if d := d.(map[string]any); d["id"] == id { + return d + } + } + t.Fatalf("dinner %s not listed", id) + return nil +} + +func TestPostDinnerReturnsBeforeGroundingCompletes(t *testing.T) { + t.Parallel() + e := newDinnersEnv(t).withRecipes(t) + e.search.Gate = make(chan struct{}) + e.model.QueueGenerate(recipestest.NormalizedReply("stovetop", 40)) + e.start(t) + + rec, created := callDinners(t, e.h, http.MethodPost, dinnersPath, `{"name":"Chicken curry"}`) + if rec.Code != http.StatusCreated { + t.Fatalf("POST = %d: %s", rec.Code, rec.Body) + } + jobID, _ := created["grounding_job_id"].(string) + if created["grounding_status"] != "pending" || created["grounding_error"] != nil || jobID == "" { + t.Fatalf("created = %v, want pending with a grounding_job_id", created) + } + // The search is held, so grounding cannot have completed. + _, job := callDinners(t, e.h, http.MethodGet, "/api/v1/jobs/"+jobID, "") + if s := job["status"]; s != "queued" && s != "running" { + t.Fatalf("job right after POST = %v, want queued or running", job) + } + if d := listedDinner(t, e.h, created["id"].(string)); d["grounding_status"] != "pending" { + t.Fatalf("dinner right after POST = %v, want pending", d) + } + + close(e.search.Gate) + job = waitJob(t, e.h, jobID) + if job["status"] != "succeeded" || job["kind"] != "ground_dinner" { + t.Fatalf("job = %v, want ground_dinner succeeded", job) + } + d := listedDinner(t, e.h, created["id"].(string)) + photo, _ := d["photo_url"].(string) + if d["grounding_status"] != "grounded" || d["recipe_id"] == nil || !strings.HasSuffix(photo, "-400.jpg") || + d["method"] != "stovetop" || d["minutes_total"] != 40.0 || d["difficulty"] != "easy" || d["grounding_error"] != nil { + t.Errorf("dinner = %v, want grounded with recipe, photo, method, minutes and difficulty", d) + } +} + +func TestPostDinnerGroundingFailure(t *testing.T) { + t.Parallel() + e := newDinnersEnv(t).withRecipes(t) + e.start(t) + + rec, created := callDinners(t, e.h, http.MethodPost, dinnersPath, `{"name":"Nothing matches this"}`) + if rec.Code != http.StatusCreated { + t.Fatalf("POST = %d: %s", rec.Code, rec.Body) + } + job := waitJob(t, e.h, created["grounding_job_id"].(string)) + if job["status"] != "failed" || job["error"] != "no_recipe_found" { + t.Errorf("job = %v, want failed with no_recipe_found", job) + } + d := listedDinner(t, e.h, created["id"].(string)) + if d["grounding_status"] != "failed" || d["grounding_error"] != "no_recipe_found" || d["recipe_id"] != nil { + t.Errorf("dinner = %v, want failed with no_recipe_found", d) + } +} + +func TestGetRecipeCandidates(t *testing.T) { + t.Parallel() + e := newDinnersEnv(t).withRecipes(t) + id := createDinner(t, e.h, `{"name":"Chicken curry"}`) // runner not started: stays pending + + rec, body := callDinners(t, e.h, http.MethodGet, dinnersPath+"/"+id+"/recipe-candidates", "") + if rec.Code != http.StatusOK { + t.Fatalf("GET = %d: %s", rec.Code, rec.Body) + } + list, _ := body["candidates"].([]any) + if len(list) != 2 { + t.Fatalf("candidates = %v, want 2", body) + } + first := list[0].(map[string]any) + want := map[string]any{ + "candidate_id": curryPage, + "title": "Chicken Curry", + "site_name": "Curry House", + "photo_url": nil, + "minutes": 40.0, + } + if len(first) != len(want) { + t.Errorf("candidate keys = %v, want exactly %v", first, want) + } + for k, v := range want { + if got, ok := first[k]; !ok || got != v { + t.Errorf("%s = %#v (present %v), want %#v", k, got, ok, v) + } + } + + rec, body = callDinners(t, e.h, http.MethodGet, dinnersPath+"/nope/recipe-candidates", "") + if rec.Code != http.StatusNotFound || errorCode(body) != "not_found" { + t.Errorf("unknown dinner = %d %v, want 404", rec.Code, body) + } +} + +func TestGetRecipeCandidatesSearchFails(t *testing.T) { + t.Parallel() + e := newDinnersEnv(t) + // The provider's text must reach neither the response nor the logs. + const leak = "provider-said-token-abc123" + e.search.Err = fmt.Errorf("%w: %s", recipes.ErrSearchRefused, leak) + id := createDinner(t, e.h, `{"name":"Chicken curry"}`) + rec, body := callDinners(t, e.h, http.MethodGet, dinnersPath+"/"+id+"/recipe-candidates", "") + if rec.Code != http.StatusBadGateway || errorCode(body) != "search_failed" { + t.Errorf("GET = %d %v, want 502 search_failed", rec.Code, body) + } + if strings.Contains(rec.Body.String(), leak) { + t.Errorf("response quotes the provider: %s", rec.Body) + } + logs := e.logs.String() + if strings.Contains(logs, leak) || !strings.Contains(logs, "reason=search_failed") { + t.Errorf("logs = %q, want reason=search_failed and no provider text", logs) + } +} + +func TestPostGroundViaCandidate(t *testing.T) { + t.Parallel() + e := newDinnersEnv(t).withRecipes(t) + id := createDinner(t, e.h, `{"name":"Chicken curry"}`) + e.model.QueueGenerate(recipestest.NormalizedReply("stovetop", 25)) + + rec, d := callDinners(t, e.h, http.MethodPost, dinnersPath+"/"+id+"/ground", `{"candidate_id":"`+curry2Page+`"}`) + if rec.Code != http.StatusOK { + t.Fatalf("POST = %d: %s", rec.Code, rec.Body) + } + photo, _ := d["photo_url"].(string) + if d["grounding_status"] != "grounded" || d["recipe_id"] == nil || photo == "" || + d["method"] != "stovetop" || d["minutes_total"] != 25.0 || d["difficulty"] != "easy" || d["updated_by"] != testSession.Subject { + t.Errorf("dinner = %v", d) + } + // The candidate is now stored, so it is offered with its photo. + _, body := callDinners(t, e.h, http.MethodGet, dinnersPath+"/"+id+"/recipe-candidates", "") + for _, c := range body["candidates"].([]any) { + if c := c.(map[string]any); c["candidate_id"] == curry2Page && c["photo_url"] != photo { + t.Errorf("grounded candidate photo_url = %v, want %s", c["photo_url"], photo) + } + } +} + +func TestPostGroundViaRecipeURL(t *testing.T) { + t.Parallel() + e := newDinnersEnv(t).withRecipes(t) + id := createDinner(t, e.h, `{"name":"Sunday stew"}`) + e.model.QueueGenerate(recipestest.NormalizedReply("oven", 150)) + + rec, d := callDinners(t, e.h, http.MethodPost, dinnersPath+"/"+id+"/ground", `{"recipe_url":"`+stewPage+`"}`) + if rec.Code != http.StatusOK { + t.Fatalf("POST = %d: %s", rec.Code, rec.Body) + } + if d["grounding_status"] != "grounded" || d["photo_url"] == nil || d["method"] != "oven" || d["minutes_total"] != 150.0 { + t.Errorf("dinner = %v", d) + } + + rec, body := callDinners(t, e.h, http.MethodPost, dinnersPath+"/"+id+"/ground", `{"recipe_url":"https://gone.example/soup"}`) + if rec.Code != http.StatusUnprocessableEntity || errorCode(body) != "ungroundable" || strings.Contains(rec.Body.String(), "gone.example") { + t.Errorf("unusable link = %d %v, want 422 ungroundable", rec.Code, body) + } +} + +func TestPostGroundViaManual(t *testing.T) { + t.Parallel() + e := newDinnersEnv(t) + id := createDinner(t, e.h, `{"name":"Family chili"}`) + + rec, d := callDinners(t, e.h, http.MethodPost, dinnersPath+"/"+id+"/ground", + `{"manual":{"ingredients":["1 lb beef","1 can beans"],"steps":["Brown.","Simmer."],"minutes":240,"method":"crockpot"}}`) + if rec.Code != http.StatusOK { + t.Fatalf("POST = %d: %s", rec.Code, rec.Body) + } + if d["grounding_status"] != "manual" || d["recipe_id"] == nil || d["photo_url"] != nil || + d["method"] != "crockpot" || d["minutes_total"] != 240.0 || d["grounding_error"] != nil { + t.Errorf("dinner = %v", d) + } + var lines int + if err := e.st.Reader.QueryRow(`SELECT count(*) FROM recipe_ingredients WHERE recipe_id = ?`, d["recipe_id"]).Scan(&lines); err != nil { + t.Fatal(err) + } + if lines != 2 { + t.Errorf("stored %d ingredient lines, want 2", lines) + } +} + +func TestPostGroundRejects(t *testing.T) { + t.Parallel() + e := newDinnersEnv(t) + id := createDinner(t, e.h, `{"name":"Pad thai"}`) + path := dinnersPath + "/" + id + "/ground" + + for _, tc := range []struct{ name, body, code string }{ + {"empty", `{}`, "invalid_dinner"}, + {"two kinds", `{"candidate_id":"https://a.example/x","recipe_url":"https://a.example/x"}`, "invalid_dinner"}, + {"candidate not a link", `{"candidate_id":"c1"}`, "invalid_dinner"}, + {"http link", `{"recipe_url":"http://a.example/x"}`, "invalid_dinner"}, + {"null", `{"recipe_url":null}`, "invalid_request"}, + {"unknown field", `{"url":"https://a.example/x"}`, "invalid_request"}, + {"unknown manual field", `{"manual":{"ingredients":["x"],"minutes":5,"method":"oven","serves":4}}`, "invalid_request"}, + {"manual without method", `{"manual":{"ingredients":["x"],"minutes":5}}`, "invalid_dinner"}, + {"manual without ingredients", `{"manual":{"ingredients":[],"minutes":5,"method":"oven"}}`, "invalid_dinner"}, + {"manual minutes too many", `{"manual":{"ingredients":["x"],"minutes":5000,"method":"oven"}}`, "invalid_dinner"}, + {"not an object", `[]`, "invalid_request"}, + } { + rec, body := callDinners(t, e.h, http.MethodPost, path, tc.body) + if rec.Code != http.StatusBadRequest || errorCode(body) != tc.code { + t.Errorf("%s: %d %v, want 400 %s", tc.name, rec.Code, body, tc.code) + } + } + if d := listedDinner(t, e.h, id); d["grounding_status"] != "pending" { + t.Errorf("dinner after rejected fixes = %v, want unchanged", d) + } + + rec, body := callDinners(t, e.h, http.MethodPost, dinnersPath+"/nope/ground", + `{"manual":{"ingredients":["x"],"minutes":5,"method":"oven"}}`) + if rec.Code != http.StatusNotFound || errorCode(body) != "not_found" { + t.Errorf("unknown dinner = %d %v, want 404", rec.Code, body) + } +} + +func TestPostDinnerEnqueueFailureAnswersFailedDinner(t *testing.T) { + t.Parallel() + e := newDinnersEnv(t) + if _, err := e.st.Writer.ExecContext(t.Context(), + `CREATE TRIGGER refuse_jobs BEFORE INSERT ON jobs BEGIN SELECT RAISE(ABORT, 'no jobs'); END`); err != nil { + t.Fatal(err) + } + rec, created := callDinners(t, e.h, http.MethodPost, dinnersPath, `{"name":"Chicken curry"}`) + if rec.Code != http.StatusCreated { + t.Fatalf("POST = %d: %s", rec.Code, rec.Body) + } + id, _ := created["id"].(string) + if id == "" || created["grounding_status"] != "failed" || created["grounding_error"] != "grounding_error" || created["grounding_job_id"] != nil { + t.Fatalf("created = %v, want its id, failed (grounding_error) and a null grounding_job_id", created) + } + if got := listedDinner(t, e.h, id); got["grounding_status"] != "failed" { + t.Errorf("listed = %v, want failed", got) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index dda300d..8eaf64c 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -36,7 +36,9 @@ type RouteMounter interface { // /api/v1/settings/recipe-search and its /test; ai serves // /api/v1/settings/ai, its /api-key, /verify, /models and /reset-prompt; // dinnerSvc serves GET and POST /api/v1/dinners, PATCH and DELETE -// /api/v1/dinners/{id} and POST /api/v1/dinners/{id}/favorite; runner serves +// /api/v1/dinners/{id}, POST /api/v1/dinners/{id}/favorite, GET +// /api/v1/dinners/{id}/recipe-candidates and POST /api/v1/dinners/{id}/ground; +// runner serves // POST /api/v1/jobs and GET /api/v1/jobs/{id}; photos serves // GET /media/photos/{id}-{size}.jpg; assets is the web build // served for every other GET. /health, /auth and the web build are public. @@ -102,6 +104,8 @@ func NewRouter(logger *slog.Logger, version string, trusted forwarded.Trusted, a apiV1.Patch("/dinners/{id}", patchDinner(logger, dinnerSvc)) apiV1.Delete("/dinners/{id}", deleteDinner(logger, dinnerSvc)) apiV1.Post("/dinners/{id}/favorite", postDinnerFavorite(logger, dinnerSvc)) + apiV1.Get("/dinners/{id}/recipe-candidates", getDinnerCandidates(logger, dinnerSvc)) + apiV1.Post("/dinners/{id}/ground", postDinnerGround(logger, dinnerSvc)) apiV1.Post("/jobs", postJob(logger, runner)) apiV1.Get("/jobs/{id}", getJob(logger, runner)) media.Get("/photos/{name}", getPhoto(logger, photos)) diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 2df926d..5f121ce 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -17,10 +17,12 @@ import ( "github.com/go-chi/chi/v5" "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/aisettings" + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm" "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/dinners" "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/household" "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/jobs" "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/recipes" + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/recipes/recipestest" "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/secretbox" "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/store" "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/migrations" @@ -140,10 +142,22 @@ func newTestSearch(t *testing.T) *recipes.SearchSettingsService { return search } -// newTestDinners returns a dinners service over a migrated temp database. +// newTestDinners returns a dinners service over a migrated temp database, +// with its own unstarted job runner and a grounder that finds nothing. func newTestDinners(t *testing.T) *dinners.Service { t.Helper() - return dinners.NewService(newTestStore(t)) + st := newTestStore(t) + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + runner, err := jobs.NewRunner(st, logger, 1) + if err != nil { + t.Fatal(err) + } + g := recipestest.NewGrounder(t, st, t.TempDir(), &recipestest.Search{}, &recipestest.Pages{}, &recipestest.Images{}, &llm.Fake{}) + svc, err := dinners.NewService(st, g, runner, logger) + if err != nil { + t.Fatal(err) + } + return svc } // newTestBox returns a secret box under a random key. diff --git a/internal/store/migrate_test.go b/internal/store/migrate_test.go index 91c4592..1c0c2a6 100644 --- a/internal/store/migrate_test.go +++ b/internal/store/migrate_test.go @@ -175,8 +175,8 @@ func TestMigrateEmbeddedMigrations(t *testing.T) { if err := Migrate(context.Background(), s.Writer, migrations.FS); err != nil { t.Fatal(err) } - if got := versions(migrationRows(t, s)); got != "0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai,0006_ai_runs_status,0007_recipe_grounding,0008_dinner_recipe_url" { - t.Errorf("versions = %q, want 0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai,0006_ai_runs_status,0007_recipe_grounding,0008_dinner_recipe_url", got) + if got := versions(migrationRows(t, s)); got != "0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai,0006_ai_runs_status,0007_recipe_grounding,0008_dinner_recipe_url,0009_dinner_grounding_error" { + t.Errorf("versions = %q, want 0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai,0006_ai_runs_status,0007_recipe_grounding,0008_dinner_recipe_url,0009_dinner_grounding_error", got) } } @@ -196,8 +196,8 @@ func TestMigrateBootstrapThenEmbeddedInitApplies(t *testing.T) { if err := Migrate(ctx, s.Writer, migrations.FS); err != nil { t.Fatal(err) } - if got := versions(migrationRows(t, s)); got != "0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai,0006_ai_runs_status,0007_recipe_grounding,0008_dinner_recipe_url" { - t.Errorf("versions = %q, want 0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai,0006_ai_runs_status,0007_recipe_grounding,0008_dinner_recipe_url", got) + if got := versions(migrationRows(t, s)); got != "0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai,0006_ai_runs_status,0007_recipe_grounding,0008_dinner_recipe_url,0009_dinner_grounding_error" { + t.Errorf("versions = %q, want 0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai,0006_ai_runs_status,0007_recipe_grounding,0008_dinner_recipe_url,0009_dinner_grounding_error", got) } if !tableExists(t, s, "dinners") { t.Error("table from 0001_init was not created") diff --git a/migrations/0009_dinner_grounding_error.sql b/migrations/0009_dinner_grounding_error.sql new file mode 100644 index 0000000..7ce9a4b --- /dev/null +++ b/migrations/0009_dinner_grounding_error.sql @@ -0,0 +1,7 @@ +-- 0009_dinner_grounding_error: a dinner whose grounding failed keeps why, +-- as one of the reason codes package dinners defines (no_recipe_found, +-- recipe_unusable, search_not_configured, search_failed, ai_failed, +-- interrupted, grounding_error). It is NULL unless grounding_status is +-- failed. + +ALTER TABLE dinners ADD COLUMN grounding_error TEXT; diff --git a/migrations/queries/dinners.sql b/migrations/queries/dinners.sql index a0a9ca1..54eaa3d 100644 --- a/migrations/queries/dinners.sql +++ b/migrations/queries/dinners.sql @@ -70,3 +70,59 @@ SET is_favorite = 1 - is_favorite, updated_at = ? WHERE id = ? RETURNING *; + +-- name: GroundDinner :one +-- Records a dinner's grounding (grounded or manual) and clears any +-- grounding_error. An unknown id returns sql.ErrNoRows. +UPDATE dinners +SET recipe_id = ?, + photo_id = ?, + method = ?, + minutes_total = ?, + difficulty = ?, + grounding_status = ?, + grounding_error = NULL, + updated_by = ?, + updated_at = ? +WHERE id = ? +RETURNING *; + +-- name: GroundPendingDinner :execrows +-- Records the background grounding of a dinner still pending; a dinner +-- grounded some other way meanwhile is left as it is (0 rows). +UPDATE dinners +SET recipe_id = ?, + photo_id = ?, + method = ?, + minutes_total = ?, + difficulty = ?, + grounding_status = 'grounded', + grounding_error = NULL, + updated_at = ? +WHERE id = ? AND grounding_status = 'pending'; + +-- name: FailPendingDinnerGrounding :execrows +-- Records why the background grounding of a dinner still pending failed; a +-- dinner grounded some other way meanwhile is left as it is (0 rows). +UPDATE dinners +SET grounding_status = 'failed', + grounding_error = ?, + updated_at = ? +WHERE id = ? AND grounding_status = 'pending'; + +-- name: FailOrphanedPendingDinnerGroundings :execrows +-- Fails every dinner still pending whose grounding job is no longer queued +-- or running: the process stopped before the job, or before its handler +-- recorded a result. Run at boot, after the job runner fails the jobs a +-- previous process left running. +UPDATE dinners +SET grounding_status = 'failed', + grounding_error = sqlc.arg(grounding_error), + updated_at = sqlc.arg(updated_at) +WHERE grounding_status = 'pending' + AND NOT EXISTS ( + SELECT 1 FROM jobs + WHERE jobs.kind = sqlc.arg(kind) + AND jobs.status IN ('queued', 'running') + AND json_extract(jobs.payload, '$.dinner_id') = dinners.id + ); diff --git a/migrations/queries/recipes.sql b/migrations/queries/recipes.sql index ddb7808..bbca037 100644 --- a/migrations/queries/recipes.sql +++ b/migrations/queries/recipes.sql @@ -69,3 +69,15 @@ ORDER BY position, id; -- name: DeleteRecipeSteps :exec DELETE FROM recipe_steps WHERE recipe_id = ?; + +-- name: CreateManualRecipe :one +-- A recipe the household typed in. Its source_url is a mealquest:manual: +-- URN rather than a page, so it never matches a fetched page. +INSERT INTO recipes ( + id, source_url, title, total_minutes, raw_jsonld, method, + minutes_total, fetched_at, created_at, updated_at +) VALUES ( + ?, ?, ?, ?, ?, ?, + ?, ?, ?, ? +) +RETURNING *;