Never show ideas for a failed ideas job, however it failed

A failed ideas_from_ingredients job could still show ideas through
GET /api/v1/jobs/{id}: a job failed at boot as interrupted by restart
keeps its last partial result, and when the write clearing the ideas
itself failed the runner recorded internal_error over the uncleared
result. Neither path can be fixed by a write.

The jobs package gains an opt-in per-kind hook, Kind.FailedResult,
which Runner.Get applies to a failed job's stored result on every read.
The ideas kind registers withoutIdeas, which empties the ideas and keeps
the rest of the result. Kinds that do not call FailedResult are
unchanged. The write-side clear stays as a tidy-up of what is stored.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

docs/ARCHITECTURE.md

@@ -332,8 +332,12 @@ automatic** (D-021).
   taking its place before those shown. It fails with `no_ideas_found`
   (every idea was dropped), `ai_failed` (the skill or `normalize_recipe`
   failed; no idea is made up instead, D-017), `search_not_configured` or
-  `search_failed`. Before it fails it publishes its result again with no
-  ideas, so a failed job shows none, even those kept before the failure.
+  `search_failed`. A failed job shows no ideas, even those kept before the
+  failure: the kind registers a `FailedResult` view, so `GET
+  /api/v1/jobs/{id}` empties the `ideas` of any failed job's result on
+  read, however it failed (including `interrupted by restart` and an
+  `internal_error` whose clearing write failed). Before it fails it also
+  publishes its result again with no ideas, to tidy what is stored.
   `GET /api/v1/suggestions/deck` returns `{suggestions: [{id, dinner_id,
   name, photo_url, difficulty, method, minutes_total, cuisine, reason,
   cited_regular_ids, status, dinner_status, decided_at, created_at}],

internal/ideas/ideas.go

@@ -99,6 +99,9 @@ func NewService(grounder *recipes.Grounder, generate Generator, runner *jobs.Run
 	if err := kind.ServiceOnly(); err != nil {
 		return nil, err
 	}
+	if err := kind.FailedResult(withoutIdeas); err != nil {
+		return nil, err
+	}
 	s.job = kind
 	return s, nil
 }
@@ -192,8 +195,10 @@ func IsQuick(method string, minutes *int) bool {
 // other grounding failure fails the job, as does ideas_from_ingredients
 // failing, with no idea made up instead. The result is published at each
 // stage and again as each idea is kept, so it fills as the ideas ground.
-// Before the job fails, its result is published again with no ideas, so a
-// failed job never shows ideas kept before the failure.
+// Before the job fails, its result is published again with no ideas, to
+// tidy the stored result. That write is not what keeps a failed job from
+// showing ideas, since it may fail and a job interrupted by restart never
+// makes it: withoutIdeas, applied whenever a failed job is read, is.
 func (s *Service) run(ctx context.Context, p FromIngredientsJob, publish func(Result) error) (Result, error) {
 	res, err := s.fill(ctx, p, publish)
 	if err != nil {
@@ -316,6 +321,18 @@ func (s *Service) ground(ctx context.Context, ideas []skills.Idea, kept func(i i
 	return failed
 }
 
+// withoutIdeas is a failed job's stored result as the job shows it: with no
+// ideas, however the job failed, so a failed job never shows ideas kept
+// before the failure.
+func withoutIdeas(stored json.RawMessage) (json.RawMessage, error) {
+	var res Result
+	if err := json.Unmarshal(stored, &res); err != nil {
+		return nil, fmt.Errorf("ideas: decode failed job's result: %w", err)
+	}
+	res.Ideas = []Idea{}
+	return res.PublicJSON()
+}
+
 // ideaOf is idea as grounded by gr.
 func ideaOf(idea skills.Idea, gr recipes.Grounded) Idea {
 	return Idea{

internal/jobs/jobs.go

@@ -135,12 +135,14 @@ type Job struct {
 type runFunc func(ctx context.Context, payload string, publish func(json.RawMessage) error) (result json.RawMessage, code, reason string)
 
 // handler is a registered kind: run runs a job, check reports whether a
-// payload decodes into the kind's payload type, and serviceOnly is set when
-// Runner.Enqueue refuses the kind.
+// payload decodes into the kind's payload type, serviceOnly is set when
+// Runner.Enqueue refuses the kind, and failedResult, when set, is what Get
+// shows of a failed job's stored result (see Kind.FailedResult).
 type handler struct {
-	run         runFunc
-	check       func(payload []byte) bool
-	serviceOnly bool
+	run          runFunc
+	check        func(payload []byte) bool
+	serviceOnly  bool
+	failedResult func(stored json.RawMessage) (json.RawMessage, error)
 }
 
 // Runner owns the job handlers and the worker pool.
@@ -307,6 +309,27 @@ func (k *Kind[P]) ServiceOnly() error {
 	return nil
 }
 
+// FailedResult makes Get return, for a failed job of the kind with a
+// stored result, show's view of that result instead of the result itself.
+// It is applied on every read, so it holds however the job failed: by its
+// handler, as interrupted by restart, or after a write meant to tidy the
+// result failed. A show error fails Get. A kind that never calls it shows a
+// failed job's last result as stored. It must be called before Start.
+func (k *Kind[P]) FailedResult(show func(stored json.RawMessage) (json.RawMessage, error)) error {
+	if show == nil {
+		return fmt.Errorf("jobs: failed result %q: nil func", k.name)
+	}
+	k.r.mu.Lock()
+	defer k.r.mu.Unlock()
+	if k.r.started {
+		return fmt.Errorf("jobs: failed result %q: runner already started", k.name)
+	}
+	h := k.r.handlers[k.name]
+	h.failedResult = show
+	k.r.handlers[k.name] = h
+	return nil
+}
+
 // Wake tells an idle worker to look for a job, as Enqueue does itself:
 // call it after committing the transaction a job was enqueued in.
 func (k *Kind[P]) Wake() { k.r.wakeOne() }
@@ -402,7 +425,8 @@ func (r *Runner) Wait() {
 	r.wg.Wait()
 }
 
-// Get returns the job with id, or ErrNotFound.
+// Get returns the job with id, or ErrNotFound. A failed job's result is
+// shown through its kind's FailedResult, when the kind set one.
 func (r *Runner) Get(ctx context.Context, id string) (Job, error) {
 	row, err := db.New(r.st.Reader).GetJob(ctx, id)
 	if errors.Is(err, sql.ErrNoRows) {
@@ -418,6 +442,21 @@ func (r *Runner) Get(ctx context.Context, id string) (Job, error) {
 	if row.Error.Valid {
 		job.Error = &row.Error.String
 	}
+	if job.Status == StatusFailed && job.Result != nil {
+		r.mu.Lock()
+		show := r.handlers[job.Kind].failedResult
+		r.mu.Unlock()
+		if show != nil {
+			shown, err := show(job.Result)
+			if err != nil {
+				return Job{}, fmt.Errorf("jobs: show failed %s result: %w", job.Kind, err)
+			}
+			if !json.Valid(shown) {
+				return Job{}, fmt.Errorf("jobs: show failed %s result: not one JSON value", job.Kind)
+			}
+			job.Result = shown
+		}
+	}
 	return job, nil
 }
 

internal/jobs/jobs_test.go

@@ -772,3 +772,47 @@ func TestServiceOnlyKind(t *testing.T) {
 		t.Error("ServiceOnly after Start succeeded, want an error")
 	}
 }
+
+func TestFailedResultAppliesOnlyToItsKindsFailedJobs(t *testing.T) {
+	t.Parallel()
+	st := migratedStore(t)
+	r := newRunner(t, st, 1)
+	h := func(_ context.Context, p countPayload, publish func(countResult) error) (countResult, error) {
+		if err := publish(countResult{Done: 5}); err != nil {
+			return countResult{}, err
+		}
+		if p.Steps == 0 {
+			return countResult{}, &Failure{Code: "stopped"}
+		}
+		return countResult{Done: p.Steps}, nil
+	}
+	hidden := register(t, r, "hidden", h, "stopped")
+	plain := register(t, r, "plain", h, "stopped")
+	if err := hidden.FailedResult(func(json.RawMessage) (json.RawMessage, error) {
+		return json.RawMessage(`{"done":0}`), nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if err := hidden.FailedResult(nil); err == nil {
+		t.Error("FailedResult(nil) succeeded, want an error")
+	}
+	start(t, r)
+	if err := hidden.FailedResult(func(s json.RawMessage) (json.RawMessage, error) { return s, nil }); err == nil {
+		t.Error("FailedResult after Start succeeded, want an error")
+	}
+
+	for _, tc := range []struct {
+		kind  *Kind[countPayload]
+		steps int
+		want  string
+	}{
+		{hidden, 0, `{"done":0}`}, // failed: shown through FailedResult
+		{hidden, 3, `{"done":3}`}, // succeeded: as stored
+		{plain, 0, `{"done":5}`},  // failed, kind without FailedResult: as stored
+	} {
+		job := waitFor(t, r, enqueue(t, tc.kind, countPayload{Steps: tc.steps}), finished)
+		if string(job.Result) != tc.want {
+			t.Errorf("%s job with %d steps: %s result = %s, want %s", job.Kind, tc.steps, job.Status, job.Result, tc.want)
+		}
+	}
+}

internal/server/ideas_test.go

@@ -2,6 +2,7 @@ package server
 
 import (
 	"context"
+	"database/sql"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -16,10 +17,12 @@ import (
 	"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/harness"
 	"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm"
 	"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/skills"
+	"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/db"
 	"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ideas"
 	"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/jobs"
 	"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/recipes"
 	"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/recipes/recipestest"
+	"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/store"
 )
 
 const ideasPath = "/api/v1/ideas/from-ingredients"
@@ -27,8 +30,9 @@ const ideasPath = "/api/v1/ideas/from-ingredients"
 // newIdeasRouter returns a router whose ideas service runs
 // ideas_from_ingredients through the real harness with the scripted model
 // and grounds with fake search, pages and images and the scripted
-// normalize. Its job runner is started when start is set.
-func newIdeasRouter(t *testing.T, model, normalize *llm.Fake, search *recipestest.Search, pages *recipestest.Pages, images *recipestest.Images, start bool) (http.Handler, *jobs.Runner) {
+// normalize, with its job runner and store. The runner is started when
+// start is set.
+func newIdeasRouter(t *testing.T, model, normalize *llm.Fake, search *recipestest.Search, pages *recipestest.Pages, images *recipestest.Images, start bool) (http.Handler, *jobs.Runner, *store.Store) {
 	t.Helper()
 	st := newTestStore(t)
 	logger := slog.New(slog.NewTextHandler(io.Discard, nil))
@@ -60,22 +64,93 @@ func newIdeasRouter(t *testing.T, model, normalize *llm.Fake, search *recipestes
 		t.Fatal(err)
 	}
 	if start {
-		ctx, cancel := context.WithCancel(context.Background())
-		if err := runner.Start(ctx); err != nil {
-			cancel()
-			t.Fatal(err)
-		}
-		t.Cleanup(func() {
-			cancel()
-			runner.Wait()
-		})
+		startRunner(t, runner)
 	}
-	return h, runner
+	return h, runner, st
+}
+
+// startRunner starts runner until the test ends.
+func startRunner(t *testing.T, runner *jobs.Runner) {
+	t.Helper()
+	ctx, cancel := context.WithCancel(context.Background())
+	if err := runner.Start(ctx); err != nil {
+		cancel()
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		cancel()
+		runner.Wait()
+	})
+}
+
+// scriptIdeas scripts ideas_from_ingredients to propose names, in order;
+// a recipe grounds those in grounds, the others ground none, and search
+// fails as fail says. It returns the router's fakes.
+func scriptIdeas(t *testing.T, names, grounds []string, fail map[string]error) (*llm.Fake, *llm.Fake, *recipestest.Search, *recipestest.Pages, *recipestest.Images) {
+	t.Helper()
+	model, normalize := &llm.Fake{}, &llm.Fake{}
+	search := &recipestest.Search{Results: map[string][]recipes.Candidate{}, Fail: fail}
+	pages := &recipestest.Pages{Bodies: map[string][]byte{}}
+	images := &recipestest.Images{JPEGs: map[string][]byte{}}
+	for i, name := range grounds {
+		page, photo := fmt.Sprintf("https://r.example/%d", i), fmt.Sprintf("https://r.example/%d.jpg", i)
+		search.Results[name] = []recipes.Candidate{{URL: page}}
+		pages.Bodies[page] = recipestest.RecipePage(name, "R Kitchen", photo, 25)
+		images.JPEGs[photo] = recipestest.JPEG(t, 400, 320, uint8(i+1))
+		normalize.QueueGenerate(recipestest.NormalizedReply("stovetop", 25))
+	}
+	list := make([]string, len(names))
+	for i, name := range names {
+		list[i] = fmt.Sprintf(`{"name":%q,"uses":["rice"],"missing":[],"est_minutes":30,"method":"stovetop","difficulty":"easy","search_query":%q}`, name, name)
+	}
+	js := `{"ideas":[` + strings.Join(list, ",") + `]}`
+	model.QueueGenerate(llm.GenerateResult{Response: llm.Response{Text: js, JSON: json.RawMessage(js)}})
+	return model, normalize, search, pages, images
+}
+
+// startIdeasJob posts a request for ideas from rice and returns the job id.
+func startIdeasJob(t *testing.T, h http.Handler) string {
+	t.Helper()
+	rec, body := callDinners(t, h, http.MethodPost, ideasPath, `{"ingredients": ["rice"], "quick_only": false}`)
+	if rec.Code != http.StatusAccepted {
+		t.Fatalf("POST = %d: %s", rec.Code, rec.Body)
+	}
+	return body["jobId"].(string)
+}
+
+// assertFailedWithoutIdeas checks that GET /jobs/{id} reports job failed
+// with code and a result showing no ideas.
+func assertFailedWithoutIdeas(t *testing.T, job map[string]any, code string) {
+	t.Helper()
+	if job["status"] != "failed" || job["error"] != code {
+		t.Fatalf("GET job = %v, want failed with %s", job, code)
+	}
+	res, _ := job["result"].(map[string]any)
+	if res == nil {
+		t.Fatalf("failed job result = %v, want one showing no ideas", job["result"])
+	}
+	if list, ok := res["ideas"].([]any); !ok || len(list) != 0 {
+		t.Fatalf("failed job ideas = %v, want none", res["ideas"])
+	}
+}
+
+// storedIdeas is how many ideas job id's stored result holds.
+func storedIdeas(t *testing.T, st *store.Store, id string) int {
+	t.Helper()
+	row, err := db.New(st.Reader).GetJob(t.Context(), id)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var res ideas.Result
+	if err := json.Unmarshal([]byte(row.Result.String), &res); err != nil {
+		t.Fatal(err)
+	}
+	return len(res.Ideas)
 }
 
 func TestPostIdeasFromIngredientsRefusesBadRequests(t *testing.T) {
 	t.Parallel()
-	h, _ := newIdeasRouter(t, &llm.Fake{}, &llm.Fake{}, &recipestest.Search{}, &recipestest.Pages{}, &recipestest.Images{}, false)
+	h, _, _ := newIdeasRouter(t, &llm.Fake{}, &llm.Fake{}, &recipestest.Search{}, &recipestest.Pages{}, &recipestest.Images{}, false)
 
 	for _, tc := range []struct{ body, message string }{
 		{`{"quick_only": false}`, "ingredients: required"},
@@ -108,7 +183,7 @@ func TestPostIdeasFromIngredientsRunsTheJob(t *testing.T) {
 	js := `{"ideas":[{"name":"Chicken Fried Rice","uses":["rice","chicken thighs","peas"],"missing":["2 eggs"],"est_minutes":25,"method":"stovetop","difficulty":"easy","search_query":"chicken fried rice"}]}`
 	model.QueueGenerate(llm.GenerateResult{Response: llm.Response{Text: js, JSON: json.RawMessage(js)}})
 	normalize.QueueGenerate(recipestest.NormalizedReply("stovetop", 25))
-	h, runner := newIdeasRouter(t, model, normalize, search, pages, images, true)
+	h, runner, _ := newIdeasRouter(t, model, normalize, search, pages, images, true)
 
 	rec, body := callDinners(t, h, http.MethodPost, ideasPath, `{"ingredients": ["Chicken thighs", "rice", "scallions"], "quick_only": true}`)
 	if rec.Code != http.StatusAccepted {
@@ -169,9 +244,6 @@ func TestPostIdeasFromIngredientsRunsTheJob(t *testing.T) {
 
 func TestFailedIdeasJobShowsNoIdeas(t *testing.T) {
 	t.Parallel()
-	idea := func(name string) string {
-		return fmt.Sprintf(`{"name":%q,"uses":["rice"],"missing":[],"est_minutes":30,"method":"stovetop","difficulty":"easy","search_query":%q}`, name, name)
-	}
 	for _, tc := range []struct {
 		name string
 		// grounds are the ideas a recipe grounds; the others ground none.
@@ -188,43 +260,82 @@ func TestFailedIdeasJobShowsNoIdeas(t *testing.T) {
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			t.Parallel()
-			model, normalize := &llm.Fake{}, &llm.Fake{}
-			search := &recipestest.Search{Results: map[string][]recipes.Candidate{}, Fail: tc.fail}
-			pages := &recipestest.Pages{Bodies: map[string][]byte{}}
-			images := &recipestest.Images{JPEGs: map[string][]byte{}}
-			for i, name := range tc.grounds {
-				page, photo := fmt.Sprintf("https://r.example/%d", i), fmt.Sprintf("https://r.example/%d.jpg", i)
-				search.Results[name] = []recipes.Candidate{{URL: page}}
-				pages.Bodies[page] = recipestest.RecipePage(name, "R Kitchen", photo, 25)
-				images.JPEGs[photo] = recipestest.JPEG(t, 400, 320, uint8(i+1))
-				normalize.QueueGenerate(recipestest.NormalizedReply("stovetop", 25))
-			}
-			list := make([]string, len(tc.ideas))
-			for i, name := range tc.ideas {
-				list[i] = idea(name)
-			}
-			js := `{"ideas":[` + strings.Join(list, ",") + `]}`
-			model.QueueGenerate(llm.GenerateResult{Response: llm.Response{Text: js, JSON: json.RawMessage(js)}})
-			h, _ := newIdeasRouter(t, model, normalize, search, pages, images, true)
-
-			rec, body := callDinners(t, h, http.MethodPost, ideasPath, `{"ingredients": ["rice"], "quick_only": false}`)
-			if rec.Code != http.StatusAccepted {
-				t.Fatalf("POST = %d: %s", rec.Code, rec.Body)
-			}
-			job := waitJob(t, h, body["jobId"].(string))
-			if job["status"] != "failed" || job["error"] != tc.code {
-				t.Fatalf("GET job = %v, want failed with %s", job, tc.code)
-			}
-			if res, _ := job["result"].(map[string]any); res == nil || len(res["ideas"].([]any)) != 0 {
-				t.Fatalf("failed job result = %v, want no ideas", job["result"])
-			}
+			model, normalize, search, pages, images := scriptIdeas(t, tc.ideas, tc.grounds, tc.fail)
+			h, _, _ := newIdeasRouter(t, model, normalize, search, pages, images, true)
+			assertFailedWithoutIdeas(t, waitJob(t, h, startIdeasJob(t, h)), tc.code)
 		})
 	}
 }
 
+// A job left running with ideas shown is failed at the next start as
+// interrupted, without its handler running again; it still shows no ideas.
+func TestIdeasJobInterruptedByRestartShowsNoIdeas(t *testing.T) {
+	t.Parallel()
+	h, runner, st := newIdeasRouter(t, &llm.Fake{}, &llm.Fake{}, &recipestest.Search{}, &recipestest.Pages{}, &recipestest.Images{}, false)
+	id := startIdeasJob(t, h)
+
+	// The previous process claimed the job and showed an idea, then stopped.
+	q := db.New(st.Writer)
+	now := store.Now().String()
+	if _, err := q.ClaimNextJob(t.Context(), sql.NullString{String: now, Valid: true}); err != nil {
+		t.Fatal(err)
+	}
+	partial, err := ideas.Result{
+		Stage:       ideas.StageGrounding,
+		Ingredients: []string{"rice"},
+		Ideas:       []ideas.Idea{{Name: "Fried Rice", Uses: []string{"rice"}, Missing: []string{}, Method: "stovetop", Difficulty: "easy"}},
+	}.PublicJSON()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if n, err := q.PublishJobResult(t.Context(), db.PublishJobResultParams{
+		Result: sql.NullString{String: string(partial), Valid: true}, UpdatedAt: now, ID: id,
+	}); err != nil || n != 1 {
+		t.Fatalf("publish partial result = %d, %v", n, err)
+	}
+
+	startRunner(t, runner)
+	_, job := callDinners(t, h, http.MethodGet, "/api/v1/jobs/"+id, "")
+	assertFailedWithoutIdeas(t, job, jobs.CodeInterrupted)
+	if res := job["result"].(map[string]any); res["stage"] != "grounding" {
+		t.Errorf("result = %v, want the rest of the partial result kept", res)
+	}
+	if n := storedIdeas(t, st, id); n != 1 {
+		t.Fatalf("stored ideas = %d, want the interrupted job's 1 still stored", n)
+	}
+}
+
+// When the write clearing a failed job's ideas itself fails, the job fails
+// with the runner's internal_error and its stored result keeps the ideas;
+// it still shows none.
+func TestIdeasJobWhoseClearingWriteFailsShowsNoIdeas(t *testing.T) {
+	t.Parallel()
+	model, normalize, search, pages, images := scriptIdeas(t,
+		[]string{"Fried Rice", "Mystery Stew", "Rice Bowls", "Rice Soup", "Rice Salad", "Rice Pilaf"},
+		[]string{"Fried Rice", "Rice Bowls", "Rice Soup", "Rice Salad"},
+		map[string]error{"Rice Pilaf": errors.New("boom")})
+	h, runner, st := newIdeasRouter(t, model, normalize, search, pages, images, false)
+	// Refuse any write that empties a running job's shown ideas.
+	if _, err := st.Writer.ExecContext(t.Context(), `
+		CREATE TRIGGER refuse_clearing_ideas BEFORE UPDATE OF result ON jobs
+		WHEN NEW.status = 'running'
+			AND json_array_length(OLD.result, '$.ideas') > 0
+			AND json_array_length(NEW.result, '$.ideas') = 0
+		BEGIN SELECT RAISE(ABORT, 'clearing refused'); END`); err != nil {
+		t.Fatal(err)
+	}
+	startRunner(t, runner)
+
+	id := startIdeasJob(t, h)
+	assertFailedWithoutIdeas(t, waitJob(t, h, id), jobs.CodeInternal)
+	if n := storedIdeas(t, st, id); n != 4 {
+		t.Fatalf("stored ideas = %d, want the 4 the refused write left", n)
+	}
+}
+
 func TestJobsEndpointRefusesIdeasKind(t *testing.T) {
 	t.Parallel()
-	h, _ := newIdeasRouter(t, &llm.Fake{}, &llm.Fake{}, &recipestest.Search{}, &recipestest.Pages{}, &recipestest.Images{}, false)
+	h, _, _ := newIdeasRouter(t, &llm.Fake{}, &llm.Fake{}, &recipestest.Search{}, &recipestest.Pages{}, &recipestest.Images{}, false)
 	rec, body := callDinners(t, h, http.MethodPost, "/api/v1/jobs", `{"kind": "ideas_from_ingredients", "payload": {"ingredients": [], "quick_only": false}}`)
 	if rec.Code != http.StatusBadRequest || errorCode(body) != "service_only_kind" {
 		t.Fatalf("POST /jobs = %d %v, want 400 service_only_kind", rec.Code, body)