Withhold unsafe recipe sources from idea job results (BASE-01) (#170)

Co-authored-by: orchestrator-agent <20+orchestrator-agent@code.cubecraftcreations.com>

internal/ideas/ideas.go

@@ -11,6 +11,7 @@ package ideas
 
 import (
 	"context"
+	"database/sql"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -123,18 +124,18 @@ func NewService(st *store.Store, grounder *recipes.Grounder, generate Generator,
 	}
 	s := &Service{st: st, grounder: grounder, runner: runner, generate: generate, quick: quick, logger: logger}
 	var err error
-	if s.job, err = register(runner, KindFromIngredients, s.run, withoutIdeas[Result]); err != nil {
+	if s.job, err = register(runner, KindFromIngredients, s.run, shownJobResult[Result]); err != nil {
 		return nil, err
 	}
-	if s.quickJob, err = register(runner, KindQuick, s.runQuick, withoutIdeas[QuickResult]); err != nil {
+	if s.quickJob, err = register(runner, KindQuick, s.runQuick, shownJobResult[QuickResult]); err != nil {
 		return nil, err
 	}
 	return s, nil
 }
 
-// register registers the service-only job name, run by run, whose failed
-// jobs show their result through failed.
-func register[P any, R jobs.Public](runner *jobs.Runner, name string, run jobs.Handler[P, R], failed func(json.RawMessage) (json.RawMessage, error)) (*jobs.Kind[P], error) {
+// register registers the service-only job name, run by run, whose jobs
+// show their result through show.
+func register[P any, R jobs.Public](runner *jobs.Runner, name string, run jobs.Handler[P, R], show func(jobs.Status, json.RawMessage) (json.RawMessage, error)) (*jobs.Kind[P], error) {
 	kind, err := jobs.Register(runner, name, run, Reasons...)
 	if err != nil {
 		return nil, err
@@ -142,7 +143,7 @@ func register[P any, R jobs.Public](runner *jobs.Runner, name string, run jobs.H
 	if err := kind.ServiceOnly(); err != nil {
 		return nil, err
 	}
-	if err := kind.FailedResult(failed); err != nil {
+	if err := kind.ShownResult(show); err != nil {
 		return nil, err
 	}
 	return kind, nil
@@ -197,11 +198,13 @@ type Idea struct {
 	// Difficulty is easy, medium or hard.
 	Difficulty string `json:"difficulty"`
 	// Quick says whether the idea is a Quick dinner (see IsQuick).
-	Quick     bool   `json:"quick"`
-	RecipeID  string `json:"recipe_id"`
-	PhotoURL  string `json:"photo_url"`
-	SourceURL string `json:"source_url"`
-	SiteName  string `json:"site_name"`
+	Quick    bool   `json:"quick"`
+	RecipeID string `json:"recipe_id"`
+	PhotoURL string `json:"photo_url"`
+	// SourceURL and SiteName are the grounded recipe's source as
+	// dinners.SourceOf gives it: each null when it could leak a secret.
+	SourceURL *string `json:"source_url"`
+	SiteName  *string `json:"site_name"`
 }
 
 // Result is the KindFromIngredients result, published as the job goes:
@@ -251,6 +254,9 @@ type shownResult[R any] interface {
 	shown() int
 	// cleared is the result showing no ideas.
 	cleared() R
+	// withSafeSources is the result with each idea's source as
+	// shownSource gives it.
+	withSafeSources() (R, error)
 }
 
 func (r Result) shown() int { return len(r.Ideas) }
@@ -260,11 +266,25 @@ func (r Result) cleared() Result {
 	return r
 }
 
+func (r Result) withSafeSources() (Result, error) {
+	ideas := make([]Idea, len(r.Ideas))
+	for i, idea := range r.Ideas {
+		src, err := shownSource(idea.SourceURL, idea.SiteName)
+		if err != nil {
+			return Result{}, fmt.Errorf("ideas: idea %d: %w", i, err)
+		}
+		idea.SourceURL, idea.SiteName = src.URL, src.SiteName
+		ideas[i] = idea
+	}
+	r.Ideas = ideas
+	return r, nil
+}
+
 // finish ends a job whose work returned res and err. A job that failed
 // has its result published again with no ideas, to tidy the stored
 // result. That write is not what keeps a failed job from showing ideas,
 // since it may fail and a job interrupted by restart never makes it:
-// withoutIdeas, applied whenever a failed job is read, is.
+// shownJobResult, applied whenever a failed job is read, is.
 func finish[R shownResult[R]](ctx context.Context, logger *slog.Logger, what string, res R, err error, publish func(R) error) (R, error) {
 	var zero R
 	if err == nil {
@@ -312,9 +332,9 @@ func (s *Service) fill(ctx context.Context, p FromIngredientsJob, publish func(R
 	for i, idea := range out.Ideas {
 		reqs[i] = recipes.GroundRequest{DishName: idea.Name, Query: idea.SearchQuery}
 	}
-	err = keepGrounded(ctx, s.grounder, reqs, skills.IdeasKept, &res.Ideas, &res.Dropped, func(i int, gr recipes.Grounded) (Idea, bool) {
-		idea := ideaOf(out.Ideas[i], gr)
-		return idea, !p.QuickOnly || idea.Quick
+	err = keepGrounded(ctx, s.grounder, reqs, skills.IdeasKept, &res.Ideas, &res.Dropped, func(i int, gr recipes.Grounded) (Idea, bool, error) {
+		idea, err := ideaOf(out.Ideas[i], gr)
+		return idea, !p.QuickOnly || idea.Quick, err
 	}, func() error { return publish(res) })
 	if err != nil {
 		return res, err
@@ -333,9 +353,9 @@ func (s *Service) fill(ctx context.Context, p FromIngredientsJob, publish func(R
 // order (an idea that grounds sooner than one before it is shown first,
 // and the earlier one is put back in its place when it grounds) and
 // publish is called. An idea no recipe grounds, or that keep refuses,
-// counts in *dropped. Any other grounding failure, or publish failing, is
-// returned (see ground).
-func keepGrounded[T any](ctx context.Context, g *recipes.Grounder, reqs []recipes.GroundRequest, want int, shown *[]T, dropped *int, keep func(i int, gr recipes.Grounded) (T, bool), publish func() error) error {
+// counts in *dropped. Any other grounding failure, or keep or publish
+// failing, is returned (see ground).
+func keepGrounded[T any](ctx context.Context, g *recipes.Grounder, reqs []recipes.GroundRequest, want int, shown *[]T, dropped *int, keep func(i int, gr recipes.Grounded) (T, bool, error), publish func() error) error {
 	// order[k] is the index in reqs of (*shown)[k], so each idea is put in
 	// its place in reqs' order as it grounds.
 	var order []int
@@ -348,7 +368,10 @@ func keepGrounded[T any](ctx context.Context, g *recipes.Grounder, reqs []recipe
 				*dropped++
 				return nil
 			}
-			idea, ok := keep(first+i, *gr)
+			idea, ok, err := keep(first+i, *gr)
+			if err != nil {
+				return err
+			}
 			if !ok {
 				*dropped++
 				return nil
@@ -406,19 +429,32 @@ func ground(ctx context.Context, g *recipes.Grounder, reqs []recipes.GroundReque
 	return failed
 }
 
-// withoutIdeas is a failed job's stored result as the job shows it: with no
-// ideas, however the job failed, so a failed job never shows ideas kept
-// before the failure.
-func withoutIdeas[R shownResult[R]](stored json.RawMessage) (json.RawMessage, error) {
+// shownJobResult is a job's stored result as the job shows it. A failed
+// job's shows no ideas, however the job failed, so a failed job never
+// shows ideas kept before the failure. Any other job's shows each idea's
+// source as shownSource gives it, so a result stored before sources were
+// made safe never shows a link or name that could leak a secret.
+func shownJobResult[R shownResult[R]](status jobs.Status, stored json.RawMessage) (json.RawMessage, error) {
 	var res R
 	if err := json.Unmarshal(stored, &res); err != nil {
-		return nil, fmt.Errorf("ideas: decode failed job's result: %w", err)
+		return nil, fmt.Errorf("ideas: decode %s job's result: %w", status, err)
 	}
-	return res.cleared().PublicJSON()
+	if status == jobs.StatusFailed {
+		return res.cleared().PublicJSON()
+	}
+	safe, err := res.withSafeSources()
+	if err != nil {
+		return nil, err
+	}
+	return safe.PublicJSON()
 }
 
 // ideaOf is idea as grounded by gr.
-func ideaOf(idea skills.Idea, gr recipes.Grounded) Idea {
+func ideaOf(idea skills.Idea, gr recipes.Grounded) (Idea, error) {
+	src, err := sourceOf(gr)
+	if err != nil {
+		return Idea{}, err
+	}
 	return Idea{
 		Name:         idea.Name,
 		Uses:         idea.Uses,
@@ -430,9 +466,38 @@ func ideaOf(idea skills.Idea, gr recipes.Grounded) Idea {
 		Quick:        IsQuick(gr.Method, gr.MinutesTotal),
 		RecipeID:     gr.RecipeID,
 		PhotoURL:     dinners.PhotoURL(gr.PhotoID),
-		SourceURL:    gr.SourceURL,
-		SiteName:     gr.SiteName,
+		SourceURL:    src.URL,
+		SiteName:     src.SiteName,
+	}, nil
+}
+
+// sourceOf is gr's recipe source as a client may see it (see
+// dinners.SourceOf): a grounded recipe's site name is stored only when
+// not empty.
+func sourceOf(gr recipes.Grounded) (dinners.Source, error) {
+	src, err := dinners.SourceOf(sql.NullString{String: gr.SiteName, Valid: gr.SiteName != ""}, gr.SourceURL)
+	if err != nil {
+		return dinners.Source{}, fmt.Errorf("ideas: recipe %s: %w", gr.RecipeID, err)
 	}
+	return src, nil
+}
+
+// shownSource is the source of an idea as stored in a job's result, with
+// link sourceURL and site name siteName, as a client may see it (see
+// dinners.SourceOf). A result stored now already holds it, and one stored
+// before sources were made safe holds the grounded recipe's own, with ""
+// for no site name, as sourceOf reads it; a null link, which only a result
+// stored now holds, is checked as "".
+func shownSource(sourceURL, siteName *string) (dinners.Source, error) {
+	var link string
+	if sourceURL != nil {
+		link = *sourceURL
+	}
+	var site sql.NullString
+	if siteName != nil {
+		site = sql.NullString{String: *siteName, Valid: *siteName != ""}
+	}
+	return dinners.SourceOf(site, link)
 }
 
 // QuickJob is the KindQuick payload: the kind of Quick dinner wanted,
@@ -478,8 +543,9 @@ type QuickIdea struct {
 	Difficulty string `json:"difficulty"`
 	RecipeID   string `json:"recipe_id"`
 	PhotoURL   string `json:"photo_url"`
-	SourceURL  string `json:"source_url"`
-	SiteName   string `json:"site_name"`
+	// SourceURL and SiteName are as in Idea.
+	SourceURL *string `json:"source_url"`
+	SiteName  *string `json:"site_name"`
 }
 
 // QuickResult is the KindQuick result, published as the job goes: its
@@ -504,6 +570,20 @@ func (r QuickResult) cleared() QuickResult {
 	return r
 }
 
+func (r QuickResult) withSafeSources() (QuickResult, error) {
+	ideas := make([]QuickIdea, len(r.Ideas))
+	for i, idea := range r.Ideas {
+		src, err := shownSource(idea.SourceURL, idea.SiteName)
+		if err != nil {
+			return QuickResult{}, fmt.Errorf("ideas: idea %d: %w", i, err)
+		}
+		idea.SourceURL, idea.SiteName = src.URL, src.SiteName
+		ideas[i] = idea
+	}
+	r.Ideas = ideas
+	return r, nil
+}
+
 // OfQuickKind reports whether a recipe cooked by method in minutes (nil
 // when unknown) is a Quick dinner of kind: for skills.QuickCrockpot,
 // cooked in a crock pot (normalize_recipe writes a slow cooker as
@@ -564,8 +644,9 @@ func (s *Service) fillQuick(ctx context.Context, p QuickJob, publish func(QuickR
 	for i, idea := range out.Ideas {
 		reqs[i] = recipes.GroundRequest{DishName: idea.Name, Query: idea.SearchQuery}
 	}
-	err = keepGrounded(ctx, s.grounder, reqs, skills.QuickIdeasKept, &res.Ideas, &res.Dropped, func(i int, gr recipes.Grounded) (QuickIdea, bool) {
-		return quickIdeaOf(out.Ideas[i], gr), OfQuickKind(p.Kind, gr.Method, gr.MinutesTotal)
+	err = keepGrounded(ctx, s.grounder, reqs, skills.QuickIdeasKept, &res.Ideas, &res.Dropped, func(i int, gr recipes.Grounded) (QuickIdea, bool, error) {
+		idea, err := quickIdeaOf(out.Ideas[i], gr)
+		return idea, OfQuickKind(p.Kind, gr.Method, gr.MinutesTotal), err
 	}, func() error { return publish(res) })
 	if err != nil {
 		return res, err
@@ -578,7 +659,11 @@ func (s *Service) fillQuick(ctx context.Context, p QuickJob, publish func(QuickR
 }
 
 // quickIdeaOf is idea as grounded by gr.
-func quickIdeaOf(idea skills.QuickIdea, gr recipes.Grounded) QuickIdea {
+func quickIdeaOf(idea skills.QuickIdea, gr recipes.Grounded) (QuickIdea, error) {
+	src, err := sourceOf(gr)
+	if err != nil {
+		return QuickIdea{}, err
+	}
 	return QuickIdea{
 		Name:         idea.Name,
 		Method:       gr.Method,
@@ -586,9 +671,9 @@ func quickIdeaOf(idea skills.QuickIdea, gr recipes.Grounded) QuickIdea {
 		Difficulty:   gr.Difficulty,
 		RecipeID:     gr.RecipeID,
 		PhotoURL:     dinners.PhotoURL(gr.PhotoID),
-		SourceURL:    gr.SourceURL,
-		SiteName:     gr.SiteName,
-	}
+		SourceURL:    src.URL,
+		SiteName:     src.SiteName,
+	}, nil
 }
 
 // Origins of a dinner made from an idea (the dinners.origin column).

internal/ideas/quick_test.go

@@ -369,7 +369,7 @@ func TestStartQuickRunsTheJob(t *testing.T) {
 	if res.Stage != StageDone || res.Kind != skills.QuickCrockpot || !reflect.DeepEqual(res.Exclude, []string{"Beef Stew"}) || len(res.Ideas) != 1 {
 		t.Fatalf("result = %+v", res)
 	}
-	if got := res.Ideas[0]; got.Name != "Pulled Pork" || got.Method != "crockpot" || got.SiteName != "R Kitchen" {
+	if got := res.Ideas[0]; got.Name != "Pulled Pork" || got.Method != "crockpot" || got.SiteName == nil || *got.SiteName != "R Kitchen" {
 		t.Errorf("idea = %+v", got)
 	}
 }