From 26928b538b5ac3a791ea284308f7e6297cc85908 Mon Sep 17 00:00:00 2001 From: dev_agent Date: Mon, 5 Oct 2026 13:17:27 -0400 Subject: [PATCH 1/2] Add SearXNG and Brave recipe search providers with ranking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit recipes.SearchProvider.Search(ctx, query, n) returns []Candidate{URL, Title, Snippet}, appending " recipe" when the query lacks the word. SearXNG calls {MEALQUEST_SEARXNG_URL}/search?q=…&format=json&categories=general directly (the configured endpoint may be internal); Brave calls the Web Search API with the key decrypted from recipe_search_settings. Redirects are not followed; 401/403, 429, other non-2xx and malformed bodies map to distinct errors. recipes.Rank fetches and extracts the top 5 candidates, rejects those without ingredients or an image, and scores the rest by harness.Similarity of the recipe title to the dish name plus completeness (ingredients, image, steps, minutes). harness.Similarity exposes the existing Jaccard score used by Similar. Maestro task 10c25cb5-3774-419c-87e1-3f87a3d1391d. Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 18 ++ internal/ai/harness/dedupe.go | 9 +- internal/ai/harness/dedupe_test.go | 16 ++ internal/recipes/rank.go | 116 ++++++++++++ internal/recipes/rank_test.go | 159 ++++++++++++++++ internal/recipes/search.go | 294 +++++++++++++++++++++++++++++ internal/recipes/search_test.go | 275 +++++++++++++++++++++++++++ 7 files changed, 886 insertions(+), 1 deletion(-) create mode 100644 internal/recipes/rank.go create mode 100644 internal/recipes/rank_test.go create mode 100644 internal/recipes/search.go create mode 100644 internal/recipes/search_test.go diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 6f31a0a..f9cd825 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -239,6 +239,24 @@ automatic** (D-021). with two is `ErrManyRecipes`; a shape schema.org does not allow — such as a non-ISO-8601 duration like "20 minutes" or an empty one, or an image `@id` no node has — is `ErrMalformedRecipe`, never guessed at. +- Recipe search (`recipes.SearchProvider`: `Search(ctx, query, n)` → + `[]Candidate{URL, Title, Snippet}`, n 1–20) appends " recipe" to a query + without that word. SearXNG is called at + `{MEALQUEST_SEARXNG_URL}/search?q=…&format=json&categories=general` — + directly, not through the Fetcher, since this configured endpoint may be + on the internal network; Brave calls the Brave Web Search API with the + key decrypted from `recipe_search_settings.api_key_ct` + (`recipes.NewSearchProvider`). Redirects are not followed; 401/403 is + `ErrSearchRefused`, 429 `ErrSearchRateLimited`, other non-2xx a + `SearchStatusError`, and a body that is not the provider's JSON, over + 1 MB or with a result lacking an absolute http(s) URL `ErrSearchResponse`. +- Ranking (`recipes.Rank`) fetches and extracts the top 5 candidates + concurrently, rejects any whose recipe lacks ingredients or an image, and + orders the rest by the token similarity of the recipe's title to the dish + name (`harness.Similarity`, 0–1) plus a quarter for each of ingredients, + image, steps and total minutes; ties keep search order. When every + candidate is rejected the error is `ErrNoUsableRecipe` joined with each + reason. - Photos are cached locally (`photos`) and processed with `golang.org/x/image`. diff --git a/internal/ai/harness/dedupe.go b/internal/ai/harness/dedupe.go index ce6c446..7b2233c 100644 --- a/internal/ai/harness/dedupe.go +++ b/internal/ai/harness/dedupe.go @@ -53,7 +53,14 @@ func Normalize(name string) string { // similarity of their normalized forms is at least SimilarThreshold. A name // with no words left after normalizing is similar to nothing. func Similar(a, b string) bool { - return similarity(tokenSet(a), tokenSet(b)) >= SimilarThreshold + return Similarity(a, b) >= SimilarThreshold +} + +// Similarity is the token-set (Jaccard) similarity of a's and b's normalized +// forms, from 0 (no shared word) to 1 (the same words). A name with no words +// left after normalizing scores 0 against everything. +func Similarity(a, b string) float64 { + return similarity(tokenSet(a), tokenSet(b)) } // tokenSet returns the distinct words of name's normalized form. diff --git a/internal/ai/harness/dedupe_test.go b/internal/ai/harness/dedupe_test.go index 5779695..99332d3 100644 --- a/internal/ai/harness/dedupe_test.go +++ b/internal/ai/harness/dedupe_test.go @@ -114,6 +114,22 @@ func TestSimilar(t *testing.T) { } } +func TestSimilarity(t *testing.T) { + for _, tc := range []struct { + a, b string + want float64 + }{ + {"Easy Chicken Curry", "chicken curry", 1}, + {"Thai Green Curry", "Chicken curry", 0.25}, + {"Beef Stew", "Chicken curry", 0}, + {"The Recipe", "chicken curry", 0}, + } { + if got := Similarity(tc.a, tc.b); got != tc.want { + t.Errorf("Similarity(%q, %q) = %v, want %v", tc.a, tc.b, got, tc.want) + } + } +} + func TestDeduper(t *testing.T) { d := NewDeduper(DedupeSources{ Regulars: []string{"Beef Tacos", "Spaghetti Bolognese"}, diff --git a/internal/recipes/rank.go b/internal/recipes/rank.go new file mode 100644 index 0000000..e025f35 --- /dev/null +++ b/internal/recipes/rank.go @@ -0,0 +1,116 @@ +package recipes + +import ( + "context" + "errors" + "fmt" + "sort" + "sync" + + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/harness" +) + +// RankDepth is how many of the top search candidates Rank fetches and +// extracts. +const RankDepth = 5 + +var ( + // ErrNoIngredients rejects a candidate whose recipe lists no + // ingredients. + ErrNoIngredients = errors.New("recipes: recipe has no ingredients") + // ErrNoImage rejects a candidate whose recipe has no image. + ErrNoImage = errors.New("recipes: recipe has no image") + // ErrNoUsableRecipe is returned when Rank rejects every candidate; it + // wraps each candidate's reason. + ErrNoUsableRecipe = errors.New("recipes: no candidate holds a usable recipe") +) + +// PageFetcher fetches a candidate's page; *Fetcher satisfies it. +type PageFetcher interface { + Fetch(ctx context.Context, rawURL string) (Page, error) +} + +// Ranked is a candidate whose page held a usable recipe, with its score. +type Ranked struct { + Candidate Candidate + Recipe Recipe + // Score is the token similarity of the recipe's title to the dish name + // (0 to 1, harness.Similarity) plus a quarter for each of ingredients, + // image, steps and total minutes present (0 to 1). + Score float64 +} + +// Rank fetches and extracts the first RankDepth candidates concurrently and +// returns those whose recipe has ingredients and an image, highest Score +// first; equal scores keep search order. When every candidate is rejected +// the error is ErrNoUsableRecipe joined with each rejection's reason. +func Rank(ctx context.Context, f PageFetcher, dish string, cands []Candidate) ([]Ranked, error) { + if len(cands) > RankDepth { + cands = cands[:RankDepth] + } + type outcome struct { + ranked Ranked + err error + } + outcomes := make([]outcome, len(cands)) + var wg sync.WaitGroup + for i, c := range cands { + wg.Add(1) + go func() { + defer wg.Done() + r, err := rankOne(ctx, f, dish, c) + outcomes[i] = outcome{r, err} + }() + } + wg.Wait() + if err := ctx.Err(); err != nil { + return nil, err + } + + var ranked []Ranked + reasons := []error{ErrNoUsableRecipe} + for i, o := range outcomes { + if o.err != nil { + reasons = append(reasons, fmt.Errorf("%s: %w", cands[i].URL, o.err)) + continue + } + ranked = append(ranked, o.ranked) + } + if len(ranked) == 0 { + return nil, errors.Join(reasons...) + } + sort.SliceStable(ranked, func(a, b int) bool { return ranked[a].Score > ranked[b].Score }) + return ranked, nil +} + +// rankOne fetches, extracts and scores one candidate, or says why it is +// rejected. +func rankOne(ctx context.Context, f PageFetcher, dish string, c Candidate) (Ranked, error) { + page, err := f.Fetch(ctx, c.URL) + if err != nil { + return Ranked{}, err + } + r, err := Extract(page.Body, page.URL) + if err != nil { + return Ranked{}, err + } + if len(r.Ingredients) == 0 { + return Ranked{}, ErrNoIngredients + } + if r.ImageURL == "" { + return Ranked{}, ErrNoImage + } + return Ranked{Candidate: c, Recipe: r, Score: harness.Similarity(r.Title, dish) + completeness(r)}, nil +} + +// completeness is a quarter for each of ingredients, image, steps and total +// minutes the recipe has. +func completeness(r Recipe) float64 { + score := 0.0 + for _, present := range []bool{len(r.Ingredients) > 0, r.ImageURL != "", len(r.Steps) > 0, r.TotalMinutes != nil} { + if present { + score += 0.25 + } + } + return score +} diff --git a/internal/recipes/rank_test.go b/internal/recipes/rank_test.go new file mode 100644 index 0000000..b70c67b --- /dev/null +++ b/internal/recipes/rank_test.go @@ -0,0 +1,159 @@ +package recipes + +import ( + "context" + "errors" + "fmt" + "sync" + "testing" +) + +var _ PageFetcher = (*Fetcher)(nil) + +// fakeFetcher serves fixture pages by URL and records what was fetched. +type fakeFetcher struct { + pages map[string][]byte + + mu sync.Mutex + fetched []string +} + +func (f *fakeFetcher) Fetch(_ context.Context, rawURL string) (Page, error) { + f.mu.Lock() + f.fetched = append(f.fetched, rawURL) + f.mu.Unlock() + body, ok := f.pages[rawURL] + if !ok { + return Page{}, &StatusError{URL: rawURL, Status: 404} + } + return Page{URL: rawURL, ContentType: "text/html", Body: body}, nil +} + +// recipePage returns a page whose Recipe has the given name and, as asked, +// ingredients, an image, steps and a total time. +func recipePage(name string, ingredients, image, steps, minutes bool) []byte { + ld := fmt.Sprintf(`{"@context":"https://schema.org","@type":"Recipe","name":%q`, name) + if ingredients { + ld += `,"recipeIngredient":["1 onion","2 cloves garlic"]` + } + if image { + ld += `,"image":"https://img.example.com/dish.jpg"` + } + if steps { + ld += `,"recipeInstructions":["Chop.","Cook."]` + } + if minutes { + ld += `,"totalTime":"PT40M"` + } + return page("", ld+"}") +} + +func TestRankOrder(t *testing.T) { + f := &fakeFetcher{pages: map[string][]byte{ + "https://a.example/thai": recipePage("Thai Green Curry", true, true, true, true), + "https://b.example/noimg": recipePage("Chicken Curry", true, false, true, true), + "https://c.example/easy": recipePage("Easy Chicken Curry", true, true, true, false), + "https://d.example/noingr": recipePage("Chicken Curry", false, true, true, true), + "https://e.example/best": recipePage("Chicken Curry", true, true, true, true), + "https://f.example/toodeep": recipePage("Chicken Curry", true, true, true, true), + }} + cands := []Candidate{ + {URL: "https://a.example/thai", Title: "Thai green curry"}, + {URL: "https://b.example/noimg"}, + {URL: "https://c.example/easy"}, + {URL: "https://d.example/noingr"}, + {URL: "https://e.example/best"}, + {URL: "https://f.example/toodeep"}, + } + got, err := Rank(context.Background(), f, "Chicken curry", cands) + if err != nil { + t.Fatal(err) + } + want := []struct { + url string + score float64 + }{ + // Same words, every field: 1 + 1. + {"https://e.example/best", 2}, + // "Easy" is a filler word, so the title matches; no total time. + {"https://c.example/easy", 1.75}, + // Shares one of four words: 0.25 + 1. + {"https://a.example/thai", 1.25}, + } + if len(got) != len(want) { + t.Fatalf("got %d ranked, want %d: %+v", len(got), len(want), got) + } + for i, w := range want { + if got[i].Candidate.URL != w.url || got[i].Score != w.score { + t.Errorf("rank %d = %s (%.2f), want %s (%.2f)", i, got[i].Candidate.URL, got[i].Score, w.url, w.score) + } + } + if got[2].Recipe.Title != "Thai Green Curry" || got[2].Candidate.Title != "Thai green curry" { + t.Errorf("ranked entry lost its recipe or candidate: %+v", got[2]) + } + for _, u := range f.fetched { + if u == "https://f.example/toodeep" { + t.Errorf("fetched a candidate past RankDepth") + } + } + if len(f.fetched) != RankDepth { + t.Errorf("fetched %d pages, want %d", len(f.fetched), RankDepth) + } +} + +func TestRankTiesKeepSearchOrder(t *testing.T) { + f := &fakeFetcher{pages: map[string][]byte{ + "https://a.example/1": recipePage("Beef Stew", true, true, true, true), + "https://b.example/2": recipePage("Beef Stew", true, true, true, true), + "https://c.example/3": recipePage("Beef Stew", true, true, true, true), + }} + cands := []Candidate{{URL: "https://a.example/1"}, {URL: "https://b.example/2"}, {URL: "https://c.example/3"}} + got, err := Rank(context.Background(), f, "beef stew", cands) + if err != nil { + t.Fatal(err) + } + for i, c := range cands { + if got[i].Candidate.URL != c.URL { + t.Errorf("rank %d = %s, want %s", i, got[i].Candidate.URL, c.URL) + } + } +} + +func TestRankAllRejected(t *testing.T) { + f := &fakeFetcher{pages: map[string][]byte{ + "https://a.example/noimg": recipePage("Chicken Curry", true, false, true, true), + "https://b.example/noingr": recipePage("Chicken Curry", false, true, true, true), + "https://c.example/blog": page(""), + }} + cands := []Candidate{ + {URL: "https://a.example/noimg"}, + {URL: "https://b.example/noingr"}, + {URL: "https://c.example/blog"}, + {URL: "https://d.example/missing"}, + } + _, err := Rank(context.Background(), f, "chicken curry", cands) + for _, want := range []error{ErrNoUsableRecipe, ErrNoImage, ErrNoIngredients, ErrNoRecipe} { + if !errors.Is(err, want) { + t.Errorf("err = %v, want it to wrap %v", err, want) + } + } + var se *StatusError + if !errors.As(err, &se) || se.Status != 404 { + t.Errorf("err = %v, want the 404 StatusError", err) + } +} + +func TestRankNoCandidates(t *testing.T) { + if _, err := Rank(context.Background(), &fakeFetcher{}, "chicken curry", nil); !errors.Is(err, ErrNoUsableRecipe) { + t.Errorf("err = %v, want ErrNoUsableRecipe", err) + } +} + +func TestRankContextCancelled(t *testing.T) { + f := &fakeFetcher{pages: map[string][]byte{"https://a.example/1": recipePage("Beef Stew", true, true, true, true)}} + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := Rank(ctx, f, "beef stew", []Candidate{{URL: "https://a.example/1"}}); !errors.Is(err, context.Canceled) { + t.Errorf("err = %v, want context.Canceled", err) + } +} diff --git a/internal/recipes/search.go b/internal/recipes/search.go new file mode 100644 index 0000000..6a17a86 --- /dev/null +++ b/internal/recipes/search.go @@ -0,0 +1,294 @@ +package recipes + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/db" + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/secretbox" +) + +const ( + // MaxSearchResults is the most candidates one search may ask for; it is + // Brave's largest page. + MaxSearchResults = 20 + // SearchTimeout bounds one search request, reading the body included. + SearchTimeout = 10 * time.Second + // MaxSearchBodyBytes caps a search response body. A larger one is an + // error, never truncated. + MaxSearchBodyBytes = 1 << 20 + // BraveEndpoint is the Brave Web Search API. + BraveEndpoint = "https://api.search.brave.com/res/v1/web/search" +) + +var ( + // ErrSearchRefused is returned for HTTP 401 or 403: Brave rejected the + // API key, or SearXNG does not allow format=json. + ErrSearchRefused = errors.New("recipes: search provider refused the request") + // ErrSearchRateLimited is returned for HTTP 429. + ErrSearchRateLimited = errors.New("recipes: search provider rate limit reached") + // ErrSearchResponse is returned for a response body that is not the + // provider's JSON shape, is over MaxSearchBodyBytes, or holds a result + // without an absolute http(s) URL. + ErrSearchResponse = errors.New("recipes: malformed search response") +) + +// SearchStatusError is returned for any other non-2xx search response, +// redirects included: a search endpoint is called directly and never +// followed elsewhere. +type SearchStatusError struct { + Provider string + Status int +} + +func (e *SearchStatusError) Error() string { + return fmt.Sprintf("recipes: %s search: HTTP %d", e.Provider, e.Status) +} + +// Candidate is one search result: a page that may hold a recipe. +type Candidate struct { + URL string + Title string + Snippet string +} + +// SearchProvider finds recipe pages on the web. +type SearchProvider interface { + // Search returns up to n candidates for query, best first. The query + // gets " recipe" appended unless it already has the word. n must be + // 1 to MaxSearchResults. + Search(ctx context.Context, query string, n int) ([]Candidate, error) +} + +// NewSearchProvider returns the provider s names. SearXNG is called at +// searxngURL, the configured MEALQUEST_SEARXNG_URL; Brave is called with the +// API key in s.ApiKeyCt, decrypted by box. +func NewSearchProvider(s db.RecipeSearchSetting, box *secretbox.Box, searxngURL string) (SearchProvider, error) { + switch s.Provider { + case "searxng": + return NewSearXNG(searxngURL) + case "brave": + if len(s.ApiKeyCt) == 0 { + return nil, errors.New("recipes: Brave search has no API key") + } + key, err := box.Decrypt(s.ApiKeyCt) + if err != nil { + return nil, fmt.Errorf("recipes: Brave API key: %w", err) + } + return NewBrave(string(key)) + default: + return nil, fmt.Errorf("recipes: unknown search provider %q", s.Provider) + } +} + +// SearXNG searches through a SearXNG instance. Its base URL is a configured +// endpoint and may be on a private network, so it is called directly rather +// than through the public-only Fetcher. +type SearXNG struct { + endpoint *url.URL + client *http.Client +} + +// NewSearXNG returns a SearXNG provider for the instance at baseURL, an +// absolute http:// or https:// URL without credentials, query or fragment. +func NewSearXNG(baseURL string) (*SearXNG, error) { + if baseURL == "" { + return nil, errors.New("recipes: SearXNG URL is not set") + } + u, err := url.Parse(baseURL) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + return nil, errors.New("recipes: SearXNG URL must be an absolute http:// or https:// URL") + } + if u.User != nil || u.RawQuery != "" || u.Fragment != "" { + return nil, errors.New("recipes: SearXNG URL must not have credentials, a query or a fragment") + } + return &SearXNG{endpoint: u.JoinPath("search"), client: newSearchClient()}, nil +} + +// Search implements SearchProvider with +// {base}/search?q=…&format=json&categories=general. +func (s *SearXNG) Search(ctx context.Context, query string, n int) ([]Candidate, error) { + q, err := searchQuery(query, n) + if err != nil { + return nil, err + } + u := *s.endpoint + u.RawQuery = url.Values{"q": {q}, "format": {"json"}, "categories": {"general"}}.Encode() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, fmt.Errorf("recipes: build SearXNG request: %w", err) + } + var body struct { + Results []struct { + URL string `json:"url"` + Title string `json:"title"` + Content string `json:"content"` + } `json:"results"` + } + if err := doSearch(s.client, req, "SearXNG", &body); err != nil { + return nil, err + } + var out []Candidate + for _, r := range body.Results { + out = append(out, Candidate{URL: r.URL, Title: r.Title, Snippet: r.Content}) + } + return candidates("SearXNG", out, n) +} + +// Brave searches through the Brave Web Search API. +type Brave struct { + key string + endpoint string + client *http.Client +} + +// NewBrave returns a Brave provider that authenticates with key, the +// decrypted API key. +func NewBrave(key string) (*Brave, error) { + if key == "" { + return nil, errors.New("recipes: Brave API key is empty") + } + for i := 0; i < len(key); i++ { + if key[i] < 0x21 || key[i] > 0x7e { + // The key is never echoed. + return nil, errors.New("recipes: Brave API key has a character not allowed in a header") + } + } + return &Brave{key: key, endpoint: BraveEndpoint, client: newSearchClient()}, nil +} + +// Search implements SearchProvider with GET BraveEndpoint?q=…&count=n. +func (b *Brave) Search(ctx context.Context, query string, n int) ([]Candidate, error) { + q, err := searchQuery(query, n) + if err != nil { + return nil, err + } + u := b.endpoint + "?" + url.Values{"q": {q}, "count": {strconv.Itoa(n)}}.Encode() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return nil, fmt.Errorf("recipes: build Brave request: %w", err) + } + req.Header.Set("X-Subscription-Token", b.key) + var body struct { + Web struct { + Results []struct { + URL string `json:"url"` + Title string `json:"title"` + Description string `json:"description"` + } `json:"results"` + } `json:"web"` + } + if err := doSearch(b.client, req, "Brave", &body); err != nil { + return nil, err + } + var out []Candidate + for _, r := range body.Web.Results { + out = append(out, Candidate{URL: r.URL, Title: r.Title, Snippet: r.Description}) + } + return candidates("Brave", out, n) +} + +func newSearchClient() *http.Client { + return &http.Client{ + Timeout: SearchTimeout, + CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }, + } +} + +// searchQuery validates n and returns query with " recipe" appended unless +// one of its words already is "recipe", in any case. +func searchQuery(query string, n int) (string, error) { + if n < 1 || n > MaxSearchResults { + return "", fmt.Errorf("recipes: search result count %d is outside 1-%d", n, MaxSearchResults) + } + q := strings.TrimSpace(query) + if q == "" { + return "", errors.New("recipes: search query is empty") + } + for _, w := range strings.Fields(q) { + if strings.EqualFold(w, "recipe") { + return q, nil + } + } + return q + " recipe", nil +} + +// doSearch sends req and decodes a 2xx JSON body into v, mapping failures to +// the search errors. +func doSearch(client *http.Client, req *http.Request, provider string, v any) error { + req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", UserAgent) + resp, err := client.Do(req) + if err != nil { + if ctxErr := req.Context().Err(); ctxErr != nil { + return ctxErr + } + return fmt.Errorf("recipes: %s search: %w", provider, err) + } + defer resp.Body.Close() + switch { + case resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden: + return fmt.Errorf("%w: %s HTTP %d", ErrSearchRefused, provider, resp.StatusCode) + case resp.StatusCode == http.StatusTooManyRequests: + return fmt.Errorf("%w: %s", ErrSearchRateLimited, provider) + case resp.StatusCode < 200 || resp.StatusCode > 299: + return &SearchStatusError{Provider: provider, Status: resp.StatusCode} + } + data, err := io.ReadAll(io.LimitReader(resp.Body, MaxSearchBodyBytes+1)) + if err != nil { + return fmt.Errorf("recipes: %s search: read body: %w", provider, err) + } + if len(data) > MaxSearchBodyBytes { + return fmt.Errorf("%w: %s body is over %d bytes", ErrSearchResponse, provider, MaxSearchBodyBytes) + } + if err := json.Unmarshal(data, v); err != nil { + return fmt.Errorf("%w: %s: %v", ErrSearchResponse, provider, err) + } + return nil +} + +// candidates cleans results — HTML tags and entities, which both providers +// leave in titles and snippets, are removed — checks every URL is absolute +// http(s) and keeps the first n. +func candidates(provider string, results []Candidate, n int) ([]Candidate, error) { + if len(results) > n { + results = results[:n] + } + out := make([]Candidate, 0, len(results)) + for i, r := range results { + u, err := url.Parse(r.URL) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + return nil, fmt.Errorf("%w: %s result %d has no absolute http(s) URL", ErrSearchResponse, provider, i) + } + out = append(out, Candidate{URL: r.URL, Title: stripTags(r.Title), Snippet: stripTags(r.Snippet)}) + } + return out, nil +} + +// stripTags drops HTML tags such as Brave's highlights, then +// decodes entities and trims space. +func stripTags(s string) string { + var b strings.Builder + inTag := false + for _, r := range s { + switch { + case r == '<': + inTag = true + case r == '>' && inTag: + inTag = false + case !inTag: + b.WriteRune(r) + } + } + return cleanText(b.String()) +} diff --git a/internal/recipes/search_test.go b/internal/recipes/search_test.go new file mode 100644 index 0000000..b3a181a --- /dev/null +++ b/internal/recipes/search_test.go @@ -0,0 +1,275 @@ +package recipes + +import ( + "context" + "crypto/rand" + "encoding/base64" + "errors" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "testing" + + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/db" + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/secretbox" +) + +const searxngBody = `{ + "query": "chicken curry recipe", + "results": [ + {"url": "https://www.bbcgoodfood.com/recipes/easy-chicken-curry", "title": "Easy chicken curry recipe | BBC Good Food", "content": "Make our easy chicken curry & rice.", "engine": "duckduckgo"}, + {"url": "https://www.recipetineats.com/chicken-curry/", "title": "Chicken Curry", "content": "A proper curry."}, + {"url": "https://example.com/third", "title": "Third", "content": ""} + ], + "number_of_results": 0 +}` + +const braveBody = `{ + "type": "search", + "web": { + "type": "search", + "results": [ + {"url": "https://www.bbcgoodfood.com/recipes/easy-chicken-curry", "title": "Easy chicken curry recipe", "description": "Make our easy chicken curry & rice.", "language": "en"}, + {"url": "https://www.recipetineats.com/chicken-curry/", "title": "Chicken Curry", "description": "A proper curry."} + ] + } +}` + +func newSearXNGServer(t *testing.T, status int, body string, got *http.Request) *SearXNG { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got != nil { + *got = *r.Clone(context.Background()) + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + w.Write([]byte(body)) + })) + t.Cleanup(srv.Close) + s, err := NewSearXNG(srv.URL + "/searx") + if err != nil { + t.Fatal(err) + } + return s +} + +func newBraveServer(t *testing.T, key string, status int, body string, got *http.Request) *Brave { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got != nil { + *got = *r.Clone(context.Background()) + } + w.WriteHeader(status) + w.Write([]byte(body)) + })) + t.Cleanup(srv.Close) + b, err := NewBrave(key) + if err != nil { + t.Fatal(err) + } + b.endpoint = srv.URL + "/res/v1/web/search" + return b +} + +func TestSearXNGSearch(t *testing.T) { + var req http.Request + s := newSearXNGServer(t, http.StatusOK, searxngBody, &req) + got, err := s.Search(context.Background(), "chicken curry", 2) + if err != nil { + t.Fatal(err) + } + if req.URL.Path != "/searx/search" { + t.Errorf("path = %q, want /searx/search", req.URL.Path) + } + q := req.URL.Query() + if q.Get("q") != "chicken curry recipe" || q.Get("format") != "json" || q.Get("categories") != "general" { + t.Errorf("query = %v", q) + } + if req.Header.Get("User-Agent") != UserAgent { + t.Errorf("User-Agent = %q", req.Header.Get("User-Agent")) + } + want := []Candidate{ + {URL: "https://www.bbcgoodfood.com/recipes/easy-chicken-curry", Title: "Easy chicken curry recipe | BBC Good Food", Snippet: "Make our easy chicken curry & rice."}, + {URL: "https://www.recipetineats.com/chicken-curry/", Title: "Chicken Curry", Snippet: "A proper curry."}, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("candidates =\n%#v\nwant\n%#v", got, want) + } +} + +func TestBraveSearch(t *testing.T) { + var req http.Request + b := newBraveServer(t, "brave-key-123", http.StatusOK, braveBody, &req) + got, err := b.Search(context.Background(), "Chicken curry RECIPE", 5) + if err != nil { + t.Fatal(err) + } + if req.Header.Get("X-Subscription-Token") != "brave-key-123" { + t.Errorf("X-Subscription-Token = %q", req.Header.Get("X-Subscription-Token")) + } + if req.Header.Get("Accept") != "application/json" { + t.Errorf("Accept = %q", req.Header.Get("Accept")) + } + q := req.URL.Query() + if q.Get("q") != "Chicken curry RECIPE" || q.Get("count") != "5" { + t.Errorf("query = %v", q) + } + want := []Candidate{ + {URL: "https://www.bbcgoodfood.com/recipes/easy-chicken-curry", Title: "Easy chicken curry recipe", Snippet: "Make our easy chicken curry & rice."}, + {URL: "https://www.recipetineats.com/chicken-curry/", Title: "Chicken Curry", Snippet: "A proper curry."}, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("candidates =\n%#v\nwant\n%#v", got, want) + } +} + +func TestSearchQuery(t *testing.T) { + for _, tc := range []struct{ in, want string }{ + {"chicken curry", "chicken curry recipe"}, + {" chicken curry ", "chicken curry recipe"}, + {"chicken curry recipe", "chicken curry recipe"}, + {"Recipe for chicken curry", "Recipe for chicken curry"}, + {"recipes for curry", "recipes for curry recipe"}, + } { + got, err := searchQuery(tc.in, 3) + if err != nil || got != tc.want { + t.Errorf("searchQuery(%q) = %q, %v; want %q", tc.in, got, err, tc.want) + } + } + for _, tc := range []struct { + q string + n int + }{{"", 3}, {" ", 3}, {"curry", 0}, {"curry", MaxSearchResults + 1}} { + if _, err := searchQuery(tc.q, tc.n); err == nil { + t.Errorf("searchQuery(%q, %d) succeeded, want error", tc.q, tc.n) + } + } +} + +func TestSearchErrors(t *testing.T) { + tests := []struct { + name string + status int + body string + check func(error) bool + }{ + {"unauthorized", http.StatusUnauthorized, `{}`, func(err error) bool { return errors.Is(err, ErrSearchRefused) }}, + {"forbidden", http.StatusForbidden, `{}`, func(err error) bool { return errors.Is(err, ErrSearchRefused) }}, + {"rate limited", http.StatusTooManyRequests, `{}`, func(err error) bool { return errors.Is(err, ErrSearchRateLimited) }}, + {"server error", http.StatusBadGateway, `{}`, func(err error) bool { + var se *SearchStatusError + return errors.As(err, &se) && se.Status == http.StatusBadGateway + }}, + {"redirect not followed", http.StatusFound, ``, func(err error) bool { + var se *SearchStatusError + return errors.As(err, &se) && se.Status == http.StatusFound + }}, + {"not JSON", http.StatusOK, ``, func(err error) bool { return errors.Is(err, ErrSearchResponse) }}, + {"result without URL", http.StatusOK, `{"results":[{"title":"x"}],"web":{"results":[{"title":"x"}]}}`, func(err error) bool { return errors.Is(err, ErrSearchResponse) }}, + {"relative URL", http.StatusOK, `{"results":[{"url":"/x"}],"web":{"results":[{"url":"/x"}]}}`, func(err error) bool { return errors.Is(err, ErrSearchResponse) }}, + {"body too large", http.StatusOK, `{"results":[],"pad":"` + strings.Repeat("x", MaxSearchBodyBytes) + `"}`, func(err error) bool { return errors.Is(err, ErrSearchResponse) }}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + providers := map[string]SearchProvider{ + "SearXNG": newSearXNGServer(t, tc.status, tc.body, nil), + "Brave": newBraveServer(t, "secret-key", tc.status, tc.body, nil), + } + for name, p := range providers { + _, err := p.Search(context.Background(), "curry", 3) + if err == nil || !tc.check(err) { + t.Errorf("%s: err = %v", name, err) + } + if err != nil && strings.Contains(err.Error(), "secret-key") { + t.Errorf("%s: error leaks the API key: %v", name, err) + } + } + }) + } +} + +func TestSearchContextCancelled(t *testing.T) { + s := newSearXNGServer(t, http.StatusOK, searxngBody, nil) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := s.Search(ctx, "curry", 3); !errors.Is(err, context.Canceled) { + t.Errorf("err = %v, want context.Canceled", err) + } +} + +func TestSearchUnreachable(t *testing.T) { + srv := httptest.NewServer(http.NotFoundHandler()) + url := srv.URL + srv.Close() + s, err := NewSearXNG(url) + if err != nil { + t.Fatal(err) + } + _, err = s.Search(context.Background(), "curry", 3) + if err == nil || !strings.Contains(err.Error(), "SearXNG search") { + t.Errorf("err = %v", err) + } +} + +func TestNewSearXNGRejects(t *testing.T) { + for _, u := range []string{"", "searxng:8080", "ftp://searxng", "http://", "http://u:p@searxng", "http://searxng?x=1", "http://searxng#f"} { + if _, err := NewSearXNG(u); err == nil { + t.Errorf("NewSearXNG(%q) succeeded, want error", u) + } + } + if _, err := NewSearXNG("http://searxng:8080"); err != nil { + t.Errorf("internal URL rejected: %v", err) + } +} + +func TestNewBraveRejects(t *testing.T) { + for _, k := range []string{"", "with space", "line\nbreak"} { + if _, err := NewBrave(k); err == nil { + t.Errorf("NewBrave(%q) succeeded, want error", k) + } + } +} + +func TestNewSearchProvider(t *testing.T) { + raw := make([]byte, secretbox.KeySize) + rand.Read(raw) + box, err := secretbox.New(base64.StdEncoding.EncodeToString(raw)) + if err != nil { + t.Fatal(err) + } + ct, err := box.Encrypt([]byte("brave-key")) + if err != nil { + t.Fatal(err) + } + + p, err := NewSearchProvider(db.RecipeSearchSetting{Provider: "brave", ApiKeyCt: ct}, box, "") + if err != nil { + t.Fatal(err) + } + if b, ok := p.(*Brave); !ok || b.key != "brave-key" || b.endpoint != BraveEndpoint { + t.Errorf("provider = %#v, want Brave with decrypted key", p) + } + + p, err = NewSearchProvider(db.RecipeSearchSetting{Provider: "searxng"}, box, "http://searxng:8080") + if err != nil { + t.Fatal(err) + } + if s, ok := p.(*SearXNG); !ok || s.endpoint.String() != "http://searxng:8080/search" { + t.Errorf("provider = %#v, want SearXNG at http://searxng:8080/search", p) + } + + for name, tc := range map[string]struct { + s db.RecipeSearchSetting + url string + }{ + "brave without key": {db.RecipeSearchSetting{Provider: "brave"}, ""}, + "brave bad key": {db.RecipeSearchSetting{Provider: "brave", ApiKeyCt: []byte("not a token at all, far too plain")}, ""}, + "searxng without URL": {db.RecipeSearchSetting{Provider: "searxng"}, ""}, + "unknown provider": {db.RecipeSearchSetting{Provider: "google"}, ""}, + } { + if _, err := NewSearchProvider(tc.s, box, tc.url); err == nil { + t.Errorf("%s: succeeded, want error", name) + } + } +} -- 2.55.0 From 01523f634b76654811230c56000638abe2f6c148 Mon Sep 17 00:00:00 2001 From: dev_agent Date: Mon, 5 Oct 2026 13:27:07 -0400 Subject: [PATCH 2/2] Reject malformed search bodies and report unranked candidates SearXNG now requires a results array and Brave requires "type": "search" with a results array in any web object, so `{}` is ErrSearchResponse instead of an empty search. Rank no longer drops failed candidates: it returns the ranked ones with ErrCandidatesUnranked joined with each unranked candidate's URL and reason, or ErrNoUsableRecipe when none rank. Adds an HTTPS ranking test through the SSRF-safe Fetcher. Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 16 +++-- internal/recipes/rank.go | 26 +++++++-- internal/recipes/rank_test.go | 100 +++++++++++++++++++++++++++++++- internal/recipes/search.go | 34 ++++++++--- internal/recipes/search_test.go | 23 +++++++- 5 files changed, 177 insertions(+), 22 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index f9cd825..44418ec 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -248,15 +248,21 @@ automatic** (D-021). key decrypted from `recipe_search_settings.api_key_ct` (`recipes.NewSearchProvider`). Redirects are not followed; 401/403 is `ErrSearchRefused`, 429 `ErrSearchRateLimited`, other non-2xx a - `SearchStatusError`, and a body that is not the provider's JSON, over - 1 MB or with a result lacking an absolute http(s) URL `ErrSearchResponse`. + `SearchStatusError`, and a body that is not the provider's JSON (SearXNG + without a `results` array; Brave without `"type": "search"` or with a + `web` object lacking `results` — Brave omits `web` when nothing matched), + over 1 MB or with a result lacking an absolute http(s) URL + `ErrSearchResponse`. - Ranking (`recipes.Rank`) fetches and extracts the top 5 candidates concurrently, rejects any whose recipe lacks ingredients or an image, and orders the rest by the token similarity of the recipe's title to the dish name (`harness.Similarity`, 0–1) plus a quarter for each of ingredients, - image, steps and total minutes; ties keep search order. When every - candidate is rejected the error is `ErrNoUsableRecipe` joined with each - reason. + image, steps and total minutes; ties keep search order. No unranked + candidate is dropped silently: when some are ranked, they are returned + with `ErrCandidatesUnranked`; when none are, the error is + `ErrNoUsableRecipe`. Either is joined with each unranked candidate's URL + and reason (fetch failure, no valid recipe, missing ingredients or + image), so the caller decides per item what a failure means. - Photos are cached locally (`photos`) and processed with `golang.org/x/image`. diff --git a/internal/recipes/rank.go b/internal/recipes/rank.go index e025f35..8d3c254 100644 --- a/internal/recipes/rank.go +++ b/internal/recipes/rank.go @@ -20,9 +20,13 @@ var ( ErrNoIngredients = errors.New("recipes: recipe has no ingredients") // ErrNoImage rejects a candidate whose recipe has no image. ErrNoImage = errors.New("recipes: recipe has no image") - // ErrNoUsableRecipe is returned when Rank rejects every candidate; it - // wraps each candidate's reason. + // ErrNoUsableRecipe is returned when Rank ranks no candidate; it is + // joined with each candidate's reason. ErrNoUsableRecipe = errors.New("recipes: no candidate holds a usable recipe") + // ErrCandidatesUnranked is returned, alongside the ranked candidates, + // when Rank ranks some candidates but not all; it is joined with each + // unranked candidate's reason. + ErrCandidatesUnranked = errors.New("recipes: some candidates were not ranked") ) // PageFetcher fetches a candidate's page; *Fetcher satisfies it. @@ -42,8 +46,15 @@ type Ranked struct { // Rank fetches and extracts the first RankDepth candidates concurrently and // returns those whose recipe has ingredients and an image, highest Score -// first; equal scores keep search order. When every candidate is rejected -// the error is ErrNoUsableRecipe joined with each rejection's reason. +// first; equal scores keep search order. +// +// Every candidate that is not ranked — its fetch failed, its page holds no +// valid recipe, or the recipe lacks ingredients or an image — is reported, +// never dropped silently: when some are ranked the error is +// ErrCandidatesUnranked, returned together with the ranked ones; when none +// are it is ErrNoUsableRecipe with no results. Either is joined with each +// unranked candidate's URL and reason. A cancelled ctx returns only its +// error. func Rank(ctx context.Context, f PageFetcher, dish string, cands []Candidate) ([]Ranked, error) { if len(cands) > RankDepth { cands = cands[:RankDepth] @@ -68,7 +79,7 @@ func Rank(ctx context.Context, f PageFetcher, dish string, cands []Candidate) ([ } var ranked []Ranked - reasons := []error{ErrNoUsableRecipe} + var reasons []error for i, o := range outcomes { if o.err != nil { reasons = append(reasons, fmt.Errorf("%s: %w", cands[i].URL, o.err)) @@ -77,9 +88,12 @@ func Rank(ctx context.Context, f PageFetcher, dish string, cands []Candidate) ([ ranked = append(ranked, o.ranked) } if len(ranked) == 0 { - return nil, errors.Join(reasons...) + return nil, errors.Join(append([]error{ErrNoUsableRecipe}, reasons...)...) } sort.SliceStable(ranked, func(a, b int) bool { return ranked[a].Score > ranked[b].Score }) + if len(reasons) > 0 { + return ranked, errors.Join(append([]error{ErrCandidatesUnranked}, reasons...)...) + } return ranked, nil } diff --git a/internal/recipes/rank_test.go b/internal/recipes/rank_test.go index b70c67b..a8d91ce 100644 --- a/internal/recipes/rank_test.go +++ b/internal/recipes/rank_test.go @@ -4,6 +4,8 @@ import ( "context" "errors" "fmt" + "net/http" + "strings" "sync" "testing" ) @@ -66,8 +68,11 @@ func TestRankOrder(t *testing.T) { {URL: "https://f.example/toodeep"}, } got, err := Rank(context.Background(), f, "Chicken curry", cands) - if err != nil { - t.Fatal(err) + if !errors.Is(err, ErrCandidatesUnranked) || !errors.Is(err, ErrNoImage) || !errors.Is(err, ErrNoIngredients) { + t.Errorf("err = %v, want ErrCandidatesUnranked with ErrNoImage and ErrNoIngredients", err) + } + if errors.Is(err, ErrNoUsableRecipe) { + t.Errorf("err = %v, want not ErrNoUsableRecipe when some are ranked", err) } want := []struct { url string @@ -119,6 +124,29 @@ func TestRankTiesKeepSearchOrder(t *testing.T) { } } +// TestRankReportsFetchFailure checks a failed fetch is reported with the +// candidates that were ranked, never swallowed. +func TestRankReportsFetchFailure(t *testing.T) { + f := &fakeFetcher{pages: map[string][]byte{ + "https://a.example/1": recipePage("Beef Stew", true, true, true, true), + }} + cands := []Candidate{{URL: "https://a.example/1"}, {URL: "https://b.example/missing"}} + got, err := Rank(context.Background(), f, "beef stew", cands) + if len(got) != 1 || got[0].Candidate.URL != "https://a.example/1" { + t.Errorf("ranked = %+v, want only https://a.example/1", got) + } + if !errors.Is(err, ErrCandidatesUnranked) { + t.Errorf("err = %v, want ErrCandidatesUnranked", err) + } + var se *StatusError + if !errors.As(err, &se) || se.Status != 404 || se.URL != "https://b.example/missing" { + t.Errorf("err = %v, want the 404 StatusError for https://b.example/missing", err) + } + if err != nil && !strings.Contains(err.Error(), "https://b.example/missing") { + t.Errorf("err = %v, want it to name the failed candidate", err) + } +} + func TestRankAllRejected(t *testing.T) { f := &fakeFetcher{pages: map[string][]byte{ "https://a.example/noimg": recipePage("Chicken Curry", true, false, true, true), @@ -157,3 +185,71 @@ func TestRankContextCancelled(t *testing.T) { t.Errorf("err = %v, want context.Canceled", err) } } + +// TestRankOverHTTP ranks fixture candidates served over HTTPS through the +// SSRF-safe Fetcher: every host, a subdomain the test server's certificate +// covers, resolves to a public test address the test +// dialer routes to one httptest server, which serves pages by host and path. +func TestRankOverHTTP(t *testing.T) { + t.Parallel() + pages := map[string][]byte{ + "best.example.com/curry": recipePage("Chicken Curry", true, true, true, true), + "easy.example.com/curry": recipePage("Easy Chicken Curry", true, true, true, false), + "thai.example.com/curry": recipePage("Thai Green Curry", true, true, true, true), + "noimg.example.com/curry": recipePage("Chicken Curry", true, false, true, true), + } + env := newTestEnv(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, ok := pages[r.Host+r.URL.Path] + if !ok { + http.Error(w, "broken", http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "text/html") + w.Write(body) + }), map[string][][]string{ + "best.example.com": {{publicIP}}, + "easy.example.com": {{publicIP}}, + "thai.example.com": {{publicIP}}, + "noimg.example.com": {{publicIP}}, + "broken.example.com": {{publicIP}}, + }) + cands := []Candidate{ + {URL: "https://thai.example.com/curry"}, + {URL: "https://broken.example.com/curry"}, + {URL: "https://easy.example.com/curry"}, + {URL: "https://noimg.example.com/curry"}, + {URL: "https://best.example.com/curry"}, + } + + got, err := Rank(t.Context(), env.f, "Chicken curry", cands) + + want := []struct { + url string + score float64 + }{ + {"https://best.example.com/curry", 2}, + {"https://easy.example.com/curry", 1.75}, + {"https://thai.example.com/curry", 1.25}, + } + if len(got) != len(want) { + t.Fatalf("got %d ranked, want %d: %+v (err %v)", len(got), len(want), got, err) + } + for i, w := range want { + if got[i].Candidate.URL != w.url || got[i].Score != w.score { + t.Errorf("rank %d = %s (%.2f), want %s (%.2f)", i, got[i].Candidate.URL, got[i].Score, w.url, w.score) + } + } + if got[0].Recipe.Title != "Chicken Curry" || got[0].Recipe.ImageURL != "https://img.example.com/dish.jpg" { + t.Errorf("top recipe = %+v", got[0].Recipe) + } + if !errors.Is(err, ErrCandidatesUnranked) || !errors.Is(err, ErrNoImage) { + t.Errorf("err = %v, want ErrCandidatesUnranked with ErrNoImage", err) + } + var se *StatusError + if !errors.As(err, &se) || se.Status != http.StatusInternalServerError { + t.Errorf("err = %v, want the 500 StatusError for broken.example.com", err) + } + if n := len(env.dials()); n != len(cands) { + t.Errorf("dialed %d times, want %d", n, len(cands)) + } +} diff --git a/internal/recipes/search.go b/internal/recipes/search.go index 6a17a86..6aec8e0 100644 --- a/internal/recipes/search.go +++ b/internal/recipes/search.go @@ -36,8 +36,10 @@ var ( // ErrSearchRateLimited is returned for HTTP 429. ErrSearchRateLimited = errors.New("recipes: search provider rate limit reached") // ErrSearchResponse is returned for a response body that is not the - // provider's JSON shape, is over MaxSearchBodyBytes, or holds a result - // without an absolute http(s) URL. + // provider's JSON shape — SearXNG without a results array, Brave without + // "type": "search" or with a web object lacking a results array — is + // over MaxSearchBodyBytes, or holds a result without an absolute http(s) + // URL. ErrSearchResponse = errors.New("recipes: malformed search response") ) @@ -126,8 +128,10 @@ func (s *SearXNG) Search(ctx context.Context, query string, n int) ([]Candidate, if err != nil { return nil, fmt.Errorf("recipes: build SearXNG request: %w", err) } + // Results is a pointer so a body without the array is told apart from + // an empty search. var body struct { - Results []struct { + Results *[]struct { URL string `json:"url"` Title string `json:"title"` Content string `json:"content"` @@ -136,8 +140,11 @@ func (s *SearXNG) Search(ctx context.Context, query string, n int) ([]Candidate, if err := doSearch(s.client, req, "SearXNG", &body); err != nil { return nil, err } + if body.Results == nil { + return nil, fmt.Errorf("%w: SearXNG body has no results array", ErrSearchResponse) + } var out []Candidate - for _, r := range body.Results { + for _, r := range *body.Results { out = append(out, Candidate{URL: r.URL, Title: r.Title, Snippet: r.Content}) } return candidates("SearXNG", out, n) @@ -177,9 +184,13 @@ func (b *Brave) Search(ctx context.Context, query string, n int) ([]Candidate, e return nil, fmt.Errorf("recipes: build Brave request: %w", err) } req.Header.Set("X-Subscription-Token", b.key) + // Brave leaves out "web" when nothing matched, so a search response is + // recognised by its "type"; a web object present must hold the results + // array. var body struct { - Web struct { - Results []struct { + Type string `json:"type"` + Web *struct { + Results *[]struct { URL string `json:"url"` Title string `json:"title"` Description string `json:"description"` @@ -189,8 +200,17 @@ func (b *Brave) Search(ctx context.Context, query string, n int) ([]Candidate, e if err := doSearch(b.client, req, "Brave", &body); err != nil { return nil, err } + if body.Type != "search" { + return nil, fmt.Errorf("%w: Brave body type is %q, not \"search\"", ErrSearchResponse, body.Type) + } + if body.Web == nil { + return nil, nil + } + if body.Web.Results == nil { + return nil, fmt.Errorf("%w: Brave web object has no results array", ErrSearchResponse) + } var out []Candidate - for _, r := range body.Web.Results { + for _, r := range *body.Web.Results { out = append(out, Candidate{URL: r.URL, Title: r.Title, Snippet: r.Description}) } return candidates("Brave", out, n) diff --git a/internal/recipes/search_test.go b/internal/recipes/search_test.go index b3a181a..8bda597 100644 --- a/internal/recipes/search_test.go +++ b/internal/recipes/search_test.go @@ -166,8 +166,11 @@ func TestSearchErrors(t *testing.T) { return errors.As(err, &se) && se.Status == http.StatusFound }}, {"not JSON", http.StatusOK, ``, func(err error) bool { return errors.Is(err, ErrSearchResponse) }}, - {"result without URL", http.StatusOK, `{"results":[{"title":"x"}],"web":{"results":[{"title":"x"}]}}`, func(err error) bool { return errors.Is(err, ErrSearchResponse) }}, - {"relative URL", http.StatusOK, `{"results":[{"url":"/x"}],"web":{"results":[{"url":"/x"}]}}`, func(err error) bool { return errors.Is(err, ErrSearchResponse) }}, + {"result without URL", http.StatusOK, `{"type":"search","results":[{"title":"x"}],"web":{"results":[{"title":"x"}]}}`, func(err error) bool { return errors.Is(err, ErrSearchResponse) }}, + {"relative URL", http.StatusOK, `{"type":"search","results":[{"url":"/x"}],"web":{"results":[{"url":"/x"}]}}`, func(err error) bool { return errors.Is(err, ErrSearchResponse) }}, + {"empty object", http.StatusOK, `{}`, func(err error) bool { return errors.Is(err, ErrSearchResponse) }}, + {"null results", http.StatusOK, `{"type":"search","results":null,"web":{"results":null}}`, func(err error) bool { return errors.Is(err, ErrSearchResponse) }}, + {"results not an array", http.StatusOK, `{"type":"search","results":{},"web":{"results":{}}}`, func(err error) bool { return errors.Is(err, ErrSearchResponse) }}, {"body too large", http.StatusOK, `{"results":[],"pad":"` + strings.Repeat("x", MaxSearchBodyBytes) + `"}`, func(err error) bool { return errors.Is(err, ErrSearchResponse) }}, } for _, tc := range tests { @@ -189,6 +192,22 @@ func TestSearchErrors(t *testing.T) { } } +// TestSearchNoMatches checks a well-formed response with nothing found is an +// empty search, not an error. +func TestSearchNoMatches(t *testing.T) { + providers := map[string]SearchProvider{ + "SearXNG": newSearXNGServer(t, http.StatusOK, `{"query":"x recipe","results":[]}`, nil), + "Brave without web": newBraveServer(t, "secret-key", http.StatusOK, `{"type":"search","query":{"original":"x recipe"}}`, nil), + "Brave empty web": newBraveServer(t, "secret-key", http.StatusOK, `{"type":"search","web":{"type":"search","results":[]}}`, nil), + } + for name, p := range providers { + got, err := p.Search(context.Background(), "x", 3) + if err != nil || len(got) != 0 { + t.Errorf("%s: got %v, %v; want no candidates and no error", name, got, err) + } + } +} + func TestSearchContextCancelled(t *testing.T) { s := newSearXNGServer(t, http.StatusOK, searxngBody, nil) ctx, cancel := context.WithCancel(context.Background()) -- 2.55.0