Add SearXNG and Brave recipe search providers with ranking
CI / ci (push) Successful in 1m35s
CI / ci (pull_request) Successful in 1m36s

recipes.SearchProvider.Search(ctx, query, n) returns []Candidate{URL,
Title, Snippet}, appending " recipe" when the query lacks the word.
SearXNG calls {MEALQUEST_SEARXNG_URL}/search?q=…&format=json&categories=general
directly (the configured endpoint may be internal); Brave calls the Web
Search API with the key decrypted from recipe_search_settings. Redirects
are not followed; 401/403, 429, other non-2xx and malformed bodies map to
distinct errors.

recipes.Rank fetches and extracts the top 5 candidates, rejects those
without ingredients or an image, and scores the rest by
harness.Similarity of the recipe title to the dish name plus
completeness (ingredients, image, steps, minutes). harness.Similarity
exposes the existing Jaccard score used by Similar.

Maestro task 10c25cb5-3774-419c-87e1-3f87a3d1391d.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-05 13:17:27 -04:00
co-authored by Claude Opus 5.5
parent ec5b586c16
commit 26928b538b
7 changed files with 886 additions and 1 deletions
+18
View File
@@ -239,6 +239,24 @@ automatic** (D-021).
with two is `ErrManyRecipes`; a shape schema.org does not allow — such as
a non-ISO-8601 duration like "20 minutes" or an empty one, or an image
`@id` no node has — is `ErrMalformedRecipe`, never guessed at.
- Recipe search (`recipes.SearchProvider`: `Search(ctx, query, n)` →
`[]Candidate{URL, Title, Snippet}`, n 1–20) appends " recipe" to a query
without that word. SearXNG is called at
`{MEALQUEST_SEARXNG_URL}/search?q=…&format=json&categories=general` —
directly, not through the Fetcher, since this configured endpoint may be
on the internal network; Brave calls the Brave Web Search API with the
key decrypted from `recipe_search_settings.api_key_ct`
(`recipes.NewSearchProvider`). Redirects are not followed; 401/403 is
`ErrSearchRefused`, 429 `ErrSearchRateLimited`, other non-2xx a
`SearchStatusError`, and a body that is not the provider's JSON, over
1 MB or with a result lacking an absolute http(s) URL `ErrSearchResponse`.
- Ranking (`recipes.Rank`) fetches and extracts the top 5 candidates
concurrently, rejects any whose recipe lacks ingredients or an image, and
orders the rest by the token similarity of the recipe's title to the dish
name (`harness.Similarity`, 0–1) plus a quarter for each of ingredients,
image, steps and total minutes; ties keep search order. When every
candidate is rejected the error is `ErrNoUsableRecipe` joined with each
reason.
- Photos are cached locally (`photos`) and processed with
`golang.org/x/image`.
+8 -1
View File
@@ -53,7 +53,14 @@ func Normalize(name string) string {
// similarity of their normalized forms is at least SimilarThreshold. A name
// with no words left after normalizing is similar to nothing.
func Similar(a, b string) bool {
return similarity(tokenSet(a), tokenSet(b)) >= SimilarThreshold
return Similarity(a, b) >= SimilarThreshold
}
// Similarity is the token-set (Jaccard) similarity of a's and b's normalized
// forms, from 0 (no shared word) to 1 (the same words). A name with no words
// left after normalizing scores 0 against everything.
func Similarity(a, b string) float64 {
return similarity(tokenSet(a), tokenSet(b))
}
// tokenSet returns the distinct words of name's normalized form.
+16
View File
@@ -114,6 +114,22 @@ func TestSimilar(t *testing.T) {
}
}
func TestSimilarity(t *testing.T) {
for _, tc := range []struct {
a, b string
want float64
}{
{"Easy Chicken Curry", "chicken curry", 1},
{"Thai Green Curry", "Chicken curry", 0.25},
{"Beef Stew", "Chicken curry", 0},
{"The Recipe", "chicken curry", 0},
} {
if got := Similarity(tc.a, tc.b); got != tc.want {
t.Errorf("Similarity(%q, %q) = %v, want %v", tc.a, tc.b, got, tc.want)
}
}
}
func TestDeduper(t *testing.T) {
d := NewDeduper(DedupeSources{
Regulars: []string{"Beef Tacos", "Spaghetti Bolognese"},
+116
View File
@@ -0,0 +1,116 @@
package recipes
import (
"context"
"errors"
"fmt"
"sort"
"sync"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/harness"
)
// RankDepth is how many of the top search candidates Rank fetches and
// extracts.
const RankDepth = 5
var (
// ErrNoIngredients rejects a candidate whose recipe lists no
// ingredients.
ErrNoIngredients = errors.New("recipes: recipe has no ingredients")
// ErrNoImage rejects a candidate whose recipe has no image.
ErrNoImage = errors.New("recipes: recipe has no image")
// ErrNoUsableRecipe is returned when Rank rejects every candidate; it
// wraps each candidate's reason.
ErrNoUsableRecipe = errors.New("recipes: no candidate holds a usable recipe")
)
// PageFetcher fetches a candidate's page; *Fetcher satisfies it.
type PageFetcher interface {
Fetch(ctx context.Context, rawURL string) (Page, error)
}
// Ranked is a candidate whose page held a usable recipe, with its score.
type Ranked struct {
Candidate Candidate
Recipe Recipe
// Score is the token similarity of the recipe's title to the dish name
// (0 to 1, harness.Similarity) plus a quarter for each of ingredients,
// image, steps and total minutes present (0 to 1).
Score float64
}
// Rank fetches and extracts the first RankDepth candidates concurrently and
// returns those whose recipe has ingredients and an image, highest Score
// first; equal scores keep search order. When every candidate is rejected
// the error is ErrNoUsableRecipe joined with each rejection's reason.
func Rank(ctx context.Context, f PageFetcher, dish string, cands []Candidate) ([]Ranked, error) {
if len(cands) > RankDepth {
cands = cands[:RankDepth]
}
type outcome struct {
ranked Ranked
err error
}
outcomes := make([]outcome, len(cands))
var wg sync.WaitGroup
for i, c := range cands {
wg.Add(1)
go func() {
defer wg.Done()
r, err := rankOne(ctx, f, dish, c)
outcomes[i] = outcome{r, err}
}()
}
wg.Wait()
if err := ctx.Err(); err != nil {
return nil, err
}
var ranked []Ranked
reasons := []error{ErrNoUsableRecipe}
for i, o := range outcomes {
if o.err != nil {
reasons = append(reasons, fmt.Errorf("%s: %w", cands[i].URL, o.err))
continue
}
ranked = append(ranked, o.ranked)
}
if len(ranked) == 0 {
return nil, errors.Join(reasons...)
}
sort.SliceStable(ranked, func(a, b int) bool { return ranked[a].Score > ranked[b].Score })
return ranked, nil
}
// rankOne fetches, extracts and scores one candidate, or says why it is
// rejected.
func rankOne(ctx context.Context, f PageFetcher, dish string, c Candidate) (Ranked, error) {
page, err := f.Fetch(ctx, c.URL)
if err != nil {
return Ranked{}, err
}
r, err := Extract(page.Body, page.URL)
if err != nil {
return Ranked{}, err
}
if len(r.Ingredients) == 0 {
return Ranked{}, ErrNoIngredients
}
if r.ImageURL == "" {
return Ranked{}, ErrNoImage
}
return Ranked{Candidate: c, Recipe: r, Score: harness.Similarity(r.Title, dish) + completeness(r)}, nil
}
// completeness is a quarter for each of ingredients, image, steps and total
// minutes the recipe has.
func completeness(r Recipe) float64 {
score := 0.0
for _, present := range []bool{len(r.Ingredients) > 0, r.ImageURL != "", len(r.Steps) > 0, r.TotalMinutes != nil} {
if present {
score += 0.25
}
}
return score
}
+159
View File
@@ -0,0 +1,159 @@
package recipes
import (
"context"
"errors"
"fmt"
"sync"
"testing"
)
var _ PageFetcher = (*Fetcher)(nil)
// fakeFetcher serves fixture pages by URL and records what was fetched.
type fakeFetcher struct {
pages map[string][]byte
mu sync.Mutex
fetched []string
}
func (f *fakeFetcher) Fetch(_ context.Context, rawURL string) (Page, error) {
f.mu.Lock()
f.fetched = append(f.fetched, rawURL)
f.mu.Unlock()
body, ok := f.pages[rawURL]
if !ok {
return Page{}, &StatusError{URL: rawURL, Status: 404}
}
return Page{URL: rawURL, ContentType: "text/html", Body: body}, nil
}
// recipePage returns a page whose Recipe has the given name and, as asked,
// ingredients, an image, steps and a total time.
func recipePage(name string, ingredients, image, steps, minutes bool) []byte {
ld := fmt.Sprintf(`{"@context":"https://schema.org","@type":"Recipe","name":%q`, name)
if ingredients {
ld += `,"recipeIngredient":["1 onion","2 cloves garlic"]`
}
if image {
ld += `,"image":"https://img.example.com/dish.jpg"`
}
if steps {
ld += `,"recipeInstructions":["Chop.","Cook."]`
}
if minutes {
ld += `,"totalTime":"PT40M"`
}
return page("", ld+"}")
}
func TestRankOrder(t *testing.T) {
f := &fakeFetcher{pages: map[string][]byte{
"https://a.example/thai": recipePage("Thai Green Curry", true, true, true, true),
"https://b.example/noimg": recipePage("Chicken Curry", true, false, true, true),
"https://c.example/easy": recipePage("Easy Chicken Curry", true, true, true, false),
"https://d.example/noingr": recipePage("Chicken Curry", false, true, true, true),
"https://e.example/best": recipePage("Chicken Curry", true, true, true, true),
"https://f.example/toodeep": recipePage("Chicken Curry", true, true, true, true),
}}
cands := []Candidate{
{URL: "https://a.example/thai", Title: "Thai green curry"},
{URL: "https://b.example/noimg"},
{URL: "https://c.example/easy"},
{URL: "https://d.example/noingr"},
{URL: "https://e.example/best"},
{URL: "https://f.example/toodeep"},
}
got, err := Rank(context.Background(), f, "Chicken curry", cands)
if err != nil {
t.Fatal(err)
}
want := []struct {
url string
score float64
}{
// Same words, every field: 1 + 1.
{"https://e.example/best", 2},
// "Easy" is a filler word, so the title matches; no total time.
{"https://c.example/easy", 1.75},
// Shares one of four words: 0.25 + 1.
{"https://a.example/thai", 1.25},
}
if len(got) != len(want) {
t.Fatalf("got %d ranked, want %d: %+v", len(got), len(want), got)
}
for i, w := range want {
if got[i].Candidate.URL != w.url || got[i].Score != w.score {
t.Errorf("rank %d = %s (%.2f), want %s (%.2f)", i, got[i].Candidate.URL, got[i].Score, w.url, w.score)
}
}
if got[2].Recipe.Title != "Thai Green Curry" || got[2].Candidate.Title != "Thai green curry" {
t.Errorf("ranked entry lost its recipe or candidate: %+v", got[2])
}
for _, u := range f.fetched {
if u == "https://f.example/toodeep" {
t.Errorf("fetched a candidate past RankDepth")
}
}
if len(f.fetched) != RankDepth {
t.Errorf("fetched %d pages, want %d", len(f.fetched), RankDepth)
}
}
func TestRankTiesKeepSearchOrder(t *testing.T) {
f := &fakeFetcher{pages: map[string][]byte{
"https://a.example/1": recipePage("Beef Stew", true, true, true, true),
"https://b.example/2": recipePage("Beef Stew", true, true, true, true),
"https://c.example/3": recipePage("Beef Stew", true, true, true, true),
}}
cands := []Candidate{{URL: "https://a.example/1"}, {URL: "https://b.example/2"}, {URL: "https://c.example/3"}}
got, err := Rank(context.Background(), f, "beef stew", cands)
if err != nil {
t.Fatal(err)
}
for i, c := range cands {
if got[i].Candidate.URL != c.URL {
t.Errorf("rank %d = %s, want %s", i, got[i].Candidate.URL, c.URL)
}
}
}
func TestRankAllRejected(t *testing.T) {
f := &fakeFetcher{pages: map[string][]byte{
"https://a.example/noimg": recipePage("Chicken Curry", true, false, true, true),
"https://b.example/noingr": recipePage("Chicken Curry", false, true, true, true),
"https://c.example/blog": page(""),
}}
cands := []Candidate{
{URL: "https://a.example/noimg"},
{URL: "https://b.example/noingr"},
{URL: "https://c.example/blog"},
{URL: "https://d.example/missing"},
}
_, err := Rank(context.Background(), f, "chicken curry", cands)
for _, want := range []error{ErrNoUsableRecipe, ErrNoImage, ErrNoIngredients, ErrNoRecipe} {
if !errors.Is(err, want) {
t.Errorf("err = %v, want it to wrap %v", err, want)
}
}
var se *StatusError
if !errors.As(err, &se) || se.Status != 404 {
t.Errorf("err = %v, want the 404 StatusError", err)
}
}
func TestRankNoCandidates(t *testing.T) {
if _, err := Rank(context.Background(), &fakeFetcher{}, "chicken curry", nil); !errors.Is(err, ErrNoUsableRecipe) {
t.Errorf("err = %v, want ErrNoUsableRecipe", err)
}
}
func TestRankContextCancelled(t *testing.T) {
f := &fakeFetcher{pages: map[string][]byte{"https://a.example/1": recipePage("Beef Stew", true, true, true, true)}}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := Rank(ctx, f, "beef stew", []Candidate{{URL: "https://a.example/1"}}); !errors.Is(err, context.Canceled) {
t.Errorf("err = %v, want context.Canceled", err)
}
}
+294
View File
@@ -0,0 +1,294 @@
package recipes
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/db"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/secretbox"
)
const (
// MaxSearchResults is the most candidates one search may ask for; it is
// Brave's largest page.
MaxSearchResults = 20
// SearchTimeout bounds one search request, reading the body included.
SearchTimeout = 10 * time.Second
// MaxSearchBodyBytes caps a search response body. A larger one is an
// error, never truncated.
MaxSearchBodyBytes = 1 << 20
// BraveEndpoint is the Brave Web Search API.
BraveEndpoint = "https://api.search.brave.com/res/v1/web/search"
)
var (
// ErrSearchRefused is returned for HTTP 401 or 403: Brave rejected the
// API key, or SearXNG does not allow format=json.
ErrSearchRefused = errors.New("recipes: search provider refused the request")
// ErrSearchRateLimited is returned for HTTP 429.
ErrSearchRateLimited = errors.New("recipes: search provider rate limit reached")
// ErrSearchResponse is returned for a response body that is not the
// provider's JSON shape, is over MaxSearchBodyBytes, or holds a result
// without an absolute http(s) URL.
ErrSearchResponse = errors.New("recipes: malformed search response")
)
// SearchStatusError is returned for any other non-2xx search response,
// redirects included: a search endpoint is called directly and never
// followed elsewhere.
type SearchStatusError struct {
Provider string
Status int
}
func (e *SearchStatusError) Error() string {
return fmt.Sprintf("recipes: %s search: HTTP %d", e.Provider, e.Status)
}
// Candidate is one search result: a page that may hold a recipe.
type Candidate struct {
URL string
Title string
Snippet string
}
// SearchProvider finds recipe pages on the web.
type SearchProvider interface {
// Search returns up to n candidates for query, best first. The query
// gets " recipe" appended unless it already has the word. n must be
// 1 to MaxSearchResults.
Search(ctx context.Context, query string, n int) ([]Candidate, error)
}
// NewSearchProvider returns the provider s names. SearXNG is called at
// searxngURL, the configured MEALQUEST_SEARXNG_URL; Brave is called with the
// API key in s.ApiKeyCt, decrypted by box.
func NewSearchProvider(s db.RecipeSearchSetting, box *secretbox.Box, searxngURL string) (SearchProvider, error) {
switch s.Provider {
case "searxng":
return NewSearXNG(searxngURL)
case "brave":
if len(s.ApiKeyCt) == 0 {
return nil, errors.New("recipes: Brave search has no API key")
}
key, err := box.Decrypt(s.ApiKeyCt)
if err != nil {
return nil, fmt.Errorf("recipes: Brave API key: %w", err)
}
return NewBrave(string(key))
default:
return nil, fmt.Errorf("recipes: unknown search provider %q", s.Provider)
}
}
// SearXNG searches through a SearXNG instance. Its base URL is a configured
// endpoint and may be on a private network, so it is called directly rather
// than through the public-only Fetcher.
type SearXNG struct {
endpoint *url.URL
client *http.Client
}
// NewSearXNG returns a SearXNG provider for the instance at baseURL, an
// absolute http:// or https:// URL without credentials, query or fragment.
func NewSearXNG(baseURL string) (*SearXNG, error) {
if baseURL == "" {
return nil, errors.New("recipes: SearXNG URL is not set")
}
u, err := url.Parse(baseURL)
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
return nil, errors.New("recipes: SearXNG URL must be an absolute http:// or https:// URL")
}
if u.User != nil || u.RawQuery != "" || u.Fragment != "" {
return nil, errors.New("recipes: SearXNG URL must not have credentials, a query or a fragment")
}
return &SearXNG{endpoint: u.JoinPath("search"), client: newSearchClient()}, nil
}
// Search implements SearchProvider with
// {base}/search?q=…&format=json&categories=general.
func (s *SearXNG) Search(ctx context.Context, query string, n int) ([]Candidate, error) {
q, err := searchQuery(query, n)
if err != nil {
return nil, err
}
u := *s.endpoint
u.RawQuery = url.Values{"q": {q}, "format": {"json"}, "categories": {"general"}}.Encode()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
if err != nil {
return nil, fmt.Errorf("recipes: build SearXNG request: %w", err)
}
var body struct {
Results []struct {
URL string `json:"url"`
Title string `json:"title"`
Content string `json:"content"`
} `json:"results"`
}
if err := doSearch(s.client, req, "SearXNG", &body); err != nil {
return nil, err
}
var out []Candidate
for _, r := range body.Results {
out = append(out, Candidate{URL: r.URL, Title: r.Title, Snippet: r.Content})
}
return candidates("SearXNG", out, n)
}
// Brave searches through the Brave Web Search API.
type Brave struct {
key string
endpoint string
client *http.Client
}
// NewBrave returns a Brave provider that authenticates with key, the
// decrypted API key.
func NewBrave(key string) (*Brave, error) {
if key == "" {
return nil, errors.New("recipes: Brave API key is empty")
}
for i := 0; i < len(key); i++ {
if key[i] < 0x21 || key[i] > 0x7e {
// The key is never echoed.
return nil, errors.New("recipes: Brave API key has a character not allowed in a header")
}
}
return &Brave{key: key, endpoint: BraveEndpoint, client: newSearchClient()}, nil
}
// Search implements SearchProvider with GET BraveEndpoint?q=…&count=n.
func (b *Brave) Search(ctx context.Context, query string, n int) ([]Candidate, error) {
q, err := searchQuery(query, n)
if err != nil {
return nil, err
}
u := b.endpoint + "?" + url.Values{"q": {q}, "count": {strconv.Itoa(n)}}.Encode()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
if err != nil {
return nil, fmt.Errorf("recipes: build Brave request: %w", err)
}
req.Header.Set("X-Subscription-Token", b.key)
var body struct {
Web struct {
Results []struct {
URL string `json:"url"`
Title string `json:"title"`
Description string `json:"description"`
} `json:"results"`
} `json:"web"`
}
if err := doSearch(b.client, req, "Brave", &body); err != nil {
return nil, err
}
var out []Candidate
for _, r := range body.Web.Results {
out = append(out, Candidate{URL: r.URL, Title: r.Title, Snippet: r.Description})
}
return candidates("Brave", out, n)
}
func newSearchClient() *http.Client {
return &http.Client{
Timeout: SearchTimeout,
CheckRedirect: func(*http.Request, []*http.Request) error {
return http.ErrUseLastResponse
},
}
}
// searchQuery validates n and returns query with " recipe" appended unless
// one of its words already is "recipe", in any case.
func searchQuery(query string, n int) (string, error) {
if n < 1 || n > MaxSearchResults {
return "", fmt.Errorf("recipes: search result count %d is outside 1-%d", n, MaxSearchResults)
}
q := strings.TrimSpace(query)
if q == "" {
return "", errors.New("recipes: search query is empty")
}
for _, w := range strings.Fields(q) {
if strings.EqualFold(w, "recipe") {
return q, nil
}
}
return q + " recipe", nil
}
// doSearch sends req and decodes a 2xx JSON body into v, mapping failures to
// the search errors.
func doSearch(client *http.Client, req *http.Request, provider string, v any) error {
req.Header.Set("Accept", "application/json")
req.Header.Set("User-Agent", UserAgent)
resp, err := client.Do(req)
if err != nil {
if ctxErr := req.Context().Err(); ctxErr != nil {
return ctxErr
}
return fmt.Errorf("recipes: %s search: %w", provider, err)
}
defer resp.Body.Close()
switch {
case resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden:
return fmt.Errorf("%w: %s HTTP %d", ErrSearchRefused, provider, resp.StatusCode)
case resp.StatusCode == http.StatusTooManyRequests:
return fmt.Errorf("%w: %s", ErrSearchRateLimited, provider)
case resp.StatusCode < 200 || resp.StatusCode > 299:
return &SearchStatusError{Provider: provider, Status: resp.StatusCode}
}
data, err := io.ReadAll(io.LimitReader(resp.Body, MaxSearchBodyBytes+1))
if err != nil {
return fmt.Errorf("recipes: %s search: read body: %w", provider, err)
}
if len(data) > MaxSearchBodyBytes {
return fmt.Errorf("%w: %s body is over %d bytes", ErrSearchResponse, provider, MaxSearchBodyBytes)
}
if err := json.Unmarshal(data, v); err != nil {
return fmt.Errorf("%w: %s: %v", ErrSearchResponse, provider, err)
}
return nil
}
// candidates cleans results — HTML tags and entities, which both providers
// leave in titles and snippets, are removed — checks every URL is absolute
// http(s) and keeps the first n.
func candidates(provider string, results []Candidate, n int) ([]Candidate, error) {
if len(results) > n {
results = results[:n]
}
out := make([]Candidate, 0, len(results))
for i, r := range results {
u, err := url.Parse(r.URL)
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
return nil, fmt.Errorf("%w: %s result %d has no absolute http(s) URL", ErrSearchResponse, provider, i)
}
out = append(out, Candidate{URL: r.URL, Title: stripTags(r.Title), Snippet: stripTags(r.Snippet)})
}
return out, nil
}
// stripTags drops HTML tags such as Brave's <strong> highlights, then
// decodes entities and trims space.
func stripTags(s string) string {
var b strings.Builder
inTag := false
for _, r := range s {
switch {
case r == '<':
inTag = true
case r == '>' && inTag:
inTag = false
case !inTag:
b.WriteRune(r)
}
}
return cleanText(b.String())
}
+275
View File
@@ -0,0 +1,275 @@
package recipes
import (
"context"
"crypto/rand"
"encoding/base64"
"errors"
"net/http"
"net/http/httptest"
"reflect"
"strings"
"testing"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/db"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/secretbox"
)
const searxngBody = `{
"query": "chicken curry recipe",
"results": [
{"url": "https://www.bbcgoodfood.com/recipes/easy-chicken-curry", "title": "Easy chicken curry recipe | BBC Good Food", "content": "Make our easy chicken curry &amp; rice.", "engine": "duckduckgo"},
{"url": "https://www.recipetineats.com/chicken-curry/", "title": "Chicken Curry", "content": "A proper curry."},
{"url": "https://example.com/third", "title": "Third", "content": ""}
],
"number_of_results": 0
}`
const braveBody = `{
"type": "search",
"web": {
"type": "search",
"results": [
{"url": "https://www.bbcgoodfood.com/recipes/easy-chicken-curry", "title": "Easy <strong>chicken curry</strong> recipe", "description": "Make our <strong>easy</strong> chicken curry &amp; rice.", "language": "en"},
{"url": "https://www.recipetineats.com/chicken-curry/", "title": "Chicken Curry", "description": "A proper curry."}
]
}
}`
func newSearXNGServer(t *testing.T, status int, body string, got *http.Request) *SearXNG {
t.Helper()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if got != nil {
*got = *r.Clone(context.Background())
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
w.Write([]byte(body))
}))
t.Cleanup(srv.Close)
s, err := NewSearXNG(srv.URL + "/searx")
if err != nil {
t.Fatal(err)
}
return s
}
func newBraveServer(t *testing.T, key string, status int, body string, got *http.Request) *Brave {
t.Helper()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if got != nil {
*got = *r.Clone(context.Background())
}
w.WriteHeader(status)
w.Write([]byte(body))
}))
t.Cleanup(srv.Close)
b, err := NewBrave(key)
if err != nil {
t.Fatal(err)
}
b.endpoint = srv.URL + "/res/v1/web/search"
return b
}
func TestSearXNGSearch(t *testing.T) {
var req http.Request
s := newSearXNGServer(t, http.StatusOK, searxngBody, &req)
got, err := s.Search(context.Background(), "chicken curry", 2)
if err != nil {
t.Fatal(err)
}
if req.URL.Path != "/searx/search" {
t.Errorf("path = %q, want /searx/search", req.URL.Path)
}
q := req.URL.Query()
if q.Get("q") != "chicken curry recipe" || q.Get("format") != "json" || q.Get("categories") != "general" {
t.Errorf("query = %v", q)
}
if req.Header.Get("User-Agent") != UserAgent {
t.Errorf("User-Agent = %q", req.Header.Get("User-Agent"))
}
want := []Candidate{
{URL: "https://www.bbcgoodfood.com/recipes/easy-chicken-curry", Title: "Easy chicken curry recipe | BBC Good Food", Snippet: "Make our easy chicken curry & rice."},
{URL: "https://www.recipetineats.com/chicken-curry/", Title: "Chicken Curry", Snippet: "A proper curry."},
}
if !reflect.DeepEqual(got, want) {
t.Errorf("candidates =\n%#v\nwant\n%#v", got, want)
}
}
func TestBraveSearch(t *testing.T) {
var req http.Request
b := newBraveServer(t, "brave-key-123", http.StatusOK, braveBody, &req)
got, err := b.Search(context.Background(), "Chicken curry RECIPE", 5)
if err != nil {
t.Fatal(err)
}
if req.Header.Get("X-Subscription-Token") != "brave-key-123" {
t.Errorf("X-Subscription-Token = %q", req.Header.Get("X-Subscription-Token"))
}
if req.Header.Get("Accept") != "application/json" {
t.Errorf("Accept = %q", req.Header.Get("Accept"))
}
q := req.URL.Query()
if q.Get("q") != "Chicken curry RECIPE" || q.Get("count") != "5" {
t.Errorf("query = %v", q)
}
want := []Candidate{
{URL: "https://www.bbcgoodfood.com/recipes/easy-chicken-curry", Title: "Easy chicken curry recipe", Snippet: "Make our easy chicken curry & rice."},
{URL: "https://www.recipetineats.com/chicken-curry/", Title: "Chicken Curry", Snippet: "A proper curry."},
}
if !reflect.DeepEqual(got, want) {
t.Errorf("candidates =\n%#v\nwant\n%#v", got, want)
}
}
func TestSearchQuery(t *testing.T) {
for _, tc := range []struct{ in, want string }{
{"chicken curry", "chicken curry recipe"},
{" chicken curry ", "chicken curry recipe"},
{"chicken curry recipe", "chicken curry recipe"},
{"Recipe for chicken curry", "Recipe for chicken curry"},
{"recipes for curry", "recipes for curry recipe"},
} {
got, err := searchQuery(tc.in, 3)
if err != nil || got != tc.want {
t.Errorf("searchQuery(%q) = %q, %v; want %q", tc.in, got, err, tc.want)
}
}
for _, tc := range []struct {
q string
n int
}{{"", 3}, {" ", 3}, {"curry", 0}, {"curry", MaxSearchResults + 1}} {
if _, err := searchQuery(tc.q, tc.n); err == nil {
t.Errorf("searchQuery(%q, %d) succeeded, want error", tc.q, tc.n)
}
}
}
func TestSearchErrors(t *testing.T) {
tests := []struct {
name string
status int
body string
check func(error) bool
}{
{"unauthorized", http.StatusUnauthorized, `{}`, func(err error) bool { return errors.Is(err, ErrSearchRefused) }},
{"forbidden", http.StatusForbidden, `{}`, func(err error) bool { return errors.Is(err, ErrSearchRefused) }},
{"rate limited", http.StatusTooManyRequests, `{}`, func(err error) bool { return errors.Is(err, ErrSearchRateLimited) }},
{"server error", http.StatusBadGateway, `{}`, func(err error) bool {
var se *SearchStatusError
return errors.As(err, &se) && se.Status == http.StatusBadGateway
}},
{"redirect not followed", http.StatusFound, ``, func(err error) bool {
var se *SearchStatusError
return errors.As(err, &se) && se.Status == http.StatusFound
}},
{"not JSON", http.StatusOK, `<html>`, func(err error) bool { return errors.Is(err, ErrSearchResponse) }},
{"result without URL", http.StatusOK, `{"results":[{"title":"x"}],"web":{"results":[{"title":"x"}]}}`, func(err error) bool { return errors.Is(err, ErrSearchResponse) }},
{"relative URL", http.StatusOK, `{"results":[{"url":"/x"}],"web":{"results":[{"url":"/x"}]}}`, func(err error) bool { return errors.Is(err, ErrSearchResponse) }},
{"body too large", http.StatusOK, `{"results":[],"pad":"` + strings.Repeat("x", MaxSearchBodyBytes) + `"}`, func(err error) bool { return errors.Is(err, ErrSearchResponse) }},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
providers := map[string]SearchProvider{
"SearXNG": newSearXNGServer(t, tc.status, tc.body, nil),
"Brave": newBraveServer(t, "secret-key", tc.status, tc.body, nil),
}
for name, p := range providers {
_, err := p.Search(context.Background(), "curry", 3)
if err == nil || !tc.check(err) {
t.Errorf("%s: err = %v", name, err)
}
if err != nil && strings.Contains(err.Error(), "secret-key") {
t.Errorf("%s: error leaks the API key: %v", name, err)
}
}
})
}
}
func TestSearchContextCancelled(t *testing.T) {
s := newSearXNGServer(t, http.StatusOK, searxngBody, nil)
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := s.Search(ctx, "curry", 3); !errors.Is(err, context.Canceled) {
t.Errorf("err = %v, want context.Canceled", err)
}
}
func TestSearchUnreachable(t *testing.T) {
srv := httptest.NewServer(http.NotFoundHandler())
url := srv.URL
srv.Close()
s, err := NewSearXNG(url)
if err != nil {
t.Fatal(err)
}
_, err = s.Search(context.Background(), "curry", 3)
if err == nil || !strings.Contains(err.Error(), "SearXNG search") {
t.Errorf("err = %v", err)
}
}
func TestNewSearXNGRejects(t *testing.T) {
for _, u := range []string{"", "searxng:8080", "ftp://searxng", "http://", "http://u:p@searxng", "http://searxng?x=1", "http://searxng#f"} {
if _, err := NewSearXNG(u); err == nil {
t.Errorf("NewSearXNG(%q) succeeded, want error", u)
}
}
if _, err := NewSearXNG("http://searxng:8080"); err != nil {
t.Errorf("internal URL rejected: %v", err)
}
}
func TestNewBraveRejects(t *testing.T) {
for _, k := range []string{"", "with space", "line\nbreak"} {
if _, err := NewBrave(k); err == nil {
t.Errorf("NewBrave(%q) succeeded, want error", k)
}
}
}
func TestNewSearchProvider(t *testing.T) {
raw := make([]byte, secretbox.KeySize)
rand.Read(raw)
box, err := secretbox.New(base64.StdEncoding.EncodeToString(raw))
if err != nil {
t.Fatal(err)
}
ct, err := box.Encrypt([]byte("brave-key"))
if err != nil {
t.Fatal(err)
}
p, err := NewSearchProvider(db.RecipeSearchSetting{Provider: "brave", ApiKeyCt: ct}, box, "")
if err != nil {
t.Fatal(err)
}
if b, ok := p.(*Brave); !ok || b.key != "brave-key" || b.endpoint != BraveEndpoint {
t.Errorf("provider = %#v, want Brave with decrypted key", p)
}
p, err = NewSearchProvider(db.RecipeSearchSetting{Provider: "searxng"}, box, "http://searxng:8080")
if err != nil {
t.Fatal(err)
}
if s, ok := p.(*SearXNG); !ok || s.endpoint.String() != "http://searxng:8080/search" {
t.Errorf("provider = %#v, want SearXNG at http://searxng:8080/search", p)
}
for name, tc := range map[string]struct {
s db.RecipeSearchSetting
url string
}{
"brave without key": {db.RecipeSearchSetting{Provider: "brave"}, ""},
"brave bad key": {db.RecipeSearchSetting{Provider: "brave", ApiKeyCt: []byte("not a token at all, far too plain")}, ""},
"searxng without URL": {db.RecipeSearchSetting{Provider: "searxng"}, ""},
"unknown provider": {db.RecipeSearchSetting{Provider: "google"}, ""},
} {
if _, err := NewSearchProvider(tc.s, box, tc.url); err == nil {
t.Errorf("%s: succeeded, want error", name)
}
}
}