diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 6f31a0a..f9cd825 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -239,6 +239,24 @@ automatic** (D-021). with two is `ErrManyRecipes`; a shape schema.org does not allow — such as a non-ISO-8601 duration like "20 minutes" or an empty one, or an image `@id` no node has — is `ErrMalformedRecipe`, never guessed at. +- Recipe search (`recipes.SearchProvider`: `Search(ctx, query, n)` → + `[]Candidate{URL, Title, Snippet}`, n 1–20) appends " recipe" to a query + without that word. SearXNG is called at + `{MEALQUEST_SEARXNG_URL}/search?q=…&format=json&categories=general` — + directly, not through the Fetcher, since this configured endpoint may be + on the internal network; Brave calls the Brave Web Search API with the + key decrypted from `recipe_search_settings.api_key_ct` + (`recipes.NewSearchProvider`). Redirects are not followed; 401/403 is + `ErrSearchRefused`, 429 `ErrSearchRateLimited`, other non-2xx a + `SearchStatusError`, and a body that is not the provider's JSON, over + 1 MB or with a result lacking an absolute http(s) URL `ErrSearchResponse`. +- Ranking (`recipes.Rank`) fetches and extracts the top 5 candidates + concurrently, rejects any whose recipe lacks ingredients or an image, and + orders the rest by the token similarity of the recipe's title to the dish + name (`harness.Similarity`, 0–1) plus a quarter for each of ingredients, + image, steps and total minutes; ties keep search order. When every + candidate is rejected the error is `ErrNoUsableRecipe` joined with each + reason. - Photos are cached locally (`photos`) and processed with `golang.org/x/image`. diff --git a/internal/ai/harness/dedupe.go b/internal/ai/harness/dedupe.go index ce6c446..7b2233c 100644 --- a/internal/ai/harness/dedupe.go +++ b/internal/ai/harness/dedupe.go @@ -53,7 +53,14 @@ func Normalize(name string) string { // similarity of their normalized forms is at least SimilarThreshold. A name // with no words left after normalizing is similar to nothing. func Similar(a, b string) bool { - return similarity(tokenSet(a), tokenSet(b)) >= SimilarThreshold + return Similarity(a, b) >= SimilarThreshold +} + +// Similarity is the token-set (Jaccard) similarity of a's and b's normalized +// forms, from 0 (no shared word) to 1 (the same words). A name with no words +// left after normalizing scores 0 against everything. +func Similarity(a, b string) float64 { + return similarity(tokenSet(a), tokenSet(b)) } // tokenSet returns the distinct words of name's normalized form. diff --git a/internal/ai/harness/dedupe_test.go b/internal/ai/harness/dedupe_test.go index 5779695..99332d3 100644 --- a/internal/ai/harness/dedupe_test.go +++ b/internal/ai/harness/dedupe_test.go @@ -114,6 +114,22 @@ func TestSimilar(t *testing.T) { } } +func TestSimilarity(t *testing.T) { + for _, tc := range []struct { + a, b string + want float64 + }{ + {"Easy Chicken Curry", "chicken curry", 1}, + {"Thai Green Curry", "Chicken curry", 0.25}, + {"Beef Stew", "Chicken curry", 0}, + {"The Recipe", "chicken curry", 0}, + } { + if got := Similarity(tc.a, tc.b); got != tc.want { + t.Errorf("Similarity(%q, %q) = %v, want %v", tc.a, tc.b, got, tc.want) + } + } +} + func TestDeduper(t *testing.T) { d := NewDeduper(DedupeSources{ Regulars: []string{"Beef Tacos", "Spaghetti Bolognese"}, diff --git a/internal/recipes/rank.go b/internal/recipes/rank.go new file mode 100644 index 0000000..e025f35 --- /dev/null +++ b/internal/recipes/rank.go @@ -0,0 +1,116 @@ +package recipes + +import ( + "context" + "errors" + "fmt" + "sort" + "sync" + + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/harness" +) + +// RankDepth is how many of the top search candidates Rank fetches and +// extracts. +const RankDepth = 5 + +var ( + // ErrNoIngredients rejects a candidate whose recipe lists no + // ingredients. + ErrNoIngredients = errors.New("recipes: recipe has no ingredients") + // ErrNoImage rejects a candidate whose recipe has no image. + ErrNoImage = errors.New("recipes: recipe has no image") + // ErrNoUsableRecipe is returned when Rank rejects every candidate; it + // wraps each candidate's reason. + ErrNoUsableRecipe = errors.New("recipes: no candidate holds a usable recipe") +) + +// PageFetcher fetches a candidate's page; *Fetcher satisfies it. +type PageFetcher interface { + Fetch(ctx context.Context, rawURL string) (Page, error) +} + +// Ranked is a candidate whose page held a usable recipe, with its score. +type Ranked struct { + Candidate Candidate + Recipe Recipe + // Score is the token similarity of the recipe's title to the dish name + // (0 to 1, harness.Similarity) plus a quarter for each of ingredients, + // image, steps and total minutes present (0 to 1). + Score float64 +} + +// Rank fetches and extracts the first RankDepth candidates concurrently and +// returns those whose recipe has ingredients and an image, highest Score +// first; equal scores keep search order. When every candidate is rejected +// the error is ErrNoUsableRecipe joined with each rejection's reason. +func Rank(ctx context.Context, f PageFetcher, dish string, cands []Candidate) ([]Ranked, error) { + if len(cands) > RankDepth { + cands = cands[:RankDepth] + } + type outcome struct { + ranked Ranked + err error + } + outcomes := make([]outcome, len(cands)) + var wg sync.WaitGroup + for i, c := range cands { + wg.Add(1) + go func() { + defer wg.Done() + r, err := rankOne(ctx, f, dish, c) + outcomes[i] = outcome{r, err} + }() + } + wg.Wait() + if err := ctx.Err(); err != nil { + return nil, err + } + + var ranked []Ranked + reasons := []error{ErrNoUsableRecipe} + for i, o := range outcomes { + if o.err != nil { + reasons = append(reasons, fmt.Errorf("%s: %w", cands[i].URL, o.err)) + continue + } + ranked = append(ranked, o.ranked) + } + if len(ranked) == 0 { + return nil, errors.Join(reasons...) + } + sort.SliceStable(ranked, func(a, b int) bool { return ranked[a].Score > ranked[b].Score }) + return ranked, nil +} + +// rankOne fetches, extracts and scores one candidate, or says why it is +// rejected. +func rankOne(ctx context.Context, f PageFetcher, dish string, c Candidate) (Ranked, error) { + page, err := f.Fetch(ctx, c.URL) + if err != nil { + return Ranked{}, err + } + r, err := Extract(page.Body, page.URL) + if err != nil { + return Ranked{}, err + } + if len(r.Ingredients) == 0 { + return Ranked{}, ErrNoIngredients + } + if r.ImageURL == "" { + return Ranked{}, ErrNoImage + } + return Ranked{Candidate: c, Recipe: r, Score: harness.Similarity(r.Title, dish) + completeness(r)}, nil +} + +// completeness is a quarter for each of ingredients, image, steps and total +// minutes the recipe has. +func completeness(r Recipe) float64 { + score := 0.0 + for _, present := range []bool{len(r.Ingredients) > 0, r.ImageURL != "", len(r.Steps) > 0, r.TotalMinutes != nil} { + if present { + score += 0.25 + } + } + return score +} diff --git a/internal/recipes/rank_test.go b/internal/recipes/rank_test.go new file mode 100644 index 0000000..b70c67b --- /dev/null +++ b/internal/recipes/rank_test.go @@ -0,0 +1,159 @@ +package recipes + +import ( + "context" + "errors" + "fmt" + "sync" + "testing" +) + +var _ PageFetcher = (*Fetcher)(nil) + +// fakeFetcher serves fixture pages by URL and records what was fetched. +type fakeFetcher struct { + pages map[string][]byte + + mu sync.Mutex + fetched []string +} + +func (f *fakeFetcher) Fetch(_ context.Context, rawURL string) (Page, error) { + f.mu.Lock() + f.fetched = append(f.fetched, rawURL) + f.mu.Unlock() + body, ok := f.pages[rawURL] + if !ok { + return Page{}, &StatusError{URL: rawURL, Status: 404} + } + return Page{URL: rawURL, ContentType: "text/html", Body: body}, nil +} + +// recipePage returns a page whose Recipe has the given name and, as asked, +// ingredients, an image, steps and a total time. +func recipePage(name string, ingredients, image, steps, minutes bool) []byte { + ld := fmt.Sprintf(`{"@context":"https://schema.org","@type":"Recipe","name":%q`, name) + if ingredients { + ld += `,"recipeIngredient":["1 onion","2 cloves garlic"]` + } + if image { + ld += `,"image":"https://img.example.com/dish.jpg"` + } + if steps { + ld += `,"recipeInstructions":["Chop.","Cook."]` + } + if minutes { + ld += `,"totalTime":"PT40M"` + } + return page("", ld+"}") +} + +func TestRankOrder(t *testing.T) { + f := &fakeFetcher{pages: map[string][]byte{ + "https://a.example/thai": recipePage("Thai Green Curry", true, true, true, true), + "https://b.example/noimg": recipePage("Chicken Curry", true, false, true, true), + "https://c.example/easy": recipePage("Easy Chicken Curry", true, true, true, false), + "https://d.example/noingr": recipePage("Chicken Curry", false, true, true, true), + "https://e.example/best": recipePage("Chicken Curry", true, true, true, true), + "https://f.example/toodeep": recipePage("Chicken Curry", true, true, true, true), + }} + cands := []Candidate{ + {URL: "https://a.example/thai", Title: "Thai green curry"}, + {URL: "https://b.example/noimg"}, + {URL: "https://c.example/easy"}, + {URL: "https://d.example/noingr"}, + {URL: "https://e.example/best"}, + {URL: "https://f.example/toodeep"}, + } + got, err := Rank(context.Background(), f, "Chicken curry", cands) + if err != nil { + t.Fatal(err) + } + want := []struct { + url string + score float64 + }{ + // Same words, every field: 1 + 1. + {"https://e.example/best", 2}, + // "Easy" is a filler word, so the title matches; no total time. + {"https://c.example/easy", 1.75}, + // Shares one of four words: 0.25 + 1. + {"https://a.example/thai", 1.25}, + } + if len(got) != len(want) { + t.Fatalf("got %d ranked, want %d: %+v", len(got), len(want), got) + } + for i, w := range want { + if got[i].Candidate.URL != w.url || got[i].Score != w.score { + t.Errorf("rank %d = %s (%.2f), want %s (%.2f)", i, got[i].Candidate.URL, got[i].Score, w.url, w.score) + } + } + if got[2].Recipe.Title != "Thai Green Curry" || got[2].Candidate.Title != "Thai green curry" { + t.Errorf("ranked entry lost its recipe or candidate: %+v", got[2]) + } + for _, u := range f.fetched { + if u == "https://f.example/toodeep" { + t.Errorf("fetched a candidate past RankDepth") + } + } + if len(f.fetched) != RankDepth { + t.Errorf("fetched %d pages, want %d", len(f.fetched), RankDepth) + } +} + +func TestRankTiesKeepSearchOrder(t *testing.T) { + f := &fakeFetcher{pages: map[string][]byte{ + "https://a.example/1": recipePage("Beef Stew", true, true, true, true), + "https://b.example/2": recipePage("Beef Stew", true, true, true, true), + "https://c.example/3": recipePage("Beef Stew", true, true, true, true), + }} + cands := []Candidate{{URL: "https://a.example/1"}, {URL: "https://b.example/2"}, {URL: "https://c.example/3"}} + got, err := Rank(context.Background(), f, "beef stew", cands) + if err != nil { + t.Fatal(err) + } + for i, c := range cands { + if got[i].Candidate.URL != c.URL { + t.Errorf("rank %d = %s, want %s", i, got[i].Candidate.URL, c.URL) + } + } +} + +func TestRankAllRejected(t *testing.T) { + f := &fakeFetcher{pages: map[string][]byte{ + "https://a.example/noimg": recipePage("Chicken Curry", true, false, true, true), + "https://b.example/noingr": recipePage("Chicken Curry", false, true, true, true), + "https://c.example/blog": page(""), + }} + cands := []Candidate{ + {URL: "https://a.example/noimg"}, + {URL: "https://b.example/noingr"}, + {URL: "https://c.example/blog"}, + {URL: "https://d.example/missing"}, + } + _, err := Rank(context.Background(), f, "chicken curry", cands) + for _, want := range []error{ErrNoUsableRecipe, ErrNoImage, ErrNoIngredients, ErrNoRecipe} { + if !errors.Is(err, want) { + t.Errorf("err = %v, want it to wrap %v", err, want) + } + } + var se *StatusError + if !errors.As(err, &se) || se.Status != 404 { + t.Errorf("err = %v, want the 404 StatusError", err) + } +} + +func TestRankNoCandidates(t *testing.T) { + if _, err := Rank(context.Background(), &fakeFetcher{}, "chicken curry", nil); !errors.Is(err, ErrNoUsableRecipe) { + t.Errorf("err = %v, want ErrNoUsableRecipe", err) + } +} + +func TestRankContextCancelled(t *testing.T) { + f := &fakeFetcher{pages: map[string][]byte{"https://a.example/1": recipePage("Beef Stew", true, true, true, true)}} + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := Rank(ctx, f, "beef stew", []Candidate{{URL: "https://a.example/1"}}); !errors.Is(err, context.Canceled) { + t.Errorf("err = %v, want context.Canceled", err) + } +} diff --git a/internal/recipes/search.go b/internal/recipes/search.go new file mode 100644 index 0000000..6a17a86 --- /dev/null +++ b/internal/recipes/search.go @@ -0,0 +1,294 @@ +package recipes + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/db" + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/secretbox" +) + +const ( + // MaxSearchResults is the most candidates one search may ask for; it is + // Brave's largest page. + MaxSearchResults = 20 + // SearchTimeout bounds one search request, reading the body included. + SearchTimeout = 10 * time.Second + // MaxSearchBodyBytes caps a search response body. A larger one is an + // error, never truncated. + MaxSearchBodyBytes = 1 << 20 + // BraveEndpoint is the Brave Web Search API. + BraveEndpoint = "https://api.search.brave.com/res/v1/web/search" +) + +var ( + // ErrSearchRefused is returned for HTTP 401 or 403: Brave rejected the + // API key, or SearXNG does not allow format=json. + ErrSearchRefused = errors.New("recipes: search provider refused the request") + // ErrSearchRateLimited is returned for HTTP 429. + ErrSearchRateLimited = errors.New("recipes: search provider rate limit reached") + // ErrSearchResponse is returned for a response body that is not the + // provider's JSON shape, is over MaxSearchBodyBytes, or holds a result + // without an absolute http(s) URL. + ErrSearchResponse = errors.New("recipes: malformed search response") +) + +// SearchStatusError is returned for any other non-2xx search response, +// redirects included: a search endpoint is called directly and never +// followed elsewhere. +type SearchStatusError struct { + Provider string + Status int +} + +func (e *SearchStatusError) Error() string { + return fmt.Sprintf("recipes: %s search: HTTP %d", e.Provider, e.Status) +} + +// Candidate is one search result: a page that may hold a recipe. +type Candidate struct { + URL string + Title string + Snippet string +} + +// SearchProvider finds recipe pages on the web. +type SearchProvider interface { + // Search returns up to n candidates for query, best first. The query + // gets " recipe" appended unless it already has the word. n must be + // 1 to MaxSearchResults. + Search(ctx context.Context, query string, n int) ([]Candidate, error) +} + +// NewSearchProvider returns the provider s names. SearXNG is called at +// searxngURL, the configured MEALQUEST_SEARXNG_URL; Brave is called with the +// API key in s.ApiKeyCt, decrypted by box. +func NewSearchProvider(s db.RecipeSearchSetting, box *secretbox.Box, searxngURL string) (SearchProvider, error) { + switch s.Provider { + case "searxng": + return NewSearXNG(searxngURL) + case "brave": + if len(s.ApiKeyCt) == 0 { + return nil, errors.New("recipes: Brave search has no API key") + } + key, err := box.Decrypt(s.ApiKeyCt) + if err != nil { + return nil, fmt.Errorf("recipes: Brave API key: %w", err) + } + return NewBrave(string(key)) + default: + return nil, fmt.Errorf("recipes: unknown search provider %q", s.Provider) + } +} + +// SearXNG searches through a SearXNG instance. Its base URL is a configured +// endpoint and may be on a private network, so it is called directly rather +// than through the public-only Fetcher. +type SearXNG struct { + endpoint *url.URL + client *http.Client +} + +// NewSearXNG returns a SearXNG provider for the instance at baseURL, an +// absolute http:// or https:// URL without credentials, query or fragment. +func NewSearXNG(baseURL string) (*SearXNG, error) { + if baseURL == "" { + return nil, errors.New("recipes: SearXNG URL is not set") + } + u, err := url.Parse(baseURL) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + return nil, errors.New("recipes: SearXNG URL must be an absolute http:// or https:// URL") + } + if u.User != nil || u.RawQuery != "" || u.Fragment != "" { + return nil, errors.New("recipes: SearXNG URL must not have credentials, a query or a fragment") + } + return &SearXNG{endpoint: u.JoinPath("search"), client: newSearchClient()}, nil +} + +// Search implements SearchProvider with +// {base}/search?q=…&format=json&categories=general. +func (s *SearXNG) Search(ctx context.Context, query string, n int) ([]Candidate, error) { + q, err := searchQuery(query, n) + if err != nil { + return nil, err + } + u := *s.endpoint + u.RawQuery = url.Values{"q": {q}, "format": {"json"}, "categories": {"general"}}.Encode() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, fmt.Errorf("recipes: build SearXNG request: %w", err) + } + var body struct { + Results []struct { + URL string `json:"url"` + Title string `json:"title"` + Content string `json:"content"` + } `json:"results"` + } + if err := doSearch(s.client, req, "SearXNG", &body); err != nil { + return nil, err + } + var out []Candidate + for _, r := range body.Results { + out = append(out, Candidate{URL: r.URL, Title: r.Title, Snippet: r.Content}) + } + return candidates("SearXNG", out, n) +} + +// Brave searches through the Brave Web Search API. +type Brave struct { + key string + endpoint string + client *http.Client +} + +// NewBrave returns a Brave provider that authenticates with key, the +// decrypted API key. +func NewBrave(key string) (*Brave, error) { + if key == "" { + return nil, errors.New("recipes: Brave API key is empty") + } + for i := 0; i < len(key); i++ { + if key[i] < 0x21 || key[i] > 0x7e { + // The key is never echoed. + return nil, errors.New("recipes: Brave API key has a character not allowed in a header") + } + } + return &Brave{key: key, endpoint: BraveEndpoint, client: newSearchClient()}, nil +} + +// Search implements SearchProvider with GET BraveEndpoint?q=…&count=n. +func (b *Brave) Search(ctx context.Context, query string, n int) ([]Candidate, error) { + q, err := searchQuery(query, n) + if err != nil { + return nil, err + } + u := b.endpoint + "?" + url.Values{"q": {q}, "count": {strconv.Itoa(n)}}.Encode() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return nil, fmt.Errorf("recipes: build Brave request: %w", err) + } + req.Header.Set("X-Subscription-Token", b.key) + var body struct { + Web struct { + Results []struct { + URL string `json:"url"` + Title string `json:"title"` + Description string `json:"description"` + } `json:"results"` + } `json:"web"` + } + if err := doSearch(b.client, req, "Brave", &body); err != nil { + return nil, err + } + var out []Candidate + for _, r := range body.Web.Results { + out = append(out, Candidate{URL: r.URL, Title: r.Title, Snippet: r.Description}) + } + return candidates("Brave", out, n) +} + +func newSearchClient() *http.Client { + return &http.Client{ + Timeout: SearchTimeout, + CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }, + } +} + +// searchQuery validates n and returns query with " recipe" appended unless +// one of its words already is "recipe", in any case. +func searchQuery(query string, n int) (string, error) { + if n < 1 || n > MaxSearchResults { + return "", fmt.Errorf("recipes: search result count %d is outside 1-%d", n, MaxSearchResults) + } + q := strings.TrimSpace(query) + if q == "" { + return "", errors.New("recipes: search query is empty") + } + for _, w := range strings.Fields(q) { + if strings.EqualFold(w, "recipe") { + return q, nil + } + } + return q + " recipe", nil +} + +// doSearch sends req and decodes a 2xx JSON body into v, mapping failures to +// the search errors. +func doSearch(client *http.Client, req *http.Request, provider string, v any) error { + req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", UserAgent) + resp, err := client.Do(req) + if err != nil { + if ctxErr := req.Context().Err(); ctxErr != nil { + return ctxErr + } + return fmt.Errorf("recipes: %s search: %w", provider, err) + } + defer resp.Body.Close() + switch { + case resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden: + return fmt.Errorf("%w: %s HTTP %d", ErrSearchRefused, provider, resp.StatusCode) + case resp.StatusCode == http.StatusTooManyRequests: + return fmt.Errorf("%w: %s", ErrSearchRateLimited, provider) + case resp.StatusCode < 200 || resp.StatusCode > 299: + return &SearchStatusError{Provider: provider, Status: resp.StatusCode} + } + data, err := io.ReadAll(io.LimitReader(resp.Body, MaxSearchBodyBytes+1)) + if err != nil { + return fmt.Errorf("recipes: %s search: read body: %w", provider, err) + } + if len(data) > MaxSearchBodyBytes { + return fmt.Errorf("%w: %s body is over %d bytes", ErrSearchResponse, provider, MaxSearchBodyBytes) + } + if err := json.Unmarshal(data, v); err != nil { + return fmt.Errorf("%w: %s: %v", ErrSearchResponse, provider, err) + } + return nil +} + +// candidates cleans results — HTML tags and entities, which both providers +// leave in titles and snippets, are removed — checks every URL is absolute +// http(s) and keeps the first n. +func candidates(provider string, results []Candidate, n int) ([]Candidate, error) { + if len(results) > n { + results = results[:n] + } + out := make([]Candidate, 0, len(results)) + for i, r := range results { + u, err := url.Parse(r.URL) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + return nil, fmt.Errorf("%w: %s result %d has no absolute http(s) URL", ErrSearchResponse, provider, i) + } + out = append(out, Candidate{URL: r.URL, Title: stripTags(r.Title), Snippet: stripTags(r.Snippet)}) + } + return out, nil +} + +// stripTags drops HTML tags such as Brave's highlights, then +// decodes entities and trims space. +func stripTags(s string) string { + var b strings.Builder + inTag := false + for _, r := range s { + switch { + case r == '<': + inTag = true + case r == '>' && inTag: + inTag = false + case !inTag: + b.WriteRune(r) + } + } + return cleanText(b.String()) +} diff --git a/internal/recipes/search_test.go b/internal/recipes/search_test.go new file mode 100644 index 0000000..b3a181a --- /dev/null +++ b/internal/recipes/search_test.go @@ -0,0 +1,275 @@ +package recipes + +import ( + "context" + "crypto/rand" + "encoding/base64" + "errors" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "testing" + + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/db" + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/secretbox" +) + +const searxngBody = `{ + "query": "chicken curry recipe", + "results": [ + {"url": "https://www.bbcgoodfood.com/recipes/easy-chicken-curry", "title": "Easy chicken curry recipe | BBC Good Food", "content": "Make our easy chicken curry & rice.", "engine": "duckduckgo"}, + {"url": "https://www.recipetineats.com/chicken-curry/", "title": "Chicken Curry", "content": "A proper curry."}, + {"url": "https://example.com/third", "title": "Third", "content": ""} + ], + "number_of_results": 0 +}` + +const braveBody = `{ + "type": "search", + "web": { + "type": "search", + "results": [ + {"url": "https://www.bbcgoodfood.com/recipes/easy-chicken-curry", "title": "Easy chicken curry recipe", "description": "Make our easy chicken curry & rice.", "language": "en"}, + {"url": "https://www.recipetineats.com/chicken-curry/", "title": "Chicken Curry", "description": "A proper curry."} + ] + } +}` + +func newSearXNGServer(t *testing.T, status int, body string, got *http.Request) *SearXNG { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got != nil { + *got = *r.Clone(context.Background()) + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + w.Write([]byte(body)) + })) + t.Cleanup(srv.Close) + s, err := NewSearXNG(srv.URL + "/searx") + if err != nil { + t.Fatal(err) + } + return s +} + +func newBraveServer(t *testing.T, key string, status int, body string, got *http.Request) *Brave { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got != nil { + *got = *r.Clone(context.Background()) + } + w.WriteHeader(status) + w.Write([]byte(body)) + })) + t.Cleanup(srv.Close) + b, err := NewBrave(key) + if err != nil { + t.Fatal(err) + } + b.endpoint = srv.URL + "/res/v1/web/search" + return b +} + +func TestSearXNGSearch(t *testing.T) { + var req http.Request + s := newSearXNGServer(t, http.StatusOK, searxngBody, &req) + got, err := s.Search(context.Background(), "chicken curry", 2) + if err != nil { + t.Fatal(err) + } + if req.URL.Path != "/searx/search" { + t.Errorf("path = %q, want /searx/search", req.URL.Path) + } + q := req.URL.Query() + if q.Get("q") != "chicken curry recipe" || q.Get("format") != "json" || q.Get("categories") != "general" { + t.Errorf("query = %v", q) + } + if req.Header.Get("User-Agent") != UserAgent { + t.Errorf("User-Agent = %q", req.Header.Get("User-Agent")) + } + want := []Candidate{ + {URL: "https://www.bbcgoodfood.com/recipes/easy-chicken-curry", Title: "Easy chicken curry recipe | BBC Good Food", Snippet: "Make our easy chicken curry & rice."}, + {URL: "https://www.recipetineats.com/chicken-curry/", Title: "Chicken Curry", Snippet: "A proper curry."}, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("candidates =\n%#v\nwant\n%#v", got, want) + } +} + +func TestBraveSearch(t *testing.T) { + var req http.Request + b := newBraveServer(t, "brave-key-123", http.StatusOK, braveBody, &req) + got, err := b.Search(context.Background(), "Chicken curry RECIPE", 5) + if err != nil { + t.Fatal(err) + } + if req.Header.Get("X-Subscription-Token") != "brave-key-123" { + t.Errorf("X-Subscription-Token = %q", req.Header.Get("X-Subscription-Token")) + } + if req.Header.Get("Accept") != "application/json" { + t.Errorf("Accept = %q", req.Header.Get("Accept")) + } + q := req.URL.Query() + if q.Get("q") != "Chicken curry RECIPE" || q.Get("count") != "5" { + t.Errorf("query = %v", q) + } + want := []Candidate{ + {URL: "https://www.bbcgoodfood.com/recipes/easy-chicken-curry", Title: "Easy chicken curry recipe", Snippet: "Make our easy chicken curry & rice."}, + {URL: "https://www.recipetineats.com/chicken-curry/", Title: "Chicken Curry", Snippet: "A proper curry."}, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("candidates =\n%#v\nwant\n%#v", got, want) + } +} + +func TestSearchQuery(t *testing.T) { + for _, tc := range []struct{ in, want string }{ + {"chicken curry", "chicken curry recipe"}, + {" chicken curry ", "chicken curry recipe"}, + {"chicken curry recipe", "chicken curry recipe"}, + {"Recipe for chicken curry", "Recipe for chicken curry"}, + {"recipes for curry", "recipes for curry recipe"}, + } { + got, err := searchQuery(tc.in, 3) + if err != nil || got != tc.want { + t.Errorf("searchQuery(%q) = %q, %v; want %q", tc.in, got, err, tc.want) + } + } + for _, tc := range []struct { + q string + n int + }{{"", 3}, {" ", 3}, {"curry", 0}, {"curry", MaxSearchResults + 1}} { + if _, err := searchQuery(tc.q, tc.n); err == nil { + t.Errorf("searchQuery(%q, %d) succeeded, want error", tc.q, tc.n) + } + } +} + +func TestSearchErrors(t *testing.T) { + tests := []struct { + name string + status int + body string + check func(error) bool + }{ + {"unauthorized", http.StatusUnauthorized, `{}`, func(err error) bool { return errors.Is(err, ErrSearchRefused) }}, + {"forbidden", http.StatusForbidden, `{}`, func(err error) bool { return errors.Is(err, ErrSearchRefused) }}, + {"rate limited", http.StatusTooManyRequests, `{}`, func(err error) bool { return errors.Is(err, ErrSearchRateLimited) }}, + {"server error", http.StatusBadGateway, `{}`, func(err error) bool { + var se *SearchStatusError + return errors.As(err, &se) && se.Status == http.StatusBadGateway + }}, + {"redirect not followed", http.StatusFound, ``, func(err error) bool { + var se *SearchStatusError + return errors.As(err, &se) && se.Status == http.StatusFound + }}, + {"not JSON", http.StatusOK, ``, func(err error) bool { return errors.Is(err, ErrSearchResponse) }}, + {"result without URL", http.StatusOK, `{"results":[{"title":"x"}],"web":{"results":[{"title":"x"}]}}`, func(err error) bool { return errors.Is(err, ErrSearchResponse) }}, + {"relative URL", http.StatusOK, `{"results":[{"url":"/x"}],"web":{"results":[{"url":"/x"}]}}`, func(err error) bool { return errors.Is(err, ErrSearchResponse) }}, + {"body too large", http.StatusOK, `{"results":[],"pad":"` + strings.Repeat("x", MaxSearchBodyBytes) + `"}`, func(err error) bool { return errors.Is(err, ErrSearchResponse) }}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + providers := map[string]SearchProvider{ + "SearXNG": newSearXNGServer(t, tc.status, tc.body, nil), + "Brave": newBraveServer(t, "secret-key", tc.status, tc.body, nil), + } + for name, p := range providers { + _, err := p.Search(context.Background(), "curry", 3) + if err == nil || !tc.check(err) { + t.Errorf("%s: err = %v", name, err) + } + if err != nil && strings.Contains(err.Error(), "secret-key") { + t.Errorf("%s: error leaks the API key: %v", name, err) + } + } + }) + } +} + +func TestSearchContextCancelled(t *testing.T) { + s := newSearXNGServer(t, http.StatusOK, searxngBody, nil) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := s.Search(ctx, "curry", 3); !errors.Is(err, context.Canceled) { + t.Errorf("err = %v, want context.Canceled", err) + } +} + +func TestSearchUnreachable(t *testing.T) { + srv := httptest.NewServer(http.NotFoundHandler()) + url := srv.URL + srv.Close() + s, err := NewSearXNG(url) + if err != nil { + t.Fatal(err) + } + _, err = s.Search(context.Background(), "curry", 3) + if err == nil || !strings.Contains(err.Error(), "SearXNG search") { + t.Errorf("err = %v", err) + } +} + +func TestNewSearXNGRejects(t *testing.T) { + for _, u := range []string{"", "searxng:8080", "ftp://searxng", "http://", "http://u:p@searxng", "http://searxng?x=1", "http://searxng#f"} { + if _, err := NewSearXNG(u); err == nil { + t.Errorf("NewSearXNG(%q) succeeded, want error", u) + } + } + if _, err := NewSearXNG("http://searxng:8080"); err != nil { + t.Errorf("internal URL rejected: %v", err) + } +} + +func TestNewBraveRejects(t *testing.T) { + for _, k := range []string{"", "with space", "line\nbreak"} { + if _, err := NewBrave(k); err == nil { + t.Errorf("NewBrave(%q) succeeded, want error", k) + } + } +} + +func TestNewSearchProvider(t *testing.T) { + raw := make([]byte, secretbox.KeySize) + rand.Read(raw) + box, err := secretbox.New(base64.StdEncoding.EncodeToString(raw)) + if err != nil { + t.Fatal(err) + } + ct, err := box.Encrypt([]byte("brave-key")) + if err != nil { + t.Fatal(err) + } + + p, err := NewSearchProvider(db.RecipeSearchSetting{Provider: "brave", ApiKeyCt: ct}, box, "") + if err != nil { + t.Fatal(err) + } + if b, ok := p.(*Brave); !ok || b.key != "brave-key" || b.endpoint != BraveEndpoint { + t.Errorf("provider = %#v, want Brave with decrypted key", p) + } + + p, err = NewSearchProvider(db.RecipeSearchSetting{Provider: "searxng"}, box, "http://searxng:8080") + if err != nil { + t.Fatal(err) + } + if s, ok := p.(*SearXNG); !ok || s.endpoint.String() != "http://searxng:8080/search" { + t.Errorf("provider = %#v, want SearXNG at http://searxng:8080/search", p) + } + + for name, tc := range map[string]struct { + s db.RecipeSearchSetting + url string + }{ + "brave without key": {db.RecipeSearchSetting{Provider: "brave"}, ""}, + "brave bad key": {db.RecipeSearchSetting{Provider: "brave", ApiKeyCt: []byte("not a token at all, far too plain")}, ""}, + "searxng without URL": {db.RecipeSearchSetting{Provider: "searxng"}, ""}, + "unknown provider": {db.RecipeSearchSetting{Provider: "google"}, ""}, + } { + if _, err := NewSearchProvider(tc.s, box, tc.url); err == nil { + t.Errorf("%s: succeeded, want error", name) + } + } +}