Maniacs_Mansion/MealQuest
11
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Feature 10: Recipe grounding and photos (#78)
CI / ci (push) Successful in 2m0s
Details

Browse Source 
Co-authored-by: dev_agent <dev_agent@cubecraftlabs.com>
This commit was merged in pull request #78.
This commit is contained in:
dev_agent
2026-10-06 07:27:58 -04:00
committed by approver_agent
parent 67e44a506a
commit f25b28b39b
58 changed files with 7621 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/recipes/extract.go
+631
View File
@@ -0,0 +1,631 @@
package recipes
import (
"encoding/json"
"errors"
"fmt"
"html"
"io"
"math"
"net/url"
"strconv"
"strings"
)
var (
// ErrNoRecipe is returned for a page with no schema.org Recipe node in
// its JSON-LD.
ErrNoRecipe = errors.New("recipes: page has no schema.org Recipe")
// ErrManyRecipes is returned for a page with more than one Recipe node:
// which one the page is about is not knowable.
ErrManyRecipes = errors.New("recipes: page has more than one schema.org Recipe")
// ErrMalformedRecipe is returned when the page's JSON-LD, or a field of
// its Recipe node, is not in a shape schema.org allows.
ErrMalformedRecipe = errors.New("recipes: malformed schema.org Recipe")
)
// Recipe is the schema.org Recipe extracted from a recipe page.
type Recipe struct {
// Title is the recipe's name.
Title string
// SiteName is the publisher's name, else the page's og:site_name, else
// its host without a leading "www.".
SiteName string
// ImageURL is the largest image, absolute; empty when the recipe has
// none.
ImageURL string
Ingredients []string
// Steps are the instructions in order, HowToSection groups flattened.
Steps []string
// PrepMinutes, CookMinutes and TotalMinutes are nil when not given.
// TotalMinutes is prep plus cook when the page gives both but no total.
PrepMinutes *int
CookMinutes *int
TotalMinutes *int
// Yield is recipeYield as text; empty when not given.
Yield string
// JSONLD is the Recipe node as found.
JSONLD json.RawMessage
}
// Extract finds the schema.org Recipe in a page's application/ld+json
// scripts. Recipe nodes are found at the top level of a script, in a
// top-level array, or in an @graph. An image or publisher given as an
// {"@id": ...} reference resolves to the node with that @id found in the
// same places, in any of the page's scripts. pageURL is the page's own
// absolute URL; relative image URLs resolve against it.
func Extract(page []byte, pageURL string) (Recipe, error) {
base, err := url.Parse(pageURL)
if err != nil || !base.IsAbs() || base.Host == "" {
return Recipe{}, fmt.Errorf("recipes: page URL %q is not absolute", pageURL)
}
doc := scanHTML(string(page))
g := graph{ids: map[string][]map[string]any{}}
for i, script := range doc.jsonLD {
v, err := decodeJSON(script)
if err != nil {
return Recipe{}, fmt.Errorf("%w: JSON-LD script %d: %v", ErrMalformedRecipe, i+1, err)
}
g.collect(v)
}
nodes := g.recipes
switch len(nodes) {
case 0:
return Recipe{}, ErrNoRecipe
case 1:
default:
return Recipe{}, fmt.Errorf("%w: found %d", ErrManyRecipes, len(nodes))
}
r, err := recipeFrom(nodes[0], g, base, doc.ogSiteName)
if err != nil {
return Recipe{}, fmt.Errorf("%w: %v", ErrMalformedRecipe, err)
}
return r, nil
}
func decodeJSON(s string) (any, error) {
dec := json.NewDecoder(strings.NewReader(s))
dec.UseNumber()
var v any
if err := dec.Decode(&v); err != nil {
return nil, err
}
if _, err := dec.Token(); err != io.EOF {
return nil, errors.New("trailing data after JSON value")
}
return v, nil
}
// graph holds the page's JSON-LD nodes: the Recipe nodes, and every node
// with an @id by that @id.
type graph struct {
recipes []map[string]any
ids map[string][]map[string]any
}
// collect gathers the nodes in v: v itself, the elements of an array, and
// the members of an @graph. A bare {"@id": ...} reference is not a node.
func (g *graph) collect(v any) {
switch n := v.(type) {
case []any:
for _, e := range n {
g.collect(e)
}
case map[string]any:
if hasType(n, "Recipe") {
g.recipes = append(g.recipes, n)
}
if id, ok := n["@id"].(string); ok && len(n) > 1 {
g.ids[id] = append(g.ids[id], n)
}
if members, ok := n["@graph"]; ok {
g.collect(members)
}
}
}
// resolve returns the node a bare {"@id": ...} reference names, and v
// itself when v is not a reference. found is false for a reference to no
// node; a reference to more than one node is an error.
func (g graph) resolve(v any) (node any, found bool, err error) {
m, ok := v.(map[string]any)
if !ok || len(m) != 1 {
return v, true, nil
}
id, ok := m["@id"].(string)
if !ok {
return v, true, nil
}
switch nodes := g.ids[id]; len(nodes) {
case 0:
return v, false, nil
case 1:
return nodes[0], true, nil
default:
return nil, false, fmt.Errorf("@id %q names %d nodes", id, len(nodes))
}
}
// hasType reports whether node's @type, a string or an array of strings,
// names the schema.org type typ, as a term or an absolute schema.org IRI.
func hasType(node map[string]any, typ string) bool {
switch t := node["@type"].(type) {
case string:
return isSchemaType(t, typ)
case []any:
for _, e := range t {
if s, ok := e.(string); ok && isSchemaType(s, typ) {
return true
}
}
}
return false
}
func isSchemaType(s, typ string) bool {
return s == typ || s == "https://schema.org/"+typ || s == "http://schema.org/"+typ
}
func recipeFrom(n map[string]any, g graph, base *url.URL, ogSiteName string) (Recipe, error) {
var r Recipe
var err error
name, ok := n["name"].(string)
if r.Title = cleanText(name); !ok || r.Title == "" {
return Recipe{}, errors.New("name is missing")
}
if r.SiteName, err = siteName(n["publisher"], g, ogSiteName, base); err != nil {
return Recipe{}, err
}
if r.ImageURL, err = imageURL(n["image"], g, base); err != nil {
return Recipe{}, err
}
if r.Ingredients, err = ingredients(n["recipeIngredient"]); err != nil {
return Recipe{}, err
}
if r.Steps, err = steps(n["recipeInstructions"]); err != nil {
return Recipe{}, err
}
if r.PrepMinutes, err = minutes(n, "prepTime"); err != nil {
return Recipe{}, err
}
if r.CookMinutes, err = minutes(n, "cookTime"); err != nil {
return Recipe{}, err
}
if r.TotalMinutes, err = minutes(n, "totalTime"); err != nil {
return Recipe{}, err
}
if r.TotalMinutes == nil && r.PrepMinutes != nil && r.CookMinutes != nil {
total := *r.PrepMinutes + *r.CookMinutes
r.TotalMinutes = &total
}
if r.Yield, err = yield(n["recipeYield"]); err != nil {
return Recipe{}, err
}
if r.JSONLD, err = json.Marshal(n); err != nil {
return Recipe{}, err
}
return r, nil
}
// cleanText decodes HTML entities, which recipe plugins often leave in
// JSON-LD text, and trims surrounding space.
func cleanText(s string) string {
return strings.TrimSpace(html.UnescapeString(s))
}
func siteName(publisher any, g graph, ogSiteName string, base *url.URL) (string, error) {
pubs, ok := publisher.([]any)
if !ok {
pubs = []any{publisher}
}
for _, p := range pubs {
p, _, err := g.resolve(p)
if err != nil {
return "", fmt.Errorf("publisher: %w", err)
}
if m, ok := p.(map[string]any); ok {
if name, ok := m["name"].(string); ok && cleanText(name) != "" {
return cleanText(name), nil
}
}
}
if ogSiteName != "" {
return ogSiteName, nil
}
return strings.TrimPrefix(base.Hostname(), "www."), nil
}
// imageURL picks the largest of the recipe's images by width × height;
// images without both dimensions count as size zero, and the first wins a
// tie, so a list of bare URLs gives its first.
func imageURL(v any, g graph, base *url.URL) (string, error) {
items, ok := v.([]any)
if !ok {
if v == nil {
return "", nil
}
items = []any{v}
}
best, bestArea := "", int64(-1)
for _, item := range items {
item, found, err := g.resolve(item)
if err != nil {
return "", fmt.Errorf("image: %w", err)
}
if !found {
return "", fmt.Errorf("image references @id %v, which no node on the page has", item.(map[string]any)["@id"])
}
var u string
var area int64
switch img := item.(type) {
case string:
u = img
case map[string]any:
if s, ok := img["url"].(string); ok {
u = s
} else if s, ok := img["contentUrl"].(string); ok {
u = s
}
w, err := dimension(img["width"])
if err != nil {
return "", fmt.Errorf("image width: %w", err)
}
h, err := dimension(img["height"])
if err != nil {
return "", fmt.Errorf("image height: %w", err)
}
area = w * h
default:
return "", fmt.Errorf("image is a %T, not a URL or ImageObject", item)
}
u = strings.TrimSpace(u)
if u == "" {
return "", errors.New("image has no URL")
}
if area > bestArea {
best, bestArea = u, area
}
}
if best == "" {
return "", nil
}
ref, err := url.Parse(best)
if err != nil {
return "", fmt.Errorf("image URL %q: %w", best, err)
}
return base.ResolveReference(ref).String(), nil
}
// dimension reads an ImageObject width or height: a number or a string of
// digits; zero when absent.
func dimension(v any) (int64, error) {
switch d := v.(type) {
case nil:
return 0, nil
case json.Number:
f, err := d.Float64()
if err != nil || f < 0 {
return 0, fmt.Errorf("%q is not a size", d)
}
return int64(f), nil
case string:
n, err := strconv.ParseInt(strings.TrimSpace(d), 10, 64)
if err != nil || n < 0 {
return 0, fmt.Errorf("%q is not a size", d)
}
return n, nil
}
return 0, fmt.Errorf("a %T is not a size", v)
}
func ingredients(v any) ([]string, error) {
items, ok := v.([]any)
if !ok {
if v == nil {
return nil, nil
}
items = []any{v}
}
var out []string
for _, item := range items {
s, ok := item.(string)
if !ok {
return nil, fmt.Errorf("recipeIngredient holds a %T, not text", item)
}
if s = cleanText(s); s != "" {
out = append(out, s)
}
}
return out, nil
}
// steps flattens recipeInstructions: one string of lines, or a list of
// strings, HowToSteps and HowToSections of those, in order.
func steps(v any) ([]string, error) {
switch in := v.(type) {
case nil:
return nil, nil
case string:
var out []string
for _, line := range strings.Split(in, "\n") {
if line = cleanText(line); line != "" {
out = append(out, line)
}
}
return out, nil
case map[string]any:
return stepList([]any{in})
case []any:
return stepList(in)
}
return nil, fmt.Errorf("recipeInstructions is a %T", v)
}
func stepList(items []any) ([]string, error) {
var out []string
for _, item := range items {
switch s := item.(type) {
case string:
if text := cleanText(s); text != "" {
out = append(out, text)
}
case map[string]any:
switch {
case hasType(s, "HowToStep"):
text, _ := s["text"].(string)
if text = cleanText(text); text == "" {
return nil, errors.New("HowToStep has no text")
}
out = append(out, text)
case hasType(s, "HowToSection"):
list, ok := s["itemListElement"].([]any)
if !ok {
return nil, errors.New("HowToSection has no itemListElement list")
}
section, err := stepList(list)
if err != nil {
return nil, err
}
out = append(out, section...)
default:
return nil, fmt.Errorf("recipeInstructions holds a %v, not a HowToStep or HowToSection", s["@type"])
}
default:
return nil, fmt.Errorf("recipeInstructions holds a %T", item)
}
}
return out, nil
}
// minutes reads a duration field: nil when absent or null; any string
// present, empty included, must be an ISO-8601 duration.
func minutes(n map[string]any, field string) (*int, error) {
v, ok := n[field]
if !ok || v == nil {
return nil, nil
}
s, ok := v.(string)
if !ok {
return nil, fmt.Errorf("%s is a %T, not an ISO-8601 duration", field, v)
}
m, err := isoMinutes(s)
if err != nil {
return nil, fmt.Errorf("%s: %w", field, err)
}
return &m, nil
}
// isoMinutes converts an ISO-8601 duration of weeks, days, hours, minutes
// and seconds to whole minutes, rounded. Years and months are refused:
// their length in minutes is not fixed.
func isoMinutes(s string) (int, error) {
bad := fmt.Errorf("%q is not an ISO-8601 duration of days, hours, minutes or seconds", s)
rest, ok := strings.CutPrefix(strings.TrimSpace(s), "P")
if !ok || rest == "" {
return 0, bad
}
const units = "WDHMS"
seconds := map[byte]float64{'W': 7 * 86400, 'D': 86400, 'H': 3600, 'M': 60, 'S': 1}
inTime, last, total := false, -1, 0.0
for rest != "" {
if rest[0] == 'T' {
if inTime || len(rest) == 1 {
return 0, bad
}
inTime, rest = true, rest[1:]
continue
}
i := 0
for i < len(rest) && (rest[i] >= '0' && rest[i] <= '9' || rest[i] == '.' || rest[i] == ',') {
i++
}
if i == 0 || i == len(rest) {
return 0, bad
}
num, err := strconv.ParseFloat(strings.Replace(rest[:i], ",", ".", 1), 64)
if err != nil {
return 0, bad
}
unit := rest[i]
pos := strings.IndexByte(units, unit)
if pos < 0 || pos <= last || inTime != (pos >= 2) {
return 0, bad
}
last, total, rest = pos, total+num*seconds[unit], rest[i+1:]
}
if last < 0 {
return 0, bad
}
return int(math.Round(total / 60)), nil
}
func yield(v any) (string, error) {
if list, ok := v.([]any); ok {
if len(list) == 0 {
return "", nil
}
v = list[0]
}
switch y := v.(type) {
case nil:
return "", nil
case string:
return cleanText(y), nil
case json.Number:
return y.String(), nil
}
return "", fmt.Errorf("recipeYield is a %T", v)
}
// htmlDoc is what Extract needs from a page's HTML.
type htmlDoc struct {
jsonLD []string
ogSiteName string
}
// scanHTML is a small targeted scan, not a full HTML parser: it walks the
// tags, skipping comments and the bodies of scripts and styles, and keeps
// the bodies of application/ld+json scripts and the first og:site_name.
func scanHTML(doc string) htmlDoc {
var out htmlDoc
lower := asciiLower(doc)
for pos := 0; ; {
lt := strings.IndexByte(doc[pos:], '<')
if lt < 0 {
return out
}
pos += lt
if strings.HasPrefix(doc[pos:], "<!--") {
end := strings.Index(doc[pos+4:], "-->")
if end < 0 {
return out
}
pos += 4 + end + 3
continue
}
name := tagName(lower[pos+1:])
if name == "" {
pos++
continue
}
end := tagEnd(doc, pos+1+len(name))
if end < 0 {
return out
}
attrs := parseAttrs(doc[pos+1+len(name) : end])
pos = end + 1
switch name {
case "meta":
if out.ogSiteName == "" && attrs["property"] == "og:site_name" {
out.ogSiteName = strings.TrimSpace(attrs["content"])
}
case "script", "style":
closeAt := strings.Index(lower[pos:], "</"+name)
if closeAt < 0 {
return out
}
typ, _, _ := strings.Cut(attrs["type"], ";")
if name == "script" && strings.EqualFold(strings.TrimSpace(typ), "application/ld+json") {
out.jsonLD = append(out.jsonLD, doc[pos:pos+closeAt])
}
pos += closeAt
}
}
}
// asciiLower lower-cases ASCII letters only, so byte offsets match doc.
func asciiLower(s string) string {
b := []byte(s)
for i, c := range b {
if 'A' <= c && c <= 'Z' {
b[i] = c + 'a' - 'A'
}
}
return string(b)
}
// tagName reads the lower-cased name of an opening tag at the start of s,
// or "" when s does not start one.
func tagName(s string) string {
i := 0
for i < len(s) && ('a' <= s[i] && s[i] <= 'z' || i > 0 && '0' <= s[i] && s[i] <= '9') {
i++
}
if i == 0 || i < len(s) && !isSpace(s[i]) && s[i] != '>' && s[i] != '/' {
return ""
}
return s[:i]
}
// tagEnd finds the '>' closing a tag whose attributes start at i, skipping
// quoted attribute values.
func tagEnd(doc string, i int) int {
var quote byte
for ; i < len(doc); i++ {
switch c := doc[i]; {
case quote != 0:
if c == quote {
quote = 0
}
case c == '"' || c == '\'':
quote = c
case c == '>':
return i
}
}
return -1
}
// parseAttrs reads a tag's attributes; names are lower-cased, values are
// entity-decoded, and the first of a repeated name wins.
func parseAttrs(s string) map[string]string {
attrs := map[string]string{}
i := 0
for i < len(s) {
for i < len(s) && (isSpace(s[i]) || s[i] == '/') {
i++
}
start := i
for i < len(s) && !isSpace(s[i]) && s[i] != '=' && s[i] != '/' {
i++
}
if i == start {
i++
continue
}
name := asciiLower(s[start:i])
for i < len(s) && isSpace(s[i]) {
i++
}
value := ""
if i < len(s) && s[i] == '=' {
i++
for i < len(s) && isSpace(s[i]) {
i++
}
if i < len(s) && (s[i] == '"' || s[i] == '\'') {
q := s[i]
end := strings.IndexByte(s[i+1:], q)
if end < 0 {
end = len(s) - i - 1
}
value = s[i+1 : i+1+end]
i += end + 2
} else {
start := i
for i < len(s) && !isSpace(s[i]) {
i++
}
value = s[start:i]
}
}
if _, seen := attrs[name]; !seen {
attrs[name] = html.UnescapeString(value)
}
}
return attrs
}
func isSpace(c byte) bool {
return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f'
}
internal/recipes/extract_test.go
+408
View File
@@ -0,0 +1,408 @@
package recipes
import (
"encoding/json"
"errors"
"os"
"path/filepath"
"strings"
"testing"
)
func readPage(t *testing.T, name string) []byte {
t.Helper()
b, err := os.ReadFile(filepath.Join("testdata", name))
if err != nil {
t.Fatal(err)
}
return b
}
func intp(n int) *int { return &n }
func fmtMinutes(m *int) string {
if m == nil {
return "nil"
}
b, _ := json.Marshal(*m)
return string(b)
}
func TestExtractRealPages(t *testing.T) {
tests := []struct {
file, url string
title, site, image, yld string
ingredients, steps int
prep, cook, total *int
firstStep string
}{
{
// A Recipe script beside VideoObject and BreadcrumbList scripts;
// image is an array of one sized ImageObject.
file: "bbcgoodfood.html", url: "https://www.bbcgoodfood.com/recipes/easy-chicken-curry",
title: "Easy chicken curry", site: "Good Food",
image: "https://images.immediate.co.uk/production/volatile/sites/30/2020/08/easy-chicken-curry-aa22a0b.jpg?resize=440,400",
ingredients: 11, steps: 4, prep: intp(5), cook: intp(45), total: intp(50), yld: "4",
firstStep: "Heat the oil in a flameproof casserole dish",
},
{
// Yoast @graph; image is a list of bare URLs, the first the
// full-size one; no publisher, so og:site_name names the site.
file: "budgetbytes.html", url: "https://www.budgetbytes.com/one-pot-creamy-cajun-chicken-pasta/",
title: "One Pot Creamy Cajun Chicken Pasta", site: "Budget Bytes",
image: "https://www.budgetbytes.com/wp-content/uploads/2018/10/One-Pot-Creamy-Cajun-Chicken-Pasta-pan.jpg",
ingredients: 17, steps: 6, prep: intp(10), cook: intp(20), total: intp(30), yld: "4",
firstStep: "Combine the ingredients for the Cajun seasoning",
},
{
// HowToSection groups (ragu, cheese sauce, assembly), flattened.
file: "recipetineats.html", url: "https://www.recipetineats.com/lasagna/",
title: "Lasagna", site: "RecipeTin Eats",
image: "https://www.recipetineats.com/tachyon/2017/05/Lasagne-recipe-3-main-square.jpg",
ingredients: 24, steps: 18, prep: intp(20), cook: intp(240), total: intp(260), yld: "10",
firstStep: "Heat oil in a large heavy based pot",
},
{
// @graph of one node; a single ImageObject; instructions as a
// list of plain strings; hours and minutes.
file: "kingarthur.html", url: "https://www.kingarthurbaking.com/recipes/classic-sandwich-bread-recipe",
title: "Classic Sandwich Bread", site: "King Arthur Baking",
image: "https://www.kingarthurbaking.com/sites/default/files/2026-02/Classic-Sandwich-Bread-12.jpg",
ingredients: 7, steps: 7, prep: intp(12), cook: intp(40), total: intp(162), yld: "16",
firstStep: "Weigh your flour",
},
{
// A top-level array; ImageObjects of different sizes.
file: "delish.html", url: "https://www.delish.com/cooking/recipe-ideas/a19636089/creamy-tuscan-chicken-recipe/",
title: "Creamy Tuscan Chicken", site: "Delish",
image: "https://hips.hearstapps.com/hmg-prod/images/1d09a24f-5b76-49f4-9ad6-496acdc03933.jpg?crop=0.502xw:1.00xh;0.250xw,0&resize=1200:*",
ingredients: 12, steps: 4, prep: intp(10), cook: intp(0), total: intp(45), yld: "4 serving(s)",
firstStep: "In a large skillet over medium heat, heat oil.",
},
{
// A bare Recipe object; image is one URL; HTML entities in steps.
file: "foodcom.html", url: "https://www.food.com/recipe/best-banana-bread-2886",
title: "Best Banana Bread", site: "Food.com",
image: "https://img.sndimg.com/food/image/upload/q_92,fl_progressive,w_1200,c_scale/v1/img/recipes/28/86/wBZxNua1T8yfDwbfo9Fz_0S9A9315.jpg",
ingredients: 8, steps: 10, prep: intp(10), cook: intp(60), total: intp(70), yld: "1 loaf, 10 serving(s)",
firstStep: "Remove odd pots and pans from oven.",
},
}
for _, tt := range tests {
t.Run(tt.file, func(t *testing.T) {
r, err := Extract(readPage(t, tt.file), tt.url)
if err != nil {
t.Fatalf("Extract: %v", err)
}
if r.Title != tt.title {
t.Errorf("Title = %q, want %q", r.Title, tt.title)
}
if r.SiteName != tt.site {
t.Errorf("SiteName = %q, want %q", r.SiteName, tt.site)
}
if r.ImageURL != tt.image {
t.Errorf("ImageURL = %q, want %q", r.ImageURL, tt.image)
}
if len(r.Ingredients) != tt.ingredients {
t.Errorf("%d ingredients, want %d", len(r.Ingredients), tt.ingredients)
}
if len(r.Steps) != tt.steps {
t.Errorf("%d steps, want %d", len(r.Steps), tt.steps)
}
if len(r.Steps) > 0 && !strings.HasPrefix(r.Steps[0], tt.firstStep) {
t.Errorf("first step = %q, want prefix %q", r.Steps[0], tt.firstStep)
}
for name, got := range map[string][2]*int{
"prep": {r.PrepMinutes, tt.prep}, "cook": {r.CookMinutes, tt.cook}, "total": {r.TotalMinutes, tt.total},
} {
if fmtMinutes(got[0]) != fmtMinutes(got[1]) {
t.Errorf("%s minutes = %s, want %s", name, fmtMinutes(got[0]), fmtMinutes(got[1]))
}
}
if r.Yield != tt.yld {
t.Errorf("Yield = %q, want %q", r.Yield, tt.yld)
}
var node map[string]any
if err := json.Unmarshal(r.JSONLD, &node); err != nil || node["name"] == nil {
t.Errorf("JSONLD is not the Recipe node: %v", err)
}
})
}
}
func TestExtractHowToSectionOrder(t *testing.T) {
r, err := Extract(readPage(t, "recipetineats.html"), "https://www.recipetineats.com/lasagna/")
if err != nil {
t.Fatal(err)
}
// Ragu (5 steps), Cheese Sauce (5), Assemble (8): each section's first
// step lands right after the previous section's last.
for i, prefix := range map[int]string{
0: "Heat oil in a large heavy based pot",
4: "The ragu is ready when the meat",
5: "Warm milk up in a saucepan",
10: "Preheat oven to 180°C/350°F.",
17: "Stand for 5 to 10 minutes",
} {
if !strings.HasPrefix(r.Steps[i], prefix) {
t.Errorf("step %d = %q, want prefix %q", i+1, r.Steps[i], prefix)
}
}
}
func TestExtractNoRecipe(t *testing.T) {
_, err := Extract(readPage(t, "budgetbytes-about.html"), "https://www.budgetbytes.com/about/")
if !errors.Is(err, ErrNoRecipe) {
t.Fatalf("err = %v, want ErrNoRecipe", err)
}
_, err = Extract([]byte("<html><head><title>x</title></head></html>"), "https://example.com/")
if !errors.Is(err, ErrNoRecipe) {
t.Fatalf("page without JSON-LD: err = %v, want ErrNoRecipe", err)
}
}
func TestExtractRefusesNonISODuration(t *testing.T) {
// Bon Appétit writes cookTime as "20 minutes".
_, err := Extract(readPage(t, "bonappetit.html"), "https://www.bonappetit.com/recipe/bas-best-chocolate-chip-cookies")
if !errors.Is(err, ErrMalformedRecipe) || !strings.Contains(err.Error(), "cookTime") {
t.Fatalf("err = %v, want ErrMalformedRecipe naming cookTime", err)
}
}
// page wraps JSON-LD scripts in a minimal HTML page.
func page(head string, jsonLD ...string) []byte {
var b strings.Builder
b.WriteString("<!DOCTYPE html><html><head>" + head)
for _, s := range jsonLD {
b.WriteString(`<script type="application/ld+json">` + s + "</script>")
}
b.WriteString("</head><body></body></html>")
return []byte(b.String())
}
func TestExtractShapes(t *testing.T) {
const pageURL = "https://www.example.com/recipes/stew/"
tests := []struct {
name string
html []byte
check func(t *testing.T, r Recipe)
}{
{
name: "type array and relative image",
html: page("", `{"@type":["Recipe","NewsArticle"],"name":"Stew","image":"/img/stew.jpg"}`),
check: func(t *testing.T, r Recipe) {
if r.ImageURL != "https://www.example.com/img/stew.jpg" {
t.Errorf("ImageURL = %q", r.ImageURL)
}
},
},
{
name: "largest image wins regardless of order",
html: page("", `{"@type":"Recipe","name":"Stew","image":[
"https://cdn.example.com/plain.jpg",
{"@type":"ImageObject","url":"small.jpg","width":300,"height":200},
{"@type":"ImageObject","contentUrl":"../big.jpg","width":"1600","height":"900"},
{"@type":"ImageObject","url":"mid.jpg","width":1200,"height":800}]}`),
check: func(t *testing.T, r Recipe) {
if r.ImageURL != "https://www.example.com/recipes/big.jpg" {
t.Errorf("ImageURL = %q", r.ImageURL)
}
},
},
{
name: "instructions as one string",
html: page("", `{"@type":"Recipe","name":"Stew","recipeInstructions":"Brown the beef.\r\n\r\nAdd the stock &amp; simmer.\nServe."}`),
check: func(t *testing.T, r Recipe) {
want := []string{"Brown the beef.", "Add the stock & simmer.", "Serve."}
if strings.Join(r.Steps, "|") != strings.Join(want, "|") {
t.Errorf("Steps = %q, want %q", r.Steps, want)
}
},
},
{
name: "sections and steps flattened in order",
html: page("", `{"@type":"Recipe","name":"Stew","recipeInstructions":[
{"@type":"HowToSection","name":"Beef","itemListElement":[
{"@type":"HowToStep","text":"one"},{"@type":"HowToStep","text":"two"}]},
{"@type":"HowToStep","text":"three"},
{"@type":"HowToSection","name":"Finish","itemListElement":["four"]}]}`),
check: func(t *testing.T, r Recipe) {
if got := strings.Join(r.Steps, " "); got != "one two three four" {
t.Errorf("Steps = %q", got)
}
},
},
{
name: "total derived from prep and cook",
html: page("", `{"@type":"Recipe","name":"Stew","prepTime":"PT15M","cookTime":"PT1H30M"}`),
check: func(t *testing.T, r Recipe) {
if fmtMinutes(r.TotalMinutes) != "105" {
t.Errorf("TotalMinutes = %s, want 105", fmtMinutes(r.TotalMinutes))
}
},
},
{
name: "total not derived from prep alone",
html: page("", `{"@type":"Recipe","name":"Stew","prepTime":"PT15M"}`),
check: func(t *testing.T, r Recipe) {
if r.TotalMinutes != nil || r.CookMinutes != nil {
t.Errorf("total %s, cook %s; want nil", fmtMinutes(r.TotalMinutes), fmtMinutes(r.CookMinutes))
}
},
},
{
name: "site name from host without publisher or og:site_name",
html: page("", `{"@type":"Recipe","name":"Stew"}`),
check: func(t *testing.T, r Recipe) {
if r.SiteName != "example.com" {
t.Errorf("SiteName = %q", r.SiteName)
}
},
},
{
name: "og:site_name with attributes in any order",
html: page(`<meta content="Our &amp; Kitchen" property="og:site_name">`, `{"@type":"Recipe","name":"Stew","publisher":{"@id":"#org"}}`),
check: func(t *testing.T, r Recipe) {
if r.SiteName != "Our & Kitchen" {
t.Errorf("SiteName = %q", r.SiteName)
}
},
},
{
name: "numeric yield and recipe in @graph beside others",
html: page("", `{"@context":"https://schema.org","@graph":[{"@type":"WebPage"},{"@type":"Recipe","name":"Stew","recipeYield":6}]}`),
check: func(t *testing.T, r Recipe) {
if r.Yield != "6" {
t.Errorf("Yield = %q", r.Yield)
}
},
},
{
name: "image and publisher resolved by @id within @graph",
html: page(`<meta property="og:site_name" content="Not this">`, `{"@context":"https://schema.org","@graph":[
{"@type":"Recipe","name":"Stew",
"image":{"@id":"https://www.example.com/#primaryimage"},
"publisher":{"@id":"https://www.example.com/#organization"}},
{"@type":"ImageObject","@id":"https://www.example.com/#primaryimage","url":"/img/stew.jpg","width":1200,"height":800},
{"@type":"Organization","@id":"https://www.example.com/#organization","name":"Our Kitchen"}]}`),
check: func(t *testing.T, r Recipe) {
if r.ImageURL != "https://www.example.com/img/stew.jpg" {
t.Errorf("ImageURL = %q", r.ImageURL)
}
if r.SiteName != "Our Kitchen" {
t.Errorf("SiteName = %q", r.SiteName)
}
},
},
{
name: "image @id resolved from another script",
html: page("", `{"@type":"Recipe","name":"Stew","image":[{"@id":"#img"}]}`,
`{"@type":"ImageObject","@id":"#img","contentUrl":"stew.jpg"}`),
check: func(t *testing.T, r Recipe) {
if r.ImageURL != "https://www.example.com/recipes/stew/stew.jpg" {
t.Errorf("ImageURL = %q", r.ImageURL)
}
},
},
{
name: "absolute schema.org type IRIs",
html: page("", `{"@type":"https://schema.org/Recipe","name":"Stew","recipeInstructions":[
{"@type":"http://schema.org/HowToStep","text":"one"}]}`),
check: func(t *testing.T, r Recipe) {
if r.Title != "Stew" || strings.Join(r.Steps, "|") != "one" {
t.Errorf("Title = %q, Steps = %q", r.Title, r.Steps)
}
},
},
{
name: "absolute type IRI in a type array",
html: page("", `{"@type":["NewsArticle","http://schema.org/Recipe"],"name":"Stew"}`),
check: func(t *testing.T, r Recipe) {
if r.Title != "Stew" {
t.Errorf("Title = %q", r.Title)
}
},
},
{
name: "null duration is absent",
html: page("", `{"@type":"Recipe","name":"Stew","prepTime":null,"cookTime":"PT10M"}`),
check: func(t *testing.T, r Recipe) {
if r.PrepMinutes != nil || r.TotalMinutes != nil || fmtMinutes(r.CookMinutes) != "10" {
t.Errorf("prep %s, cook %s, total %s", fmtMinutes(r.PrepMinutes), fmtMinutes(r.CookMinutes), fmtMinutes(r.TotalMinutes))
}
},
},
{
name: "JSON-LD text inside comments and other scripts is ignored",
html: []byte(`<html><head><!-- <script type="application/ld+json">{"@type":"Recipe","name":"Old"}</script> -->` +
`<script>var s = '<script type="application/ld+json">';</script>` +
`<SCRIPT TYPE='application/ld+json; charset=utf-8'>{"@type":"Recipe","name":"Stew"}</SCRIPT></head></html>`),
check: func(t *testing.T, r Recipe) {
if r.Title != "Stew" {
t.Errorf("Title = %q", r.Title)
}
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
r, err := Extract(tt.html, pageURL)
if err != nil {
t.Fatalf("Extract: %v", err)
}
tt.check(t, r)
})
}
}
func TestExtractErrors(t *testing.T) {
tests := []struct {
name string
html []byte
url string
want error
}{
{"two recipes", page("", `{"@type":"Recipe","name":"A"}`, `{"@type":"Recipe","name":"B"}`), "https://example.com/", ErrManyRecipes},
{"invalid JSON", page("", `{"@type":"Recipe",`), "https://example.com/", ErrMalformedRecipe},
{"no name", page("", `{"@type":"Recipe"}`), "https://example.com/", ErrMalformedRecipe},
{"bad step type", page("", `{"@type":"Recipe","name":"A","recipeInstructions":[{"@type":"Thing"}]}`), "https://example.com/", ErrMalformedRecipe},
{"step without text", page("", `{"@type":"Recipe","name":"A","recipeInstructions":[{"@type":"HowToStep","name":"x"}]}`), "https://example.com/", ErrMalformedRecipe},
{"bad image width", page("", `{"@type":"Recipe","name":"A","image":{"url":"a.jpg","width":"wide"}}`), "https://example.com/", ErrMalformedRecipe},
{"empty prepTime", page("", `{"@type":"Recipe","name":"A","prepTime":""}`), "https://example.com/", ErrMalformedRecipe},
{"blank cookTime", page("", `{"@type":"Recipe","name":"A","cookTime":" "}`), "https://example.com/", ErrMalformedRecipe},
{"empty totalTime", page("", `{"@type":"Recipe","name":"A","totalTime":""}`), "https://example.com/", ErrMalformedRecipe},
{"image @id not on page", page("", `{"@type":"Recipe","name":"A","image":{"@id":"#missing"}}`), "https://example.com/", ErrMalformedRecipe},
{"image @id on two nodes", page("", `{"@type":"Recipe","name":"A","image":{"@id":"#img"}}`,
`{"@id":"#img","url":"a.jpg"}`, `{"@id":"#img","url":"b.jpg"}`), "https://example.com/", ErrMalformedRecipe},
{"non-text ingredient", page("", `{"@type":"Recipe","name":"A","recipeIngredient":[{"name":"salt"}]}`), "https://example.com/", ErrMalformedRecipe},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := Extract(tt.html, tt.url)
if !errors.Is(err, tt.want) {
t.Fatalf("err = %v, want %v", err, tt.want)
}
})
}
if _, err := Extract(page("", `{"@type":"Recipe","name":"A"}`), "/relative"); err == nil {
t.Fatal("relative page URL accepted")
}
}
func TestISOMinutes(t *testing.T) {
good := map[string]int{
"PT0S": 0, "PT45M": 45, "PT1H": 60, "PT2H42M": 162, "PT240M": 240, "P1D": 1440,
"P1DT2H": 1560, "PT90S": 2, "PT1.5H": 90, "PT0,5H": 30, "P1W": 10080, " PT5M ": 5,
}
for in, want := range good {
got, err := isoMinutes(in)
if err != nil || got != want {
t.Errorf("isoMinutes(%q) = %d, %v; want %d", in, got, err, want)
}
}
for _, in := range []string{"", "P", "PT", "20 minutes", "PT5", "P1Y", "P1M", "PT1D", "P5H", "PT5M1H", "PT5M5M", "P1DT", "PTxM", "PT1..5M"} {
if got, err := isoMinutes(in); err == nil {
t.Errorf("isoMinutes(%q) = %d, want error", in, got)
}
}
}
internal/recipes/fetcher.go
+575
View File
@@ -0,0 +1,575 @@
// Package recipes fetches and holds real recipe pages that ground dinners.
package recipes
import (
"bytes"
"context"
"crypto/tls"
"errors"
"fmt"
"io"
"mime"
"net"
"net/http"
"net/netip"
"net/url"
"slices"
"strings"
"sync"
"time"
)
const (
// UserAgent identifies MealQuest to the recipe sites it fetches.
UserAgent = "MealQuest/1.0 (household meal planner; recipe fetcher)"
// MaxBodyBytes caps a fetched body. A larger page is an error, never
// silently truncated: a cut-off recipe page would parse as a wrong one.
MaxBodyBytes = 2 << 20
// MaxImageBytes caps a fetched image body, as MaxBodyBytes caps a page.
MaxImageBytes = 8 << 20
// MaxRedirects is how many redirects a fetch follows; each hop is
// re-checked for scheme and address.
MaxRedirects = 3
// Timeout bounds a whole fetch: rate-limit waits, every redirect hop
// and reading the body.
Timeout = 10 * time.Second
// HostInterval is the least time between two requests to one host.
HostInterval = time.Second
// CacheTTL is how long a fetched page is served from the cache.
CacheTTL = time.Hour
// CacheSize is how many pages the cache holds; the oldest is evicted
// first.
CacheSize = 64
)
var (
// ErrNotHTTPS is returned for a URL, or a redirect, that is not
// https://.
ErrNotHTTPS = errors.New("recipes: only https:// URLs are fetched")
// ErrBlockedAddress is returned when a host resolves to an address
// that is not public, such as loopback or a private network.
ErrBlockedAddress = errors.New("recipes: address is not public")
// ErrTooManyRedirects is returned after MaxRedirects redirects.
ErrTooManyRedirects = errors.New("recipes: too many redirects")
// ErrTooLarge is returned for a body over MaxBodyBytes.
ErrTooLarge = errors.New("recipes: body exceeds size cap")
// ErrTimeout is returned when a fetch runs past Timeout.
ErrTimeout = errors.New("recipes: fetch timed out")
// ErrNotImage is returned by FetchImage for a response whose
// Content-Type is not one of ImageContentTypes.
ErrNotImage = errors.New("recipes: response is not a supported image")
)
// ImageContentTypes are the media types FetchImage accepts.
var ImageContentTypes = []string{"image/jpeg", "image/png", "image/webp", "image/gif"}
// fetchKind is what a request asks for: its Accept header, the body cap and
// the media types allowed, where nil allows any.
type fetchKind struct {
accept string
maxBytes int64
contentTypes []string
}
var (
pageKind = fetchKind{accept: "text/html,application/xhtml+xml", maxBytes: MaxBodyBytes}
imageKind = fetchKind{accept: strings.Join(ImageContentTypes, ","), maxBytes: MaxImageBytes, contentTypes: ImageContentTypes}
)
// StatusError is returned for a final response whose status is not 2xx.
type StatusError struct {
URL string
Status int
}
func (e *StatusError) Error() string {
return fmt.Sprintf("recipes: GET %s: HTTP %d", e.URL, e.Status)
}
// Page is a fetched page.
type Page struct {
// URL is the final URL, after redirects.
URL string
ContentType string
Body []byte
}
// Resolver looks up a host's addresses; *net.Resolver satisfies it.
type Resolver interface {
LookupIPAddr(ctx context.Context, host string) ([]net.IPAddr, error)
}
// dialFunc dials an address that has already passed the public-address
// check.
type dialFunc func(ctx context.Context, network, address string) (net.Conn, error)
// Fetcher retrieves recipe pages without reaching private networks. Its
// dialer resolves each host itself, refuses the connection unless every
// answer is a public address, and dials the checked address, so a second DNS
// answer cannot swap in a private one. Keep-alives are off: every request,
// redirect hops included, dials and checks afresh. Proxies from the
// environment are not used, since a proxy would dial on the Fetcher's
// behalf. Requests are HTTP/1.1, so each connection carries exactly one
// request and its first write is that request. A Fetcher is safe for
// concurrent use.
type Fetcher struct {
client *http.Client
resolver Resolver
dial dialFunc
tlsConfig *tls.Config
// timeout bounds a fetch; tests shorten it.
timeout time.Duration
// now is the clock; tests replace it.
now func() time.Time
// sleep waits d or until ctx is done; tests replace it.
sleep func(ctx context.Context, d time.Duration) error
mu sync.Mutex
// lastSent is when each host's latest request began to be written to
// its connection; a request whose write fails before any byte is sent
// hands its slot back.
lastSent map[string]time.Time
// released is closed, then replaced, whenever a slot is handed back,
// waking requests waiting for their turn.
released chan struct{}
cache map[string]cacheEntry
}
type cacheEntry struct {
page Page
fetched time.Time
}
// NewFetcher returns a Fetcher that resolves through the system resolver.
func NewFetcher() *Fetcher {
return newFetcher(net.DefaultResolver, (&net.Dialer{}).DialContext, nil)
}
// newFetcher returns a Fetcher over resolver and dial; tlsConfig, when not
// nil, replaces the default TLS settings so tests can trust their servers.
func newFetcher(resolver Resolver, dial dialFunc, tlsConfig *tls.Config) *Fetcher {
f := &Fetcher{
resolver: resolver,
dial: dial,
tlsConfig: tlsConfig,
timeout: Timeout,
now: time.Now,
sleep: sleep,
lastSent: map[string]time.Time{},
released: make(chan struct{}),
cache: map[string]cacheEntry{},
}
transport := &http.Transport{
Proxy: nil,
DialTLSContext: f.dialTLS,
DisableKeepAlives: true,
}
f.client = &http.Client{
Transport: &requestTransport{next: transport},
CheckRedirect: checkRedirect,
}
return f
}
// Fetch returns the page at rawURL, from the cache when it was fetched
// within CacheTTL. Only 2xx responses are returned and cached. The
// returned Body is the caller's own copy.
func (f *Fetcher) Fetch(ctx context.Context, rawURL string) (Page, error) {
u, err := url.Parse(rawURL)
if err != nil {
return Page{}, fmt.Errorf("recipes: parse URL: %w", err)
}
if err := checkURL(u); err != nil {
return Page{}, err
}
key := u.String()
if page, ok := f.cached(key); ok {
return page, nil
}
page, err := f.fetch(ctx, key, pageKind)
if err != nil {
return Page{}, err
}
f.store(key, page)
return page, nil
}
// FetchImage returns the image at rawURL under the same safeguards as
// Fetch, but with a body cap of MaxImageBytes, and only for a response
// whose Content-Type is one of ImageContentTypes. Images are not cached.
func (f *Fetcher) FetchImage(ctx context.Context, rawURL string) (Page, error) {
u, err := url.Parse(rawURL)
if err != nil {
return Page{}, fmt.Errorf("recipes: parse URL: %w", err)
}
if err := checkURL(u); err != nil {
return Page{}, err
}
return f.fetch(ctx, u.String(), imageKind)
}
// fetch gets rawURL as kind within the Fetcher's timeout.
func (f *Fetcher) fetch(ctx context.Context, rawURL string, kind fetchKind) (Page, error) {
fetchCtx, cancel := context.WithTimeout(ctx, f.timeout)
defer cancel()
page, err := f.get(fetchCtx, rawURL, kind)
if err != nil {
if ctx.Err() == nil && errors.Is(fetchCtx.Err(), context.DeadlineExceeded) {
return Page{}, fmt.Errorf("%w after %s: %s", ErrTimeout, f.timeout, rawURL)
}
return Page{}, err
}
return page, nil
}
func (f *Fetcher) get(ctx context.Context, rawURL string, kind fetchKind) (Page, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
if err != nil {
return Page{}, fmt.Errorf("recipes: build request: %w", err)
}
req.Header.Set("User-Agent", UserAgent)
req.Header.Set("Accept", kind.accept)
resp, err := f.client.Do(req)
if err != nil {
return Page{}, err
}
defer resp.Body.Close()
final := resp.Request.URL.String()
if resp.StatusCode < 200 || resp.StatusCode > 299 {
return Page{}, &StatusError{URL: final, Status: resp.StatusCode}
}
contentType := resp.Header.Get("Content-Type")
if kind.contentTypes != nil {
mediaType, _, err := mime.ParseMediaType(contentType)
if err != nil || !slices.Contains(kind.contentTypes, mediaType) {
return Page{}, fmt.Errorf("%w: %s has Content-Type %q", ErrNotImage, final, contentType)
}
}
if resp.ContentLength > kind.maxBytes {
return Page{}, fmt.Errorf("%w: %s declares %d bytes", ErrTooLarge, final, resp.ContentLength)
}
body, err := io.ReadAll(io.LimitReader(resp.Body, kind.maxBytes+1))
if err != nil {
return Page{}, fmt.Errorf("recipes: read %s: %w", final, err)
}
if int64(len(body)) > kind.maxBytes {
return Page{}, fmt.Errorf("%w: %s is over %d bytes", ErrTooLarge, final, kind.maxBytes)
}
return Page{URL: final, ContentType: contentType, Body: body}, nil
}
// cached returns the page cached under key if it is younger than CacheTTL.
func (f *Fetcher) cached(key string) (Page, bool) {
f.mu.Lock()
defer f.mu.Unlock()
e, ok := f.cache[key]
if !ok {
return Page{}, false
}
if f.now().Sub(e.fetched) >= CacheTTL {
delete(f.cache, key)
return Page{}, false
}
page := e.page
page.Body = bytes.Clone(page.Body)
return page, true
}
// store caches a copy of page under key, evicting the oldest entry when
// full.
func (f *Fetcher) store(key string, page Page) {
page.Body = bytes.Clone(page.Body)
f.mu.Lock()
defer f.mu.Unlock()
if _, ok := f.cache[key]; !ok && len(f.cache) >= CacheSize {
var oldest string
var oldestAt time.Time
for k, e := range f.cache {
if oldest == "" || e.fetched.Before(oldestAt) {
oldest, oldestAt = k, e.fetched
}
}
delete(f.cache, oldest)
}
f.cache[key] = cacheEntry{page: page, fetched: f.now()}
}
// waitTurn blocks until a request to host may be sent, keeping requests to
// one host at least HostInterval apart. A wait holds no reservation, and a
// request ending with ctx before its turn takes no slot. The request let
// through takes the host's slot and must be sent at once, or call release
// if sending fails, so that it delays no other request. Fetch calls it from
// turnConn's first Write, so the slot is taken after DNS, dialing and the
// TLS handshake, as the request is written.
func (f *Fetcher) waitTurn(ctx context.Context, host string) (release func(), err error) {
for {
if err := ctx.Err(); err != nil {
return nil, err
}
f.mu.Lock()
now := f.now()
last, sent := f.lastSent[host]
wait := last.Add(HostInterval).Sub(now)
if !sent || wait <= 0 {
f.lastSent[host] = now
f.mu.Unlock()
return func() { f.release(host, now, last, sent) }, nil
}
released := f.released
f.mu.Unlock()
if err := f.waitOrRelease(ctx, wait, released); err != nil {
return nil, err
}
}
}
// release hands back the slot taken at taken for a request that was not
// sent, restoring the host's previous send time, and wakes waiting requests.
func (f *Fetcher) release(host string, taken, prev time.Time, hadPrev bool) {
f.mu.Lock()
defer f.mu.Unlock()
if !f.lastSent[host].Equal(taken) {
// A later request already holds the slot.
return
}
if hadPrev {
f.lastSent[host] = prev
} else {
delete(f.lastSent, host)
}
close(f.released)
f.released = make(chan struct{})
}
// waitOrRelease sleeps for d, returning early without error when a slot is
// handed back so the caller can look again.
func (f *Fetcher) waitOrRelease(ctx context.Context, d time.Duration, released <-chan struct{}) error {
sleepCtx, cancel := context.WithCancel(ctx)
defer cancel()
go func() {
select {
case <-released:
cancel()
case <-sleepCtx.Done():
}
}()
err := f.sleep(sleepCtx, d)
select {
case <-released:
return nil
default:
return err
}
}
// dialPublic resolves address's host once, refuses unless every answer is
// public, then dials the checked addresses in turn.
func (f *Fetcher) dialPublic(ctx context.Context, network, address string) (net.Conn, error) {
host, port, err := net.SplitHostPort(address)
if err != nil {
return nil, fmt.Errorf("recipes: dial address %q: %w", address, err)
}
var addrs []netip.Addr
if ip, err := netip.ParseAddr(host); err == nil {
addrs = []netip.Addr{ip}
} else {
answers, err := f.resolver.LookupIPAddr(ctx, host)
if err != nil {
return nil, fmt.Errorf("recipes: resolve %s: %w", host, err)
}
for _, a := range answers {
ip, ok := netip.AddrFromSlice(a.IP)
if !ok {
return nil, fmt.Errorf("recipes: resolve %s: malformed address %v", host, a.IP)
}
addrs = append(addrs, ip)
}
}
if len(addrs) == 0 {
return nil, fmt.Errorf("recipes: resolve %s: no addresses", host)
}
for _, ip := range addrs {
if !IsPublic(ip) {
return nil, fmt.Errorf("%w: %s resolves to %s", ErrBlockedAddress, host, ip)
}
}
var errs []error
for _, ip := range addrs {
conn, err := f.dial(ctx, network, net.JoinHostPort(ip.Unmap().WithZone("").String(), port))
if err == nil {
return conn, nil
}
errs = append(errs, err)
}
return nil, errors.Join(errs...)
}
// requestCtxKey carries a request's own context into its dial. The
// transport dials with a context that keeps the request's values but not
// its cancellation; the connection and the rate-limit wait must end with
// the request.
type requestCtxKey struct{}
// requestTransport hands each request's context, redirect hops included,
// to dialTLS.
type requestTransport struct {
next http.RoundTripper
}
func (t *requestTransport) RoundTrip(req *http.Request) (*http.Response, error) {
ctx := req.Context()
return t.next.RoundTrip(req.WithContext(context.WithValue(ctx, requestCtxKey{}, ctx)))
}
// dialTLS connects to address through dialPublic and completes the TLS
// handshake, all under the request's context, and returns a turnConn whose
// first write waits for the host's rate-limit slot. ALPN offers only
// HTTP/1.1, so the connection carries this one request.
func (f *Fetcher) dialTLS(dialCtx context.Context, network, address string) (net.Conn, error) {
ctx, ok := dialCtx.Value(requestCtxKey{}).(context.Context)
if !ok {
return nil, fmt.Errorf("recipes: dial %s outside a request", address)
}
host, _, err := net.SplitHostPort(address)
if err != nil {
return nil, fmt.Errorf("recipes: dial address %q: %w", address, err)
}
conn, err := f.dialPublic(ctx, network, address)
if err != nil {
return nil, err
}
cfg := &tls.Config{}
if f.tlsConfig != nil {
cfg = f.tlsConfig.Clone()
}
cfg.ServerName = host
cfg.NextProtos = []string{"http/1.1"}
tc := tls.Client(conn, cfg)
if err := tc.HandshakeContext(ctx); err != nil {
conn.Close()
return nil, fmt.Errorf("recipes: TLS handshake with %s: %w", host, err)
}
return &turnConn{Conn: tc, f: f, ctx: ctx, host: strings.ToLower(host)}, nil
}
// turnConn is a connection ready to carry one request. Its first Write —
// the start of the request — waits for the host's rate-limit slot, so two
// requests to one host are written at least HostInterval apart however
// long each spent resolving, dialing and in the TLS handshake. A first
// Write that sends nothing hands the slot back.
type turnConn struct {
net.Conn
f *Fetcher
ctx context.Context
host string
// started is set by the first Write; the transport writes a request
// from one goroutine, so it needs no lock.
started bool
}
func (c *turnConn) Write(p []byte) (int, error) {
if c.started {
return c.Conn.Write(p)
}
release, err := c.f.waitTurn(c.ctx, c.host)
if err != nil {
return 0, err
}
c.started = true
n, err := c.Conn.Write(p)
if n == 0 && err != nil {
release()
}
return n, err
}
// checkURL refuses anything but an absolute https:// URL with a host.
func checkURL(u *url.URL) error {
if u.Scheme != "https" {
return fmt.Errorf("%w: %q", ErrNotHTTPS, u.Redacted())
}
if u.Hostname() == "" {
return fmt.Errorf("recipes: URL %q has no host", u.Redacted())
}
return nil
}
// checkRedirect applies checkURL to every hop and stops after
// MaxRedirects. The hop's address is checked when it is dialed.
func checkRedirect(req *http.Request, via []*http.Request) error {
if len(via) > MaxRedirects {
return fmt.Errorf("%w: more than %d", ErrTooManyRedirects, MaxRedirects)
}
return checkURL(req.URL)
}
// blockedIPv4 are the IPv4 special-purpose ranges that are not globally
// reachable (IANA IPv4 Special-Purpose Address Registry). IPv4-mapped IPv6
// addresses are unmapped before the check, so these cover them too.
var blockedIPv4 = []netip.Prefix{
netip.MustParsePrefix("0.0.0.0/8"), // "this network"
netip.MustParsePrefix("10.0.0.0/8"), // RFC 1918
netip.MustParsePrefix("100.64.0.0/10"), // carrier-grade NAT
netip.MustParsePrefix("127.0.0.0/8"), // loopback
netip.MustParsePrefix("169.254.0.0/16"), // link-local
netip.MustParsePrefix("172.16.0.0/12"), // RFC 1918
netip.MustParsePrefix("192.0.0.0/24"), // IETF protocol assignments
netip.MustParsePrefix("192.0.2.0/24"), // documentation, TEST-NET-1
netip.MustParsePrefix("192.88.99.0/24"), // 6to4 relay anycast (deprecated)
netip.MustParsePrefix("192.168.0.0/16"), // RFC 1918
netip.MustParsePrefix("198.18.0.0/15"), // benchmarking
netip.MustParsePrefix("198.51.100.0/24"), // documentation, TEST-NET-2
netip.MustParsePrefix("203.0.113.0/24"), // documentation, TEST-NET-3
netip.MustParsePrefix("224.0.0.0/4"), // multicast
netip.MustParsePrefix("240.0.0.0/4"), // reserved, broadcast
}
// globalIPv6 is the only IPv6 range dialed: global unicast. Everything
// outside it is refused, including ::, ::1, IPv4-compatible and
// IPv4-translated forms, NAT64 (64:ff9b::/96), discard-only (100::/64),
// unique local (fc00::/7), link-local (fe80::/10) and multicast (ff00::/8).
var globalIPv6 = netip.MustParsePrefix("2000::/3")
// blockedIPv6 are the special-purpose ranges inside global unicast that are
// not public or embed an arbitrary IPv4 address.
var blockedIPv6 = []netip.Prefix{
netip.MustParsePrefix("2001::/23"), // IETF protocol assignments, Teredo, benchmarking
netip.MustParsePrefix("2001:db8::/32"), // documentation
netip.MustParsePrefix("2002::/16"), // 6to4, embeds any IPv4
netip.MustParsePrefix("3fff::/20"), // documentation
}
// IsPublic reports whether ip is a public unicast address the Fetcher may
// dial. IPv4 addresses are public unless in a special-purpose range; IPv6
// addresses must be global unicast and outside its special-purpose ranges.
func IsPublic(ip netip.Addr) bool {
if !ip.IsValid() {
return false
}
ip = ip.Unmap().WithZone("")
blocked := blockedIPv4
if ip.Is6() {
if !globalIPv6.Contains(ip) {
return false
}
blocked = blockedIPv6
}
for _, p := range blocked {
if p.Contains(ip) {
return false
}
}
return true
}
// sleep waits d or until ctx is done, returning ctx's error if it is.
func sleep(ctx context.Context, d time.Duration) error {
t := time.NewTimer(d)
defer t.Stop()
select {
case <-t.C:
return nil
case <-ctx.Done():
return ctx.Err()
}
}
internal/recipes/fetcher_test.go
+721
View File
@@ -0,0 +1,721 @@
package recipes
import (
"context"
"crypto/tls"
"crypto/x509"
"errors"
"fmt"
"net"
"net/http"
"net/http/httptest"
"net/netip"
"strings"
"sync"
"testing"
"time"
)
// stubResolver answers each host from a list of answers, one per lookup;
// the last answer repeats.
type stubResolver struct {
mu sync.Mutex
answers map[string][][]string
calls map[string]int
}
func (r *stubResolver) LookupIPAddr(_ context.Context, host string) ([]net.IPAddr, error) {
r.mu.Lock()
defer r.mu.Unlock()
seq, ok := r.answers[host]
if !ok {
return nil, fmt.Errorf("stub: no such host %s", host)
}
n := r.calls[host]
r.calls[host]++
if n >= len(seq) {
n = len(seq) - 1
}
var out []net.IPAddr
for _, s := range seq[n] {
out = append(out, net.IPAddr{IP: net.ParseIP(s)})
}
return out, nil
}
func (r *stubResolver) lookups(host string) int {
r.mu.Lock()
defer r.mu.Unlock()
return r.calls[host]
}
// testEnv is a Fetcher whose public test addresses all reach one httptest
// TLS server, and which records every address it dials.
type testEnv struct {
f *Fetcher
resolver *stubResolver
server *httptest.Server
mu sync.Mutex
dialed []string
}
// publicIP, publicIP2 and publicIPv6 are public addresses; the test dialer
// routes them to the test server and never reaches the real hosts.
const (
publicIP = "93.184.215.14"
publicIP2 = "93.184.215.20"
publicIPv6 = "2606:2800:21f::10"
)
func newTestEnv(t *testing.T, handler http.Handler, answers map[string][][]string) *testEnv {
t.Helper()
server := httptest.NewTLSServer(handler)
t.Cleanup(server.Close)
pool := x509.NewCertPool()
pool.AddCert(server.Certificate())
env := &testEnv{
resolver: &stubResolver{answers: answers, calls: map[string]int{}},
server: server,
}
dial := func(ctx context.Context, network, address string) (net.Conn, error) {
env.mu.Lock()
env.dialed = append(env.dialed, address)
env.mu.Unlock()
host, _, err := net.SplitHostPort(address)
if err != nil {
return nil, err
}
if host == publicIP || host == publicIP2 || host == publicIPv6 {
var d net.Dialer
return d.DialContext(ctx, network, server.Listener.Addr().String())
}
return nil, fmt.Errorf("test dial: no route to %s", address)
}
env.f = newFetcher(env.resolver, dial, &tls.Config{RootCAs: pool})
// Real requests in these tests are not rate-limited unless a test
// installs a fake clock.
env.f.sleep = func(context.Context, time.Duration) error { return nil }
return env
}
func (e *testEnv) dials() []string {
e.mu.Lock()
defer e.mu.Unlock()
return append([]string(nil), e.dialed...)
}
func publicAnswers() map[string][][]string {
return map[string][][]string{"example.com": {{publicIP}}}
}
func TestFetchReturnsPage(t *testing.T) {
t.Parallel()
var gotUA string
env := newTestEnv(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotUA = r.Header.Get("User-Agent")
w.Header().Set("Content-Type", "text/html")
fmt.Fprint(w, "<html>tacos</html>")
}), publicAnswers())
page, err := env.f.Fetch(t.Context(), "https://example.com/tacos")
if err != nil {
t.Fatalf("Fetch: %v", err)
}
if string(page.Body) != "<html>tacos</html>" || page.ContentType != "text/html" || page.URL != "https://example.com/tacos" {
t.Errorf("page = %+v", page)
}
if gotUA != UserAgent {
t.Errorf("User-Agent = %q, want %q", gotUA, UserAgent)
}
if got := env.dials(); len(got) != 1 || got[0] != publicIP+":443" {
t.Errorf("dialed %v, want [%s:443]", got, publicIP)
}
}
func TestFetchDialsPublicIPv6(t *testing.T) {
t.Parallel()
env := newTestEnv(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, "ok")
}), map[string][][]string{"example.com": {{publicIPv6}}})
if _, err := env.f.Fetch(t.Context(), "https://example.com/"); err != nil {
t.Fatalf("Fetch: %v", err)
}
if got := env.dials(); len(got) != 1 || got[0] != "["+publicIPv6+"]:443" {
t.Errorf("dialed %v, want [[%s]:443]", got, publicIPv6)
}
}
func TestFetchRejectsNonHTTPS(t *testing.T) {
t.Parallel()
env := newTestEnv(t, http.NotFoundHandler(), publicAnswers())
for _, raw := range []string{"http://example.com/tacos", "ftp://example.com/", "example.com/tacos", "https:///tacos"} {
_, err := env.f.Fetch(t.Context(), raw)
if err == nil {
t.Errorf("Fetch(%q) succeeded", raw)
}
if strings.HasPrefix(raw, "http://") && !errors.Is(err, ErrNotHTTPS) {
t.Errorf("Fetch(%q) error = %v, want ErrNotHTTPS", raw, err)
}
}
if got := env.dials(); len(got) != 0 {
t.Errorf("dialed %v, want nothing", got)
}
if n := env.resolver.lookups("example.com"); n != 0 {
t.Errorf("resolved example.com %d times, want 0", n)
}
}
// blockedIPs has an address in every refused range, including IPv4-mapped
// forms of the IPv4 ranges.
var blockedIPs = []string{
"127.0.0.1", "127.255.255.254", "::1", "::ffff:127.0.0.1", // loopback
"10.0.0.1", "172.16.0.1", "172.31.255.255", "192.168.1.1", // RFC 1918
"::ffff:10.1.2.3", "::ffff:172.20.0.1", "::ffff:192.168.0.10",
"169.254.169.254", "::ffff:169.254.169.254", "fe80::1", // link-local
"100.64.0.1", "100.127.255.254", "::ffff:100.100.0.1", // carrier-grade NAT
"224.0.0.1", "239.255.255.250", "ff02::1", "::ffff:224.0.0.251", // multicast
"0.0.0.0", "0.1.2.3", "::", "::ffff:0.0.0.0", // unspecified
"fc00::1", "fd12:3456:789a::1", // unique local
"255.255.255.255", "240.0.0.1", "fec0::1", // reserved, site-local
"192.0.0.1", "192.0.0.255", // IETF protocol assignments
"192.0.2.1", "198.51.100.7", "203.0.113.10", "::ffff:203.0.113.10", // documentation
"192.88.99.1", // 6to4 relay anycast
"198.18.0.1", "198.19.255.255", "::ffff:198.18.0.1", // benchmarking
"2001:db8::1", "3fff::1", // IPv6 documentation
"2001:2::1", "100::1", // IPv6 benchmarking, discard-only
"::ffff:0:a00:1", // IPv4-translated
"64:ff9b::a00:1", "64:ff9b:1::a00:1", "2002:a00:1::1", "2001:0:a00:1::1", "::a00:1",
}
func TestIsPublic(t *testing.T) {
t.Parallel()
for _, s := range blockedIPs {
if IsPublic(netip.MustParseAddr(s)) {
t.Errorf("IsPublic(%s) = true, want false", s)
}
}
for _, s := range []string{
"8.8.8.8", publicIP, "::ffff:8.8.8.8", "100.63.255.255", "100.128.0.1", "172.32.0.1",
"192.0.1.255", "192.0.3.1", "192.88.98.255", "198.17.255.255", "198.20.0.1",
"198.51.99.255", "203.0.112.255", "203.0.114.1", "223.255.255.254",
"2606:4700::1111", publicIPv6, "2001:4860:4860::8888", "2a00:1450::1",
} {
if !IsPublic(netip.MustParseAddr(s)) {
t.Errorf("IsPublic(%s) = false, want true", s)
}
}
if IsPublic(netip.Addr{}) {
t.Error("IsPublic(zero Addr) = true, want false")
}
}
func TestFetchRefusesEachBlockedRange(t *testing.T) {
t.Parallel()
for _, ip := range blockedIPs {
t.Run(ip, func(t *testing.T) {
t.Parallel()
env := newTestEnv(t, http.NotFoundHandler(), map[string][][]string{"example.com": {{ip}}})
_, err := env.f.Fetch(t.Context(), "https://example.com/")
if !errors.Is(err, ErrBlockedAddress) {
t.Fatalf("Fetch error = %v, want ErrBlockedAddress", err)
}
if got := env.dials(); len(got) != 0 {
t.Errorf("dialed %v, want nothing", got)
}
})
}
}
func TestFetchRefusesBlockedLiteralHost(t *testing.T) {
t.Parallel()
env := newTestEnv(t, http.NotFoundHandler(), nil)
for _, raw := range []string{"https://127.0.0.1/", "https://[::1]/", "https://[::ffff:10.0.0.1]/", "https://169.254.169.254/latest/meta-data/", "https://[fe80::1%25eth0]/"} {
if _, err := env.f.Fetch(t.Context(), raw); !errors.Is(err, ErrBlockedAddress) {
t.Errorf("Fetch(%q) error = %v, want ErrBlockedAddress", raw, err)
}
}
if got := env.dials(); len(got) != 0 {
t.Errorf("dialed %v, want nothing", got)
}
}
func TestFetchRefusesMixedAnswer(t *testing.T) {
t.Parallel()
env := newTestEnv(t, http.NotFoundHandler(), map[string][][]string{"example.com": {{publicIP, "10.0.0.1"}}})
if _, err := env.f.Fetch(t.Context(), "https://example.com/"); !errors.Is(err, ErrBlockedAddress) {
t.Fatalf("Fetch error = %v, want ErrBlockedAddress", err)
}
if got := env.dials(); len(got) != 0 {
t.Errorf("dialed %v, want nothing", got)
}
}
func TestFetchRefusesRedirectToPrivateHost(t *testing.T) {
t.Parallel()
answers := map[string][][]string{
"example.com": {{publicIP}},
"evil.example.com": {{"10.0.0.5"}},
}
for _, target := range []string{"https://evil.example.com/admin", "https://127.0.0.1/admin", "https://[::ffff:192.168.0.1]/"} {
t.Run(target, func(t *testing.T) {
t.Parallel()
env := newTestEnv(t, http.RedirectHandler(target, http.StatusFound), answers)
_, err := env.f.Fetch(t.Context(), "https://example.com/tacos")
if !errors.Is(err, ErrBlockedAddress) {
t.Fatalf("Fetch error = %v, want ErrBlockedAddress", err)
}
if got := env.dials(); len(got) != 1 || got[0] != publicIP+":443" {
t.Errorf("dialed %v, want only the first hop", got)
}
})
}
}
func TestFetchRefusesRedirectToHTTP(t *testing.T) {
t.Parallel()
env := newTestEnv(t, http.RedirectHandler("http://example.com/plain", http.StatusMovedPermanently), publicAnswers())
if _, err := env.f.Fetch(t.Context(), "https://example.com/"); !errors.Is(err, ErrNotHTTPS) {
t.Fatalf("Fetch error = %v, want ErrNotHTTPS", err)
}
}
// chainHandler redirects /hop/N to /hop/N-1 until /hop/0, which serves a
// page.
var chainHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var n int
if _, err := fmt.Sscanf(r.URL.Path, "/hop/%d", &n); err != nil {
http.NotFound(w, r)
return
}
if n == 0 {
fmt.Fprint(w, "arrived")
return
}
http.Redirect(w, r, fmt.Sprintf("https://recipes.example.com/hop/%d", n-1), http.StatusFound)
})
func TestFetchFollowsAtMostThreeRedirects(t *testing.T) {
t.Parallel()
answers := map[string][][]string{
"example.com": {{publicIP}},
"recipes.example.com": {{publicIP2}},
}
env := newTestEnv(t, chainHandler, answers)
page, err := env.f.Fetch(t.Context(), "https://example.com/hop/3")
if err != nil {
t.Fatalf("Fetch with 3 redirects: %v", err)
}
if page.URL != "https://recipes.example.com/hop/0" || string(page.Body) != "arrived" {
t.Errorf("page = %+v", page)
}
// Every hop is dialed, so every hop's address is checked.
if got := env.dials(); len(got) != 4 || got[0] != publicIP+":443" || got[3] != publicIP2+":443" {
t.Errorf("dialed %v, want 4 checked hops", got)
}
if _, err := env.f.Fetch(t.Context(), "https://example.com/hop/4"); !errors.Is(err, ErrTooManyRedirects) {
t.Fatalf("Fetch with 4 redirects error = %v, want ErrTooManyRedirects", err)
}
}
// TestFetchRefusesDNSSwitchToPrivate covers DNS rebinding: the host's first
// answer is public and every later one private. The first fetch dials the
// address it checked; the next lookup's private answer is refused, and the
// private address is never dialed.
func TestFetchRefusesDNSSwitchToPrivate(t *testing.T) {
t.Parallel()
answers := map[string][][]string{"example.com": {{publicIP}, {"127.0.0.1"}}}
env := newTestEnv(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, "ok")
}), answers)
if _, err := env.f.Fetch(t.Context(), "https://example.com/one"); err != nil {
t.Fatalf("first Fetch: %v", err)
}
if n := env.resolver.lookups("example.com"); n != 1 {
t.Fatalf("first Fetch resolved %d times, want 1 (checked IP must be dialed IP)", n)
}
_, err := env.f.Fetch(t.Context(), "https://example.com/two")
if !errors.Is(err, ErrBlockedAddress) {
t.Fatalf("second Fetch error = %v, want ErrBlockedAddress", err)
}
if got := env.dials(); len(got) != 1 || got[0] != publicIP+":443" {
t.Errorf("dialed %v, want only %s:443", got, publicIP)
}
}
func TestFetchSizeCap(t *testing.T) {
t.Parallel()
env := newTestEnv(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
size := MaxBodyBytes
if r.URL.Path != "/exact" {
size++
}
if r.URL.Path == "/declared" {
w.Header().Set("Content-Length", fmt.Sprint(size))
}
w.Write([]byte(strings.Repeat("a", size)))
}), publicAnswers())
page, err := env.f.Fetch(t.Context(), "https://example.com/exact")
if err != nil {
t.Fatalf("Fetch at the cap: %v", err)
}
if len(page.Body) != MaxBodyBytes {
t.Errorf("body is %d bytes, want %d", len(page.Body), MaxBodyBytes)
}
for _, path := range []string{"/declared", "/chunked"} {
if _, err := env.f.Fetch(t.Context(), "https://example.com"+path); !errors.Is(err, ErrTooLarge) {
t.Errorf("Fetch %s error = %v, want ErrTooLarge", path, err)
}
}
}
func TestFetchTimeout(t *testing.T) {
t.Parallel()
env := newTestEnv(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/slow-body" {
w.Write([]byte("start"))
w.(http.Flusher).Flush()
}
<-r.Context().Done()
}), publicAnswers())
env.f.timeout = 100 * time.Millisecond
for _, path := range []string{"/slow-headers", "/slow-body"} {
start := time.Now()
_, err := env.f.Fetch(t.Context(), "https://example.com"+path)
if !errors.Is(err, ErrTimeout) {
t.Errorf("Fetch %s error = %v, want ErrTimeout", path, err)
}
if elapsed := time.Since(start); elapsed > 5*time.Second {
t.Errorf("Fetch %s took %s", path, elapsed)
}
}
if Timeout != 10*time.Second {
t.Errorf("Timeout = %s, want 10s", Timeout)
}
}
func TestFetchCallerCancelIsNotTimeout(t *testing.T) {
t.Parallel()
env := newTestEnv(t, http.NotFoundHandler(), publicAnswers())
ctx, cancel := context.WithCancel(t.Context())
cancel()
_, err := env.f.Fetch(ctx, "https://example.com/")
if !errors.Is(err, context.Canceled) || errors.Is(err, ErrTimeout) {
t.Fatalf("Fetch error = %v, want context.Canceled", err)
}
}
// fakeClock is a clock whose sleep advances it and records the wait.
type fakeClock struct {
mu sync.Mutex
t time.Time
sleeps []time.Duration
}
func (c *fakeClock) now() time.Time {
c.mu.Lock()
defer c.mu.Unlock()
return c.t
}
func (c *fakeClock) sleep(_ context.Context, d time.Duration) error {
c.mu.Lock()
defer c.mu.Unlock()
c.sleeps = append(c.sleeps, d)
c.t = c.t.Add(d)
return nil
}
func (c *fakeClock) advance(d time.Duration) {
c.mu.Lock()
defer c.mu.Unlock()
c.t = c.t.Add(d)
}
func (c *fakeClock) waits() []time.Duration {
c.mu.Lock()
defer c.mu.Unlock()
return append([]time.Duration(nil), c.sleeps...)
}
func installClock(f *Fetcher) *fakeClock {
c := &fakeClock{t: time.Date(2026, 10, 5, 18, 0, 0, 0, time.UTC)}
f.now = c.now
f.sleep = c.sleep
return c
}
func TestFetchCachesByURL(t *testing.T) {
t.Parallel()
var mu sync.Mutex
hits := map[string]int{}
env := newTestEnv(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
mu.Lock()
hits[r.URL.Path]++
n := hits[r.URL.Path]
mu.Unlock()
if r.URL.Path == "/missing" {
http.NotFound(w, r)
return
}
fmt.Fprintf(w, "%s #%d", r.URL.Path, n)
}), publicAnswers())
clock := installClock(env.f)
fetch := func(path string) string {
t.Helper()
page, err := env.f.Fetch(t.Context(), "https://example.com"+path)
if err != nil {
t.Fatalf("Fetch %s: %v", path, err)
}
return string(page.Body)
}
if got := fetch("/a"); got != "/a #1" {
t.Errorf("first /a = %q", got)
}
if got := fetch("/a"); got != "/a #1" {
t.Errorf("cached /a = %q, want the first response", got)
}
if got := fetch("/b"); got != "/b #1" {
t.Errorf("/b = %q, want its own response", got)
}
clock.advance(CacheTTL)
if got := fetch("/a"); got != "/a #2" {
t.Errorf("/a after CacheTTL = %q, want a refetch", got)
}
for range 2 {
var se *StatusError
if _, err := env.f.Fetch(t.Context(), "https://example.com/missing"); !errors.As(err, &se) || se.Status != http.StatusNotFound {
t.Fatalf("Fetch /missing error = %v, want StatusError 404", err)
}
}
mu.Lock()
defer mu.Unlock()
if hits["/missing"] != 2 {
t.Errorf("/missing was requested %d times, want 2 (errors are not cached)", hits["/missing"])
}
}
func TestCacheEvictsOldest(t *testing.T) {
t.Parallel()
f := newFetcher(&stubResolver{}, nil, nil)
clock := installClock(f)
for i := range CacheSize + 1 {
f.store(fmt.Sprintf("https://example.com/%d", i), Page{})
clock.advance(time.Second)
}
if len(f.cache) != CacheSize {
t.Errorf("cache holds %d pages, want %d", len(f.cache), CacheSize)
}
if _, ok := f.cached("https://example.com/0"); ok {
t.Error("oldest page was not evicted")
}
if _, ok := f.cached(fmt.Sprintf("https://example.com/%d", CacheSize)); !ok {
t.Error("newest page is missing")
}
}
func TestFetchRateLimitsPerHost(t *testing.T) {
t.Parallel()
answers := map[string][][]string{
"example.com": {{publicIP}},
"recipes.example.com": {{publicIP2}},
}
env := newTestEnv(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, "ok")
}), answers)
clock := installClock(env.f)
for _, raw := range []string{"https://example.com/1", "https://recipes.example.com/1", "https://example.com/2", "https://example.com/3"} {
if _, err := env.f.Fetch(t.Context(), raw); err != nil {
t.Fatalf("Fetch %s: %v", raw, err)
}
}
// example.com/2 waits a full interval; example.com/3 waits another.
// recipes.example.com has its own slot and never waits.
got := clock.waits()
if len(got) != 2 || got[0] != HostInterval || got[1] != HostInterval {
t.Errorf("waits = %v, want [%s %s]", got, HostInterval, HostInterval)
}
}
func TestFetchCacheIsNotSharedWithCallers(t *testing.T) {
t.Parallel()
env := newTestEnv(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, "tacos")
}), publicAnswers())
for range 2 {
page, err := env.f.Fetch(t.Context(), "https://example.com/tacos")
if err != nil {
t.Fatalf("Fetch: %v", err)
}
if string(page.Body) != "tacos" {
t.Fatalf("body = %q, want the cached page unchanged", page.Body)
}
// Overwrite the returned body: neither the fetched nor the cached
// copy may reach the cache.
copy(page.Body, "XXXXX")
}
}
// TestFetchCanceledWaitHoldsNoSlot covers a burst of fetches whose
// rate-limit waits end with their context: they must not push back the
// next request that is allowed through.
func TestFetchCanceledWaitHoldsNoSlot(t *testing.T) {
t.Parallel()
env := newTestEnv(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, "ok")
}), publicAnswers())
clock := installClock(env.f)
if _, err := env.f.Fetch(t.Context(), "https://example.com/first"); err != nil {
t.Fatalf("first Fetch: %v", err)
}
env.f.sleep = func(context.Context, time.Duration) error { return context.DeadlineExceeded }
for i := range 5 {
if _, err := env.f.Fetch(t.Context(), fmt.Sprintf("https://example.com/burst/%d", i)); !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("burst Fetch %d error = %v, want context.DeadlineExceeded", i, err)
}
}
env.f.sleep = clock.sleep
clock.advance(HostInterval)
if _, err := env.f.Fetch(t.Context(), "https://example.com/after"); err != nil {
t.Fatalf("Fetch after the burst: %v", err)
}
if got := clock.waits(); len(got) != 0 {
t.Errorf("waits = %v, want none: canceled waits held slots", got)
}
}
// TestFetchUnsentRequestHoldsNoSlot covers requests to a host with no
// previous request that are never sent — canceled before Fetch, or failing
// to connect — and checks that the next request is not delayed.
func TestFetchUnsentRequestHoldsNoSlot(t *testing.T) {
t.Parallel()
const unroutableIP = "93.184.215.99"
answers := map[string][][]string{"example.com": {{unroutableIP}, {publicIP}}}
env := newTestEnv(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, "ok")
}), answers)
clock := installClock(env.f)
ctx, cancel := context.WithCancel(t.Context())
cancel()
if _, err := env.f.Fetch(ctx, "https://example.com/canceled"); !errors.Is(err, context.Canceled) {
t.Fatalf("canceled Fetch error = %v, want context.Canceled", err)
}
if _, err := env.f.Fetch(t.Context(), "https://example.com/unroutable"); err == nil || !strings.Contains(err.Error(), "no route") {
t.Fatalf("unroutable Fetch error = %v, want a dial failure", err)
}
if _, err := env.f.Fetch(t.Context(), "https://example.com/next"); err != nil {
t.Fatalf("next Fetch: %v", err)
}
if got := clock.waits(); len(got) != 0 {
t.Errorf("waits = %v, want none: unsent requests held slots", got)
}
if got := env.dials(); len(got) != 2 {
t.Errorf("dials = %v, want the unroutable and the next request only", got)
}
}
// TestReleaseWakesWaiter checks that a request waiting behind a slot that
// is handed back goes ahead at once instead of sleeping out the interval.
func TestReleaseWakesWaiter(t *testing.T) {
t.Parallel()
f := newFetcher(&stubResolver{}, nil, nil)
release, err := f.waitTurn(t.Context(), "example.com")
if err != nil {
t.Fatalf("first waitTurn: %v", err)
}
done := make(chan error, 1)
start := time.Now()
go func() {
_, err := f.waitTurn(t.Context(), "example.com")
done <- err
}()
time.Sleep(50 * time.Millisecond)
release()
select {
case err := <-done:
if err != nil {
t.Fatalf("waiting waitTurn: %v", err)
}
if waited := time.Since(start); waited >= HostInterval {
t.Errorf("waiter took %s, want less than %s", waited, HostInterval)
}
case <-time.After(HostInterval / 2):
t.Fatal("waiter still waiting after the slot was handed back")
}
}
// stallingResolver holds its first lookup until unstall is closed, then
// answers from the stub.
type stallingResolver struct {
*stubResolver
once sync.Once
stalled chan struct{}
unstall chan struct{}
}
func (r *stallingResolver) LookupIPAddr(ctx context.Context, host string) ([]net.IPAddr, error) {
first := false
r.once.Do(func() { first = true })
if first {
close(r.stalled)
<-r.unstall
}
return r.stubResolver.LookupIPAddr(ctx, host)
}
// TestFetchRateLimitsSendsNotSetup covers a request that stalls in DNS
// while a later request to the same host is sent: the stalled request must
// still be sent a full HostInterval after the other one, because the slot
// is taken when the request is written, not when it starts.
func TestFetchRateLimitsSendsNotSetup(t *testing.T) {
t.Parallel()
var clock *fakeClock
var mu sync.Mutex
sentAt := map[string]time.Time{}
env := newTestEnv(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
mu.Lock()
sentAt[r.URL.Path] = clock.now()
mu.Unlock()
fmt.Fprint(w, "ok")
}), publicAnswers())
resolver := &stallingResolver{
stubResolver: env.resolver,
stalled: make(chan struct{}),
unstall: make(chan struct{}),
}
env.f.resolver = resolver
clock = installClock(env.f)
slow := make(chan error, 1)
go func() {
_, err := env.f.Fetch(t.Context(), "https://example.com/slow")
slow <- err
}()
<-resolver.stalled
if _, err := env.f.Fetch(t.Context(), "https://example.com/fast"); err != nil {
t.Fatalf("fast Fetch: %v", err)
}
close(resolver.unstall)
if err := <-slow; err != nil {
t.Fatalf("slow Fetch: %v", err)
}
mu.Lock()
defer mu.Unlock()
if gap := sentAt["/slow"].Sub(sentAt["/fast"]); gap < HostInterval {
t.Errorf("slow request sent %s after fast, want at least %s", gap, HostInterval)
}
if got := clock.waits(); len(got) != 1 || got[0] != HostInterval {
t.Errorf("waits = %v, want [%s]: only the stalled request waits", got, HostInterval)
}
}
internal/recipes/ground.go
+471
View File
@@ -0,0 +1,471 @@
package recipes
import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"strings"
"sync"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/skills"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/db"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/store"
)
// GroundConcurrency is how many items GroundMany grounds at once.
const GroundConcurrency = 3
var (
// ErrNoCandidates is a reason a dish is ungroundable: the search found
// no pages at all.
ErrNoCandidates = errors.New("recipes: search found no candidates")
// ErrTotalTimeNotPositive rejects a recipe whose page gives a total
// time of zero minutes, which normalize_recipe refuses.
ErrTotalTimeNotPositive = errors.New("recipes: recipe total time is not positive")
)
// ErrUngroundable is returned when no recipe page could ground a dish or a
// pasted URL. Reasons says why each page tried was rejected, or why there
// were none; errors.Is matches any of them.
type ErrUngroundable struct {
// Target is the dish name or the URL.
Target string
Reasons []error
}
func (e *ErrUngroundable) Error() string {
parts := make([]string, len(e.Reasons))
for i, r := range e.Reasons {
parts[i] = r.Error()
}
return fmt.Sprintf("recipes: cannot ground %q: %s", e.Target, strings.Join(parts, "; "))
}
func (e *ErrUngroundable) Unwrap() []error { return e.Reasons }
// Grounded is a recipe stored for a dish: the recipe and its cached photo,
// with normalize_recipe's method, minutes and difficulty.
type Grounded struct {
RecipeID string `json:"recipe_id"`
PhotoID string `json:"photo_id"`
// Method is stovetop, oven, crockpot, grill or other.
Method string `json:"method"`
// MinutesTotal is prep to table, nil when unknown.
MinutesTotal *int `json:"minutes_total"`
// Difficulty is easy, medium or hard.
Difficulty string `json:"difficulty"`
SourceURL string `json:"source_url"`
SiteName string `json:"site_name"`
}
// SearchSource returns the household's current search provider;
// *SearchSettingsService satisfies it.
type SearchSource interface {
Provider(ctx context.Context) (SearchProvider, error)
}
// Normalizer runs normalize_recipe on a recipe; scope is its source URL. In
// production it is skills.NormalizeRecipe with the household's
// harness.Config.
type Normalizer func(ctx context.Context, scope string, in skills.RecipeText) (skills.NormalizedRecipe, error)
// Grounder grounds dishes in recipe pages: it searches, fetches, extracts
// and ranks candidates, caches the best one's photo, normalizes its
// ingredients and stores it all. It is safe for concurrent use.
type Grounder struct {
st *store.Store
search SearchSource
pages PageFetcher
photos *PhotoCache
normalize Normalizer
}
// NewGrounder returns a Grounder storing recipes in st, searching with
// search, fetching pages with pages, caching photos with photos and
// normalizing with normalize.
func NewGrounder(st *store.Store, search SearchSource, pages PageFetcher, photos *PhotoCache, normalize Normalizer) (*Grounder, error) {
switch {
case st == nil:
return nil, errors.New("recipes: nil store")
case search == nil:
return nil, errors.New("recipes: nil search source")
case pages == nil:
return nil, errors.New("recipes: nil page fetcher")
case photos == nil:
return nil, errors.New("recipes: nil photo cache")
case normalize == nil:
return nil, errors.New("recipes: nil normalizer")
}
return &Grounder{st: st, search: search, pages: pages, photos: photos, normalize: normalize}, nil
}
// Ground searches for query and grounds dishName in the best recipe found.
// The candidates are ranked as Rank does, except that a candidate already
// stored is scored on its stored recipe and never fetched. They are tried
// best first: a stored candidate is returned as it is, and otherwise the
// first whose photo can be cached and whose ingredients normalize is stored
// — the recipe, its ingredients, steps and photo in one transaction — and
// returned. A stored recipe is reused only when it ranks above every
// candidate that grounds.
//
// When no candidate grounds the dish the error is an *ErrUngroundable with
// each reason. A failure that is not the candidates' — the search, the
// model provider, the database — is returned as it is.
func (g *Grounder) Ground(ctx context.Context, dishName, query string) (Grounded, error) {
if strings.TrimSpace(dishName) == "" {
return Grounded{}, errors.New("recipes: ground: dish name is empty")
}
provider, err := g.search.Provider(ctx)
if err != nil {
return Grounded{}, fmt.Errorf("recipes: ground %q: %w", dishName, err)
}
cands, err := provider.Search(ctx, query, RankDepth)
if err != nil {
return Grounded{}, fmt.Errorf("recipes: ground %q: search: %w", dishName, err)
}
if len(cands) == 0 {
return Grounded{}, &ErrUngroundable{Target: dishName, Reasons: []error{ErrNoCandidates}}
}
cands = cands[:min(len(cands), RankDepth)]
stored := make(map[string]storedRecipe)
for _, c := range cands {
sr, ok, err := g.storedRecipe(ctx, c.URL)
if err != nil {
return Grounded{}, fmt.Errorf("recipes: ground %q: %w", dishName, err)
}
if ok {
stored[c.URL] = sr
}
}
ranked, rankErr := rank(ctx, dishName, cands, func(ctx context.Context, c Candidate) (Recipe, error) {
if sr, ok := stored[c.URL]; ok {
return sr.recipe, nil
}
return fetchRecipe(ctx, g.pages, c.URL)
})
if err := ctx.Err(); err != nil {
return Grounded{}, err
}
var reasons []error
if rankErr != nil {
reasons = append(reasons, rankErr)
}
for _, r := range ranked {
if sr, ok := stored[r.Candidate.URL]; ok {
return sr.grounded, nil
}
gr, reason, err := g.ground(ctx, r.Candidate.URL, r.Recipe)
if err != nil {
return Grounded{}, fmt.Errorf("recipes: ground %q: %w", dishName, err)
}
if reason == nil {
return gr, nil
}
reasons = append(reasons, fmt.Errorf("%s: %w", r.Candidate.URL, reason))
}
return Grounded{}, &ErrUngroundable{Target: dishName, Reasons: reasons}
}
// GroundURL grounds the recipe on the page at rawURL, a pasted URL, as
// Ground does a candidate. An already stored rawURL is returned without
// fetching it. A page that cannot be fetched, holds no usable recipe, or
// whose photo or ingredients fail is an *ErrUngroundable.
func (g *Grounder) GroundURL(ctx context.Context, rawURL string) (Grounded, error) {
if gr, ok, err := g.stored(ctx, rawURL); err != nil || ok {
return gr, err
}
ungroundable := func(err error) (Grounded, error) {
if ctxErr := ctx.Err(); ctxErr != nil {
return Grounded{}, ctxErr
}
return Grounded{}, &ErrUngroundable{Target: rawURL, Reasons: []error{err}}
}
page, err := g.pages.Fetch(ctx, rawURL)
if err != nil {
return ungroundable(err)
}
r, err := Extract(page.Body, page.URL)
if err != nil {
return ungroundable(err)
}
if err := usable(r); err != nil {
return ungroundable(err)
}
gr, reason, err := g.ground(ctx, rawURL, r)
if err != nil {
return Grounded{}, fmt.Errorf("recipes: ground %s: %w", rawURL, err)
}
if reason != nil {
return ungroundable(reason)
}
return gr, nil
}
// GroundRequest is one dish for GroundMany.
type GroundRequest struct {
DishName string
Query string
}
// GroundResult is one GroundMany outcome: Grounded, or Err when the item
// failed.
type GroundResult struct {
Grounded Grounded
Err error
}
// GroundMany grounds each request with Ground, GroundConcurrency at a time,
// and returns their results in request order. One item's failure does not
// affect the others.
func (g *Grounder) GroundMany(ctx context.Context, reqs []GroundRequest) []GroundResult {
results := make([]GroundResult, len(reqs))
slots := make(chan struct{}, GroundConcurrency)
var wg sync.WaitGroup
for i, req := range reqs {
wg.Add(1)
go func() {
defer wg.Done()
select {
case slots <- struct{}{}:
case <-ctx.Done():
results[i] = GroundResult{Err: ctx.Err()}
return
}
defer func() { <-slots }()
gr, err := g.Ground(ctx, req.DishName, req.Query)
results[i] = GroundResult{Grounded: gr, Err: err}
}()
}
wg.Wait()
return results
}
// stored returns the grounding of the recipe stored for sourceURL, and
// false when there is none.
func (g *Grounder) stored(ctx context.Context, sourceURL string) (Grounded, bool, error) {
row, ok, err := g.storedRow(ctx, sourceURL)
if err != nil || !ok {
return Grounded{}, false, err
}
return groundedFrom(row), true, nil
}
// storedRow returns the recipe row stored for sourceURL, and false when
// there is none.
func (g *Grounder) storedRow(ctx context.Context, sourceURL string) (db.Recipe, bool, error) {
row, err := db.New(g.st.Reader).GetRecipeBySourceURL(ctx, sourceURL)
if errors.Is(err, sql.ErrNoRows) {
return db.Recipe{}, false, nil
}
if err != nil {
return db.Recipe{}, false, fmt.Errorf("recipes: look up recipe %s: %w", sourceURL, err)
}
if !row.PhotoID.Valid || !row.Method.Valid || !row.Difficulty.Valid {
return db.Recipe{}, false, fmt.Errorf("recipes: stored recipe %s for %s has no photo, method or difficulty", row.ID, sourceURL)
}
return row, true, nil
}
// storedRecipe is a recipe already stored: its grounding, and the recipe
// rebuilt from what was stored so it can be ranked without its page.
type storedRecipe struct {
grounded Grounded
recipe Recipe
}
// storedRecipe returns the recipe stored for sourceURL with its photo,
// ingredients and steps, and false when there is none.
func (g *Grounder) storedRecipe(ctx context.Context, sourceURL string) (storedRecipe, bool, error) {
row, ok, err := g.storedRow(ctx, sourceURL)
if err != nil || !ok {
return storedRecipe{}, false, err
}
gr := groundedFrom(row)
q := db.New(g.st.Reader)
photo, err := q.GetPhoto(ctx, gr.PhotoID)
if err != nil {
return storedRecipe{}, false, fmt.Errorf("recipes: load photo %s of recipe %s: %w", gr.PhotoID, gr.RecipeID, err)
}
ings, err := q.ListRecipeIngredients(ctx, gr.RecipeID)
if err != nil {
return storedRecipe{}, false, fmt.Errorf("recipes: load ingredients of recipe %s: %w", gr.RecipeID, err)
}
steps, err := q.ListRecipeSteps(ctx, gr.RecipeID)
if err != nil {
return storedRecipe{}, false, fmt.Errorf("recipes: load steps of recipe %s: %w", gr.RecipeID, err)
}
r := Recipe{
Title: row.Title,
SiteName: row.SiteName.String,
ImageURL: photo.SourceUrl,
PrepMinutes: intFrom(row.PrepMinutes),
CookMinutes: intFrom(row.CookMinutes),
TotalMinutes: intFrom(row.TotalMinutes),
Yield: row.Yield.String,
JSONLD: json.RawMessage(row.RawJsonld),
}
for _, ing := range ings {
r.Ingredients = append(r.Ingredients, ing.RawText)
}
for _, st := range steps {
r.Steps = append(r.Steps, st.Text)
}
return storedRecipe{grounded: gr, recipe: r}, true, nil
}
// ground downloads r's photo, normalizes it and stores it as sourceURL's
// recipe. A problem with this recipe is returned as reason, so the caller
// can try another; err is a failure that is not the recipe's.
//
// The order is fixed: the photo is downloaded and validated first, so a
// recipe without a usable photo never costs a normalize_recipe call; then
// normalize_recipe runs; then save records the photo (CachePhoto) and the
// recipe in one transaction.
func (g *Grounder) ground(ctx context.Context, sourceURL string, r Recipe) (gr Grounded, reason, err error) {
photo, err := g.photos.DownloadPhoto(ctx, r.ImageURL)
if err != nil {
if ctxErr := ctx.Err(); ctxErr != nil {
return Grounded{}, nil, ctxErr
}
return Grounded{}, fmt.Errorf("photo: %w", err), nil
}
norm, err := g.normalize(ctx, sourceURL, skills.RecipeText{
Title: r.Title,
Ingredients: r.Ingredients,
Steps: r.Steps,
PrepMinutes: r.PrepMinutes,
CookMinutes: r.CookMinutes,
TotalMinutes: r.TotalMinutes,
})
if errors.Is(err, llm.ErrBadOutput) {
return Grounded{}, fmt.Errorf("normalize: %w", err), nil
}
if err != nil {
return Grounded{}, nil, fmt.Errorf("normalize: %w", err)
}
if len(norm.Ingredients) != len(r.Ingredients) {
return Grounded{}, nil, fmt.Errorf("normalize: %d ingredients for %d lines", len(norm.Ingredients), len(r.Ingredients))
}
gr, err = g.save(ctx, sourceURL, r, photo, norm)
return gr, nil, err
}
// save stores the recipe, its ingredients and steps and its photo in one
// transaction. A photo's files written for a transaction that does not
// commit are removed.
func (g *Grounder) save(ctx context.Context, sourceURL string, r Recipe, p DownloadedPhoto, norm skills.NormalizedRecipe) (Grounded, error) {
var (
row db.Recipe
photo db.Photo
created bool
)
err := g.st.WithTx(ctx, func(tx *sql.Tx) error {
var err error
photo, created, err = g.photos.CachePhoto(ctx, tx, p)
if err != nil {
return err
}
q := db.New(tx)
now := store.Now().String()
row, err = q.UpsertRecipe(ctx, db.UpsertRecipeParams{
ID: store.NewID(),
SourceUrl: sourceURL,
SiteName: sql.NullString{String: r.SiteName, Valid: r.SiteName != ""},
Title: r.Title,
TotalMinutes: nullInt(r.TotalMinutes),
PrepMinutes: nullInt(r.PrepMinutes),
CookMinutes: nullInt(r.CookMinutes),
Yield: sql.NullString{String: r.Yield, Valid: r.Yield != ""},
RawJsonld: string(r.JSONLD),
PhotoID: sql.NullString{String: photo.ID, Valid: true},
Method: sql.NullString{String: norm.Method, Valid: true},
Difficulty: sql.NullString{String: norm.Difficulty, Valid: true},
MinutesTotal: nullInt(norm.MinutesTotal),
FetchedAt: now,
CreatedAt: now,
UpdatedAt: now,
})
if err != nil {
return fmt.Errorf("recipes: store recipe: %w", err)
}
if err := q.DeleteRecipeIngredients(ctx, row.ID); err != nil {
return fmt.Errorf("recipes: replace ingredients: %w", err)
}
if err := q.DeleteRecipeSteps(ctx, row.ID); err != nil {
return fmt.Errorf("recipes: replace steps: %w", err)
}
for i, line := range r.Ingredients {
ing := norm.Ingredients[i]
err := q.CreateRecipeIngredient(ctx, db.CreateRecipeIngredientParams{
ID: store.NewID(),
RecipeID: row.ID,
Position: int64(i),
RawText: line,
Name: sql.NullString{String: ing.Name, Valid: true},
Qty: sql.NullString{String: ing.Qty, Valid: ing.Qty != ""},
Unit: sql.NullString{String: ing.Unit, Valid: ing.Unit != ""},
Note: sql.NullString{String: ing.Note, Valid: ing.Note != ""},
CreatedAt: now,
UpdatedAt: now,
})
if err != nil {
return fmt.Errorf("recipes: store ingredient %d: %w", i+1, err)
}
}
for i, text := range r.Steps {
err := q.CreateRecipeStep(ctx, db.CreateRecipeStepParams{
ID: store.NewID(),
RecipeID: row.ID,
Position: int64(i),
Text: text,
CreatedAt: now,
UpdatedAt: now,
})
if err != nil {
return fmt.Errorf("recipes: store step %d: %w", i+1, err)
}
}
return nil
})
if err != nil {
if created {
err = errors.Join(err, g.photos.RemovePhotoFiles(photo.ID))
}
return Grounded{}, err
}
return groundedFrom(row), nil
}
func groundedFrom(row db.Recipe) Grounded {
gr := Grounded{
RecipeID: row.ID,
PhotoID: row.PhotoID.String,
Method: row.Method.String,
Difficulty: row.Difficulty.String,
SourceURL: row.SourceUrl,
SiteName: row.SiteName.String,
}
if row.MinutesTotal.Valid {
m := int(row.MinutesTotal.Int64)
gr.MinutesTotal = &m
}
return gr
}
func intFrom(v sql.NullInt64) *int {
if !v.Valid {
return nil
}
n := int(v.Int64)
return &n
}
func nullInt(v *int) sql.NullInt64 {
if v == nil {
return sql.NullInt64{}
}
return sql.NullInt64{Int64: int64(*v), Valid: true}
}
internal/recipes/ground_test.go
+518
View File
@@ -0,0 +1,518 @@
package recipes
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"sync"
"testing"
"time"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/harness"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/skills"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/db"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/store"
)
// fakeSearch is a SearchSource and SearchProvider answering each query
// from results, and failing a query in errs. With gate set, every Search
// reports its query on started and waits for gate to close.
type fakeSearch struct {
results map[string][]Candidate
errs map[string]error
started chan string
gate chan struct{}
mu sync.Mutex
calls int
active int
maxActive int
}
func (s *fakeSearch) Provider(context.Context) (SearchProvider, error) { return s, nil }
func (s *fakeSearch) Search(_ context.Context, query string, n int) ([]Candidate, error) {
s.mu.Lock()
s.calls++
s.active++
s.maxActive = max(s.maxActive, s.active)
s.mu.Unlock()
defer func() {
s.mu.Lock()
s.active--
s.mu.Unlock()
}()
if s.gate != nil {
s.started <- query
<-s.gate
}
if err := s.errs[query]; err != nil {
return nil, err
}
c := s.results[query]
return c[:min(n, len(c))], nil
}
func (s *fakeSearch) callCount() int {
s.mu.Lock()
defer s.mu.Unlock()
return s.calls
}
func (f *fakeFetcher) fetchCount() int {
f.mu.Lock()
defer f.mu.Unlock()
return len(f.fetched)
}
// groundPage is a recipe page for name with two ingredients, two steps, a
// 40-minute total and the image at https://example.com{imagePath}.
func groundPage(name, imagePath string) []byte {
return page(`<meta property="og:site_name" content="Example Kitchen">`, fmt.Sprintf(`{
"@context": "https://schema.org",
"@type": "Recipe",
"name": %q,
"image": "https://example.com%s",
"recipeIngredient": ["1 onion", "2 cloves garlic, minced"],
"recipeInstructions": ["Chop the onion.", "Fry with the garlic."],
"totalTime": "PT40M",
"recipeYield": "4"
}`, name, imagePath))
}
// normalizedReply is a usable normalize_recipe reply for groundPage.
func normalizedReply() llm.GenerateResult {
js := `{"ingredients":[
{"name":"onion","qty":"1","unit":"","note":""},
{"name":"garlic","qty":"2","unit":"cloves","note":"minced"}
],"method":"stovetop","difficulty":null,"minutes_total":40}`
return llm.GenerateResult{Response: llm.Response{Text: js, JSON: json.RawMessage(js)}}
}
// badReply is a normalize_recipe reply that fails validation.
func badReply() llm.GenerateResult {
js := `{"ingredients":[],"method":"stovetop","difficulty":null,"minutes_total":40}`
return llm.GenerateResult{Response: llm.Response{Text: js, JSON: json.RawMessage(js)}}
}
type groundEnv struct {
g *Grounder
st *store.Store
dataDir string
images *photoServer
pages *fakeFetcher
search *fakeSearch
llm *llm.Fake
}
// newGroundEnv returns a Grounder over pages, images served at
// https://example.com, search and a fake model, normalizing through the
// real harness and normalize_recipe skill.
func newGroundEnv(t *testing.T, pages map[string][]byte, images map[string]photoFile, search *fakeSearch) *groundEnv {
t.Helper()
photos, st, dataDir, imgSrv := newTestPhotoCache(t, images)
model := &llm.Fake{}
guard, err := harness.NewGuard(harness.DefaultMaxCalls)
if err != nil {
t.Fatal(err)
}
cfg := harness.Config{
Settings: llm.Settings{Provider: "anthropic", Model: "claude-test"},
NewProvider: func(llm.Settings, string) (llm.Provider, error) { return model, nil },
Store: st,
Guard: guard,
}
normalize := func(ctx context.Context, scope string, in skills.RecipeText) (skills.NormalizedRecipe, error) {
return skills.NormalizeRecipe(ctx, cfg, scope, in)
}
fetcher := &fakeFetcher{pages: pages}
g, err := NewGrounder(st, search, fetcher, photos, normalize)
if err != nil {
t.Fatal(err)
}
return &groundEnv{g: g, st: st, dataDir: dataDir, images: imgSrv, pages: fetcher, search: search, llm: model}
}
func jpegFile(t *testing.T, w, h int) photoFile {
return photoFile{"image/jpeg", encodeJPEG(t, gradient(w, h))}
}
func countRows(t *testing.T, st *store.Store, table string) int {
t.Helper()
var n int
if err := st.Reader.QueryRowContext(context.Background(), "SELECT count(*) FROM "+table).Scan(&n); err != nil {
t.Fatal(err)
}
return n
}
func TestGroundHappyPath(t *testing.T) {
t.Parallel()
search := &fakeSearch{results: map[string][]Candidate{
"chicken curry": {
{URL: "https://a.example/missing"},
{URL: "https://b.example/curry"},
},
}}
env := newGroundEnv(t,
map[string][]byte{"https://b.example/curry": groundPage("Chicken Curry", "/img/curry.jpg")},
map[string]photoFile{"/img/curry.jpg": jpegFile(t, 800, 600)},
search)
env.llm.QueueGenerate(normalizedReply())
gr, err := env.g.Ground(t.Context(), "Chicken Curry", "chicken curry")
if err != nil {
t.Fatalf("Ground: %v", err)
}
if gr.RecipeID == "" || gr.PhotoID == "" {
t.Fatalf("Grounded = %+v, want recipe and photo ids", gr)
}
want := Grounded{
RecipeID: gr.RecipeID, PhotoID: gr.PhotoID, Method: "stovetop", MinutesTotal: gr.MinutesTotal,
Difficulty: "easy", SourceURL: "https://b.example/curry", SiteName: "Example Kitchen",
}
if gr != want || gr.MinutesTotal == nil || *gr.MinutesTotal != 40 {
t.Errorf("Grounded = %+v (minutes %v), want %+v with 40 minutes", gr, gr.MinutesTotal, want)
}
q := db.New(env.st.Reader)
rec, err := q.GetRecipe(t.Context(), gr.RecipeID)
if err != nil {
t.Fatal(err)
}
if rec.Title != "Chicken Curry" || rec.PhotoID.String != gr.PhotoID || rec.Method.String != "stovetop" ||
rec.Difficulty.String != "easy" || rec.MinutesTotal.Int64 != 40 || rec.TotalMinutes.Int64 != 40 || rec.Yield.String != "4" {
t.Errorf("recipe row = %+v", rec)
}
ings, err := q.ListRecipeIngredients(t.Context(), gr.RecipeID)
if err != nil {
t.Fatal(err)
}
if len(ings) != 2 || ings[1].RawText != "2 cloves garlic, minced" || ings[1].Name.String != "garlic" ||
ings[1].Qty.String != "2" || ings[1].Unit.String != "cloves" || ings[1].Note.String != "minced" || ings[0].Unit.Valid {
t.Errorf("ingredients = %+v", ings)
}
steps, err := q.ListRecipeSteps(t.Context(), gr.RecipeID)
if err != nil {
t.Fatal(err)
}
if len(steps) != 2 || steps[0].Text != "Chop the onion." || steps[1].Position != 1 {
t.Errorf("steps = %+v", steps)
}
photo, err := q.GetPhoto(t.Context(), gr.PhotoID)
if err != nil {
t.Fatal(err)
}
if photo.SourceUrl != "https://example.com/img/curry.jpg" {
t.Errorf("photo row = %+v", photo)
}
if _, err := os.Stat(filepath.Join(env.dataDir, filepath.FromSlash(photo.LocalPath))); err != nil {
t.Errorf("photo file: %v", err)
}
}
func TestGroundUngroundable(t *testing.T) {
t.Parallel()
search := &fakeSearch{results: map[string][]Candidate{
"mystery stew": {
{URL: "https://a.example/missing"},
{URL: "https://b.example/no-recipe"},
{URL: "https://c.example/tiny-photo"},
},
}}
env := newGroundEnv(t,
map[string][]byte{
"https://b.example/no-recipe": page(""),
"https://c.example/tiny-photo": groundPage("Mystery Stew", "/img/tiny.jpg"),
},
map[string]photoFile{"/img/tiny.jpg": jpegFile(t, 200, 200)},
search)
_, err := env.g.Ground(t.Context(), "Mystery Stew", "mystery stew")
var ungroundable *ErrUngroundable
if !errors.As(err, &ungroundable) {
t.Fatalf("Ground error = %v, want *ErrUngroundable", err)
}
if ungroundable.Target != "Mystery Stew" {
t.Errorf("Target = %q", ungroundable.Target)
}
for _, reason := range []error{ErrCandidatesUnranked, ErrNoRecipe, ErrPhotoTooSmall} {
if !errors.Is(err, reason) {
t.Errorf("error %v does not report %v", err, reason)
}
}
if n := len(env.llm.Requests()); n != 0 {
t.Errorf("%d model calls, want none before a photo is cached", n)
}
for _, table := range []string{"recipes", "recipe_ingredients", "photos"} {
if n := countRows(t, env.st, table); n != 0 {
t.Errorf("%d %s rows for an ungroundable dish", n, table)
}
}
if names := photoFiles(t, env.dataDir); len(names) != 0 {
t.Errorf("photo files stored for an ungroundable dish: %v", names)
}
_, err = env.g.Ground(t.Context(), "Nothing At All", "nothing at all")
if !errors.As(err, &ungroundable) || !errors.Is(err, ErrNoCandidates) {
t.Errorf("Ground with no search results error = %v, want *ErrUngroundable with ErrNoCandidates", err)
}
}
func TestGroundTriesNextCandidateAfterBadOutput(t *testing.T) {
t.Parallel()
search := &fakeSearch{results: map[string][]Candidate{
"curry": {{URL: "https://a.example/curry"}, {URL: "https://b.example/curry"}},
}}
env := newGroundEnv(t,
map[string][]byte{
"https://a.example/curry": groundPage("Curry", "/img/a.jpg"),
"https://b.example/curry": groundPage("Curry", "/img/b.jpg"),
},
map[string]photoFile{"/img/a.jpg": jpegFile(t, 640, 480), "/img/b.jpg": jpegFile(t, 480, 640)},
search)
// The first candidate's reply and repair are unusable; the second's is
// fine.
env.llm.QueueGenerate(badReply(), badReply(), normalizedReply())
gr, err := env.g.Ground(t.Context(), "Curry", "curry")
if err != nil {
t.Fatalf("Ground: %v", err)
}
if gr.SourceURL != "https://b.example/curry" {
t.Errorf("grounded in %s, want the second candidate", gr.SourceURL)
}
if n := countRows(t, env.st, "photos"); n != 1 {
t.Errorf("%d photos rows, want only the grounded recipe's", n)
}
}
func TestGroundSearchFailureIsNotUngroundable(t *testing.T) {
t.Parallel()
env := newGroundEnv(t, nil, nil, &fakeSearch{errs: map[string]error{"curry": ErrSearchRateLimited}})
_, err := env.g.Ground(t.Context(), "Curry", "curry")
var ungroundable *ErrUngroundable
if !errors.Is(err, ErrSearchRateLimited) || errors.As(err, &ungroundable) {
t.Errorf("Ground error = %v, want ErrSearchRateLimited and not *ErrUngroundable", err)
}
}
func TestGroundReusesStoredRecipe(t *testing.T) {
t.Parallel()
search := &fakeSearch{results: map[string][]Candidate{
"curry": {{URL: "https://b.example/curry"}},
"chicken curry": {{URL: "https://a.example/other"}, {URL: "https://b.example/curry"}},
}}
env := newGroundEnv(t,
map[string][]byte{"https://b.example/curry": groundPage("Curry", "/img/curry.jpg")},
map[string]photoFile{"/img/curry.jpg": jpegFile(t, 640, 480)},
search)
env.llm.QueueGenerate(normalizedReply())
first, err := env.g.Ground(t.Context(), "Curry", "curry")
if err != nil {
t.Fatalf("first Ground: %v", err)
}
pages, images, calls := env.pages.fetchCount(), env.images.requests.Load(), len(env.llm.Requests())
// The stored page is gone and the model has no reply left: only the
// stored recipe can ground the dish.
delete(env.pages.pages, "https://b.example/curry")
again, err := env.g.Ground(t.Context(), "Chicken Curry", "chicken curry")
if err != nil {
t.Fatalf("second Ground: %v", err)
}
if again.RecipeID != first.RecipeID || again.PhotoID != first.PhotoID || again.Method != first.Method ||
again.Difficulty != first.Difficulty || *again.MinutesTotal != *first.MinutesTotal || again.SiteName != first.SiteName {
t.Errorf("reused grounding = %+v, want %+v", again, first)
}
if env.images.requests.Load() != images || len(env.llm.Requests()) != calls {
t.Errorf("reuse fetched images %d→%d, model calls %d→%d; want none",
images, env.images.requests.Load(), calls, len(env.llm.Requests()))
}
// Ranking fetches only the candidate not stored; the stored one ranks
// on its stored recipe.
env.pages.mu.Lock()
fetched := append([]string(nil), env.pages.fetched[pages:]...)
env.pages.mu.Unlock()
if len(fetched) != 1 || fetched[0] != "https://a.example/other" {
t.Errorf("reuse fetched %v, want only https://a.example/other", fetched)
}
pages = env.pages.fetchCount()
byURL, err := env.g.GroundURL(t.Context(), "https://b.example/curry")
if err != nil || byURL.RecipeID != first.RecipeID {
t.Errorf("GroundURL of the stored page = %+v, %v; want the stored recipe", byURL, err)
}
if env.pages.fetchCount() != pages {
t.Error("GroundURL refetched a stored page")
}
if n := countRows(t, env.st, "recipes"); n != 1 {
t.Errorf("%d recipes rows, want 1", n)
}
}
func TestGroundStoredRecipeDoesNotOutrankRelevantOne(t *testing.T) {
t.Parallel()
search := &fakeSearch{results: map[string][]Candidate{
"brownies": {{URL: "https://b.example/brownies"}},
"chicken curry": {{URL: "https://b.example/brownies"}, {URL: "https://a.example/curry"}},
}}
env := newGroundEnv(t,
map[string][]byte{
"https://a.example/curry": groundPage("Chicken Curry", "/img/curry.jpg"),
"https://b.example/brownies": groundPage("Fudgy Brownies", "/img/brownies.jpg"),
},
map[string]photoFile{
"/img/curry.jpg": jpegFile(t, 640, 480),
"/img/brownies.jpg": jpegFile(t, 640, 480),
},
search)
env.llm.QueueGenerate(normalizedReply(), normalizedReply())
brownies, err := env.g.Ground(t.Context(), "Brownies", "brownies")
if err != nil {
t.Fatalf("Ground brownies: %v", err)
}
// Search lists the unrelated stored recipe first and the relevant new
// one second: ranking, not search order or storage, grounds the
// relevant one.
curry, err := env.g.Ground(t.Context(), "Chicken Curry", "chicken curry")
if err != nil {
t.Fatalf("Ground chicken curry: %v", err)
}
if curry.SourceURL != "https://a.example/curry" || curry.RecipeID == brownies.RecipeID {
t.Errorf("grounded Chicken Curry in %s (recipe %s), want https://a.example/curry, not the stored brownies %s",
curry.SourceURL, curry.RecipeID, brownies.RecipeID)
}
if n := countRows(t, env.st, "recipes"); n != 2 {
t.Errorf("%d recipes rows, want 2", n)
}
}
func TestGroundURL(t *testing.T) {
t.Parallel()
env := newGroundEnv(t,
map[string][]byte{
"https://b.example/pasted": groundPage("Grandma's Slow Cooker Chili", "/img/chili.jpg"),
"https://b.example/no-image": recipePage("Chili", true, false, true, true),
"https://b.example/no-time": page("", `{"@type":"Recipe","name":"Chili","image":"https://example.com/img/chili.jpg",
"recipeIngredient":["1 onion"],"totalTime":"PT0M"}`),
},
map[string]photoFile{"/img/chili.jpg": jpegFile(t, 640, 480)},
&fakeSearch{})
env.llm.QueueGenerate(normalizedReply())
gr, err := env.g.GroundURL(t.Context(), "https://b.example/pasted")
if err != nil {
t.Fatalf("GroundURL: %v", err)
}
if gr.SourceURL != "https://b.example/pasted" || gr.Method != "crockpot" || gr.PhotoID == "" {
t.Errorf("Grounded = %+v, want the pasted URL, crockpot from the title", gr)
}
if env.search.callCount() != 0 {
t.Error("GroundURL searched")
}
for rawURL, reason := range map[string]error{
"https://b.example/no-image": ErrNoImage,
"https://b.example/no-time": ErrTotalTimeNotPositive,
"https://b.example/missing": nil,
} {
_, err := env.g.GroundURL(t.Context(), rawURL)
var ungroundable *ErrUngroundable
if !errors.As(err, &ungroundable) || ungroundable.Target != rawURL {
t.Errorf("GroundURL(%s) error = %v, want *ErrUngroundable", rawURL, err)
}
var status *StatusError
if reason != nil && !errors.Is(err, reason) || reason == nil && !errors.As(err, &status) {
t.Errorf("GroundURL(%s) error = %v, want reason %v", rawURL, err, reason)
}
}
}
func TestGroundManyIsolatesFailures(t *testing.T) {
t.Parallel()
search := &fakeSearch{
results: map[string][]Candidate{
"curry": {{URL: "https://a.example/curry"}},
"nothing": {{URL: "https://a.example/missing"}},
"chili": {{URL: "https://b.example/chili"}},
},
errs: map[string]error{"broken": ErrSearchRefused},
}
env := newGroundEnv(t,
map[string][]byte{
"https://a.example/curry": groundPage("Curry", "/img/curry.jpg"),
"https://b.example/chili": groundPage("Chili", "/img/chili.jpg"),
},
map[string]photoFile{"/img/curry.jpg": jpegFile(t, 640, 480), "/img/chili.jpg": jpegFile(t, 480, 640)},
search)
env.llm.QueueGenerate(normalizedReply(), normalizedReply())
results := env.g.GroundMany(t.Context(), []GroundRequest{
{DishName: "Curry", Query: "curry"},
{DishName: "Nothing", Query: "nothing"},
{DishName: "Broken", Query: "broken"},
{DishName: "Chili", Query: "chili"},
})
if len(results) != 4 {
t.Fatalf("%d results, want 4", len(results))
}
if results[0].Err != nil || results[0].Grounded.SourceURL != "https://a.example/curry" {
t.Errorf("curry = %+v", results[0])
}
var ungroundable *ErrUngroundable
if !errors.As(results[1].Err, &ungroundable) {
t.Errorf("nothing error = %v, want *ErrUngroundable", results[1].Err)
}
if !errors.Is(results[2].Err, ErrSearchRefused) {
t.Errorf("broken error = %v, want ErrSearchRefused", results[2].Err)
}
if results[3].Err != nil || results[3].Grounded.SourceURL != "https://b.example/chili" {
t.Errorf("chili = %+v", results[3])
}
if n := countRows(t, env.st, "recipes"); n != 2 {
t.Errorf("%d recipes rows, want 2", n)
}
}
func TestGroundManyLimitsConcurrency(t *testing.T) {
t.Parallel()
search := &fakeSearch{started: make(chan string, 5), gate: make(chan struct{})}
env := newGroundEnv(t, nil, nil, search)
reqs := make([]GroundRequest, 5)
for i := range reqs {
reqs[i] = GroundRequest{DishName: fmt.Sprintf("Dish %d", i), Query: fmt.Sprintf("dish %d", i)}
}
done := make(chan []GroundResult)
go func() { done <- env.g.GroundMany(t.Context(), reqs) }()
for range GroundConcurrency {
select {
case <-search.started:
case <-time.After(5 * time.Second):
t.Fatal("fewer than GroundConcurrency items started")
}
}
select {
case q := <-search.started:
t.Fatalf("%s started while %d were in flight", q, GroundConcurrency)
case <-time.After(100 * time.Millisecond):
}
close(search.gate)
results := <-done
for i, r := range results {
if !errors.Is(r.Err, ErrNoCandidates) {
t.Errorf("item %d error = %v, want ErrNoCandidates", i, r.Err)
}
}
if search.maxActive != GroundConcurrency || search.callCount() != 5 {
t.Errorf("max in flight %d of %d calls, want %d of 5", search.maxActive, search.callCount(), GroundConcurrency)
}
}
internal/recipes/photo.go
+269
View File
@@ -0,0 +1,269 @@
package recipes
import (
"bytes"
"context"
"crypto/sha256"
"database/sql"
"encoding/hex"
"errors"
"fmt"
"image"
"image/gif"
"image/jpeg"
"image/png"
"io/fs"
"math"
"os"
"path"
"path/filepath"
"slices"
"strconv"
"github.com/oklog/ulid/v2"
"golang.org/x/image/draw"
"golang.org/x/image/webp"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/db"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/store"
)
const (
// PhotoDir is where cached photos are stored, relative to the data
// directory.
PhotoDir = "media/photos"
// PhotoMinShortSide is the least a photo's shorter side may measure, in
// pixels.
PhotoMinShortSide = 300
// PhotoMaxPixels bounds the pixels a photo may decode to, so a small
// compressed file cannot claim gigabytes of memory.
PhotoMaxPixels = 40_000_000
// PhotoQuality is the JPEG quality of every stored size.
PhotoQuality = 82
)
// PhotoSizes are the long-side lengths, in pixels, each photo is stored at.
var PhotoSizes = []int{1200, 400}
var (
// ErrPhotoTooSmall is returned for an image whose shorter side is under
// PhotoMinShortSide.
ErrPhotoTooSmall = errors.New("recipes: photo is too small")
// ErrPhotoTooLarge is returned for an image over PhotoMaxPixels.
ErrPhotoTooLarge = errors.New("recipes: photo has too many pixels")
// ErrPhotoFormat is returned for bytes that are not a JPEG, PNG, WebP
// or GIF image.
ErrPhotoFormat = errors.New("recipes: photo is not a JPEG, PNG, WebP or GIF image")
)
// photoDecoders are the formats a photo may be in. A GIF decodes to its
// first frame.
var photoDecoders = map[string]func([]byte) (image.Image, error){
"jpeg": func(b []byte) (image.Image, error) { return jpeg.Decode(bytes.NewReader(b)) },
"png": func(b []byte) (image.Image, error) { return png.Decode(bytes.NewReader(b)) },
"gif": func(b []byte) (image.Image, error) { return gif.Decode(bytes.NewReader(b)) },
"webp": func(b []byte) (image.Image, error) { return webp.Decode(bytes.NewReader(b)) },
}
// PhotoCache downloads photos through a Fetcher and stores each at every
// PhotoSizes length as {id}-{size}.jpg under PhotoDir, with a photos row.
// Photos are deduplicated by the SHA-256 of the downloaded bytes. A
// PhotoCache is safe for concurrent use.
type PhotoCache struct {
fetcher *Fetcher
store *store.Store
// dir is the absolute photo directory.
dir string
}
// NewPhotoCache returns a PhotoCache storing photos under dataDir, creating
// the photo directory if it is absent.
func NewPhotoCache(fetcher *Fetcher, st *store.Store, dataDir string) (*PhotoCache, error) {
if fetcher == nil {
return nil, errors.New("recipes: nil fetcher")
}
if st == nil {
return nil, errors.New("recipes: nil store")
}
dir, err := filepath.Abs(filepath.Join(dataDir, filepath.FromSlash(PhotoDir)))
if err != nil {
return nil, fmt.Errorf("recipes: resolve photo directory: %w", err)
}
if err := os.MkdirAll(dir, 0o750); err != nil {
return nil, fmt.Errorf("recipes: create photo directory: %w", err)
}
return &PhotoCache{fetcher: fetcher, store: st, dir: dir}, nil
}
// DownloadedPhoto is an image DownloadPhoto has downloaded and decoded,
// ready for CachePhoto.
type DownloadedPhoto struct {
// url is the image's final URL, after redirects.
url string
sha256 string
img image.Image
}
// DownloadPhoto downloads the image at rawURL and decodes it, refusing one
// whose shorter side is under PhotoMinShortSide. It records nothing, so a
// caller can try another image before opening a transaction.
func (c *PhotoCache) DownloadPhoto(ctx context.Context, rawURL string) (DownloadedPhoto, error) {
page, err := c.fetcher.FetchImage(ctx, rawURL)
if err != nil {
return DownloadedPhoto{}, err
}
img, err := decodePhoto(page.Body)
if err != nil {
return DownloadedPhoto{}, fmt.Errorf("%w: %s", err, page.URL)
}
digest := sha256.Sum256(page.Body)
return DownloadedPhoto{url: page.URL, sha256: hex.EncodeToString(digest[:]), img: img}, nil
}
// CachePhoto records p in tx, the caller's transaction, so its photos row
// commits or rolls back with the rows that refer to it. An image whose
// bytes are already recorded returns that row and created false. Otherwise
// p is scaled so its long side is each of PhotoSizes, encoded as JPEG and
// recorded, and created is true: if tx then does not commit, the caller
// removes the files with RemovePhotoFiles. The row's width and height are
// the original image's, and its local_path is the largest size's file
// relative to the data directory.
//
// The store's writer has one connection, so transactions run one at a
// time and an image is never recorded twice.
func (c *PhotoCache) CachePhoto(ctx context.Context, tx *sql.Tx, p DownloadedPhoto) (photo db.Photo, created bool, err error) {
q := db.New(tx)
existing, err := q.GetPhotoBySHA256(ctx, p.sha256)
if err == nil {
return existing, false, nil
}
if !errors.Is(err, sql.ErrNoRows) {
return db.Photo{}, false, fmt.Errorf("recipes: look up photo: %w", err)
}
id := store.NewID()
for _, size := range PhotoSizes {
if err := writeJPEG(c.path(id, size), scaleLongSide(p.img, size)); err != nil {
return db.Photo{}, false, errors.Join(err, c.RemovePhotoFiles(id))
}
}
now := store.Now().String()
bounds := p.img.Bounds()
photo, err = q.CreatePhoto(ctx, db.CreatePhotoParams{
ID: id,
SourceUrl: p.url,
LocalPath: path.Join(PhotoDir, photoFileName(id, PhotoSizes[0])),
Width: int64(bounds.Dx()),
Height: int64(bounds.Dy()),
Sha256: p.sha256,
FetchedAt: now,
CreatedAt: now,
UpdatedAt: now,
})
if err != nil {
return db.Photo{}, false, errors.Join(fmt.Errorf("recipes: record photo: %w", err), c.RemovePhotoFiles(id))
}
return photo, true, nil
}
// RemovePhotoFiles removes the files of photo id at every PhotoSizes
// length; a file already absent is not an error.
func (c *PhotoCache) RemovePhotoFiles(id string) error {
var errs []error
for _, size := range PhotoSizes {
if err := os.Remove(c.path(id, size)); err != nil && !errors.Is(err, fs.ErrNotExist) {
errs = append(errs, err)
}
}
return errors.Join(errs...)
}
// OpenPhoto opens the stored file of photo id at size. An id that is not a
// canonical ULID, a size not in PhotoSizes, an id with no photos row, or a
// photo whose file is not stored is an error matching fs.ErrNotExist.
func (c *PhotoCache) OpenPhoto(ctx context.Context, id string, size int) (*os.File, error) {
parsed, err := ulid.ParseStrict(id)
if err != nil || parsed.String() != id || !slices.Contains(PhotoSizes, size) {
return nil, fmt.Errorf("recipes: photo %q at size %d: %w", id, size, fs.ErrNotExist)
}
if _, err := db.New(c.store.Reader).GetPhoto(ctx, id); errors.Is(err, sql.ErrNoRows) {
return nil, fmt.Errorf("recipes: photo %q: %w", id, fs.ErrNotExist)
} else if err != nil {
return nil, fmt.Errorf("recipes: look up photo %q: %w", id, err)
}
return os.Open(c.path(id, size))
}
func (c *PhotoCache) path(id string, size int) string {
return filepath.Join(c.dir, photoFileName(id, size))
}
func photoFileName(id string, size int) string {
return id + "-" + strconv.Itoa(size) + ".jpg"
}
// decodePhoto decodes a JPEG, PNG, WebP or GIF image, checking its pixel
// count from the header before decoding the pixels, and the shorter side of
// the decoded image: a GIF's first frame may be smaller than its screen.
func decodePhoto(b []byte) (image.Image, error) {
cfg, format, err := image.DecodeConfig(bytes.NewReader(b))
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrPhotoFormat, err)
}
decode, ok := photoDecoders[format]
if !ok {
return nil, fmt.Errorf("%w: format %q", ErrPhotoFormat, format)
}
if int64(cfg.Width)*int64(cfg.Height) > PhotoMaxPixels {
return nil, fmt.Errorf("%w: %dx%d", ErrPhotoTooLarge, cfg.Width, cfg.Height)
}
img, err := decode(b)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrPhotoFormat, err)
}
if size := img.Bounds().Size(); min(size.X, size.Y) < PhotoMinShortSide {
return nil, fmt.Errorf("%w: %dx%d, shorter side under %d px", ErrPhotoTooSmall, size.X, size.Y, PhotoMinShortSide)
}
return img, nil
}
// scaleLongSide returns img scaled so its longer side is long pixels,
// keeping its aspect ratio. Transparent areas become white, since JPEG has
// no alpha.
func scaleLongSide(img image.Image, long int) *image.RGBA {
b := img.Bounds()
w, h := b.Dx(), b.Dy()
dw, dh := long, long
if w >= h {
dh = max(1, int(math.Round(float64(h)*float64(long)/float64(w))))
} else {
dw = max(1, int(math.Round(float64(w)*float64(long)/float64(h))))
}
dst := image.NewRGBA(image.Rect(0, 0, dw, dh))
draw.Draw(dst, dst.Bounds(), image.White, image.Point{}, draw.Src)
draw.CatmullRom.Scale(dst, dst.Bounds(), img, b, draw.Over, nil)
return dst
}
// writeJPEG encodes img at PhotoQuality into a temporary file beside path
// and renames it into place, so a reader never sees a partial file.
func writeJPEG(path string, img image.Image) error {
f, err := os.CreateTemp(filepath.Dir(path), ".photo-*.tmp")
if err != nil {
return fmt.Errorf("recipes: create photo file: %w", err)
}
tmp := f.Name()
err = jpeg.Encode(f, img, &jpeg.Options{Quality: PhotoQuality})
if err == nil {
err = f.Sync()
}
err = errors.Join(err, f.Close())
if err == nil {
err = os.Rename(tmp, path)
}
if err != nil {
return errors.Join(fmt.Errorf("recipes: write photo %s: %w", filepath.Base(path), err), os.Remove(tmp))
}
return nil
}
internal/recipes/photo_test.go
+495
View File
@@ -0,0 +1,495 @@
package recipes
import (
"bytes"
"context"
"database/sql"
"errors"
"image"
"image/color"
"image/gif"
"image/jpeg"
"image/png"
"io/fs"
"net/http"
"net/url"
"os"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/db"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/store"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/migrations"
)
// photoServer serves each path's body with its Content-Type and counts
// requests.
type photoServer struct {
files map[string]photoFile
requests atomic.Int32
}
type photoFile struct {
contentType string
body []byte
}
func (s *photoServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
s.requests.Add(1)
f, ok := s.files[r.URL.Path]
if !ok {
http.NotFound(w, r)
return
}
w.Header().Set("Content-Type", f.contentType)
w.Write(f.body)
}
// newTestPhotoCache returns a PhotoCache over files, a migrated temp
// database and a temp data directory.
func newTestPhotoCache(t *testing.T, files map[string]photoFile) (*PhotoCache, *store.Store, string, *photoServer) {
t.Helper()
srv := &photoServer{files: files}
env := newTestEnv(t, srv, publicAnswers())
installClock(env.f)
dataDir := t.TempDir()
st, err := store.Open(t.Context(), dataDir)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
if err := st.Close(); err != nil {
t.Error(err)
}
})
if err := store.Migrate(t.Context(), st.Writer, migrations.FS); err != nil {
t.Fatal(err)
}
cache, err := NewPhotoCache(env.f, st, dataDir)
if err != nil {
t.Fatal(err)
}
return cache, st, dataDir, srv
}
// cachePhoto downloads the image at rawURL and records it in its own
// transaction, as a caller of DownloadPhoto and CachePhoto does.
func cachePhoto(t *testing.T, cache *PhotoCache, rawURL string) (db.Photo, error) {
t.Helper()
p, err := cache.DownloadPhoto(t.Context(), rawURL)
if err != nil {
return db.Photo{}, err
}
var photo db.Photo
err = cache.store.WithTx(t.Context(), func(tx *sql.Tx) error {
photo, _, err = cache.CachePhoto(t.Context(), tx, p)
return err
})
return photo, err
}
// gradient is a w×h opaque image.
func gradient(w, h int) *image.RGBA {
img := image.NewRGBA(image.Rect(0, 0, w, h))
for y := range h {
for x := range w {
img.Set(x, y, color.RGBA{uint8(x * 255 / w), uint8(y * 255 / h), 128, 255})
}
}
return img
}
func encodeJPEG(t *testing.T, img image.Image) []byte {
t.Helper()
var b bytes.Buffer
if err := jpeg.Encode(&b, img, nil); err != nil {
t.Fatal(err)
}
return b.Bytes()
}
func encodePNG(t *testing.T, img image.Image) []byte {
t.Helper()
var b bytes.Buffer
if err := png.Encode(&b, img); err != nil {
t.Fatal(err)
}
return b.Bytes()
}
// twoFrameGIF is a w×h animated GIF whose first frame is red and second
// blue.
func twoFrameGIF(t *testing.T, w, h int) []byte {
t.Helper()
frame := func(c color.Color) *image.Paletted {
p := image.NewPaletted(image.Rect(0, 0, w, h), color.Palette{c})
return p
}
var b bytes.Buffer
anim := &gif.GIF{
Image: []*image.Paletted{frame(color.RGBA{255, 0, 0, 255}), frame(color.RGBA{0, 0, 255, 255})},
Delay: []int{10, 10},
}
if err := gif.EncodeAll(&b, anim); err != nil {
t.Fatal(err)
}
return b.Bytes()
}
func readTestdata(t *testing.T, name string) []byte {
t.Helper()
b, err := os.ReadFile(filepath.Join("testdata", name))
if err != nil {
t.Fatal(err)
}
return b
}
// storedSize decodes the stored JPEG of id at size and returns it.
func storedSize(t *testing.T, dataDir, id string, size int) image.Image {
t.Helper()
b, err := os.ReadFile(filepath.Join(dataDir, "media", "photos", photoFileName(id, size)))
if err != nil {
t.Fatalf("read stored photo: %v", err)
}
img, format, err := image.Decode(bytes.NewReader(b))
if err != nil || format != "jpeg" {
t.Fatalf("stored photo is %q, %v; want a JPEG", format, err)
}
return img
}
func TestCachePhotoFormatsAndSizes(t *testing.T) {
t.Parallel()
files := map[string]photoFile{
"/landscape.jpg": {"image/jpeg", encodeJPEG(t, gradient(800, 600))},
"/portrait.png": {"image/png", encodePNG(t, gradient(600, 900))},
"/photo.webp": {"image/webp", readTestdata(t, "photo.webp")},
"/anim.gif": {"image/gif; charset=binary", twoFrameGIF(t, 300, 450)},
"/square.png": {"image/png", encodePNG(t, gradient(1500, 1500))},
}
cache, _, dataDir, _ := newTestPhotoCache(t, files)
tests := []struct {
path string
width, height int64
// sizes maps each stored size to its expected dimensions.
sizes map[int]image.Point
}{
{"/landscape.jpg", 800, 600, map[int]image.Point{1200: {1200, 900}, 400: {400, 300}}},
{"/portrait.png", 600, 900, map[int]image.Point{1200: {800, 1200}, 400: {267, 400}}},
{"/photo.webp", 640, 480, map[int]image.Point{1200: {1200, 900}, 400: {400, 300}}},
{"/anim.gif", 300, 450, map[int]image.Point{1200: {800, 1200}, 400: {267, 400}}},
{"/square.png", 1500, 1500, map[int]image.Point{1200: {1200, 1200}, 400: {400, 400}}},
}
for _, tc := range tests {
photo, err := cachePhoto(t, cache, "https://example.com"+tc.path)
if err != nil {
t.Fatalf("CachePhoto %s: %v", tc.path, err)
}
if photo.Width != tc.width || photo.Height != tc.height {
t.Errorf("%s: row is %dx%d, want %dx%d", tc.path, photo.Width, photo.Height, tc.width, tc.height)
}
if photo.SourceUrl != "https://example.com"+tc.path || len(photo.Sha256) != 64 {
t.Errorf("%s: row = %+v", tc.path, photo)
}
if want := "media/photos/" + photo.ID + "-1200.jpg"; photo.LocalPath != want {
t.Errorf("%s: local_path = %q, want %q", tc.path, photo.LocalPath, want)
}
for size, want := range tc.sizes {
img := storedSize(t, dataDir, photo.ID, size)
if got := img.Bounds().Size(); got != want {
t.Errorf("%s at %d: stored %v, want %v", tc.path, size, got, want)
}
}
}
}
func TestCachePhotoGIFUsesFirstFrame(t *testing.T) {
t.Parallel()
cache, _, dataDir, _ := newTestPhotoCache(t, map[string]photoFile{
"/anim.gif": {"image/gif", twoFrameGIF(t, 400, 400)},
})
photo, err := cachePhoto(t, cache, "https://example.com/anim.gif")
if err != nil {
t.Fatalf("CachePhoto: %v", err)
}
r, g, b, _ := storedSize(t, dataDir, photo.ID, 400).At(200, 200).RGBA()
if r>>8 < 200 || g>>8 > 50 || b>>8 > 50 {
t.Errorf("centre pixel = (%d,%d,%d), want the first frame's red", r>>8, g>>8, b>>8)
}
}
func TestCachePhotoRejectsSmallGIFFirstFrame(t *testing.T) {
t.Parallel()
// A 400x400 screen whose first frame is 1x1: the frame is what would be
// stored, so it is what must measure PhotoMinShortSide.
var b bytes.Buffer
anim := &gif.GIF{
Image: []*image.Paletted{image.NewPaletted(image.Rect(0, 0, 1, 1), color.Palette{color.Black})},
Delay: []int{10},
Config: image.Config{ColorModel: color.Palette{color.Black}, Width: 400, Height: 400},
}
if err := gif.EncodeAll(&b, anim); err != nil {
t.Fatal(err)
}
cache, st, dataDir, _ := newTestPhotoCache(t, map[string]photoFile{
"/speck.gif": {"image/gif", b.Bytes()},
})
if _, err := cachePhoto(t, cache, "https://example.com/speck.gif"); !errors.Is(err, ErrPhotoTooSmall) {
t.Errorf("CachePhoto error = %v, want ErrPhotoTooSmall", err)
}
if n := countPhotos(t, st); n != 0 {
t.Errorf("%d photos recorded for a refused image", n)
}
if names := photoFiles(t, dataDir); len(names) != 0 {
t.Errorf("files stored for a refused image: %v", names)
}
}
func TestCachePhotoFillsTransparencyWhite(t *testing.T) {
t.Parallel()
cache, _, dataDir, _ := newTestPhotoCache(t, map[string]photoFile{
"/clear.png": {"image/png", encodePNG(t, image.NewNRGBA(image.Rect(0, 0, 400, 400)))},
})
photo, err := cachePhoto(t, cache, "https://example.com/clear.png")
if err != nil {
t.Fatalf("CachePhoto: %v", err)
}
r, g, b, _ := storedSize(t, dataDir, photo.ID, 400).At(200, 200).RGBA()
if r>>8 < 250 || g>>8 < 250 || b>>8 < 250 {
t.Errorf("transparent pixel stored as (%d,%d,%d), want white", r>>8, g>>8, b>>8)
}
}
func countPhotos(t *testing.T, st *store.Store) int {
t.Helper()
var n int
if err := st.Reader.QueryRowContext(context.Background(), "SELECT count(*) FROM photos").Scan(&n); err != nil {
t.Fatal(err)
}
return n
}
func photoFiles(t *testing.T, dataDir string) []string {
t.Helper()
entries, err := os.ReadDir(filepath.Join(dataDir, "media", "photos"))
if err != nil {
t.Fatal(err)
}
var names []string
for _, e := range entries {
names = append(names, e.Name())
}
return names
}
func TestCachePhotoRejectsTooSmall(t *testing.T) {
t.Parallel()
cache, st, dataDir, _ := newTestPhotoCache(t, map[string]photoFile{
"/narrow.png": {"image/png", encodePNG(t, gradient(299, 1000))},
"/short.jpg": {"image/jpeg", encodeJPEG(t, gradient(1000, 299))},
"/edge.png": {"image/png", encodePNG(t, gradient(300, 1000))},
})
for _, p := range []string{"/narrow.png", "/short.jpg"} {
if _, err := cachePhoto(t, cache, "https://example.com"+p); !errors.Is(err, ErrPhotoTooSmall) {
t.Errorf("CachePhoto %s error = %v, want ErrPhotoTooSmall", p, err)
}
}
if n := countPhotos(t, st); n != 0 {
t.Errorf("%d photos recorded for refused images", n)
}
if names := photoFiles(t, dataDir); len(names) != 0 {
t.Errorf("files stored for refused images: %v", names)
}
if _, err := cachePhoto(t, cache, "https://example.com/edge.png"); err != nil {
t.Errorf("CachePhoto with a 300 px short side: %v", err)
}
}
func TestCachePhotoRejectsNonImages(t *testing.T) {
t.Parallel()
cache, st, _, _ := newTestPhotoCache(t, map[string]photoFile{
"/page.html": {"text/html", encodePNG(t, gradient(400, 400))},
"/fake.jpg": {"image/jpeg", []byte("<html>not a picture</html>")},
})
if _, err := cachePhoto(t, cache, "https://example.com/page.html"); !errors.Is(err, ErrNotImage) {
t.Errorf("text/html error = %v, want ErrNotImage", err)
}
if _, err := cachePhoto(t, cache, "https://example.com/fake.jpg"); !errors.Is(err, ErrPhotoFormat) {
t.Errorf("undecodable image error = %v, want ErrPhotoFormat", err)
}
if n := countPhotos(t, st); n != 0 {
t.Errorf("%d photos recorded", n)
}
}
func TestCachePhotoDeduplicatesBySHA256(t *testing.T) {
t.Parallel()
body := encodeJPEG(t, gradient(640, 480))
cache, st, dataDir, srv := newTestPhotoCache(t, map[string]photoFile{
"/a.jpg": {"image/jpeg", body},
"/b.jpg": {"image/jpeg", body},
"/other.jpg": {"image/jpeg", encodeJPEG(t, gradient(480, 640))},
})
first, err := cachePhoto(t, cache, "https://example.com/a.jpg")
if err != nil {
t.Fatalf("CachePhoto a: %v", err)
}
again, err := cachePhoto(t, cache, "https://example.com/b.jpg")
if err != nil {
t.Fatalf("CachePhoto b: %v", err)
}
if again != first {
t.Errorf("same bytes at another URL = %+v, want the first row %+v", again, first)
}
if n := srv.requests.Load(); n != 2 {
t.Errorf("%d downloads, want 2: each URL is fetched to hash it", n)
}
other, err := cachePhoto(t, cache, "https://example.com/other.jpg")
if err != nil {
t.Fatalf("CachePhoto other: %v", err)
}
if other.ID == first.ID || other.Sha256 == first.Sha256 {
t.Errorf("different bytes share a photo: %+v and %+v", first, other)
}
if n := countPhotos(t, st); n != 2 {
t.Errorf("%d photos recorded, want 2", n)
}
if names := photoFiles(t, dataDir); len(names) != 4 {
t.Errorf("stored files %v, want two sizes of two photos", names)
}
}
func TestOpenPhoto(t *testing.T) {
t.Parallel()
cache, _, dataDir, _ := newTestPhotoCache(t, map[string]photoFile{
"/a.png": {"image/png", encodePNG(t, gradient(400, 300))},
})
photo, err := cachePhoto(t, cache, "https://example.com/a.png")
if err != nil {
t.Fatalf("CachePhoto: %v", err)
}
for _, size := range PhotoSizes {
f, err := cache.OpenPhoto(t.Context(), photo.ID, size)
if err != nil {
t.Fatalf("OpenPhoto %d: %v", size, err)
}
f.Close()
}
// A file with no photos row is not a stored photo.
orphan := store.NewID()
if err := os.WriteFile(filepath.Join(dataDir, "media", "photos", orphan+"-400.jpg"), []byte("jpeg"), 0o600); err != nil {
t.Fatal(err)
}
for _, tc := range []struct {
id string
size int
}{
{photo.ID, 800},
{store.NewID(), 400},
{orphan, 400},
{strings.ToLower(photo.ID), 400},
{"../" + photo.ID, 400},
{"", 400},
} {
if _, err := cache.OpenPhoto(t.Context(), tc.id, tc.size); !errors.Is(err, fs.ErrNotExist) {
t.Errorf("OpenPhoto(%q, %d) error = %v, want fs.ErrNotExist", tc.id, tc.size, err)
}
}
}
func TestFetchImageSizeCap(t *testing.T) {
t.Parallel()
env := newTestEnv(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
size := MaxImageBytes
if r.URL.Path != "/exact" {
size++
}
w.Header().Set("Content-Type", "image/jpeg")
w.Write(bytes.Repeat([]byte("a"), size))
}), publicAnswers())
installClock(env.f)
page, err := env.f.FetchImage(t.Context(), "https://example.com/exact")
if err != nil {
t.Fatalf("FetchImage at the cap: %v", err)
}
if len(page.Body) != MaxImageBytes {
t.Errorf("body is %d bytes, want %d", len(page.Body), MaxImageBytes)
}
if _, err := env.f.FetchImage(t.Context(), "https://example.com/over"); !errors.Is(err, ErrTooLarge) {
t.Errorf("FetchImage over the cap error = %v, want ErrTooLarge", err)
}
}
func TestFetchImageContentTypes(t *testing.T) {
t.Parallel()
var accept atomic.Value
env := newTestEnv(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
accept.Store(r.Header.Get("Accept"))
w.Header().Set("Content-Type", r.URL.Query().Get("type"))
w.Write([]byte("x"))
}), publicAnswers())
installClock(env.f)
typeURL := func(ct string) string { return "https://example.com/?type=" + url.QueryEscape(ct) }
for _, ct := range []string{"image/jpeg", "image/png", "image/webp", "image/gif", "IMAGE/PNG; q=1"} {
if _, err := env.f.FetchImage(t.Context(), typeURL(ct)); err != nil {
t.Errorf("FetchImage %q: %v", ct, err)
}
}
if got := accept.Load(); got != "image/jpeg,image/png,image/webp,image/gif" {
t.Errorf("Accept = %q", got)
}
for _, ct := range []string{"text/html", "image/svg+xml", "image/avif", "application/octet-stream", ""} {
if _, err := env.f.FetchImage(t.Context(), typeURL(ct)); !errors.Is(err, ErrNotImage) {
t.Errorf("FetchImage %q error = %v, want ErrNotImage", ct, err)
}
}
// Pages still fetch whatever the Content-Type.
if _, err := env.f.Fetch(t.Context(), typeURL("text/plain")); err != nil {
t.Errorf("Fetch: %v", err)
}
}
func TestCachePhotoRollsBackWithTransaction(t *testing.T) {
t.Parallel()
cache, st, dataDir, _ := newTestPhotoCache(t, map[string]photoFile{
"/a.jpg": {"image/jpeg", encodeJPEG(t, gradient(640, 480))},
})
p, err := cache.DownloadPhoto(t.Context(), "https://example.com/a.jpg")
if err != nil {
t.Fatal(err)
}
errAbort := errors.New("abort")
var photo db.Photo
var created bool
err = st.WithTx(t.Context(), func(tx *sql.Tx) error {
photo, created, err = cache.CachePhoto(t.Context(), tx, p)
if err != nil {
return err
}
return errAbort
})
if !errors.Is(err, errAbort) || !created {
t.Fatalf("WithTx error = %v, created = %v; want the abort after a new photo", err, created)
}
if n := countPhotos(t, st); n != 0 {
t.Errorf("%d photos rows after rollback", n)
}
if names := photoFiles(t, dataDir); len(names) != len(PhotoSizes) {
t.Errorf("files before removal = %v, want one per size", names)
}
if err := cache.RemovePhotoFiles(photo.ID); err != nil {
t.Fatal(err)
}
if names := photoFiles(t, dataDir); len(names) != 0 {
t.Errorf("files after RemovePhotoFiles = %v", names)
}
if err := cache.RemovePhotoFiles(photo.ID); err != nil {
t.Errorf("RemovePhotoFiles of absent files: %v", err)
}
}
internal/recipes/rank.go
+155
View File
@@ -0,0 +1,155 @@
package recipes
import (
"context"
"errors"
"fmt"
"sort"
"sync"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/harness"
)
// RankDepth is how many of the top search candidates Rank fetches and
// extracts.
const RankDepth = 5
var (
// ErrNoIngredients rejects a candidate whose recipe lists no
// ingredients.
ErrNoIngredients = errors.New("recipes: recipe has no ingredients")
// ErrNoImage rejects a candidate whose recipe has no image.
ErrNoImage = errors.New("recipes: recipe has no image")
// ErrNoUsableRecipe is returned when Rank ranks no candidate; it is
// joined with each candidate's reason.
ErrNoUsableRecipe = errors.New("recipes: no candidate holds a usable recipe")
// ErrCandidatesUnranked is returned, alongside the ranked candidates,
// when Rank ranks some candidates but not all; it is joined with each
// unranked candidate's reason.
ErrCandidatesUnranked = errors.New("recipes: some candidates were not ranked")
)
// PageFetcher fetches a candidate's page; *Fetcher satisfies it.
type PageFetcher interface {
Fetch(ctx context.Context, rawURL string) (Page, error)
}
// Ranked is a candidate whose page held a usable recipe, with its score.
type Ranked struct {
Candidate Candidate
Recipe Recipe
// Score is the token similarity of the recipe's title to the dish name
// (0 to 1, harness.Similarity) plus a quarter for each of ingredients,
// image, steps and total minutes present (0 to 1).
Score float64
}
// Rank fetches and extracts the first RankDepth candidates concurrently and
// returns those whose recipe is usable — it has ingredients and an image,
// and any total time it gives is positive — highest Score first; equal
// scores keep search order.
//
// Every candidate that is not ranked — its fetch failed, its page holds no
// valid recipe, or the recipe is not usable — is reported, never dropped
// silently: when some are ranked the error is
// ErrCandidatesUnranked, returned together with the ranked ones; when none
// are it is ErrNoUsableRecipe with no results. Either is joined with each
// unranked candidate's URL and reason. A cancelled ctx returns only its
// error.
func Rank(ctx context.Context, f PageFetcher, dish string, cands []Candidate) ([]Ranked, error) {
return rank(ctx, dish, cands, func(ctx context.Context, c Candidate) (Recipe, error) {
return fetchRecipe(ctx, f, c.URL)
})
}
// rank is Rank with each candidate's recipe got by load rather than always
// fetched.
func rank(ctx context.Context, dish string, cands []Candidate, load func(context.Context, Candidate) (Recipe, error)) ([]Ranked, error) {
if len(cands) > RankDepth {
cands = cands[:RankDepth]
}
type outcome struct {
ranked Ranked
err error
}
outcomes := make([]outcome, len(cands))
var wg sync.WaitGroup
for i, c := range cands {
wg.Add(1)
go func() {
defer wg.Done()
r, err := rankOne(ctx, load, dish, c)
outcomes[i] = outcome{r, err}
}()
}
wg.Wait()
if err := ctx.Err(); err != nil {
return nil, err
}
var ranked []Ranked
var reasons []error
for i, o := range outcomes {
if o.err != nil {
reasons = append(reasons, fmt.Errorf("%s: %w", cands[i].URL, o.err))
continue
}
ranked = append(ranked, o.ranked)
}
if len(ranked) == 0 {
return nil, errors.Join(append([]error{ErrNoUsableRecipe}, reasons...)...)
}
sort.SliceStable(ranked, func(a, b int) bool { return ranked[a].Score > ranked[b].Score })
if len(reasons) > 0 {
return ranked, errors.Join(append([]error{ErrCandidatesUnranked}, reasons...)...)
}
return ranked, nil
}
// rankOne loads and scores one candidate's recipe, or says why it is
// rejected.
func rankOne(ctx context.Context, load func(context.Context, Candidate) (Recipe, error), dish string, c Candidate) (Ranked, error) {
r, err := load(ctx, c)
if err != nil {
return Ranked{}, err
}
if err := usable(r); err != nil {
return Ranked{}, err
}
return Ranked{Candidate: c, Recipe: r, Score: harness.Similarity(r.Title, dish) + completeness(r)}, nil
}
// fetchRecipe fetches the page at rawURL and extracts its recipe.
func fetchRecipe(ctx context.Context, f PageFetcher, rawURL string) (Recipe, error) {
page, err := f.Fetch(ctx, rawURL)
if err != nil {
return Recipe{}, err
}
return Extract(page.Body, page.URL)
}
// usable says why r cannot ground a dish: it has no ingredients or no
// image, or a total time that is not positive.
func usable(r Recipe) error {
switch {
case len(r.Ingredients) == 0:
return ErrNoIngredients
case r.ImageURL == "":
return ErrNoImage
case r.TotalMinutes != nil && *r.TotalMinutes <= 0:
return ErrTotalTimeNotPositive
}
return nil
}
// completeness is a quarter for each of ingredients, image, steps and total
// minutes the recipe has.
func completeness(r Recipe) float64 {
score := 0.0
for _, present := range []bool{len(r.Ingredients) > 0, r.ImageURL != "", len(r.Steps) > 0, r.TotalMinutes != nil} {
if present {
score += 0.25
}
}
return score
}
internal/recipes/rank_test.go
+255
View File
@@ -0,0 +1,255 @@
package recipes
import (
"context"
"errors"
"fmt"
"net/http"
"strings"
"sync"
"testing"
)
var _ PageFetcher = (*Fetcher)(nil)
// fakeFetcher serves fixture pages by URL and records what was fetched.
type fakeFetcher struct {
pages map[string][]byte
mu sync.Mutex
fetched []string
}
func (f *fakeFetcher) Fetch(_ context.Context, rawURL string) (Page, error) {
f.mu.Lock()
f.fetched = append(f.fetched, rawURL)
f.mu.Unlock()
body, ok := f.pages[rawURL]
if !ok {
return Page{}, &StatusError{URL: rawURL, Status: 404}
}
return Page{URL: rawURL, ContentType: "text/html", Body: body}, nil
}
// recipePage returns a page whose Recipe has the given name and, as asked,
// ingredients, an image, steps and a total time.
func recipePage(name string, ingredients, image, steps, minutes bool) []byte {
ld := fmt.Sprintf(`{"@context":"https://schema.org","@type":"Recipe","name":%q`, name)
if ingredients {
ld += `,"recipeIngredient":["1 onion","2 cloves garlic"]`
}
if image {
ld += `,"image":"https://img.example.com/dish.jpg"`
}
if steps {
ld += `,"recipeInstructions":["Chop.","Cook."]`
}
if minutes {
ld += `,"totalTime":"PT40M"`
}
return page("", ld+"}")
}
func TestRankOrder(t *testing.T) {
f := &fakeFetcher{pages: map[string][]byte{
"https://a.example/thai": recipePage("Thai Green Curry", true, true, true, true),
"https://b.example/noimg": recipePage("Chicken Curry", true, false, true, true),
"https://c.example/easy": recipePage("Easy Chicken Curry", true, true, true, false),
"https://d.example/noingr": recipePage("Chicken Curry", false, true, true, true),
"https://e.example/best": recipePage("Chicken Curry", true, true, true, true),
"https://f.example/toodeep": recipePage("Chicken Curry", true, true, true, true),
}}
cands := []Candidate{
{URL: "https://a.example/thai", Title: "Thai green curry"},
{URL: "https://b.example/noimg"},
{URL: "https://c.example/easy"},
{URL: "https://d.example/noingr"},
{URL: "https://e.example/best"},
{URL: "https://f.example/toodeep"},
}
got, err := Rank(context.Background(), f, "Chicken curry", cands)
if !errors.Is(err, ErrCandidatesUnranked) || !errors.Is(err, ErrNoImage) || !errors.Is(err, ErrNoIngredients) {
t.Errorf("err = %v, want ErrCandidatesUnranked with ErrNoImage and ErrNoIngredients", err)
}
if errors.Is(err, ErrNoUsableRecipe) {
t.Errorf("err = %v, want not ErrNoUsableRecipe when some are ranked", err)
}
want := []struct {
url string
score float64
}{
// Same words, every field: 1 + 1.
{"https://e.example/best", 2},
// "Easy" is a filler word, so the title matches; no total time.
{"https://c.example/easy", 1.75},
// Shares one of four words: 0.25 + 1.
{"https://a.example/thai", 1.25},
}
if len(got) != len(want) {
t.Fatalf("got %d ranked, want %d: %+v", len(got), len(want), got)
}
for i, w := range want {
if got[i].Candidate.URL != w.url || got[i].Score != w.score {
t.Errorf("rank %d = %s (%.2f), want %s (%.2f)", i, got[i].Candidate.URL, got[i].Score, w.url, w.score)
}
}
if got[2].Recipe.Title != "Thai Green Curry" || got[2].Candidate.Title != "Thai green curry" {
t.Errorf("ranked entry lost its recipe or candidate: %+v", got[2])
}
for _, u := range f.fetched {
if u == "https://f.example/toodeep" {
t.Errorf("fetched a candidate past RankDepth")
}
}
if len(f.fetched) != RankDepth {
t.Errorf("fetched %d pages, want %d", len(f.fetched), RankDepth)
}
}
func TestRankTiesKeepSearchOrder(t *testing.T) {
f := &fakeFetcher{pages: map[string][]byte{
"https://a.example/1": recipePage("Beef Stew", true, true, true, true),
"https://b.example/2": recipePage("Beef Stew", true, true, true, true),
"https://c.example/3": recipePage("Beef Stew", true, true, true, true),
}}
cands := []Candidate{{URL: "https://a.example/1"}, {URL: "https://b.example/2"}, {URL: "https://c.example/3"}}
got, err := Rank(context.Background(), f, "beef stew", cands)
if err != nil {
t.Fatal(err)
}
for i, c := range cands {
if got[i].Candidate.URL != c.URL {
t.Errorf("rank %d = %s, want %s", i, got[i].Candidate.URL, c.URL)
}
}
}
// TestRankReportsFetchFailure checks a failed fetch is reported with the
// candidates that were ranked, never swallowed.
func TestRankReportsFetchFailure(t *testing.T) {
f := &fakeFetcher{pages: map[string][]byte{
"https://a.example/1": recipePage("Beef Stew", true, true, true, true),
}}
cands := []Candidate{{URL: "https://a.example/1"}, {URL: "https://b.example/missing"}}
got, err := Rank(context.Background(), f, "beef stew", cands)
if len(got) != 1 || got[0].Candidate.URL != "https://a.example/1" {
t.Errorf("ranked = %+v, want only https://a.example/1", got)
}
if !errors.Is(err, ErrCandidatesUnranked) {
t.Errorf("err = %v, want ErrCandidatesUnranked", err)
}
var se *StatusError
if !errors.As(err, &se) || se.Status != 404 || se.URL != "https://b.example/missing" {
t.Errorf("err = %v, want the 404 StatusError for https://b.example/missing", err)
}
if err != nil && !strings.Contains(err.Error(), "https://b.example/missing") {
t.Errorf("err = %v, want it to name the failed candidate", err)
}
}
func TestRankAllRejected(t *testing.T) {
f := &fakeFetcher{pages: map[string][]byte{
"https://a.example/noimg": recipePage("Chicken Curry", true, false, true, true),
"https://b.example/noingr": recipePage("Chicken Curry", false, true, true, true),
"https://c.example/blog": page(""),
}}
cands := []Candidate{
{URL: "https://a.example/noimg"},
{URL: "https://b.example/noingr"},
{URL: "https://c.example/blog"},
{URL: "https://d.example/missing"},
}
_, err := Rank(context.Background(), f, "chicken curry", cands)
for _, want := range []error{ErrNoUsableRecipe, ErrNoImage, ErrNoIngredients, ErrNoRecipe} {
if !errors.Is(err, want) {
t.Errorf("err = %v, want it to wrap %v", err, want)
}
}
var se *StatusError
if !errors.As(err, &se) || se.Status != 404 {
t.Errorf("err = %v, want the 404 StatusError", err)
}
}
func TestRankNoCandidates(t *testing.T) {
if _, err := Rank(context.Background(), &fakeFetcher{}, "chicken curry", nil); !errors.Is(err, ErrNoUsableRecipe) {
t.Errorf("err = %v, want ErrNoUsableRecipe", err)
}
}
func TestRankContextCancelled(t *testing.T) {
f := &fakeFetcher{pages: map[string][]byte{"https://a.example/1": recipePage("Beef Stew", true, true, true, true)}}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := Rank(ctx, f, "beef stew", []Candidate{{URL: "https://a.example/1"}}); !errors.Is(err, context.Canceled) {
t.Errorf("err = %v, want context.Canceled", err)
}
}
// TestRankOverHTTP ranks fixture candidates served over HTTPS through the
// SSRF-safe Fetcher: every host, a subdomain the test server's certificate
// covers, resolves to a public test address the test
// dialer routes to one httptest server, which serves pages by host and path.
func TestRankOverHTTP(t *testing.T) {
t.Parallel()
pages := map[string][]byte{
"best.example.com/curry": recipePage("Chicken Curry", true, true, true, true),
"easy.example.com/curry": recipePage("Easy Chicken Curry", true, true, true, false),
"thai.example.com/curry": recipePage("Thai Green Curry", true, true, true, true),
"noimg.example.com/curry": recipePage("Chicken Curry", true, false, true, true),
}
env := newTestEnv(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, ok := pages[r.Host+r.URL.Path]
if !ok {
http.Error(w, "broken", http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "text/html")
w.Write(body)
}), map[string][][]string{
"best.example.com": {{publicIP}},
"easy.example.com": {{publicIP}},
"thai.example.com": {{publicIP}},
"noimg.example.com": {{publicIP}},
"broken.example.com": {{publicIP}},
})
cands := []Candidate{
{URL: "https://thai.example.com/curry"},
{URL: "https://broken.example.com/curry"},
{URL: "https://easy.example.com/curry"},
{URL: "https://noimg.example.com/curry"},
{URL: "https://best.example.com/curry"},
}
got, err := Rank(t.Context(), env.f, "Chicken curry", cands)
want := []struct {
url string
score float64
}{
{"https://best.example.com/curry", 2},
{"https://easy.example.com/curry", 1.75},
{"https://thai.example.com/curry", 1.25},
}
if len(got) != len(want) {
t.Fatalf("got %d ranked, want %d: %+v (err %v)", len(got), len(want), got, err)
}
for i, w := range want {
if got[i].Candidate.URL != w.url || got[i].Score != w.score {
t.Errorf("rank %d = %s (%.2f), want %s (%.2f)", i, got[i].Candidate.URL, got[i].Score, w.url, w.score)
}
}
if got[0].Recipe.Title != "Chicken Curry" || got[0].Recipe.ImageURL != "https://img.example.com/dish.jpg" {
t.Errorf("top recipe = %+v", got[0].Recipe)
}
if !errors.Is(err, ErrCandidatesUnranked) || !errors.Is(err, ErrNoImage) {
t.Errorf("err = %v, want ErrCandidatesUnranked with ErrNoImage", err)
}
var se *StatusError
if !errors.As(err, &se) || se.Status != http.StatusInternalServerError {
t.Errorf("err = %v, want the 500 StatusError for broken.example.com", err)
}
if n := len(env.dials()); n != len(cands) {
t.Errorf("dialed %d times, want %d", n, len(cands))
}
}
internal/recipes/search.go
+314
View File
@@ -0,0 +1,314 @@
package recipes
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/db"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/secretbox"
)
const (
// MaxSearchResults is the most candidates one search may ask for; it is
// Brave's largest page.
MaxSearchResults = 20
// SearchTimeout bounds one search request, reading the body included.
SearchTimeout = 10 * time.Second
// MaxSearchBodyBytes caps a search response body. A larger one is an
// error, never truncated.
MaxSearchBodyBytes = 1 << 20
// BraveEndpoint is the Brave Web Search API.
BraveEndpoint = "https://api.search.brave.com/res/v1/web/search"
)
var (
// ErrSearchRefused is returned for HTTP 401 or 403: Brave rejected the
// API key, or SearXNG does not allow format=json.
ErrSearchRefused = errors.New("recipes: search provider refused the request")
// ErrSearchRateLimited is returned for HTTP 429.
ErrSearchRateLimited = errors.New("recipes: search provider rate limit reached")
// ErrSearchResponse is returned for a response body that is not the
// provider's JSON shape — SearXNG without a results array, Brave without
// "type": "search" or with a web object lacking a results array — is
// over MaxSearchBodyBytes, or holds a result without an absolute http(s)
// URL.
ErrSearchResponse = errors.New("recipes: malformed search response")
)
// SearchStatusError is returned for any other non-2xx search response,
// redirects included: a search endpoint is called directly and never
// followed elsewhere.
type SearchStatusError struct {
Provider string
Status int
}
func (e *SearchStatusError) Error() string {
return fmt.Sprintf("recipes: %s search: HTTP %d", e.Provider, e.Status)
}
// Candidate is one search result: a page that may hold a recipe.
type Candidate struct {
URL string
Title string
Snippet string
}
// SearchProvider finds recipe pages on the web.
type SearchProvider interface {
// Search returns up to n candidates for query, best first. The query
// gets " recipe" appended unless it already has the word. n must be
// 1 to MaxSearchResults.
Search(ctx context.Context, query string, n int) ([]Candidate, error)
}
// NewSearchProvider returns the provider s names. SearXNG is called at
// s.BaseUrl; Brave is called with the API key in s.ApiKeyCt, decrypted by
// box.
func NewSearchProvider(s db.RecipeSearchSetting, box *secretbox.Box) (SearchProvider, error) {
switch s.Provider {
case "searxng":
return NewSearXNG(s.BaseUrl.String)
case "brave":
if len(s.ApiKeyCt) == 0 {
return nil, errors.New("recipes: Brave search has no API key")
}
key, err := box.Decrypt(s.ApiKeyCt)
if err != nil {
return nil, fmt.Errorf("recipes: Brave API key: %w", err)
}
return NewBrave(string(key))
default:
return nil, fmt.Errorf("recipes: unknown search provider %q", s.Provider)
}
}
// SearXNG searches through a SearXNG instance. Its base URL is a configured
// endpoint and may be on a private network, so it is called directly rather
// than through the public-only Fetcher.
type SearXNG struct {
endpoint *url.URL
client *http.Client
}
// NewSearXNG returns a SearXNG provider for the instance at baseURL, an
// absolute http:// or https:// URL without credentials, query or fragment.
func NewSearXNG(baseURL string) (*SearXNG, error) {
if baseURL == "" {
return nil, errors.New("recipes: SearXNG URL is not set")
}
u, err := url.Parse(baseURL)
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
return nil, errors.New("recipes: SearXNG URL must be an absolute http:// or https:// URL")
}
if u.User != nil || u.RawQuery != "" || u.Fragment != "" {
return nil, errors.New("recipes: SearXNG URL must not have credentials, a query or a fragment")
}
return &SearXNG{endpoint: u.JoinPath("search"), client: newSearchClient()}, nil
}
// Search implements SearchProvider with
// {base}/search?q=…&format=json&categories=general.
func (s *SearXNG) Search(ctx context.Context, query string, n int) ([]Candidate, error) {
q, err := searchQuery(query, n)
if err != nil {
return nil, err
}
u := *s.endpoint
u.RawQuery = url.Values{"q": {q}, "format": {"json"}, "categories": {"general"}}.Encode()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
if err != nil {
return nil, fmt.Errorf("recipes: build SearXNG request: %w", err)
}
// Results is a pointer so a body without the array is told apart from
// an empty search.
var body struct {
Results *[]struct {
URL string `json:"url"`
Title string `json:"title"`
Content string `json:"content"`
} `json:"results"`
}
if err := doSearch(s.client, req, "SearXNG", &body); err != nil {
return nil, err
}
if body.Results == nil {
return nil, fmt.Errorf("%w: SearXNG body has no results array", ErrSearchResponse)
}
var out []Candidate
for _, r := range *body.Results {
out = append(out, Candidate{URL: r.URL, Title: r.Title, Snippet: r.Content})
}
return candidates("SearXNG", out, n)
}
// Brave searches through the Brave Web Search API.
type Brave struct {
key string
endpoint string
client *http.Client
}
// NewBrave returns a Brave provider that authenticates with key, the
// decrypted API key.
func NewBrave(key string) (*Brave, error) {
if key == "" {
return nil, errors.New("recipes: Brave API key is empty")
}
for i := 0; i < len(key); i++ {
if key[i] < 0x21 || key[i] > 0x7e {
// The key is never echoed.
return nil, errors.New("recipes: Brave API key has a character not allowed in a header")
}
}
return &Brave{key: key, endpoint: BraveEndpoint, client: newSearchClient()}, nil
}
// Search implements SearchProvider with GET BraveEndpoint?q=…&count=n.
func (b *Brave) Search(ctx context.Context, query string, n int) ([]Candidate, error) {
q, err := searchQuery(query, n)
if err != nil {
return nil, err
}
u := b.endpoint + "?" + url.Values{"q": {q}, "count": {strconv.Itoa(n)}}.Encode()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
if err != nil {
return nil, fmt.Errorf("recipes: build Brave request: %w", err)
}
req.Header.Set("X-Subscription-Token", b.key)
// Brave leaves out "web" when nothing matched, so a search response is
// recognised by its "type"; a web object present must hold the results
// array.
var body struct {
Type string `json:"type"`
Web *struct {
Results *[]struct {
URL string `json:"url"`
Title string `json:"title"`
Description string `json:"description"`
} `json:"results"`
} `json:"web"`
}
if err := doSearch(b.client, req, "Brave", &body); err != nil {
return nil, err
}
if body.Type != "search" {
return nil, fmt.Errorf("%w: Brave body type is %q, not \"search\"", ErrSearchResponse, body.Type)
}
if body.Web == nil {
return nil, nil
}
if body.Web.Results == nil {
return nil, fmt.Errorf("%w: Brave web object has no results array", ErrSearchResponse)
}
var out []Candidate
for _, r := range *body.Web.Results {
out = append(out, Candidate{URL: r.URL, Title: r.Title, Snippet: r.Description})
}
return candidates("Brave", out, n)
}
func newSearchClient() *http.Client {
return &http.Client{
Timeout: SearchTimeout,
CheckRedirect: func(*http.Request, []*http.Request) error {
return http.ErrUseLastResponse
},
}
}
// searchQuery validates n and returns query with " recipe" appended unless
// one of its words already is "recipe", in any case.
func searchQuery(query string, n int) (string, error) {
if n < 1 || n > MaxSearchResults {
return "", fmt.Errorf("recipes: search result count %d is outside 1-%d", n, MaxSearchResults)
}
q := strings.TrimSpace(query)
if q == "" {
return "", errors.New("recipes: search query is empty")
}
for _, w := range strings.Fields(q) {
if strings.EqualFold(w, "recipe") {
return q, nil
}
}
return q + " recipe", nil
}
// doSearch sends req and decodes a 2xx JSON body into v, mapping failures to
// the search errors.
func doSearch(client *http.Client, req *http.Request, provider string, v any) error {
req.Header.Set("Accept", "application/json")
req.Header.Set("User-Agent", UserAgent)
resp, err := client.Do(req)
if err != nil {
if ctxErr := req.Context().Err(); ctxErr != nil {
return ctxErr
}
return fmt.Errorf("recipes: %s search: %w", provider, err)
}
defer resp.Body.Close()
switch {
case resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden:
return fmt.Errorf("%w: %s HTTP %d", ErrSearchRefused, provider, resp.StatusCode)
case resp.StatusCode == http.StatusTooManyRequests:
return fmt.Errorf("%w: %s", ErrSearchRateLimited, provider)
case resp.StatusCode < 200 || resp.StatusCode > 299:
return &SearchStatusError{Provider: provider, Status: resp.StatusCode}
}
data, err := io.ReadAll(io.LimitReader(resp.Body, MaxSearchBodyBytes+1))
if err != nil {
return fmt.Errorf("recipes: %s search: read body: %w", provider, err)
}
if len(data) > MaxSearchBodyBytes {
return fmt.Errorf("%w: %s body is over %d bytes", ErrSearchResponse, provider, MaxSearchBodyBytes)
}
if err := json.Unmarshal(data, v); err != nil {
return fmt.Errorf("%w: %s: %v", ErrSearchResponse, provider, err)
}
return nil
}
// candidates cleans results — HTML tags and entities, which both providers
// leave in titles and snippets, are removed — checks every URL is absolute
// http(s) and keeps the first n.
func candidates(provider string, results []Candidate, n int) ([]Candidate, error) {
if len(results) > n {
results = results[:n]
}
out := make([]Candidate, 0, len(results))
for i, r := range results {
u, err := url.Parse(r.URL)
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
return nil, fmt.Errorf("%w: %s result %d has no absolute http(s) URL", ErrSearchResponse, provider, i)
}
out = append(out, Candidate{URL: r.URL, Title: stripTags(r.Title), Snippet: stripTags(r.Snippet)})
}
return out, nil
}
// stripTags drops HTML tags such as Brave's <strong> highlights, then
// decodes entities and trims space.
func stripTags(s string) string {
var b strings.Builder
inTag := false
for _, r := range s {
switch {
case r == '<':
inTag = true
case r == '>' && inTag:
inTag = false
case !inTag:
b.WriteRune(r)
}
}
return cleanText(b.String())
}
internal/recipes/search_test.go
+292
View File
@@ -0,0 +1,292 @@
package recipes
import (
"context"
"crypto/rand"
"database/sql"
"encoding/base64"
"errors"
"net/http"
"net/http/httptest"
"reflect"
"strings"
"testing"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/db"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/secretbox"
)
const searxngBody = `{
"query": "chicken curry recipe",
"results": [
{"url": "https://www.bbcgoodfood.com/recipes/easy-chicken-curry", "title": "Easy chicken curry recipe | BBC Good Food", "content": "Make our easy chicken curry &amp; rice.", "engine": "duckduckgo"},
{"url": "https://www.recipetineats.com/chicken-curry/", "title": "Chicken Curry", "content": "A proper curry."},
{"url": "https://example.com/third", "title": "Third", "content": ""}
],
"number_of_results": 0
}`
const braveBody = `{
"type": "search",
"web": {
"type": "search",
"results": [
{"url": "https://www.bbcgoodfood.com/recipes/easy-chicken-curry", "title": "Easy <strong>chicken curry</strong> recipe", "description": "Make our <strong>easy</strong> chicken curry &amp; rice.", "language": "en"},
{"url": "https://www.recipetineats.com/chicken-curry/", "title": "Chicken Curry", "description": "A proper curry."}
]
}
}`
func newSearXNGServer(t *testing.T, status int, body string, got *http.Request) *SearXNG {
t.Helper()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if got != nil {
*got = *r.Clone(context.Background())
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
w.Write([]byte(body))
}))
t.Cleanup(srv.Close)
s, err := NewSearXNG(srv.URL + "/searx")
if err != nil {
t.Fatal(err)
}
return s
}
func newBraveServer(t *testing.T, key string, status int, body string, got *http.Request) *Brave {
t.Helper()
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if got != nil {
*got = *r.Clone(context.Background())
}
w.WriteHeader(status)
w.Write([]byte(body))
}))
t.Cleanup(srv.Close)
b, err := NewBrave(key)
if err != nil {
t.Fatal(err)
}
b.endpoint = srv.URL + "/res/v1/web/search"
return b
}
func TestSearXNGSearch(t *testing.T) {
var req http.Request
s := newSearXNGServer(t, http.StatusOK, searxngBody, &req)
got, err := s.Search(context.Background(), "chicken curry", 2)
if err != nil {
t.Fatal(err)
}
if req.URL.Path != "/searx/search" {
t.Errorf("path = %q, want /searx/search", req.URL.Path)
}
q := req.URL.Query()
if q.Get("q") != "chicken curry recipe" || q.Get("format") != "json" || q.Get("categories") != "general" {
t.Errorf("query = %v", q)
}
if req.Header.Get("User-Agent") != UserAgent {
t.Errorf("User-Agent = %q", req.Header.Get("User-Agent"))
}
want := []Candidate{
{URL: "https://www.bbcgoodfood.com/recipes/easy-chicken-curry", Title: "Easy chicken curry recipe | BBC Good Food", Snippet: "Make our easy chicken curry & rice."},
{URL: "https://www.recipetineats.com/chicken-curry/", Title: "Chicken Curry", Snippet: "A proper curry."},
}
if !reflect.DeepEqual(got, want) {
t.Errorf("candidates =\n%#v\nwant\n%#v", got, want)
}
}
func TestBraveSearch(t *testing.T) {
var req http.Request
b := newBraveServer(t, "brave-key-123", http.StatusOK, braveBody, &req)
got, err := b.Search(context.Background(), "Chicken curry RECIPE", 5)
if err != nil {
t.Fatal(err)
}
if req.Header.Get("X-Subscription-Token") != "brave-key-123" {
t.Errorf("X-Subscription-Token = %q", req.Header.Get("X-Subscription-Token"))
}
if req.Header.Get("Accept") != "application/json" {
t.Errorf("Accept = %q", req.Header.Get("Accept"))
}
q := req.URL.Query()
if q.Get("q") != "Chicken curry RECIPE" || q.Get("count") != "5" {
t.Errorf("query = %v", q)
}
want := []Candidate{
{URL: "https://www.bbcgoodfood.com/recipes/easy-chicken-curry", Title: "Easy chicken curry recipe", Snippet: "Make our easy chicken curry & rice."},
{URL: "https://www.recipetineats.com/chicken-curry/", Title: "Chicken Curry", Snippet: "A proper curry."},
}
if !reflect.DeepEqual(got, want) {
t.Errorf("candidates =\n%#v\nwant\n%#v", got, want)
}
}
func TestSearchQuery(t *testing.T) {
for _, tc := range []struct{ in, want string }{
{"chicken curry", "chicken curry recipe"},
{" chicken curry ", "chicken curry recipe"},
{"chicken curry recipe", "chicken curry recipe"},
{"Recipe for chicken curry", "Recipe for chicken curry"},
{"recipes for curry", "recipes for curry recipe"},
} {
got, err := searchQuery(tc.in, 3)
if err != nil || got != tc.want {
t.Errorf("searchQuery(%q) = %q, %v; want %q", tc.in, got, err, tc.want)
}
}
for _, tc := range []struct {
q string
n int
}{{"", 3}, {" ", 3}, {"curry", 0}, {"curry", MaxSearchResults + 1}} {
if _, err := searchQuery(tc.q, tc.n); err == nil {
t.Errorf("searchQuery(%q, %d) succeeded, want error", tc.q, tc.n)
}
}
}
func TestSearchErrors(t *testing.T) {
tests := []struct {
name string
status int
body string
check func(error) bool
}{
{"unauthorized", http.StatusUnauthorized, `{}`, func(err error) bool { return errors.Is(err, ErrSearchRefused) }},
{"forbidden", http.StatusForbidden, `{}`, func(err error) bool { return errors.Is(err, ErrSearchRefused) }},
{"rate limited", http.StatusTooManyRequests, `{}`, func(err error) bool { return errors.Is(err, ErrSearchRateLimited) }},
{"server error", http.StatusBadGateway, `{}`, func(err error) bool {
var se *SearchStatusError
return errors.As(err, &se) && se.Status == http.StatusBadGateway
}},
{"redirect not followed", http.StatusFound, ``, func(err error) bool {
var se *SearchStatusError
return errors.As(err, &se) && se.Status == http.StatusFound
}},
{"not JSON", http.StatusOK, `<html>`, func(err error) bool { return errors.Is(err, ErrSearchResponse) }},
{"result without URL", http.StatusOK, `{"type":"search","results":[{"title":"x"}],"web":{"results":[{"title":"x"}]}}`, func(err error) bool { return errors.Is(err, ErrSearchResponse) }},
{"relative URL", http.StatusOK, `{"type":"search","results":[{"url":"/x"}],"web":{"results":[{"url":"/x"}]}}`, func(err error) bool { return errors.Is(err, ErrSearchResponse) }},
{"empty object", http.StatusOK, `{}`, func(err error) bool { return errors.Is(err, ErrSearchResponse) }},
{"null results", http.StatusOK, `{"type":"search","results":null,"web":{"results":null}}`, func(err error) bool { return errors.Is(err, ErrSearchResponse) }},
{"results not an array", http.StatusOK, `{"type":"search","results":{},"web":{"results":{}}}`, func(err error) bool { return errors.Is(err, ErrSearchResponse) }},
{"body too large", http.StatusOK, `{"results":[],"pad":"` + strings.Repeat("x", MaxSearchBodyBytes) + `"}`, func(err error) bool { return errors.Is(err, ErrSearchResponse) }},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
providers := map[string]SearchProvider{
"SearXNG": newSearXNGServer(t, tc.status, tc.body, nil),
"Brave": newBraveServer(t, "secret-key", tc.status, tc.body, nil),
}
for name, p := range providers {
_, err := p.Search(context.Background(), "curry", 3)
if err == nil || !tc.check(err) {
t.Errorf("%s: err = %v", name, err)
}
if err != nil && strings.Contains(err.Error(), "secret-key") {
t.Errorf("%s: error leaks the API key: %v", name, err)
}
}
})
}
}
// TestSearchNoMatches checks a well-formed response with nothing found is an
// empty search, not an error.
func TestSearchNoMatches(t *testing.T) {
providers := map[string]SearchProvider{
"SearXNG": newSearXNGServer(t, http.StatusOK, `{"query":"x recipe","results":[]}`, nil),
"Brave without web": newBraveServer(t, "secret-key", http.StatusOK, `{"type":"search","query":{"original":"x recipe"}}`, nil),
"Brave empty web": newBraveServer(t, "secret-key", http.StatusOK, `{"type":"search","web":{"type":"search","results":[]}}`, nil),
}
for name, p := range providers {
got, err := p.Search(context.Background(), "x", 3)
if err != nil || len(got) != 0 {
t.Errorf("%s: got %v, %v; want no candidates and no error", name, got, err)
}
}
}
func TestSearchContextCancelled(t *testing.T) {
s := newSearXNGServer(t, http.StatusOK, searxngBody, nil)
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := s.Search(ctx, "curry", 3); !errors.Is(err, context.Canceled) {
t.Errorf("err = %v, want context.Canceled", err)
}
}
func TestSearchUnreachable(t *testing.T) {
srv := httptest.NewServer(http.NotFoundHandler())
url := srv.URL
srv.Close()
s, err := NewSearXNG(url)
if err != nil {
t.Fatal(err)
}
_, err = s.Search(context.Background(), "curry", 3)
if err == nil || !strings.Contains(err.Error(), "SearXNG search") {
t.Errorf("err = %v", err)
}
}
func TestNewSearXNGRejects(t *testing.T) {
for _, u := range []string{"", "searxng:8080", "ftp://searxng", "http://", "http://u:p@searxng", "http://searxng?x=1", "http://searxng#f"} {
if _, err := NewSearXNG(u); err == nil {
t.Errorf("NewSearXNG(%q) succeeded, want error", u)
}
}
if _, err := NewSearXNG("http://searxng:8080"); err != nil {
t.Errorf("internal URL rejected: %v", err)
}
}
func TestNewBraveRejects(t *testing.T) {
for _, k := range []string{"", "with space", "line\nbreak"} {
if _, err := NewBrave(k); err == nil {
t.Errorf("NewBrave(%q) succeeded, want error", k)
}
}
}
func TestNewSearchProvider(t *testing.T) {
raw := make([]byte, secretbox.KeySize)
rand.Read(raw)
box, err := secretbox.New(base64.StdEncoding.EncodeToString(raw))
if err != nil {
t.Fatal(err)
}
ct, err := box.Encrypt([]byte("brave-key"))
if err != nil {
t.Fatal(err)
}
p, err := NewSearchProvider(db.RecipeSearchSetting{Provider: "brave", ApiKeyCt: ct}, box)
if err != nil {
t.Fatal(err)
}
if b, ok := p.(*Brave); !ok || b.key != "brave-key" || b.endpoint != BraveEndpoint {
t.Errorf("provider = %#v, want Brave with decrypted key", p)
}
p, err = NewSearchProvider(db.RecipeSearchSetting{Provider: "searxng", BaseUrl: sql.NullString{String: "http://searxng:8080", Valid: true}}, box)
if err != nil {
t.Fatal(err)
}
if s, ok := p.(*SearXNG); !ok || s.endpoint.String() != "http://searxng:8080/search" {
t.Errorf("provider = %#v, want SearXNG at http://searxng:8080/search", p)
}
for name, s := range map[string]db.RecipeSearchSetting{
"brave without key": {Provider: "brave"},
"brave bad key": {Provider: "brave", ApiKeyCt: []byte("not a token at all, far too plain")},
"searxng without URL": {Provider: "searxng"},
"unknown provider": {Provider: "google"},
} {
if _, err := NewSearchProvider(s, box); err == nil {
t.Errorf("%s: succeeded, want error", name)
}
}
}
internal/recipes/searchsettings.go
+245
View File
@@ -0,0 +1,245 @@
package recipes
import (
"context"
"database/sql"
"errors"
"fmt"
"strings"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/db"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/secretbox"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/store"
)
const (
// SearchTestQuery is what TestSearch searches for.
SearchTestQuery = "chicken tikka masala"
// MinBraveKeyLength is the shortest Brave API key accepted, so its last
// four characters never give away most of it.
MinBraveKeyLength = 8
)
// ErrSearchNotConfigured is returned when no recipe search provider has
// been chosen yet.
var ErrSearchNotConfigured = errors.New("recipes: recipe search is not configured")
// SearchSettings is the household's recipe search configuration as the API
// shows it. The Brave API key is never part of it: only whether one is
// stored and its last four characters.
type SearchSettings struct {
// Provider is "searxng" or "brave", nil until first chosen.
Provider *string `json:"provider"`
// BaseURL is the SearXNG instance's URL, nil when not set.
BaseURL *string `json:"base_url"`
APIKeySet bool `json:"api_key_set"`
APIKeyLast4 *string `json:"api_key_last4"`
// UpdatedAt is nil until the settings are first saved.
UpdatedAt *string `json:"updated_at"`
}
// SearchSettingsPatch changes the settings. A nil field is left as it is;
// an empty BaseURL or APIKey removes the stored one.
type SearchSettingsPatch struct {
Provider *string `json:"provider"`
BaseURL *string `json:"base_url"`
// APIKey is the Brave API key in plain text. It is stored encrypted
// and never returned.
APIKey *string `json:"api_key"`
}
// SearchSettingsProblem is one invalid field of a SearchSettingsPatch,
// named by its JSON key.
type SearchSettingsProblem struct {
Field string
Reason string
}
// SearchSettingsError lists every problem with a SearchSettingsPatch, or
// with the settings it would leave.
type SearchSettingsError struct {
Problems []SearchSettingsProblem
}
func (e *SearchSettingsError) Error() string {
parts := make([]string, len(e.Problems))
for i, p := range e.Problems {
parts[i] = p.Field + ": " + p.Reason
}
return "invalid recipe search settings: " + strings.Join(parts, "; ")
}
// SearchSettingsService reads and changes the recipe_search_settings row
// and builds the search provider it names. Brave API keys are sealed with
// its secret box.
type SearchSettingsService struct {
st *store.Store
box *secretbox.Box
// newProvider is NewSearchProvider; tests point Brave elsewhere.
newProvider func(db.RecipeSearchSetting, *secretbox.Box) (SearchProvider, error)
}
// NewSearchSettingsService returns the service over st, sealing keys with
// box.
func NewSearchSettingsService(st *store.Store, box *secretbox.Box) (*SearchSettingsService, error) {
if st == nil {
return nil, errors.New("recipes: nil store")
}
if box == nil {
return nil, errors.New("recipes: nil secret box")
}
return &SearchSettingsService{st: st, box: box, newProvider: NewSearchProvider}, nil
}
// Get returns the settings; before any are saved every field is unset.
func (s *SearchSettingsService) Get(ctx context.Context) (SearchSettings, error) {
row, err := db.New(s.st.Reader).GetRecipeSearchSettings(ctx)
if errors.Is(err, sql.ErrNoRows) {
return SearchSettings{}, nil
}
if err != nil {
return SearchSettings{}, fmt.Errorf("recipes: read recipe search settings: %w", err)
}
return searchSettingsFrom(row), nil
}
// Update applies patch and returns the settings it leaves. The first update
// must choose a provider. The result must be usable: SearXNG needs a base
// URL and Brave an API key. Every problem is reported in a
// *SearchSettingsError.
func (s *SearchSettingsService) Update(ctx context.Context, patch SearchSettingsPatch) (SearchSettings, error) {
var saved db.RecipeSearchSetting
err := s.st.WithTx(ctx, func(tx *sql.Tx) error {
q := db.New(tx)
row, err := q.GetRecipeSearchSettings(ctx)
exists := err == nil
if err != nil && !errors.Is(err, sql.ErrNoRows) {
return fmt.Errorf("recipes: read recipe search settings: %w", err)
}
next, err := s.apply(row, exists, patch)
if err != nil {
return err
}
now := store.Now().String()
created := now
if exists {
created = row.CreatedAt
}
saved, err = q.UpsertRecipeSearchSettings(ctx, db.UpsertRecipeSearchSettingsParams{
Provider: next.Provider,
BaseUrl: next.BaseUrl,
ApiKeyCt: next.ApiKeyCt,
ApiKeyLast4: next.ApiKeyLast4,
CreatedAt: created,
UpdatedAt: now,
})
if err != nil {
return fmt.Errorf("recipes: save recipe search settings: %w", err)
}
return nil
})
if err != nil {
return SearchSettings{}, err
}
return searchSettingsFrom(saved), nil
}
// apply returns row with patch applied, validated.
func (s *SearchSettingsService) apply(row db.RecipeSearchSetting, exists bool, patch SearchSettingsPatch) (db.RecipeSearchSetting, error) {
var problems []SearchSettingsProblem
if patch.Provider != nil {
switch *patch.Provider {
case "searxng", "brave":
row.Provider = *patch.Provider
default:
problems = append(problems, SearchSettingsProblem{"provider", `must be "searxng" or "brave"`})
}
} else if !exists {
problems = append(problems, SearchSettingsProblem{"provider", "is required until recipe search is configured"})
}
if patch.BaseURL != nil {
row.BaseUrl = sql.NullString{}
if u := *patch.BaseURL; u != "" {
if _, err := NewSearXNG(u); err != nil {
problems = append(problems, SearchSettingsProblem{"base_url", strings.TrimPrefix(err.Error(), "recipes: ")})
} else {
row.BaseUrl = sql.NullString{String: u, Valid: true}
}
}
}
if patch.APIKey != nil {
row.ApiKeyCt, row.ApiKeyLast4 = nil, sql.NullString{}
if key := *patch.APIKey; key != "" {
// The key is never echoed in a problem.
if _, err := NewBrave(key); err != nil {
problems = append(problems, SearchSettingsProblem{"api_key", "has a character not allowed in a header"})
} else if len(key) < MinBraveKeyLength {
problems = append(problems, SearchSettingsProblem{"api_key", fmt.Sprintf("must be at least %d characters", MinBraveKeyLength)})
} else {
ct, err := s.box.Encrypt([]byte(key))
if err != nil {
return db.RecipeSearchSetting{}, fmt.Errorf("recipes: seal Brave API key: %w", err)
}
row.ApiKeyCt = ct
row.ApiKeyLast4 = sql.NullString{String: key[len(key)-4:], Valid: true}
}
}
}
if len(problems) == 0 {
switch {
case row.Provider == "searxng" && !row.BaseUrl.Valid:
problems = append(problems, SearchSettingsProblem{"base_url", "is required for SearXNG"})
case row.Provider == "brave" && len(row.ApiKeyCt) == 0:
problems = append(problems, SearchSettingsProblem{"api_key", "is required for Brave"})
}
}
if len(problems) > 0 {
return db.RecipeSearchSetting{}, &SearchSettingsError{Problems: problems}
}
return row, nil
}
// Provider returns the search provider the settings name, or
// ErrSearchNotConfigured.
func (s *SearchSettingsService) Provider(ctx context.Context) (SearchProvider, error) {
row, err := db.New(s.st.Reader).GetRecipeSearchSettings(ctx)
if errors.Is(err, sql.ErrNoRows) {
return nil, ErrSearchNotConfigured
}
if err != nil {
return nil, fmt.Errorf("recipes: read recipe search settings: %w", err)
}
return s.newProvider(row, s.box)
}
// TestSearch searches for SearchTestQuery with the configured provider and
// returns how many candidates it found, up to MaxSearchResults.
func (s *SearchSettingsService) TestSearch(ctx context.Context) (int, error) {
p, err := s.Provider(ctx)
if err != nil {
return 0, err
}
cands, err := p.Search(ctx, SearchTestQuery, MaxSearchResults)
if err != nil {
return 0, err
}
return len(cands), nil
}
func searchSettingsFrom(row db.RecipeSearchSetting) SearchSettings {
provider, updated := row.Provider, row.UpdatedAt
return SearchSettings{
Provider: &provider,
BaseURL: nullString(row.BaseUrl),
APIKeySet: len(row.ApiKeyCt) > 0,
APIKeyLast4: nullString(row.ApiKeyLast4),
UpdatedAt: &updated,
}
}
func nullString(s sql.NullString) *string {
if !s.Valid {
return nil
}
return &s.String
}
internal/recipes/searchsettings_test.go
+172
View File
@@ -0,0 +1,172 @@
package recipes
import (
"crypto/rand"
"encoding/base64"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"strings"
"testing"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/db"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/secretbox"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/store"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/migrations"
)
const testBraveKey = "BSA-secret-brave-key-9f3c"
func newTestSearchSettings(t *testing.T) *SearchSettingsService {
t.Helper()
st, err := store.Open(t.Context(), t.TempDir())
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
if err := st.Close(); err != nil {
t.Error(err)
}
})
if err := store.Migrate(t.Context(), st.Writer, migrations.FS); err != nil {
t.Fatal(err)
}
key := make([]byte, secretbox.KeySize)
rand.Read(key)
box, err := secretbox.New(base64.StdEncoding.EncodeToString(key))
if err != nil {
t.Fatal(err)
}
svc, err := NewSearchSettingsService(st, box)
if err != nil {
t.Fatal(err)
}
return svc
}
func ptr(s string) *string { return &s }
// noKey fails the test when v, marshalled as the API would, holds the
// Brave key.
func noKey(t *testing.T, what string, v any) {
t.Helper()
b, err := json.Marshal(v)
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(b), testBraveKey) || strings.Contains(string(b), testBraveKey[:12]) {
t.Errorf("%s holds the Brave key: %s", what, b)
}
}
func TestSearchSettingsUpdate(t *testing.T) {
t.Parallel()
svc := newTestSearchSettings(t)
got, err := svc.Get(t.Context())
if err != nil || got != (SearchSettings{}) {
t.Fatalf("Get before any update = %+v, %v; want every field unset", got, err)
}
if _, err := svc.Provider(t.Context()); !errors.Is(err, ErrSearchNotConfigured) {
t.Errorf("Provider before any update error = %v, want ErrSearchNotConfigured", err)
}
for name, patch := range map[string]SearchSettingsPatch{
"no provider": {BaseURL: ptr("http://searxng:8080")},
"unknown provider": {Provider: ptr("google")},
"searxng without URL": {Provider: ptr("searxng")},
"bad URL": {Provider: ptr("searxng"), BaseURL: ptr("ftp://searxng")},
"brave without key": {Provider: ptr("brave")},
"short key": {Provider: ptr("brave"), APIKey: ptr("abc12")},
"key with space": {Provider: ptr("brave"), APIKey: ptr("abc 12345678")},
} {
var invalid *SearchSettingsError
if _, err := svc.Update(t.Context(), patch); !errors.As(err, &invalid) {
t.Errorf("%s: error = %v, want *SearchSettingsError", name, err)
}
}
got, err = svc.Update(t.Context(), SearchSettingsPatch{Provider: ptr("searxng"), BaseURL: ptr("http://searxng:8080")})
if err != nil {
t.Fatal(err)
}
if *got.Provider != "searxng" || *got.BaseURL != "http://searxng:8080" || got.APIKeySet || got.APIKeyLast4 != nil || got.UpdatedAt == nil {
t.Errorf("SearXNG settings = %+v", got)
}
got, err = svc.Update(t.Context(), SearchSettingsPatch{Provider: ptr("brave"), APIKey: ptr(testBraveKey)})
if err != nil {
t.Fatal(err)
}
if *got.Provider != "brave" || !got.APIKeySet || *got.APIKeyLast4 != "9f3c" || *got.BaseURL != "http://searxng:8080" {
t.Errorf("Brave settings = %+v", got)
}
noKey(t, "Update result", got)
got, err = svc.Get(t.Context())
if err != nil {
t.Fatal(err)
}
noKey(t, "Get result", got)
row, err := db.New(svc.st.Reader).GetRecipeSearchSettings(t.Context())
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(row.ApiKeyCt), testBraveKey) {
t.Error("Brave key stored in plain text")
}
if _, err := svc.Update(t.Context(), SearchSettingsPatch{APIKey: ptr("")}); err == nil {
t.Error("removing the key while Brave is chosen succeeded")
}
got, err = svc.Update(t.Context(), SearchSettingsPatch{Provider: ptr("searxng"), APIKey: ptr("")})
if err != nil || got.APIKeySet || got.APIKeyLast4 != nil {
t.Errorf("switch to SearXNG removing the key = %+v, %v", got, err)
}
}
func TestSearchSettingsTestSearch(t *testing.T) {
t.Parallel()
searx := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Query().Get("q") != SearchTestQuery+" recipe" {
t.Errorf("SearXNG query = %q", r.URL.Query().Get("q"))
}
w.Write([]byte(`{"results":[{"url":"https://a.example/1"},{"url":"https://b.example/2"}]}`))
}))
t.Cleanup(searx.Close)
svc := newTestSearchSettings(t)
if _, err := svc.TestSearch(t.Context()); !errors.Is(err, ErrSearchNotConfigured) {
t.Errorf("TestSearch unconfigured error = %v, want ErrSearchNotConfigured", err)
}
if _, err := svc.Update(t.Context(), SearchSettingsPatch{Provider: ptr("searxng"), BaseURL: ptr(searx.URL)}); err != nil {
t.Fatal(err)
}
if n, err := svc.TestSearch(t.Context()); err != nil || n != 2 {
t.Errorf("TestSearch = %d, %v; want 2", n, err)
}
// Brave refuses the key; the error must not carry it.
brave := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("X-Subscription-Token") != testBraveKey {
t.Errorf("Brave token = %q", r.Header.Get("X-Subscription-Token"))
}
http.Error(w, "bad key "+testBraveKey, http.StatusUnauthorized)
}))
t.Cleanup(brave.Close)
svc.newProvider = func(s db.RecipeSearchSetting, box *secretbox.Box) (SearchProvider, error) {
p, err := NewSearchProvider(s, box)
if b, ok := p.(*Brave); ok {
b.endpoint = brave.URL
}
return p, err
}
if _, err := svc.Update(t.Context(), SearchSettingsPatch{Provider: ptr("brave"), APIKey: ptr(testBraveKey)}); err != nil {
t.Fatal(err)
}
_, err := svc.TestSearch(t.Context())
if !errors.Is(err, ErrSearchRefused) {
t.Fatalf("TestSearch error = %v, want ErrSearchRefused", err)
}
if strings.Contains(err.Error(), testBraveKey) {
t.Errorf("TestSearch error holds the Brave key: %v", err)
}
}
internal/recipes/testdata/bbcgoodfood.html
Vendored
+13
View File
@@ -0,0 +1,13 @@
<!DOCTYPE html>
<!-- Trimmed from https://www.bbcgoodfood.com/recipes/easy-chicken-curry (fetched 2026-10-05): head metadata and JSON-LD only. -->
<html><head>
<title>Easy chicken curry recipe | Good Food</title>
<meta property="og:title" content="Easy chicken curry" data-testid="meta-og-title"/>
<meta property="og:site_name" content="Good Food" data-testid="meta-og-site-name"/>
<meta property="og:url" content="https://www.bbcgoodfood.com/recipes/easy-chicken-curry" data-testid="meta-og-url"/>
<meta property="og:image" content="https://images.immediate.co.uk/production/volatile/sites/30/2020/08/easy-chicken-curry-aa22a0b.jpg?resize=440%2C230" data-testid="meta-og-image"/>
<script data-testid="schema" type="application/ld+json">{"@context":"https://schema.org","@type":"VideoObject","name":"Easy Chicken Curry","contentUrl":"https://media.delight.video/0c18e53b610964fac175fe0e757adc30557e79b7/cb0e0397ffd102279027b8d892036a85e98afa8b/MEDIA/v0/HD/media.mp4","description":"This easy staple chicken curry is a fantastic recipe for family dinners. It's made with just a handful of ingredients and is enriched with creamy yogurt","embedUrl":"https://www.bbcgoodfood.com/recipes/easy-chicken-curry","thumbnailUrl":"https://media.delight.video/0c18e53b610964fac175fe0e757adc30557e79b7/cb0e0397ffd102279027b8d892036a85e98afa8b/MEDIA/v0/THUMBS/media.0000005.jpg","uploadDate":"2022-07-29T10:34:28.279Z","duration":"PT0M57S","customPosterUrl":"https://media.delight.video/0c18e53b610964fac175fe0e757adc30557e79b7/cb0e0397ffd102279027b8d892036a85e98afa8b/POSTER_USER/v0/cb0e0397ffd102279027b8d892036a85e98afa8b.jpeg"}</script>
<script data-testid="page-schema" type="application/ld+json">{"@context":"https://schema.org","@id":"https://www.bbcgoodfood.com/recipes/easy-chicken-curry#Recipe","@type":"Recipe","description":"This easy staple chicken curry is a fantastic recipe for family dinners. It's made with just a handful of ingredients and is enriched with creamy yogurt","image":[{"@type":"ImageObject","url":"https://images.immediate.co.uk/production/volatile/sites/30/2020/08/easy-chicken-curry-aa22a0b.jpg?resize=440,400","width":440,"height":400}],"mainEntityOfPage":{"@type":"WebPage","@id":"https://www.bbcgoodfood.com/recipes/easy-chicken-curry"},"name":"Easy chicken curry","url":"https://www.bbcgoodfood.com/recipes/easy-chicken-curry","author":[{"@type":"Person","name":"Esther Clark","url":"https://www.bbcgoodfood.com/author/estherclark"}],"dateModified":"2026-08-13T14:40:21+01:00","datePublished":"2019-01-25T15:26:05+00:00","headline":"Easy chicken curry","keywords":"1 of 5-a-day, Chicken, Curry, Esther Clark, Family, Family friendly, Gluten free","publisher":{"@type":"Organization","name":"Good Food","url":"https://www.bbcgoodfood.com","logo":{"@type":"ImageObject","height":325,"url":"https://images.immediate.co.uk/production/volatile/sites/30/2026/02/goodfood.logo-f37096c.png?resize=1312,325","width":1312}},"cookTime":"PT45M","nutrition":{"@type":"NutritionInformation","calories":"354 calories","fatContent":"23 grams fat","saturatedFatContent":"4 grams saturated fat","carbohydrateContent":"10 grams carbohydrates","sugarContent":"8 grams sugar","fiberContent":"3 grams fiber","proteinContent":"24 grams protein","sodiumContent":"0.6 milligram of sodium"},"prepTime":"PT5M","recipeCategory":"Dinner, Main course","recipeIngredient":["2 tbsp sunflower oil","1 onion thinly sliced","2 garlic cloves crushed","thumb-sized piece of ginger grated","6 chicken thighs boneless and skinless","3 tbsp medium spice paste (tikka works well)","400g can chopped tomatoes","100g Greek yogurt","1 small bunch of coriander leaves chopped","50g ground almonds","naan breads or cooked basmati rice, to serve"],"recipeInstructions":[{"@type":"HowToStep","text":"Heat the oil in a flameproof casserole dish or large frying pan over a medium heat. Add the onion and a generous pinch of salt and fry for 8-10 mins, or until the onion has turned golden brown and sticky."},{"@type":"HowToStep","text":"Add the garlic and ginger, cooking for a further minute."},{"@type":"HowToStep","text":"Chop the chicken into chunky 3cm pieces, add to the pan and fry for 5 mins before stirring through the spice paste and tomatoes, along with 250ml water. Bring to the boil, lower to a simmer and cook on a gentle heat uncovered for 25-30 mins or until rich and slightly reduced."},{"@type":"HowToStep","text":"Stir though the yogurt, coriander and ground almonds, season and serve with warm naan or fluffy basmati rice."}],"recipeYield":4,"suitableForDiet":"https://schema.org/GlutenFreeDiet","totalTime":"PT50M"}</script>
<script type="application/ld+json">{"@context":"https://schema.org/","@type":"BreadcrumbList","itemListElement":[{"@type":"ListItem","position":1,"name":"Home","item":"https://www.bbcgoodfood.com/"},{"@type":"ListItem","position":2,"name":"Recipes","item":"https://www.bbcgoodfood.com/recipes"},{"@type":"ListItem","position":3,"name":"Easy chicken curry"}]}</script>
<script type="application/ld+json">{"@context":"https://schema.org/","@id":"https://www.bbcgoodfood.com/recipes/easy-chicken-curry#Recipe","aggregateRating":{"@type":"AggregateRating","ratingValue":5,"reviewCount":363,"bestRating":5,"worstRating":1}}</script>
</head><body><main>(page body trimmed)</main></body></html>
internal/recipes/testdata/bonappetit.html
Vendored
+10
View File
File diff suppressed because one or more lines are too long
internal/recipes/testdata/budgetbytes-about.html
Vendored
+10
View File
@@ -0,0 +1,10 @@
<!DOCTYPE html>
<!-- Trimmed from https://www.budgetbytes.com/about/ (fetched 2026-10-05): head metadata and JSON-LD only. -->
<html><head>
<title>The Budget Bytes Story - BudgetBytes.com</title>
<meta property="og:title" content="The Budget Bytes Story - BudgetBytes.com" />
<meta property="og:url" content="https://www.budgetbytes.com/about/" />
<meta property="og:site_name" content="Budget Bytes" />
<meta property="og:image" content="https://www.budgetbytes.com/wp-content/uploads/2025/04/Budget-Bytes-Group-Shot_Web.jpg" />
<script type="application/ld+json" class="yoast-schema-graph">{"@context":"https:\/\/schema.org","@graph":[{"@type":"WebPage","@id":"https:\/\/www.budgetbytes.com\/about\/","url":"https:\/\/www.budgetbytes.com\/about\/","name":"The Budget Bytes Story - BudgetBytes.com","isPartOf":{"@id":"https:\/\/www.budgetbytes.com\/#website"},"primaryImageOfPage":{"@id":"https:\/\/www.budgetbytes.com\/about\/#primaryimage"},"image":{"@id":"https:\/\/www.budgetbytes.com\/about\/#primaryimage"},"thumbnailUrl":"https:\/\/www.budgetbytes.com\/wp-content\/uploads\/2025\/04\/Budget-Bytes-Group-Shot_Web.jpg","datePublished":"2013-05-02T18:19:55+00:00","dateModified":"2026-07-20T15:36:17+00:00","description":"Beth Moncel created Budget Bytes in 2009 to help people learn how to cook and create delicious recipes designed for small budgets.","inLanguage":"en-US","potentialAction":[{"@type":"ReadAction","target":["https:\/\/www.budgetbytes.com\/about\/"]}]},{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/www.budgetbytes.com\/about\/#primaryimage","url":"https:\/\/www.budgetbytes.com\/wp-content\/uploads\/2025\/04\/Budget-Bytes-Group-Shot_Web.jpg","contentUrl":"https:\/\/www.budgetbytes.com\/wp-content\/uploads\/2025\/04\/Budget-Bytes-Group-Shot_Web.jpg","width":1200,"height":800,"caption":"The Budget Bytes team that develops recipes out of our Nashville studio. Left to right: Rebecca Denton (Photographer), Jess Rice (Recipe Development Manager), Jennie Alley (Senior Recipe Developer), and Melissa Nolan (Senior Recipe Developer). We develop, test, and photograph each and every recipe on Budget Bytes from Nashville Tennessee."},{"@type":"WebSite","@id":"https:\/\/www.budgetbytes.com\/#website","url":"https:\/\/www.budgetbytes.com\/","name":"Budget Bytes","description":"Delicious Recipes Designed for Small Budgets","publisher":{"@id":"https:\/\/www.budgetbytes.com\/#organization"},"potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/www.budgetbytes.com\/?s={search_term_string}"},"query-input":{"@type":"PropertyValueSpecification","valueRequired":true,"valueName":"search_term_string"}}],"inLanguage":"en-US"},{"@type":"Organization","@id":"https:\/\/www.budgetbytes.com\/#organization","name":"Budget Bytes","url":"https:\/\/www.budgetbytes.com\/","logo":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/www.budgetbytes.com\/#\/schema\/logo\/image\/","url":"https:\/\/www.budgetbytes.com\/wp-content\/uploads\/2024\/11\/budgetbytes_square_icon_transparent-300x300-1.png","contentUrl":"https:\/\/www.budgetbytes.com\/wp-content\/uploads\/2024\/11\/budgetbytes_square_icon_transparent-300x300-1.png","width":300,"height":300,"caption":"Budget Bytes"},"image":{"@id":"https:\/\/www.budgetbytes.com\/#\/schema\/logo\/image\/"},"sameAs":["https:\/\/www.facebook.com\/budgetbytes1\/?ref=hl","https:\/\/x.com\/Budget_Bytes","https:\/\/instagram.com\/budgetbytes\/","https:\/\/www.pinterest.com\/budgetbytes\/","https:\/\/www.youtube.com\/channel\/UC17vdVOZBVxTSDcldUBBlRg"]}]}</script>
</head><body><main>(page body trimmed)</main></body></html>
internal/recipes/testdata/budgetbytes.html
Vendored
+10
View File
File diff suppressed because one or more lines are too long
internal/recipes/testdata/delish.html
Vendored
+10
View File
File diff suppressed because one or more lines are too long
internal/recipes/testdata/foodcom.html
Vendored
+9
View File
@@ -0,0 +1,9 @@
<!DOCTYPE html>
<!-- Trimmed from https://www.food.com/recipe/best-banana-bread-2886 (fetched 2026-10-05): head metadata and JSON-LD only; the 80 KB review array removed from the Recipe node. -->
<html><head>
<title>Best Banana Bread Recipe - Food.com</title>
<meta name="og:title" content="Best Banana Bread Recipe - Food.com"/>
<meta name="og:image" content="https://img.sndimg.com/food/image/upload/w_555,h_416,c_fit,fl_progressive,q_95/v1/img/recipes/28/86/wBZxNua1T8yfDwbfo9Fz_0S9A9315.jpg"/>
<meta name="og:url" content=""/>
<script type="application/ld+json">{"@context":"http://schema.org","@type":"Recipe","mainEntityOfPage":"true","name":"Best Banana Bread","author":"lkadlec","cookTime":"PT1H","prepTime":"PT10M","totalTime":"PT1H10M","datePublished":"1999-09-26T20:49Z","description":"Make and share this Best Banana Bread recipe from Food.com.","image":"https://img.sndimg.com/food/image/upload/q_92,fl_progressive,w_1200,c_scale/v1/img/recipes/28/86/wBZxNua1T8yfDwbfo9Fz_0S9A9315.jpg","recipeCategory":"Quick Breads","keywords":"Breads,Fruit,Oven,< 4 Hours","recipeIngredient":["1/2 cup butter, softened ","1 cup granulated sugar","2 eggs, beaten ","3 bananas, finely crushed (for serious and extreme moist and delicious, try 4 bananas)","1 1/2 cups all-purpose flour","1 teaspoon baking soda","1/2 teaspoon salt","1/2 teaspoon vanilla (optional)"],"aggregateRating":{"@type":"AggregateRating","ratingValue":"5.0","reviewCount":"2355"},"nutrition":{"@type":"NutritionInformation","calories":"272.8","fatContent":"10.5","saturatedFatContent":"6.2","cholesterolContent":"61.6","sodiumContent":"338.3","carbohydrateContent":"42.5","fiberContent":"1.4","sugarContent":"24.4","proteinContent":"3.7"},"recipeInstructions":[{"@type":"HowToStep","text":"Remove odd pots and pans from oven."},{"@type":"HowToStep","text":"Preheat oven to 350&ordm; / 180&ordm;."},{"@type":"HowToStep","text":"Cream together butter and sugar."},{"@type":"HowToStep","text":"Add eggs and crushed bananas."},{"@type":"HowToStep","text":"Combine well."},{"@type":"HowToStep","text":"Sift together flour, soda and salt. Add to creamed mixture. Add vanilla."},{"@type":"HowToStep","text":"Mix just until combined. Do not overmix."},{"@type":"HowToStep","text":"Pour into greased and floured loaf pan."},{"@type":"HowToStep","text":"Bake at 350&ordm; / 180&ordm; for 55 minutes."},{"@type":"HowToStep","text":"Keeps well, refrigerated."}],"recipeYield":"1 loaf, 10 serving(s)","publisher":{"@type":"Organization","name":"Food.com","logo":{"@type":"ImageObject","url":"https://geniuskitchen.sndimg.com/fdc-new/img/FDC-Logo.png"},"url":"https://www.food.com"},"video":[{"@type":"VideoObject","name":"Best Banana Bread","duration":"42S","thumbnailUrl":"https://img.sndimg.com/food/image/upload/w_1024,h_576,c_fit/v1/img/mpx/5458761_16x9.jpg","contentUrl":"https://www.food.com/recipe/best-banana-bread-2886"}]}</script>
</head><body><main>(page body trimmed)</main></body></html>
internal/recipes/testdata/kingarthur.html
Vendored
+79
View File
@@ -0,0 +1,79 @@
<!DOCTYPE html>
<!-- Trimmed from https://www.kingarthurbaking.com/recipes/classic-sandwich-bread-recipe (fetched 2026-10-05): head metadata and JSON-LD only. -->
<html><head>
<title>Classic Sandwich Bread Recipe | King Arthur Baking</title>
<meta property="og:site_name" content="King Arthur Baking" />
<meta property="og:url" content="https://www.kingarthurbaking.com/recipes/classic-sandwich-bread-recipe" />
<meta property="og:title" content="Classic Sandwich Bread" />
<meta property="og:image" content="https://www.kingarthurbaking.com/sites/default/files/2026-02/Classic-Sandwich-Bread-12.jpg" />
<script type="application/ld+json">{
"@context": "https://schema.org",
"@graph": [
{
"@type": "Recipe",
"keywords": "Sandwich bread;;Tangzhong",
"name": "Classic Sandwich Bread",
"description": "Made with all of the ingredients that make homemade white bread superior to storebought — butter, milk, and high-quality flour — this sandwich bread recipe is our version of a classic, perfected. The soft and tender interior makes this loaf ideal for sandwiches, toast, or even French toast. It\u0027s a timeless recipe you\u0027ll come back to time and time again.",
"recipeYield": [
"16",
"1 loaf"
],
"datePublished": "December 31, 2007 at 7:00pm",
"prepTime": "PT12M",
"cookTime": "PT40M",
"totalTime": "PT2H42M",
"recipeInstructions": [
"Weigh your flour; or measure it by gently spooning it into a cup, then sweeping off any excess.",
"In a large bowl, combine all of the ingredients. Mix and knead everything together — by hand, mixer, or bread machine set on the dough cycle — until you\u0027ve made a smooth dough. If you\u0027re kneading in a stand mixer, it should take 5 to 7 minutes at medium-low speed, and the dough should barely clean the sides of the bowl, perhaps sticking a bit at the bottom. In a bread machine (or by hand), knead the dough for about 6 to 8 minutes, until it forms a smooth ball. ",
"Transfer the dough to a lightly greased bowl, cover the bowl, and allow the dough to rise until puffy though not necessarily doubled in bulk, about 1 to 2 hours, depending on the warmth of your kitchen. If you\u0027re using a bread machine, allow the machine to complete its cycle, then leave the dough in the machine until it\u0027s doubled in bulk, perhaps an additional 30 minutes or so.",
"Gently deflate the dough and transfer it to a lightly oiled work surface. Shape the dough into an 8\u0022 log.",
"Place the log in a lightly greased 8 1/2\u0022 x 4 1/2\u0022 loaf pan, cover the pan loosely with lightly greased plastic wrap, and allow the bread to rise for about 60 minutes, until it\u0027s domed about 1\u0022 above the edge of the pan. A finger pressed into the dough should leave a mark that rebounds slowly. Towards the end of the rise, preheat your oven to 350°F.",
"Bake the bread for 30 to 35 minutes, until it\u0027s light golden brown. Test it for doneness by removing it from the pan and thumping it on the bottom (it should sound hollow), or by measuring its interior temperature with a digital thermometer (it should register 190°F at the center of the loaf).",
"Remove the bread from the oven, and cool it on a rack before slicing. Store the bread in a plastic bag at room temperature for several days; freeze for longer storage."
],
"recipeIngredient": [
"3 cups (360g) King Arthur Unbleached All-Purpose Flour",
"1/2 cup (113g) milk, (skim, 1%, 2% or whole, your choice)*",
"1/2 to 2/3 cup (113g to 152g) hot water, enough to make a soft, smooth dough* (see \u0022tips,\u0022 below)",
"4 tablespoons (57g) melted butter or 1/4 cup (50g) vegetable oil",
"2 tablespoons (25g) granulated sugar",
"1 1/4 teaspoons (8g) table salt",
"2 teaspoons instant yeast"
],
"publisher": {
"@type": "Organization",
"name": "King Arthur Baking",
"url": "https://www.kingarthurbaking.com/"
},
"author": {
"@type": "Person",
"name": "PJ Hamel",
"url": "https://www.kingarthurbaking.com/author/pj-hamel"
},
"image": {
"@type": "ImageObject",
"url": "https://www.kingarthurbaking.com/sites/default/files/2026-02/Classic-Sandwich-Bread-12.jpg"
},
"nutrition": {
"@type": "NutritionInformation",
"servingSize": "43g",
"calories": "120 calories",
"carbohydrateContent": "19g",
"cholesterolContent": "10mg",
"fiberContent": "1g",
"proteinContent": "3g",
"sodiumContent": "190mg",
"sugarContent": "2g",
"fatContent": "3g",
"saturatedFatContent": "2g",
"transFatContent": "0g"
},
"aggregateRating": {
"@type": "AggregateRating",
"ratingValue": "4.60",
"ratingCount": "681"
}
}
]
}</script>
</head><body><main>(page body trimmed)</main></body></html>
internal/recipes/testdata/photo.webp
Vendored
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.3 KiB

internal/recipes/testdata/recipetineats.html
Vendored
+11
View File
File diff suppressed because one or more lines are too long