From 9e05d16a46cdc9a90c7b22662dfdb6f3416cc501 Mon Sep 17 00:00:00 2001 From: dev_agent Date: Mon, 5 Oct 2026 12:49:54 -0400 Subject: [PATCH] Add schema.org Recipe JSON-LD extractor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit recipes.Extract(page, url) scans a page's application/ld+json scripts and og:site_name with a small targeted scanner (no new dependency) and returns the single Recipe node: top level, top-level array or @graph, @type as a string or array. It picks the largest image and resolves it against the page URL, flattens HowToStep/HowToSection instructions in order, converts ISO-8601 durations to minutes (total derived from prep + cook when absent), and names the site from publisher, og:site_name or host. No Recipe is ErrNoRecipe; two is ErrManyRecipes; shapes schema.org does not allow, such as a "20 minutes" cookTime, are ErrMalformedRecipe. testdata holds trimmed real pages from BBC Good Food, Budget Bytes, RecipeTin Eats (HowToSection), King Arthur Baking, Delish (top-level array), Food.com, Bon Appétit (non-ISO duration) and a Budget Bytes page with no Recipe. Maestro task 4c4570be-e0d2-43ad-90ea-0a4a9eeeb0b9. Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 15 +- internal/recipes/extract.go | 580 ++++++++++++++++++ internal/recipes/extract_test.go | 347 +++++++++++ internal/recipes/testdata/bbcgoodfood.html | 13 + internal/recipes/testdata/bonappetit.html | 10 + .../recipes/testdata/budgetbytes-about.html | 10 + internal/recipes/testdata/budgetbytes.html | 10 + internal/recipes/testdata/delish.html | 10 + internal/recipes/testdata/foodcom.html | 9 + internal/recipes/testdata/kingarthur.html | 79 +++ internal/recipes/testdata/recipetineats.html | 11 + 11 files changed, 1093 insertions(+), 1 deletion(-) create mode 100644 internal/recipes/extract.go create mode 100644 internal/recipes/extract_test.go create mode 100644 internal/recipes/testdata/bbcgoodfood.html create mode 100644 internal/recipes/testdata/bonappetit.html create mode 100644 internal/recipes/testdata/budgetbytes-about.html create mode 100644 internal/recipes/testdata/budgetbytes.html create mode 100644 internal/recipes/testdata/delish.html create mode 100644 internal/recipes/testdata/foodcom.html create mode 100644 internal/recipes/testdata/kingarthur.html create mode 100644 internal/recipes/testdata/recipetineats.html diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index b71272f..8c11051 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -223,7 +223,20 @@ automatic** (D-021). 1 s apart, counted from requests actually sent (an abandoned wait holds no slot); 2xx pages cached by URL for an hour (64 pages, oldest evicted), and callers get their own copy of a cached body. -- Recipe data is parsed from schema.org `Recipe` JSON-LD. +- Recipe data is parsed from schema.org `Recipe` JSON-LD + (`recipes.Extract`): a small targeted scan of the page's + `application/ld+json` scripts and `og:site_name` (no HTML parser + dependency), finding the Recipe node at a script's top level, in a + top-level array or in an `@graph`, with `@type` a string or an array. The + largest image (by ImageObject width × height; the first of equals) is + resolved to an absolute URL; instructions — one string of lines, or a + list of strings, HowToSteps and HowToSections — are flattened in order; + ISO-8601 prep/cook/total times become minutes, total derived as prep + + cook when only those are given; the site name is the publisher's, else + `og:site_name`, else the host. A page with no Recipe is `ErrNoRecipe`, + with two is `ErrManyRecipes`; a shape schema.org does not allow — such as + a non-ISO-8601 duration like "20 minutes" — is `ErrMalformedRecipe`, + never guessed at. - Photos are cached locally (`photos`) and processed with `golang.org/x/image`. diff --git a/internal/recipes/extract.go b/internal/recipes/extract.go new file mode 100644 index 0000000..122aac0 --- /dev/null +++ b/internal/recipes/extract.go @@ -0,0 +1,580 @@ +package recipes + +import ( + "encoding/json" + "errors" + "fmt" + "html" + "io" + "math" + "net/url" + "strconv" + "strings" +) + +var ( + // ErrNoRecipe is returned for a page with no schema.org Recipe node in + // its JSON-LD. + ErrNoRecipe = errors.New("recipes: page has no schema.org Recipe") + // ErrManyRecipes is returned for a page with more than one Recipe node: + // which one the page is about is not knowable. + ErrManyRecipes = errors.New("recipes: page has more than one schema.org Recipe") + // ErrMalformedRecipe is returned when the page's JSON-LD, or a field of + // its Recipe node, is not in a shape schema.org allows. + ErrMalformedRecipe = errors.New("recipes: malformed schema.org Recipe") +) + +// Recipe is the schema.org Recipe extracted from a recipe page. +type Recipe struct { + // Title is the recipe's name. + Title string + // SiteName is the publisher's name, else the page's og:site_name, else + // its host without a leading "www.". + SiteName string + // ImageURL is the largest image, absolute; empty when the recipe has + // none. + ImageURL string + Ingredients []string + // Steps are the instructions in order, HowToSection groups flattened. + Steps []string + // PrepMinutes, CookMinutes and TotalMinutes are nil when not given. + // TotalMinutes is prep plus cook when the page gives both but no total. + PrepMinutes *int + CookMinutes *int + TotalMinutes *int + // Yield is recipeYield as text; empty when not given. + Yield string + // JSONLD is the Recipe node as found. + JSONLD json.RawMessage +} + +// Extract finds the schema.org Recipe in a page's application/ld+json +// scripts. Recipe nodes are found at the top level of a script, in a +// top-level array, or in an @graph. pageURL is the page's own absolute URL; +// relative image URLs resolve against it. +func Extract(page []byte, pageURL string) (Recipe, error) { + base, err := url.Parse(pageURL) + if err != nil || !base.IsAbs() || base.Host == "" { + return Recipe{}, fmt.Errorf("recipes: page URL %q is not absolute", pageURL) + } + doc := scanHTML(string(page)) + var nodes []map[string]any + for i, script := range doc.jsonLD { + v, err := decodeJSON(script) + if err != nil { + return Recipe{}, fmt.Errorf("%w: JSON-LD script %d: %v", ErrMalformedRecipe, i+1, err) + } + collectRecipes(v, &nodes) + } + switch len(nodes) { + case 0: + return Recipe{}, ErrNoRecipe + case 1: + default: + return Recipe{}, fmt.Errorf("%w: found %d", ErrManyRecipes, len(nodes)) + } + r, err := recipeFrom(nodes[0], base, doc.ogSiteName) + if err != nil { + return Recipe{}, fmt.Errorf("%w: %v", ErrMalformedRecipe, err) + } + return r, nil +} + +func decodeJSON(s string) (any, error) { + dec := json.NewDecoder(strings.NewReader(s)) + dec.UseNumber() + var v any + if err := dec.Decode(&v); err != nil { + return nil, err + } + if _, err := dec.Token(); err != io.EOF { + return nil, errors.New("trailing data after JSON value") + } + return v, nil +} + +// collectRecipes appends every Recipe node in v: v itself, the elements of +// an array, and the members of an @graph. +func collectRecipes(v any, out *[]map[string]any) { + switch n := v.(type) { + case []any: + for _, e := range n { + collectRecipes(e, out) + } + case map[string]any: + if hasType(n, "Recipe") { + *out = append(*out, n) + } + if g, ok := n["@graph"]; ok { + collectRecipes(g, out) + } + } +} + +// hasType reports whether node's @type, a string or an array of strings, +// names typ. +func hasType(node map[string]any, typ string) bool { + switch t := node["@type"].(type) { + case string: + return t == typ + case []any: + for _, e := range t { + if s, ok := e.(string); ok && s == typ { + return true + } + } + } + return false +} + +func recipeFrom(n map[string]any, base *url.URL, ogSiteName string) (Recipe, error) { + var r Recipe + var err error + name, ok := n["name"].(string) + if r.Title = cleanText(name); !ok || r.Title == "" { + return Recipe{}, errors.New("name is missing") + } + r.SiteName = siteName(n["publisher"], ogSiteName, base) + if r.ImageURL, err = imageURL(n["image"], base); err != nil { + return Recipe{}, err + } + if r.Ingredients, err = ingredients(n["recipeIngredient"]); err != nil { + return Recipe{}, err + } + if r.Steps, err = steps(n["recipeInstructions"]); err != nil { + return Recipe{}, err + } + if r.PrepMinutes, err = minutes(n, "prepTime"); err != nil { + return Recipe{}, err + } + if r.CookMinutes, err = minutes(n, "cookTime"); err != nil { + return Recipe{}, err + } + if r.TotalMinutes, err = minutes(n, "totalTime"); err != nil { + return Recipe{}, err + } + if r.TotalMinutes == nil && r.PrepMinutes != nil && r.CookMinutes != nil { + total := *r.PrepMinutes + *r.CookMinutes + r.TotalMinutes = &total + } + if r.Yield, err = yield(n["recipeYield"]); err != nil { + return Recipe{}, err + } + if r.JSONLD, err = json.Marshal(n); err != nil { + return Recipe{}, err + } + return r, nil +} + +// cleanText decodes HTML entities, which recipe plugins often leave in +// JSON-LD text, and trims surrounding space. +func cleanText(s string) string { + return strings.TrimSpace(html.UnescapeString(s)) +} + +func siteName(publisher any, ogSiteName string, base *url.URL) string { + pubs, ok := publisher.([]any) + if !ok { + pubs = []any{publisher} + } + for _, p := range pubs { + if m, ok := p.(map[string]any); ok { + if name, ok := m["name"].(string); ok && cleanText(name) != "" { + return cleanText(name) + } + } + } + if ogSiteName != "" { + return ogSiteName + } + return strings.TrimPrefix(base.Hostname(), "www.") +} + +// imageURL picks the largest of the recipe's images by width × height; +// images without both dimensions count as size zero, and the first wins a +// tie, so a list of bare URLs gives its first. +func imageURL(v any, base *url.URL) (string, error) { + items, ok := v.([]any) + if !ok { + if v == nil { + return "", nil + } + items = []any{v} + } + best, bestArea := "", int64(-1) + for _, item := range items { + var u string + var area int64 + switch img := item.(type) { + case string: + u = img + case map[string]any: + if s, ok := img["url"].(string); ok { + u = s + } else if s, ok := img["contentUrl"].(string); ok { + u = s + } + w, err := dimension(img["width"]) + if err != nil { + return "", fmt.Errorf("image width: %w", err) + } + h, err := dimension(img["height"]) + if err != nil { + return "", fmt.Errorf("image height: %w", err) + } + area = w * h + default: + return "", fmt.Errorf("image is a %T, not a URL or ImageObject", item) + } + u = strings.TrimSpace(u) + if u == "" { + return "", errors.New("image has no URL") + } + if area > bestArea { + best, bestArea = u, area + } + } + if best == "" { + return "", nil + } + ref, err := url.Parse(best) + if err != nil { + return "", fmt.Errorf("image URL %q: %w", best, err) + } + return base.ResolveReference(ref).String(), nil +} + +// dimension reads an ImageObject width or height: a number or a string of +// digits; zero when absent. +func dimension(v any) (int64, error) { + switch d := v.(type) { + case nil: + return 0, nil + case json.Number: + f, err := d.Float64() + if err != nil || f < 0 { + return 0, fmt.Errorf("%q is not a size", d) + } + return int64(f), nil + case string: + n, err := strconv.ParseInt(strings.TrimSpace(d), 10, 64) + if err != nil || n < 0 { + return 0, fmt.Errorf("%q is not a size", d) + } + return n, nil + } + return 0, fmt.Errorf("a %T is not a size", v) +} + +func ingredients(v any) ([]string, error) { + items, ok := v.([]any) + if !ok { + if v == nil { + return nil, nil + } + items = []any{v} + } + var out []string + for _, item := range items { + s, ok := item.(string) + if !ok { + return nil, fmt.Errorf("recipeIngredient holds a %T, not text", item) + } + if s = cleanText(s); s != "" { + out = append(out, s) + } + } + return out, nil +} + +// steps flattens recipeInstructions: one string of lines, or a list of +// strings, HowToSteps and HowToSections of those, in order. +func steps(v any) ([]string, error) { + switch in := v.(type) { + case nil: + return nil, nil + case string: + var out []string + for _, line := range strings.Split(in, "\n") { + if line = cleanText(line); line != "" { + out = append(out, line) + } + } + return out, nil + case map[string]any: + return stepList([]any{in}) + case []any: + return stepList(in) + } + return nil, fmt.Errorf("recipeInstructions is a %T", v) +} + +func stepList(items []any) ([]string, error) { + var out []string + for _, item := range items { + switch s := item.(type) { + case string: + if text := cleanText(s); text != "" { + out = append(out, text) + } + case map[string]any: + switch { + case hasType(s, "HowToStep"): + text, _ := s["text"].(string) + if text = cleanText(text); text == "" { + return nil, errors.New("HowToStep has no text") + } + out = append(out, text) + case hasType(s, "HowToSection"): + list, ok := s["itemListElement"].([]any) + if !ok { + return nil, errors.New("HowToSection has no itemListElement list") + } + section, err := stepList(list) + if err != nil { + return nil, err + } + out = append(out, section...) + default: + return nil, fmt.Errorf("recipeInstructions holds a %v, not a HowToStep or HowToSection", s["@type"]) + } + default: + return nil, fmt.Errorf("recipeInstructions holds a %T", item) + } + } + return out, nil +} + +func minutes(n map[string]any, field string) (*int, error) { + v, ok := n[field] + if !ok || v == nil { + return nil, nil + } + s, ok := v.(string) + if !ok { + return nil, fmt.Errorf("%s is a %T, not an ISO-8601 duration", field, v) + } + if strings.TrimSpace(s) == "" { + return nil, nil + } + m, err := isoMinutes(s) + if err != nil { + return nil, fmt.Errorf("%s: %w", field, err) + } + return &m, nil +} + +// isoMinutes converts an ISO-8601 duration of weeks, days, hours, minutes +// and seconds to whole minutes, rounded. Years and months are refused: +// their length in minutes is not fixed. +func isoMinutes(s string) (int, error) { + bad := fmt.Errorf("%q is not an ISO-8601 duration of days, hours, minutes or seconds", s) + rest, ok := strings.CutPrefix(strings.TrimSpace(s), "P") + if !ok || rest == "" { + return 0, bad + } + const units = "WDHMS" + seconds := map[byte]float64{'W': 7 * 86400, 'D': 86400, 'H': 3600, 'M': 60, 'S': 1} + inTime, last, total := false, -1, 0.0 + for rest != "" { + if rest[0] == 'T' { + if inTime || len(rest) == 1 { + return 0, bad + } + inTime, rest = true, rest[1:] + continue + } + i := 0 + for i < len(rest) && (rest[i] >= '0' && rest[i] <= '9' || rest[i] == '.' || rest[i] == ',') { + i++ + } + if i == 0 || i == len(rest) { + return 0, bad + } + num, err := strconv.ParseFloat(strings.Replace(rest[:i], ",", ".", 1), 64) + if err != nil { + return 0, bad + } + unit := rest[i] + pos := strings.IndexByte(units, unit) + if pos < 0 || pos <= last || inTime != (pos >= 2) { + return 0, bad + } + last, total, rest = pos, total+num*seconds[unit], rest[i+1:] + } + if last < 0 { + return 0, bad + } + return int(math.Round(total / 60)), nil +} + +func yield(v any) (string, error) { + if list, ok := v.([]any); ok { + if len(list) == 0 { + return "", nil + } + v = list[0] + } + switch y := v.(type) { + case nil: + return "", nil + case string: + return cleanText(y), nil + case json.Number: + return y.String(), nil + } + return "", fmt.Errorf("recipeYield is a %T", v) +} + +// htmlDoc is what Extract needs from a page's HTML. +type htmlDoc struct { + jsonLD []string + ogSiteName string +} + +// scanHTML is a small targeted scan, not a full HTML parser: it walks the +// tags, skipping comments and the bodies of scripts and styles, and keeps +// the bodies of application/ld+json scripts and the first og:site_name. +func scanHTML(doc string) htmlDoc { + var out htmlDoc + lower := asciiLower(doc) + for pos := 0; ; { + lt := strings.IndexByte(doc[pos:], '<') + if lt < 0 { + return out + } + pos += lt + if strings.HasPrefix(doc[pos:], "") + if end < 0 { + return out + } + pos += 4 + end + 3 + continue + } + name := tagName(lower[pos+1:]) + if name == "" { + pos++ + continue + } + end := tagEnd(doc, pos+1+len(name)) + if end < 0 { + return out + } + attrs := parseAttrs(doc[pos+1+len(name) : end]) + pos = end + 1 + switch name { + case "meta": + if out.ogSiteName == "" && attrs["property"] == "og:site_name" { + out.ogSiteName = strings.TrimSpace(attrs["content"]) + } + case "script", "style": + closeAt := strings.Index(lower[pos:], " 0 && '0' <= s[i] && s[i] <= '9') { + i++ + } + if i == 0 || i < len(s) && !isSpace(s[i]) && s[i] != '>' && s[i] != '/' { + return "" + } + return s[:i] +} + +// tagEnd finds the '>' closing a tag whose attributes start at i, skipping +// quoted attribute values. +func tagEnd(doc string, i int) int { + var quote byte + for ; i < len(doc); i++ { + switch c := doc[i]; { + case quote != 0: + if c == quote { + quote = 0 + } + case c == '"' || c == '\'': + quote = c + case c == '>': + return i + } + } + return -1 +} + +// parseAttrs reads a tag's attributes; names are lower-cased, values are +// entity-decoded, and the first of a repeated name wins. +func parseAttrs(s string) map[string]string { + attrs := map[string]string{} + i := 0 + for i < len(s) { + for i < len(s) && (isSpace(s[i]) || s[i] == '/') { + i++ + } + start := i + for i < len(s) && !isSpace(s[i]) && s[i] != '=' && s[i] != '/' { + i++ + } + if i == start { + i++ + continue + } + name := asciiLower(s[start:i]) + for i < len(s) && isSpace(s[i]) { + i++ + } + value := "" + if i < len(s) && s[i] == '=' { + i++ + for i < len(s) && isSpace(s[i]) { + i++ + } + if i < len(s) && (s[i] == '"' || s[i] == '\'') { + q := s[i] + end := strings.IndexByte(s[i+1:], q) + if end < 0 { + end = len(s) - i - 1 + } + value = s[i+1 : i+1+end] + i += end + 2 + } else { + start := i + for i < len(s) && !isSpace(s[i]) { + i++ + } + value = s[start:i] + } + } + if _, seen := attrs[name]; !seen { + attrs[name] = html.UnescapeString(value) + } + } + return attrs +} + +func isSpace(c byte) bool { + return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' +} diff --git a/internal/recipes/extract_test.go b/internal/recipes/extract_test.go new file mode 100644 index 0000000..4d593e5 --- /dev/null +++ b/internal/recipes/extract_test.go @@ -0,0 +1,347 @@ +package recipes + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +func readPage(t *testing.T, name string) []byte { + t.Helper() + b, err := os.ReadFile(filepath.Join("testdata", name)) + if err != nil { + t.Fatal(err) + } + return b +} + +func intp(n int) *int { return &n } + +func fmtMinutes(m *int) string { + if m == nil { + return "nil" + } + b, _ := json.Marshal(*m) + return string(b) +} + +func TestExtractRealPages(t *testing.T) { + tests := []struct { + file, url string + title, site, image, yld string + ingredients, steps int + prep, cook, total *int + firstStep string + }{ + { + // A Recipe script beside VideoObject and BreadcrumbList scripts; + // image is an array of one sized ImageObject. + file: "bbcgoodfood.html", url: "https://www.bbcgoodfood.com/recipes/easy-chicken-curry", + title: "Easy chicken curry", site: "Good Food", + image: "https://images.immediate.co.uk/production/volatile/sites/30/2020/08/easy-chicken-curry-aa22a0b.jpg?resize=440,400", + ingredients: 11, steps: 4, prep: intp(5), cook: intp(45), total: intp(50), yld: "4", + firstStep: "Heat the oil in a flameproof casserole dish", + }, + { + // Yoast @graph; image is a list of bare URLs, the first the + // full-size one; no publisher, so og:site_name names the site. + file: "budgetbytes.html", url: "https://www.budgetbytes.com/one-pot-creamy-cajun-chicken-pasta/", + title: "One Pot Creamy Cajun Chicken Pasta", site: "Budget Bytes", + image: "https://www.budgetbytes.com/wp-content/uploads/2018/10/One-Pot-Creamy-Cajun-Chicken-Pasta-pan.jpg", + ingredients: 17, steps: 6, prep: intp(10), cook: intp(20), total: intp(30), yld: "4", + firstStep: "Combine the ingredients for the Cajun seasoning", + }, + { + // HowToSection groups (ragu, cheese sauce, assembly), flattened. + file: "recipetineats.html", url: "https://www.recipetineats.com/lasagna/", + title: "Lasagna", site: "RecipeTin Eats", + image: "https://www.recipetineats.com/tachyon/2017/05/Lasagne-recipe-3-main-square.jpg", + ingredients: 24, steps: 18, prep: intp(20), cook: intp(240), total: intp(260), yld: "10", + firstStep: "Heat oil in a large heavy based pot", + }, + { + // @graph of one node; a single ImageObject; instructions as a + // list of plain strings; hours and minutes. + file: "kingarthur.html", url: "https://www.kingarthurbaking.com/recipes/classic-sandwich-bread-recipe", + title: "Classic Sandwich Bread", site: "King Arthur Baking", + image: "https://www.kingarthurbaking.com/sites/default/files/2026-02/Classic-Sandwich-Bread-12.jpg", + ingredients: 7, steps: 7, prep: intp(12), cook: intp(40), total: intp(162), yld: "16", + firstStep: "Weigh your flour", + }, + { + // A top-level array; ImageObjects of different sizes. + file: "delish.html", url: "https://www.delish.com/cooking/recipe-ideas/a19636089/creamy-tuscan-chicken-recipe/", + title: "Creamy Tuscan Chicken", site: "Delish", + image: "https://hips.hearstapps.com/hmg-prod/images/1d09a24f-5b76-49f4-9ad6-496acdc03933.jpg?crop=0.502xw:1.00xh;0.250xw,0&resize=1200:*", + ingredients: 12, steps: 4, prep: intp(10), cook: intp(0), total: intp(45), yld: "4 serving(s)", + firstStep: "In a large skillet over medium heat, heat oil.", + }, + { + // A bare Recipe object; image is one URL; HTML entities in steps. + file: "foodcom.html", url: "https://www.food.com/recipe/best-banana-bread-2886", + title: "Best Banana Bread", site: "Food.com", + image: "https://img.sndimg.com/food/image/upload/q_92,fl_progressive,w_1200,c_scale/v1/img/recipes/28/86/wBZxNua1T8yfDwbfo9Fz_0S9A9315.jpg", + ingredients: 8, steps: 10, prep: intp(10), cook: intp(60), total: intp(70), yld: "1 loaf, 10 serving(s)", + firstStep: "Remove odd pots and pans from oven.", + }, + } + for _, tt := range tests { + t.Run(tt.file, func(t *testing.T) { + r, err := Extract(readPage(t, tt.file), tt.url) + if err != nil { + t.Fatalf("Extract: %v", err) + } + if r.Title != tt.title { + t.Errorf("Title = %q, want %q", r.Title, tt.title) + } + if r.SiteName != tt.site { + t.Errorf("SiteName = %q, want %q", r.SiteName, tt.site) + } + if r.ImageURL != tt.image { + t.Errorf("ImageURL = %q, want %q", r.ImageURL, tt.image) + } + if len(r.Ingredients) != tt.ingredients { + t.Errorf("%d ingredients, want %d", len(r.Ingredients), tt.ingredients) + } + if len(r.Steps) != tt.steps { + t.Errorf("%d steps, want %d", len(r.Steps), tt.steps) + } + if len(r.Steps) > 0 && !strings.HasPrefix(r.Steps[0], tt.firstStep) { + t.Errorf("first step = %q, want prefix %q", r.Steps[0], tt.firstStep) + } + for name, got := range map[string][2]*int{ + "prep": {r.PrepMinutes, tt.prep}, "cook": {r.CookMinutes, tt.cook}, "total": {r.TotalMinutes, tt.total}, + } { + if fmtMinutes(got[0]) != fmtMinutes(got[1]) { + t.Errorf("%s minutes = %s, want %s", name, fmtMinutes(got[0]), fmtMinutes(got[1])) + } + } + if r.Yield != tt.yld { + t.Errorf("Yield = %q, want %q", r.Yield, tt.yld) + } + var node map[string]any + if err := json.Unmarshal(r.JSONLD, &node); err != nil || node["name"] == nil { + t.Errorf("JSONLD is not the Recipe node: %v", err) + } + }) + } +} + +func TestExtractHowToSectionOrder(t *testing.T) { + r, err := Extract(readPage(t, "recipetineats.html"), "https://www.recipetineats.com/lasagna/") + if err != nil { + t.Fatal(err) + } + // Ragu (5 steps), Cheese Sauce (5), Assemble (8): each section's first + // step lands right after the previous section's last. + for i, prefix := range map[int]string{ + 0: "Heat oil in a large heavy based pot", + 4: "The ragu is ready when the meat", + 5: "Warm milk up in a saucepan", + 10: "Preheat oven to 180°C/350°F.", + 17: "Stand for 5 to 10 minutes", + } { + if !strings.HasPrefix(r.Steps[i], prefix) { + t.Errorf("step %d = %q, want prefix %q", i+1, r.Steps[i], prefix) + } + } +} + +func TestExtractNoRecipe(t *testing.T) { + _, err := Extract(readPage(t, "budgetbytes-about.html"), "https://www.budgetbytes.com/about/") + if !errors.Is(err, ErrNoRecipe) { + t.Fatalf("err = %v, want ErrNoRecipe", err) + } + _, err = Extract([]byte("x"), "https://example.com/") + if !errors.Is(err, ErrNoRecipe) { + t.Fatalf("page without JSON-LD: err = %v, want ErrNoRecipe", err) + } +} + +func TestExtractRefusesNonISODuration(t *testing.T) { + // Bon Appétit writes cookTime as "20 minutes". + _, err := Extract(readPage(t, "bonappetit.html"), "https://www.bonappetit.com/recipe/bas-best-chocolate-chip-cookies") + if !errors.Is(err, ErrMalformedRecipe) || !strings.Contains(err.Error(), "cookTime") { + t.Fatalf("err = %v, want ErrMalformedRecipe naming cookTime", err) + } +} + +// page wraps JSON-LD scripts in a minimal HTML page. +func page(head string, jsonLD ...string) []byte { + var b strings.Builder + b.WriteString("" + head) + for _, s := range jsonLD { + b.WriteString(`") + } + b.WriteString("") + return []byte(b.String()) +} + +func TestExtractShapes(t *testing.T) { + const pageURL = "https://www.example.com/recipes/stew/" + tests := []struct { + name string + html []byte + check func(t *testing.T, r Recipe) + }{ + { + name: "type array and relative image", + html: page("", `{"@type":["Recipe","NewsArticle"],"name":"Stew","image":"/img/stew.jpg"}`), + check: func(t *testing.T, r Recipe) { + if r.ImageURL != "https://www.example.com/img/stew.jpg" { + t.Errorf("ImageURL = %q", r.ImageURL) + } + }, + }, + { + name: "largest image wins regardless of order", + html: page("", `{"@type":"Recipe","name":"Stew","image":[ + "https://cdn.example.com/plain.jpg", + {"@type":"ImageObject","url":"small.jpg","width":300,"height":200}, + {"@type":"ImageObject","contentUrl":"../big.jpg","width":"1600","height":"900"}, + {"@type":"ImageObject","url":"mid.jpg","width":1200,"height":800}]}`), + check: func(t *testing.T, r Recipe) { + if r.ImageURL != "https://www.example.com/recipes/big.jpg" { + t.Errorf("ImageURL = %q", r.ImageURL) + } + }, + }, + { + name: "instructions as one string", + html: page("", `{"@type":"Recipe","name":"Stew","recipeInstructions":"Brown the beef.\r\n\r\nAdd the stock & simmer.\nServe."}`), + check: func(t *testing.T, r Recipe) { + want := []string{"Brown the beef.", "Add the stock & simmer.", "Serve."} + if strings.Join(r.Steps, "|") != strings.Join(want, "|") { + t.Errorf("Steps = %q, want %q", r.Steps, want) + } + }, + }, + { + name: "sections and steps flattened in order", + html: page("", `{"@type":"Recipe","name":"Stew","recipeInstructions":[ + {"@type":"HowToSection","name":"Beef","itemListElement":[ + {"@type":"HowToStep","text":"one"},{"@type":"HowToStep","text":"two"}]}, + {"@type":"HowToStep","text":"three"}, + {"@type":"HowToSection","name":"Finish","itemListElement":["four"]}]}`), + check: func(t *testing.T, r Recipe) { + if got := strings.Join(r.Steps, " "); got != "one two three four" { + t.Errorf("Steps = %q", got) + } + }, + }, + { + name: "total derived from prep and cook", + html: page("", `{"@type":"Recipe","name":"Stew","prepTime":"PT15M","cookTime":"PT1H30M"}`), + check: func(t *testing.T, r Recipe) { + if fmtMinutes(r.TotalMinutes) != "105" { + t.Errorf("TotalMinutes = %s, want 105", fmtMinutes(r.TotalMinutes)) + } + }, + }, + { + name: "total not derived from prep alone", + html: page("", `{"@type":"Recipe","name":"Stew","prepTime":"PT15M"}`), + check: func(t *testing.T, r Recipe) { + if r.TotalMinutes != nil || r.CookMinutes != nil { + t.Errorf("total %s, cook %s; want nil", fmtMinutes(r.TotalMinutes), fmtMinutes(r.CookMinutes)) + } + }, + }, + { + name: "site name from host without publisher or og:site_name", + html: page("", `{"@type":"Recipe","name":"Stew"}`), + check: func(t *testing.T, r Recipe) { + if r.SiteName != "example.com" { + t.Errorf("SiteName = %q", r.SiteName) + } + }, + }, + { + name: "og:site_name with attributes in any order", + html: page(``, `{"@type":"Recipe","name":"Stew","publisher":{"@id":"#org"}}`), + check: func(t *testing.T, r Recipe) { + if r.SiteName != "Our & Kitchen" { + t.Errorf("SiteName = %q", r.SiteName) + } + }, + }, + { + name: "numeric yield and recipe in @graph beside others", + html: page("", `{"@context":"https://schema.org","@graph":[{"@type":"WebPage"},{"@type":"Recipe","name":"Stew","recipeYield":6}]}`), + check: func(t *testing.T, r Recipe) { + if r.Yield != "6" { + t.Errorf("Yield = %q", r.Yield) + } + }, + }, + { + name: "JSON-LD text inside comments and other scripts is ignored", + html: []byte(`` + + `` + + ``), + check: func(t *testing.T, r Recipe) { + if r.Title != "Stew" { + t.Errorf("Title = %q", r.Title) + } + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r, err := Extract(tt.html, pageURL) + if err != nil { + t.Fatalf("Extract: %v", err) + } + tt.check(t, r) + }) + } +} + +func TestExtractErrors(t *testing.T) { + tests := []struct { + name string + html []byte + url string + want error + }{ + {"two recipes", page("", `{"@type":"Recipe","name":"A"}`, `{"@type":"Recipe","name":"B"}`), "https://example.com/", ErrManyRecipes}, + {"invalid JSON", page("", `{"@type":"Recipe",`), "https://example.com/", ErrMalformedRecipe}, + {"no name", page("", `{"@type":"Recipe"}`), "https://example.com/", ErrMalformedRecipe}, + {"bad step type", page("", `{"@type":"Recipe","name":"A","recipeInstructions":[{"@type":"Thing"}]}`), "https://example.com/", ErrMalformedRecipe}, + {"step without text", page("", `{"@type":"Recipe","name":"A","recipeInstructions":[{"@type":"HowToStep","name":"x"}]}`), "https://example.com/", ErrMalformedRecipe}, + {"bad image width", page("", `{"@type":"Recipe","name":"A","image":{"url":"a.jpg","width":"wide"}}`), "https://example.com/", ErrMalformedRecipe}, + {"non-text ingredient", page("", `{"@type":"Recipe","name":"A","recipeIngredient":[{"name":"salt"}]}`), "https://example.com/", ErrMalformedRecipe}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := Extract(tt.html, tt.url) + if !errors.Is(err, tt.want) { + t.Fatalf("err = %v, want %v", err, tt.want) + } + }) + } + if _, err := Extract(page("", `{"@type":"Recipe","name":"A"}`), "/relative"); err == nil { + t.Fatal("relative page URL accepted") + } +} + +func TestISOMinutes(t *testing.T) { + good := map[string]int{ + "PT0S": 0, "PT45M": 45, "PT1H": 60, "PT2H42M": 162, "PT240M": 240, "P1D": 1440, + "P1DT2H": 1560, "PT90S": 2, "PT1.5H": 90, "PT0,5H": 30, "P1W": 10080, " PT5M ": 5, + } + for in, want := range good { + got, err := isoMinutes(in) + if err != nil || got != want { + t.Errorf("isoMinutes(%q) = %d, %v; want %d", in, got, err, want) + } + } + for _, in := range []string{"", "P", "PT", "20 minutes", "PT5", "P1Y", "P1M", "PT1D", "P5H", "PT5M1H", "PT5M5M", "P1DT", "PTxM", "PT1..5M"} { + if got, err := isoMinutes(in); err == nil { + t.Errorf("isoMinutes(%q) = %d, want error", in, got) + } + } +} diff --git a/internal/recipes/testdata/bbcgoodfood.html b/internal/recipes/testdata/bbcgoodfood.html new file mode 100644 index 0000000..47f911e --- /dev/null +++ b/internal/recipes/testdata/bbcgoodfood.html @@ -0,0 +1,13 @@ + + + +Easy chicken curry recipe | Good Food + + + + + + + + +
(page body trimmed)
diff --git a/internal/recipes/testdata/bonappetit.html b/internal/recipes/testdata/bonappetit.html new file mode 100644 index 0000000..0fd897a --- /dev/null +++ b/internal/recipes/testdata/bonappetit.html @@ -0,0 +1,10 @@ + + + +BA’s Best Chocolate Chip Cookies Recipe | Bon Appétit + + + + + +
(page body trimmed)
diff --git a/internal/recipes/testdata/budgetbytes-about.html b/internal/recipes/testdata/budgetbytes-about.html new file mode 100644 index 0000000..9ffda42 --- /dev/null +++ b/internal/recipes/testdata/budgetbytes-about.html @@ -0,0 +1,10 @@ + + + +The Budget Bytes Story - BudgetBytes.com + + + + + +
(page body trimmed)
diff --git a/internal/recipes/testdata/budgetbytes.html b/internal/recipes/testdata/budgetbytes.html new file mode 100644 index 0000000..248815a --- /dev/null +++ b/internal/recipes/testdata/budgetbytes.html @@ -0,0 +1,10 @@ + + + +Creamy Cajun Chicken Pasta (with Video) - Budget Bytes + + + + + +
(page body trimmed)
diff --git a/internal/recipes/testdata/delish.html b/internal/recipes/testdata/delish.html new file mode 100644 index 0000000..65711b3 --- /dev/null +++ b/internal/recipes/testdata/delish.html @@ -0,0 +1,10 @@ + + + +Best Creamy Tuscan Chicken Recipe + + + + + +
(page body trimmed)
diff --git a/internal/recipes/testdata/foodcom.html b/internal/recipes/testdata/foodcom.html new file mode 100644 index 0000000..fb2757d --- /dev/null +++ b/internal/recipes/testdata/foodcom.html @@ -0,0 +1,9 @@ + + + +Best Banana Bread Recipe - Food.com + + + + +
(page body trimmed)
diff --git a/internal/recipes/testdata/kingarthur.html b/internal/recipes/testdata/kingarthur.html new file mode 100644 index 0000000..aae9a22 --- /dev/null +++ b/internal/recipes/testdata/kingarthur.html @@ -0,0 +1,79 @@ + + + +Classic Sandwich Bread Recipe | King Arthur Baking + + + + + +
(page body trimmed)
diff --git a/internal/recipes/testdata/recipetineats.html b/internal/recipes/testdata/recipetineats.html new file mode 100644 index 0000000..0e657b0 --- /dev/null +++ b/internal/recipes/testdata/recipetineats.html @@ -0,0 +1,11 @@ + + + +Lasagna! - RecipeTin Eats + + + + + + +
(page body trimmed)