Co-authored-by: dev_agent <dev_agent@cubecraftlabs.com>
This commit was merged in pull request #66.
This commit is contained in:
+16
-1
@@ -223,7 +223,22 @@ automatic** (D-021).
|
||||
1 s apart, counted from requests actually sent (an abandoned wait holds no
|
||||
slot); 2xx pages cached by URL for an hour (64 pages, oldest evicted), and
|
||||
callers get their own copy of a cached body.
|
||||
- Recipe data is parsed from schema.org `Recipe` JSON-LD.
|
||||
- Recipe data is parsed from schema.org `Recipe` JSON-LD
|
||||
(`recipes.Extract`): a small targeted scan of the page's
|
||||
`application/ld+json` scripts and `og:site_name` (no HTML parser
|
||||
dependency), finding the Recipe node at a script's top level, in a
|
||||
top-level array or in an `@graph`, with `@type` a string or an array of
|
||||
terms or absolute schema.org IRIs (`https://schema.org/Recipe`). An image
|
||||
or publisher given as an `{"@id": ...}` reference resolves to the node
|
||||
with that `@id` anywhere in the page's JSON-LD. The largest image (by ImageObject width × height; the first of equals) is
|
||||
resolved to an absolute URL; instructions — one string of lines, or a
|
||||
list of strings, HowToSteps and HowToSections — are flattened in order;
|
||||
ISO-8601 prep/cook/total times become minutes, total derived as prep +
|
||||
cook when only those are given; the site name is the publisher's, else
|
||||
`og:site_name`, else the host. A page with no Recipe is `ErrNoRecipe`,
|
||||
with two is `ErrManyRecipes`; a shape schema.org does not allow — such as
|
||||
a non-ISO-8601 duration like "20 minutes" or an empty one, or an image
|
||||
`@id` no node has — is `ErrMalformedRecipe`, never guessed at.
|
||||
- Photos are cached locally (`photos`) and processed with
|
||||
`golang.org/x/image`.
|
||||
|
||||
|
||||
@@ -0,0 +1,631 @@
|
||||
package recipes
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"html"
|
||||
"io"
|
||||
"math"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrNoRecipe is returned for a page with no schema.org Recipe node in
|
||||
// its JSON-LD.
|
||||
ErrNoRecipe = errors.New("recipes: page has no schema.org Recipe")
|
||||
// ErrManyRecipes is returned for a page with more than one Recipe node:
|
||||
// which one the page is about is not knowable.
|
||||
ErrManyRecipes = errors.New("recipes: page has more than one schema.org Recipe")
|
||||
// ErrMalformedRecipe is returned when the page's JSON-LD, or a field of
|
||||
// its Recipe node, is not in a shape schema.org allows.
|
||||
ErrMalformedRecipe = errors.New("recipes: malformed schema.org Recipe")
|
||||
)
|
||||
|
||||
// Recipe is the schema.org Recipe extracted from a recipe page.
|
||||
type Recipe struct {
|
||||
// Title is the recipe's name.
|
||||
Title string
|
||||
// SiteName is the publisher's name, else the page's og:site_name, else
|
||||
// its host without a leading "www.".
|
||||
SiteName string
|
||||
// ImageURL is the largest image, absolute; empty when the recipe has
|
||||
// none.
|
||||
ImageURL string
|
||||
Ingredients []string
|
||||
// Steps are the instructions in order, HowToSection groups flattened.
|
||||
Steps []string
|
||||
// PrepMinutes, CookMinutes and TotalMinutes are nil when not given.
|
||||
// TotalMinutes is prep plus cook when the page gives both but no total.
|
||||
PrepMinutes *int
|
||||
CookMinutes *int
|
||||
TotalMinutes *int
|
||||
// Yield is recipeYield as text; empty when not given.
|
||||
Yield string
|
||||
// JSONLD is the Recipe node as found.
|
||||
JSONLD json.RawMessage
|
||||
}
|
||||
|
||||
// Extract finds the schema.org Recipe in a page's application/ld+json
|
||||
// scripts. Recipe nodes are found at the top level of a script, in a
|
||||
// top-level array, or in an @graph. An image or publisher given as an
|
||||
// {"@id": ...} reference resolves to the node with that @id found in the
|
||||
// same places, in any of the page's scripts. pageURL is the page's own
|
||||
// absolute URL; relative image URLs resolve against it.
|
||||
func Extract(page []byte, pageURL string) (Recipe, error) {
|
||||
base, err := url.Parse(pageURL)
|
||||
if err != nil || !base.IsAbs() || base.Host == "" {
|
||||
return Recipe{}, fmt.Errorf("recipes: page URL %q is not absolute", pageURL)
|
||||
}
|
||||
doc := scanHTML(string(page))
|
||||
g := graph{ids: map[string][]map[string]any{}}
|
||||
for i, script := range doc.jsonLD {
|
||||
v, err := decodeJSON(script)
|
||||
if err != nil {
|
||||
return Recipe{}, fmt.Errorf("%w: JSON-LD script %d: %v", ErrMalformedRecipe, i+1, err)
|
||||
}
|
||||
g.collect(v)
|
||||
}
|
||||
nodes := g.recipes
|
||||
switch len(nodes) {
|
||||
case 0:
|
||||
return Recipe{}, ErrNoRecipe
|
||||
case 1:
|
||||
default:
|
||||
return Recipe{}, fmt.Errorf("%w: found %d", ErrManyRecipes, len(nodes))
|
||||
}
|
||||
r, err := recipeFrom(nodes[0], g, base, doc.ogSiteName)
|
||||
if err != nil {
|
||||
return Recipe{}, fmt.Errorf("%w: %v", ErrMalformedRecipe, err)
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
func decodeJSON(s string) (any, error) {
|
||||
dec := json.NewDecoder(strings.NewReader(s))
|
||||
dec.UseNumber()
|
||||
var v any
|
||||
if err := dec.Decode(&v); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := dec.Token(); err != io.EOF {
|
||||
return nil, errors.New("trailing data after JSON value")
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// graph holds the page's JSON-LD nodes: the Recipe nodes, and every node
|
||||
// with an @id by that @id.
|
||||
type graph struct {
|
||||
recipes []map[string]any
|
||||
ids map[string][]map[string]any
|
||||
}
|
||||
|
||||
// collect gathers the nodes in v: v itself, the elements of an array, and
|
||||
// the members of an @graph. A bare {"@id": ...} reference is not a node.
|
||||
func (g *graph) collect(v any) {
|
||||
switch n := v.(type) {
|
||||
case []any:
|
||||
for _, e := range n {
|
||||
g.collect(e)
|
||||
}
|
||||
case map[string]any:
|
||||
if hasType(n, "Recipe") {
|
||||
g.recipes = append(g.recipes, n)
|
||||
}
|
||||
if id, ok := n["@id"].(string); ok && len(n) > 1 {
|
||||
g.ids[id] = append(g.ids[id], n)
|
||||
}
|
||||
if members, ok := n["@graph"]; ok {
|
||||
g.collect(members)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// resolve returns the node a bare {"@id": ...} reference names, and v
|
||||
// itself when v is not a reference. found is false for a reference to no
|
||||
// node; a reference to more than one node is an error.
|
||||
func (g graph) resolve(v any) (node any, found bool, err error) {
|
||||
m, ok := v.(map[string]any)
|
||||
if !ok || len(m) != 1 {
|
||||
return v, true, nil
|
||||
}
|
||||
id, ok := m["@id"].(string)
|
||||
if !ok {
|
||||
return v, true, nil
|
||||
}
|
||||
switch nodes := g.ids[id]; len(nodes) {
|
||||
case 0:
|
||||
return v, false, nil
|
||||
case 1:
|
||||
return nodes[0], true, nil
|
||||
default:
|
||||
return nil, false, fmt.Errorf("@id %q names %d nodes", id, len(nodes))
|
||||
}
|
||||
}
|
||||
|
||||
// hasType reports whether node's @type, a string or an array of strings,
|
||||
// names the schema.org type typ, as a term or an absolute schema.org IRI.
|
||||
func hasType(node map[string]any, typ string) bool {
|
||||
switch t := node["@type"].(type) {
|
||||
case string:
|
||||
return isSchemaType(t, typ)
|
||||
case []any:
|
||||
for _, e := range t {
|
||||
if s, ok := e.(string); ok && isSchemaType(s, typ) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isSchemaType(s, typ string) bool {
|
||||
return s == typ || s == "https://schema.org/"+typ || s == "http://schema.org/"+typ
|
||||
}
|
||||
|
||||
func recipeFrom(n map[string]any, g graph, base *url.URL, ogSiteName string) (Recipe, error) {
|
||||
var r Recipe
|
||||
var err error
|
||||
name, ok := n["name"].(string)
|
||||
if r.Title = cleanText(name); !ok || r.Title == "" {
|
||||
return Recipe{}, errors.New("name is missing")
|
||||
}
|
||||
if r.SiteName, err = siteName(n["publisher"], g, ogSiteName, base); err != nil {
|
||||
return Recipe{}, err
|
||||
}
|
||||
if r.ImageURL, err = imageURL(n["image"], g, base); err != nil {
|
||||
return Recipe{}, err
|
||||
}
|
||||
if r.Ingredients, err = ingredients(n["recipeIngredient"]); err != nil {
|
||||
return Recipe{}, err
|
||||
}
|
||||
if r.Steps, err = steps(n["recipeInstructions"]); err != nil {
|
||||
return Recipe{}, err
|
||||
}
|
||||
if r.PrepMinutes, err = minutes(n, "prepTime"); err != nil {
|
||||
return Recipe{}, err
|
||||
}
|
||||
if r.CookMinutes, err = minutes(n, "cookTime"); err != nil {
|
||||
return Recipe{}, err
|
||||
}
|
||||
if r.TotalMinutes, err = minutes(n, "totalTime"); err != nil {
|
||||
return Recipe{}, err
|
||||
}
|
||||
if r.TotalMinutes == nil && r.PrepMinutes != nil && r.CookMinutes != nil {
|
||||
total := *r.PrepMinutes + *r.CookMinutes
|
||||
r.TotalMinutes = &total
|
||||
}
|
||||
if r.Yield, err = yield(n["recipeYield"]); err != nil {
|
||||
return Recipe{}, err
|
||||
}
|
||||
if r.JSONLD, err = json.Marshal(n); err != nil {
|
||||
return Recipe{}, err
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// cleanText decodes HTML entities, which recipe plugins often leave in
|
||||
// JSON-LD text, and trims surrounding space.
|
||||
func cleanText(s string) string {
|
||||
return strings.TrimSpace(html.UnescapeString(s))
|
||||
}
|
||||
|
||||
func siteName(publisher any, g graph, ogSiteName string, base *url.URL) (string, error) {
|
||||
pubs, ok := publisher.([]any)
|
||||
if !ok {
|
||||
pubs = []any{publisher}
|
||||
}
|
||||
for _, p := range pubs {
|
||||
p, _, err := g.resolve(p)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("publisher: %w", err)
|
||||
}
|
||||
if m, ok := p.(map[string]any); ok {
|
||||
if name, ok := m["name"].(string); ok && cleanText(name) != "" {
|
||||
return cleanText(name), nil
|
||||
}
|
||||
}
|
||||
}
|
||||
if ogSiteName != "" {
|
||||
return ogSiteName, nil
|
||||
}
|
||||
return strings.TrimPrefix(base.Hostname(), "www."), nil
|
||||
}
|
||||
|
||||
// imageURL picks the largest of the recipe's images by width × height;
|
||||
// images without both dimensions count as size zero, and the first wins a
|
||||
// tie, so a list of bare URLs gives its first.
|
||||
func imageURL(v any, g graph, base *url.URL) (string, error) {
|
||||
items, ok := v.([]any)
|
||||
if !ok {
|
||||
if v == nil {
|
||||
return "", nil
|
||||
}
|
||||
items = []any{v}
|
||||
}
|
||||
best, bestArea := "", int64(-1)
|
||||
for _, item := range items {
|
||||
item, found, err := g.resolve(item)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("image: %w", err)
|
||||
}
|
||||
if !found {
|
||||
return "", fmt.Errorf("image references @id %v, which no node on the page has", item.(map[string]any)["@id"])
|
||||
}
|
||||
var u string
|
||||
var area int64
|
||||
switch img := item.(type) {
|
||||
case string:
|
||||
u = img
|
||||
case map[string]any:
|
||||
if s, ok := img["url"].(string); ok {
|
||||
u = s
|
||||
} else if s, ok := img["contentUrl"].(string); ok {
|
||||
u = s
|
||||
}
|
||||
w, err := dimension(img["width"])
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("image width: %w", err)
|
||||
}
|
||||
h, err := dimension(img["height"])
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("image height: %w", err)
|
||||
}
|
||||
area = w * h
|
||||
default:
|
||||
return "", fmt.Errorf("image is a %T, not a URL or ImageObject", item)
|
||||
}
|
||||
u = strings.TrimSpace(u)
|
||||
if u == "" {
|
||||
return "", errors.New("image has no URL")
|
||||
}
|
||||
if area > bestArea {
|
||||
best, bestArea = u, area
|
||||
}
|
||||
}
|
||||
if best == "" {
|
||||
return "", nil
|
||||
}
|
||||
ref, err := url.Parse(best)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("image URL %q: %w", best, err)
|
||||
}
|
||||
return base.ResolveReference(ref).String(), nil
|
||||
}
|
||||
|
||||
// dimension reads an ImageObject width or height: a number or a string of
|
||||
// digits; zero when absent.
|
||||
func dimension(v any) (int64, error) {
|
||||
switch d := v.(type) {
|
||||
case nil:
|
||||
return 0, nil
|
||||
case json.Number:
|
||||
f, err := d.Float64()
|
||||
if err != nil || f < 0 {
|
||||
return 0, fmt.Errorf("%q is not a size", d)
|
||||
}
|
||||
return int64(f), nil
|
||||
case string:
|
||||
n, err := strconv.ParseInt(strings.TrimSpace(d), 10, 64)
|
||||
if err != nil || n < 0 {
|
||||
return 0, fmt.Errorf("%q is not a size", d)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
return 0, fmt.Errorf("a %T is not a size", v)
|
||||
}
|
||||
|
||||
func ingredients(v any) ([]string, error) {
|
||||
items, ok := v.([]any)
|
||||
if !ok {
|
||||
if v == nil {
|
||||
return nil, nil
|
||||
}
|
||||
items = []any{v}
|
||||
}
|
||||
var out []string
|
||||
for _, item := range items {
|
||||
s, ok := item.(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("recipeIngredient holds a %T, not text", item)
|
||||
}
|
||||
if s = cleanText(s); s != "" {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// steps flattens recipeInstructions: one string of lines, or a list of
|
||||
// strings, HowToSteps and HowToSections of those, in order.
|
||||
func steps(v any) ([]string, error) {
|
||||
switch in := v.(type) {
|
||||
case nil:
|
||||
return nil, nil
|
||||
case string:
|
||||
var out []string
|
||||
for _, line := range strings.Split(in, "\n") {
|
||||
if line = cleanText(line); line != "" {
|
||||
out = append(out, line)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
case map[string]any:
|
||||
return stepList([]any{in})
|
||||
case []any:
|
||||
return stepList(in)
|
||||
}
|
||||
return nil, fmt.Errorf("recipeInstructions is a %T", v)
|
||||
}
|
||||
|
||||
func stepList(items []any) ([]string, error) {
|
||||
var out []string
|
||||
for _, item := range items {
|
||||
switch s := item.(type) {
|
||||
case string:
|
||||
if text := cleanText(s); text != "" {
|
||||
out = append(out, text)
|
||||
}
|
||||
case map[string]any:
|
||||
switch {
|
||||
case hasType(s, "HowToStep"):
|
||||
text, _ := s["text"].(string)
|
||||
if text = cleanText(text); text == "" {
|
||||
return nil, errors.New("HowToStep has no text")
|
||||
}
|
||||
out = append(out, text)
|
||||
case hasType(s, "HowToSection"):
|
||||
list, ok := s["itemListElement"].([]any)
|
||||
if !ok {
|
||||
return nil, errors.New("HowToSection has no itemListElement list")
|
||||
}
|
||||
section, err := stepList(list)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, section...)
|
||||
default:
|
||||
return nil, fmt.Errorf("recipeInstructions holds a %v, not a HowToStep or HowToSection", s["@type"])
|
||||
}
|
||||
default:
|
||||
return nil, fmt.Errorf("recipeInstructions holds a %T", item)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// minutes reads a duration field: nil when absent or null; any string
|
||||
// present, empty included, must be an ISO-8601 duration.
|
||||
func minutes(n map[string]any, field string) (*int, error) {
|
||||
v, ok := n[field]
|
||||
if !ok || v == nil {
|
||||
return nil, nil
|
||||
}
|
||||
s, ok := v.(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%s is a %T, not an ISO-8601 duration", field, v)
|
||||
}
|
||||
m, err := isoMinutes(s)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", field, err)
|
||||
}
|
||||
return &m, nil
|
||||
}
|
||||
|
||||
// isoMinutes converts an ISO-8601 duration of weeks, days, hours, minutes
|
||||
// and seconds to whole minutes, rounded. Years and months are refused:
|
||||
// their length in minutes is not fixed.
|
||||
func isoMinutes(s string) (int, error) {
|
||||
bad := fmt.Errorf("%q is not an ISO-8601 duration of days, hours, minutes or seconds", s)
|
||||
rest, ok := strings.CutPrefix(strings.TrimSpace(s), "P")
|
||||
if !ok || rest == "" {
|
||||
return 0, bad
|
||||
}
|
||||
const units = "WDHMS"
|
||||
seconds := map[byte]float64{'W': 7 * 86400, 'D': 86400, 'H': 3600, 'M': 60, 'S': 1}
|
||||
inTime, last, total := false, -1, 0.0
|
||||
for rest != "" {
|
||||
if rest[0] == 'T' {
|
||||
if inTime || len(rest) == 1 {
|
||||
return 0, bad
|
||||
}
|
||||
inTime, rest = true, rest[1:]
|
||||
continue
|
||||
}
|
||||
i := 0
|
||||
for i < len(rest) && (rest[i] >= '0' && rest[i] <= '9' || rest[i] == '.' || rest[i] == ',') {
|
||||
i++
|
||||
}
|
||||
if i == 0 || i == len(rest) {
|
||||
return 0, bad
|
||||
}
|
||||
num, err := strconv.ParseFloat(strings.Replace(rest[:i], ",", ".", 1), 64)
|
||||
if err != nil {
|
||||
return 0, bad
|
||||
}
|
||||
unit := rest[i]
|
||||
pos := strings.IndexByte(units, unit)
|
||||
if pos < 0 || pos <= last || inTime != (pos >= 2) {
|
||||
return 0, bad
|
||||
}
|
||||
last, total, rest = pos, total+num*seconds[unit], rest[i+1:]
|
||||
}
|
||||
if last < 0 {
|
||||
return 0, bad
|
||||
}
|
||||
return int(math.Round(total / 60)), nil
|
||||
}
|
||||
|
||||
func yield(v any) (string, error) {
|
||||
if list, ok := v.([]any); ok {
|
||||
if len(list) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
v = list[0]
|
||||
}
|
||||
switch y := v.(type) {
|
||||
case nil:
|
||||
return "", nil
|
||||
case string:
|
||||
return cleanText(y), nil
|
||||
case json.Number:
|
||||
return y.String(), nil
|
||||
}
|
||||
return "", fmt.Errorf("recipeYield is a %T", v)
|
||||
}
|
||||
|
||||
// htmlDoc is what Extract needs from a page's HTML.
|
||||
type htmlDoc struct {
|
||||
jsonLD []string
|
||||
ogSiteName string
|
||||
}
|
||||
|
||||
// scanHTML is a small targeted scan, not a full HTML parser: it walks the
|
||||
// tags, skipping comments and the bodies of scripts and styles, and keeps
|
||||
// the bodies of application/ld+json scripts and the first og:site_name.
|
||||
func scanHTML(doc string) htmlDoc {
|
||||
var out htmlDoc
|
||||
lower := asciiLower(doc)
|
||||
for pos := 0; ; {
|
||||
lt := strings.IndexByte(doc[pos:], '<')
|
||||
if lt < 0 {
|
||||
return out
|
||||
}
|
||||
pos += lt
|
||||
if strings.HasPrefix(doc[pos:], "<!--") {
|
||||
end := strings.Index(doc[pos+4:], "-->")
|
||||
if end < 0 {
|
||||
return out
|
||||
}
|
||||
pos += 4 + end + 3
|
||||
continue
|
||||
}
|
||||
name := tagName(lower[pos+1:])
|
||||
if name == "" {
|
||||
pos++
|
||||
continue
|
||||
}
|
||||
end := tagEnd(doc, pos+1+len(name))
|
||||
if end < 0 {
|
||||
return out
|
||||
}
|
||||
attrs := parseAttrs(doc[pos+1+len(name) : end])
|
||||
pos = end + 1
|
||||
switch name {
|
||||
case "meta":
|
||||
if out.ogSiteName == "" && attrs["property"] == "og:site_name" {
|
||||
out.ogSiteName = strings.TrimSpace(attrs["content"])
|
||||
}
|
||||
case "script", "style":
|
||||
closeAt := strings.Index(lower[pos:], "</"+name)
|
||||
if closeAt < 0 {
|
||||
return out
|
||||
}
|
||||
typ, _, _ := strings.Cut(attrs["type"], ";")
|
||||
if name == "script" && strings.EqualFold(strings.TrimSpace(typ), "application/ld+json") {
|
||||
out.jsonLD = append(out.jsonLD, doc[pos:pos+closeAt])
|
||||
}
|
||||
pos += closeAt
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// asciiLower lower-cases ASCII letters only, so byte offsets match doc.
|
||||
func asciiLower(s string) string {
|
||||
b := []byte(s)
|
||||
for i, c := range b {
|
||||
if 'A' <= c && c <= 'Z' {
|
||||
b[i] = c + 'a' - 'A'
|
||||
}
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// tagName reads the lower-cased name of an opening tag at the start of s,
|
||||
// or "" when s does not start one.
|
||||
func tagName(s string) string {
|
||||
i := 0
|
||||
for i < len(s) && ('a' <= s[i] && s[i] <= 'z' || i > 0 && '0' <= s[i] && s[i] <= '9') {
|
||||
i++
|
||||
}
|
||||
if i == 0 || i < len(s) && !isSpace(s[i]) && s[i] != '>' && s[i] != '/' {
|
||||
return ""
|
||||
}
|
||||
return s[:i]
|
||||
}
|
||||
|
||||
// tagEnd finds the '>' closing a tag whose attributes start at i, skipping
|
||||
// quoted attribute values.
|
||||
func tagEnd(doc string, i int) int {
|
||||
var quote byte
|
||||
for ; i < len(doc); i++ {
|
||||
switch c := doc[i]; {
|
||||
case quote != 0:
|
||||
if c == quote {
|
||||
quote = 0
|
||||
}
|
||||
case c == '"' || c == '\'':
|
||||
quote = c
|
||||
case c == '>':
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// parseAttrs reads a tag's attributes; names are lower-cased, values are
|
||||
// entity-decoded, and the first of a repeated name wins.
|
||||
func parseAttrs(s string) map[string]string {
|
||||
attrs := map[string]string{}
|
||||
i := 0
|
||||
for i < len(s) {
|
||||
for i < len(s) && (isSpace(s[i]) || s[i] == '/') {
|
||||
i++
|
||||
}
|
||||
start := i
|
||||
for i < len(s) && !isSpace(s[i]) && s[i] != '=' && s[i] != '/' {
|
||||
i++
|
||||
}
|
||||
if i == start {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
name := asciiLower(s[start:i])
|
||||
for i < len(s) && isSpace(s[i]) {
|
||||
i++
|
||||
}
|
||||
value := ""
|
||||
if i < len(s) && s[i] == '=' {
|
||||
i++
|
||||
for i < len(s) && isSpace(s[i]) {
|
||||
i++
|
||||
}
|
||||
if i < len(s) && (s[i] == '"' || s[i] == '\'') {
|
||||
q := s[i]
|
||||
end := strings.IndexByte(s[i+1:], q)
|
||||
if end < 0 {
|
||||
end = len(s) - i - 1
|
||||
}
|
||||
value = s[i+1 : i+1+end]
|
||||
i += end + 2
|
||||
} else {
|
||||
start := i
|
||||
for i < len(s) && !isSpace(s[i]) {
|
||||
i++
|
||||
}
|
||||
value = s[start:i]
|
||||
}
|
||||
}
|
||||
if _, seen := attrs[name]; !seen {
|
||||
attrs[name] = html.UnescapeString(value)
|
||||
}
|
||||
}
|
||||
return attrs
|
||||
}
|
||||
|
||||
func isSpace(c byte) bool {
|
||||
return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f'
|
||||
}
|
||||
@@ -0,0 +1,408 @@
|
||||
package recipes
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func readPage(t *testing.T, name string) []byte {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile(filepath.Join("testdata", name))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func intp(n int) *int { return &n }
|
||||
|
||||
func fmtMinutes(m *int) string {
|
||||
if m == nil {
|
||||
return "nil"
|
||||
}
|
||||
b, _ := json.Marshal(*m)
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func TestExtractRealPages(t *testing.T) {
|
||||
tests := []struct {
|
||||
file, url string
|
||||
title, site, image, yld string
|
||||
ingredients, steps int
|
||||
prep, cook, total *int
|
||||
firstStep string
|
||||
}{
|
||||
{
|
||||
// A Recipe script beside VideoObject and BreadcrumbList scripts;
|
||||
// image is an array of one sized ImageObject.
|
||||
file: "bbcgoodfood.html", url: "https://www.bbcgoodfood.com/recipes/easy-chicken-curry",
|
||||
title: "Easy chicken curry", site: "Good Food",
|
||||
image: "https://images.immediate.co.uk/production/volatile/sites/30/2020/08/easy-chicken-curry-aa22a0b.jpg?resize=440,400",
|
||||
ingredients: 11, steps: 4, prep: intp(5), cook: intp(45), total: intp(50), yld: "4",
|
||||
firstStep: "Heat the oil in a flameproof casserole dish",
|
||||
},
|
||||
{
|
||||
// Yoast @graph; image is a list of bare URLs, the first the
|
||||
// full-size one; no publisher, so og:site_name names the site.
|
||||
file: "budgetbytes.html", url: "https://www.budgetbytes.com/one-pot-creamy-cajun-chicken-pasta/",
|
||||
title: "One Pot Creamy Cajun Chicken Pasta", site: "Budget Bytes",
|
||||
image: "https://www.budgetbytes.com/wp-content/uploads/2018/10/One-Pot-Creamy-Cajun-Chicken-Pasta-pan.jpg",
|
||||
ingredients: 17, steps: 6, prep: intp(10), cook: intp(20), total: intp(30), yld: "4",
|
||||
firstStep: "Combine the ingredients for the Cajun seasoning",
|
||||
},
|
||||
{
|
||||
// HowToSection groups (ragu, cheese sauce, assembly), flattened.
|
||||
file: "recipetineats.html", url: "https://www.recipetineats.com/lasagna/",
|
||||
title: "Lasagna", site: "RecipeTin Eats",
|
||||
image: "https://www.recipetineats.com/tachyon/2017/05/Lasagne-recipe-3-main-square.jpg",
|
||||
ingredients: 24, steps: 18, prep: intp(20), cook: intp(240), total: intp(260), yld: "10",
|
||||
firstStep: "Heat oil in a large heavy based pot",
|
||||
},
|
||||
{
|
||||
// @graph of one node; a single ImageObject; instructions as a
|
||||
// list of plain strings; hours and minutes.
|
||||
file: "kingarthur.html", url: "https://www.kingarthurbaking.com/recipes/classic-sandwich-bread-recipe",
|
||||
title: "Classic Sandwich Bread", site: "King Arthur Baking",
|
||||
image: "https://www.kingarthurbaking.com/sites/default/files/2026-02/Classic-Sandwich-Bread-12.jpg",
|
||||
ingredients: 7, steps: 7, prep: intp(12), cook: intp(40), total: intp(162), yld: "16",
|
||||
firstStep: "Weigh your flour",
|
||||
},
|
||||
{
|
||||
// A top-level array; ImageObjects of different sizes.
|
||||
file: "delish.html", url: "https://www.delish.com/cooking/recipe-ideas/a19636089/creamy-tuscan-chicken-recipe/",
|
||||
title: "Creamy Tuscan Chicken", site: "Delish",
|
||||
image: "https://hips.hearstapps.com/hmg-prod/images/1d09a24f-5b76-49f4-9ad6-496acdc03933.jpg?crop=0.502xw:1.00xh;0.250xw,0&resize=1200:*",
|
||||
ingredients: 12, steps: 4, prep: intp(10), cook: intp(0), total: intp(45), yld: "4 serving(s)",
|
||||
firstStep: "In a large skillet over medium heat, heat oil.",
|
||||
},
|
||||
{
|
||||
// A bare Recipe object; image is one URL; HTML entities in steps.
|
||||
file: "foodcom.html", url: "https://www.food.com/recipe/best-banana-bread-2886",
|
||||
title: "Best Banana Bread", site: "Food.com",
|
||||
image: "https://img.sndimg.com/food/image/upload/q_92,fl_progressive,w_1200,c_scale/v1/img/recipes/28/86/wBZxNua1T8yfDwbfo9Fz_0S9A9315.jpg",
|
||||
ingredients: 8, steps: 10, prep: intp(10), cook: intp(60), total: intp(70), yld: "1 loaf, 10 serving(s)",
|
||||
firstStep: "Remove odd pots and pans from oven.",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.file, func(t *testing.T) {
|
||||
r, err := Extract(readPage(t, tt.file), tt.url)
|
||||
if err != nil {
|
||||
t.Fatalf("Extract: %v", err)
|
||||
}
|
||||
if r.Title != tt.title {
|
||||
t.Errorf("Title = %q, want %q", r.Title, tt.title)
|
||||
}
|
||||
if r.SiteName != tt.site {
|
||||
t.Errorf("SiteName = %q, want %q", r.SiteName, tt.site)
|
||||
}
|
||||
if r.ImageURL != tt.image {
|
||||
t.Errorf("ImageURL = %q, want %q", r.ImageURL, tt.image)
|
||||
}
|
||||
if len(r.Ingredients) != tt.ingredients {
|
||||
t.Errorf("%d ingredients, want %d", len(r.Ingredients), tt.ingredients)
|
||||
}
|
||||
if len(r.Steps) != tt.steps {
|
||||
t.Errorf("%d steps, want %d", len(r.Steps), tt.steps)
|
||||
}
|
||||
if len(r.Steps) > 0 && !strings.HasPrefix(r.Steps[0], tt.firstStep) {
|
||||
t.Errorf("first step = %q, want prefix %q", r.Steps[0], tt.firstStep)
|
||||
}
|
||||
for name, got := range map[string][2]*int{
|
||||
"prep": {r.PrepMinutes, tt.prep}, "cook": {r.CookMinutes, tt.cook}, "total": {r.TotalMinutes, tt.total},
|
||||
} {
|
||||
if fmtMinutes(got[0]) != fmtMinutes(got[1]) {
|
||||
t.Errorf("%s minutes = %s, want %s", name, fmtMinutes(got[0]), fmtMinutes(got[1]))
|
||||
}
|
||||
}
|
||||
if r.Yield != tt.yld {
|
||||
t.Errorf("Yield = %q, want %q", r.Yield, tt.yld)
|
||||
}
|
||||
var node map[string]any
|
||||
if err := json.Unmarshal(r.JSONLD, &node); err != nil || node["name"] == nil {
|
||||
t.Errorf("JSONLD is not the Recipe node: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractHowToSectionOrder(t *testing.T) {
|
||||
r, err := Extract(readPage(t, "recipetineats.html"), "https://www.recipetineats.com/lasagna/")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Ragu (5 steps), Cheese Sauce (5), Assemble (8): each section's first
|
||||
// step lands right after the previous section's last.
|
||||
for i, prefix := range map[int]string{
|
||||
0: "Heat oil in a large heavy based pot",
|
||||
4: "The ragu is ready when the meat",
|
||||
5: "Warm milk up in a saucepan",
|
||||
10: "Preheat oven to 180°C/350°F.",
|
||||
17: "Stand for 5 to 10 minutes",
|
||||
} {
|
||||
if !strings.HasPrefix(r.Steps[i], prefix) {
|
||||
t.Errorf("step %d = %q, want prefix %q", i+1, r.Steps[i], prefix)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractNoRecipe(t *testing.T) {
|
||||
_, err := Extract(readPage(t, "budgetbytes-about.html"), "https://www.budgetbytes.com/about/")
|
||||
if !errors.Is(err, ErrNoRecipe) {
|
||||
t.Fatalf("err = %v, want ErrNoRecipe", err)
|
||||
}
|
||||
_, err = Extract([]byte("<html><head><title>x</title></head></html>"), "https://example.com/")
|
||||
if !errors.Is(err, ErrNoRecipe) {
|
||||
t.Fatalf("page without JSON-LD: err = %v, want ErrNoRecipe", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractRefusesNonISODuration(t *testing.T) {
|
||||
// Bon Appétit writes cookTime as "20 minutes".
|
||||
_, err := Extract(readPage(t, "bonappetit.html"), "https://www.bonappetit.com/recipe/bas-best-chocolate-chip-cookies")
|
||||
if !errors.Is(err, ErrMalformedRecipe) || !strings.Contains(err.Error(), "cookTime") {
|
||||
t.Fatalf("err = %v, want ErrMalformedRecipe naming cookTime", err)
|
||||
}
|
||||
}
|
||||
|
||||
// page wraps JSON-LD scripts in a minimal HTML page.
|
||||
func page(head string, jsonLD ...string) []byte {
|
||||
var b strings.Builder
|
||||
b.WriteString("<!DOCTYPE html><html><head>" + head)
|
||||
for _, s := range jsonLD {
|
||||
b.WriteString(`<script type="application/ld+json">` + s + "</script>")
|
||||
}
|
||||
b.WriteString("</head><body></body></html>")
|
||||
return []byte(b.String())
|
||||
}
|
||||
|
||||
func TestExtractShapes(t *testing.T) {
|
||||
const pageURL = "https://www.example.com/recipes/stew/"
|
||||
tests := []struct {
|
||||
name string
|
||||
html []byte
|
||||
check func(t *testing.T, r Recipe)
|
||||
}{
|
||||
{
|
||||
name: "type array and relative image",
|
||||
html: page("", `{"@type":["Recipe","NewsArticle"],"name":"Stew","image":"/img/stew.jpg"}`),
|
||||
check: func(t *testing.T, r Recipe) {
|
||||
if r.ImageURL != "https://www.example.com/img/stew.jpg" {
|
||||
t.Errorf("ImageURL = %q", r.ImageURL)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "largest image wins regardless of order",
|
||||
html: page("", `{"@type":"Recipe","name":"Stew","image":[
|
||||
"https://cdn.example.com/plain.jpg",
|
||||
{"@type":"ImageObject","url":"small.jpg","width":300,"height":200},
|
||||
{"@type":"ImageObject","contentUrl":"../big.jpg","width":"1600","height":"900"},
|
||||
{"@type":"ImageObject","url":"mid.jpg","width":1200,"height":800}]}`),
|
||||
check: func(t *testing.T, r Recipe) {
|
||||
if r.ImageURL != "https://www.example.com/recipes/big.jpg" {
|
||||
t.Errorf("ImageURL = %q", r.ImageURL)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "instructions as one string",
|
||||
html: page("", `{"@type":"Recipe","name":"Stew","recipeInstructions":"Brown the beef.\r\n\r\nAdd the stock & simmer.\nServe."}`),
|
||||
check: func(t *testing.T, r Recipe) {
|
||||
want := []string{"Brown the beef.", "Add the stock & simmer.", "Serve."}
|
||||
if strings.Join(r.Steps, "|") != strings.Join(want, "|") {
|
||||
t.Errorf("Steps = %q, want %q", r.Steps, want)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "sections and steps flattened in order",
|
||||
html: page("", `{"@type":"Recipe","name":"Stew","recipeInstructions":[
|
||||
{"@type":"HowToSection","name":"Beef","itemListElement":[
|
||||
{"@type":"HowToStep","text":"one"},{"@type":"HowToStep","text":"two"}]},
|
||||
{"@type":"HowToStep","text":"three"},
|
||||
{"@type":"HowToSection","name":"Finish","itemListElement":["four"]}]}`),
|
||||
check: func(t *testing.T, r Recipe) {
|
||||
if got := strings.Join(r.Steps, " "); got != "one two three four" {
|
||||
t.Errorf("Steps = %q", got)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "total derived from prep and cook",
|
||||
html: page("", `{"@type":"Recipe","name":"Stew","prepTime":"PT15M","cookTime":"PT1H30M"}`),
|
||||
check: func(t *testing.T, r Recipe) {
|
||||
if fmtMinutes(r.TotalMinutes) != "105" {
|
||||
t.Errorf("TotalMinutes = %s, want 105", fmtMinutes(r.TotalMinutes))
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "total not derived from prep alone",
|
||||
html: page("", `{"@type":"Recipe","name":"Stew","prepTime":"PT15M"}`),
|
||||
check: func(t *testing.T, r Recipe) {
|
||||
if r.TotalMinutes != nil || r.CookMinutes != nil {
|
||||
t.Errorf("total %s, cook %s; want nil", fmtMinutes(r.TotalMinutes), fmtMinutes(r.CookMinutes))
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "site name from host without publisher or og:site_name",
|
||||
html: page("", `{"@type":"Recipe","name":"Stew"}`),
|
||||
check: func(t *testing.T, r Recipe) {
|
||||
if r.SiteName != "example.com" {
|
||||
t.Errorf("SiteName = %q", r.SiteName)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "og:site_name with attributes in any order",
|
||||
html: page(`<meta content="Our & Kitchen" property="og:site_name">`, `{"@type":"Recipe","name":"Stew","publisher":{"@id":"#org"}}`),
|
||||
check: func(t *testing.T, r Recipe) {
|
||||
if r.SiteName != "Our & Kitchen" {
|
||||
t.Errorf("SiteName = %q", r.SiteName)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "numeric yield and recipe in @graph beside others",
|
||||
html: page("", `{"@context":"https://schema.org","@graph":[{"@type":"WebPage"},{"@type":"Recipe","name":"Stew","recipeYield":6}]}`),
|
||||
check: func(t *testing.T, r Recipe) {
|
||||
if r.Yield != "6" {
|
||||
t.Errorf("Yield = %q", r.Yield)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "image and publisher resolved by @id within @graph",
|
||||
html: page(`<meta property="og:site_name" content="Not this">`, `{"@context":"https://schema.org","@graph":[
|
||||
{"@type":"Recipe","name":"Stew",
|
||||
"image":{"@id":"https://www.example.com/#primaryimage"},
|
||||
"publisher":{"@id":"https://www.example.com/#organization"}},
|
||||
{"@type":"ImageObject","@id":"https://www.example.com/#primaryimage","url":"/img/stew.jpg","width":1200,"height":800},
|
||||
{"@type":"Organization","@id":"https://www.example.com/#organization","name":"Our Kitchen"}]}`),
|
||||
check: func(t *testing.T, r Recipe) {
|
||||
if r.ImageURL != "https://www.example.com/img/stew.jpg" {
|
||||
t.Errorf("ImageURL = %q", r.ImageURL)
|
||||
}
|
||||
if r.SiteName != "Our Kitchen" {
|
||||
t.Errorf("SiteName = %q", r.SiteName)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "image @id resolved from another script",
|
||||
html: page("", `{"@type":"Recipe","name":"Stew","image":[{"@id":"#img"}]}`,
|
||||
`{"@type":"ImageObject","@id":"#img","contentUrl":"stew.jpg"}`),
|
||||
check: func(t *testing.T, r Recipe) {
|
||||
if r.ImageURL != "https://www.example.com/recipes/stew/stew.jpg" {
|
||||
t.Errorf("ImageURL = %q", r.ImageURL)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "absolute schema.org type IRIs",
|
||||
html: page("", `{"@type":"https://schema.org/Recipe","name":"Stew","recipeInstructions":[
|
||||
{"@type":"http://schema.org/HowToStep","text":"one"}]}`),
|
||||
check: func(t *testing.T, r Recipe) {
|
||||
if r.Title != "Stew" || strings.Join(r.Steps, "|") != "one" {
|
||||
t.Errorf("Title = %q, Steps = %q", r.Title, r.Steps)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "absolute type IRI in a type array",
|
||||
html: page("", `{"@type":["NewsArticle","http://schema.org/Recipe"],"name":"Stew"}`),
|
||||
check: func(t *testing.T, r Recipe) {
|
||||
if r.Title != "Stew" {
|
||||
t.Errorf("Title = %q", r.Title)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "null duration is absent",
|
||||
html: page("", `{"@type":"Recipe","name":"Stew","prepTime":null,"cookTime":"PT10M"}`),
|
||||
check: func(t *testing.T, r Recipe) {
|
||||
if r.PrepMinutes != nil || r.TotalMinutes != nil || fmtMinutes(r.CookMinutes) != "10" {
|
||||
t.Errorf("prep %s, cook %s, total %s", fmtMinutes(r.PrepMinutes), fmtMinutes(r.CookMinutes), fmtMinutes(r.TotalMinutes))
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "JSON-LD text inside comments and other scripts is ignored",
|
||||
html: []byte(`<html><head><!-- <script type="application/ld+json">{"@type":"Recipe","name":"Old"}</script> -->` +
|
||||
`<script>var s = '<script type="application/ld+json">';</script>` +
|
||||
`<SCRIPT TYPE='application/ld+json; charset=utf-8'>{"@type":"Recipe","name":"Stew"}</SCRIPT></head></html>`),
|
||||
check: func(t *testing.T, r Recipe) {
|
||||
if r.Title != "Stew" {
|
||||
t.Errorf("Title = %q", r.Title)
|
||||
}
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r, err := Extract(tt.html, pageURL)
|
||||
if err != nil {
|
||||
t.Fatalf("Extract: %v", err)
|
||||
}
|
||||
tt.check(t, r)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractErrors(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
html []byte
|
||||
url string
|
||||
want error
|
||||
}{
|
||||
{"two recipes", page("", `{"@type":"Recipe","name":"A"}`, `{"@type":"Recipe","name":"B"}`), "https://example.com/", ErrManyRecipes},
|
||||
{"invalid JSON", page("", `{"@type":"Recipe",`), "https://example.com/", ErrMalformedRecipe},
|
||||
{"no name", page("", `{"@type":"Recipe"}`), "https://example.com/", ErrMalformedRecipe},
|
||||
{"bad step type", page("", `{"@type":"Recipe","name":"A","recipeInstructions":[{"@type":"Thing"}]}`), "https://example.com/", ErrMalformedRecipe},
|
||||
{"step without text", page("", `{"@type":"Recipe","name":"A","recipeInstructions":[{"@type":"HowToStep","name":"x"}]}`), "https://example.com/", ErrMalformedRecipe},
|
||||
{"bad image width", page("", `{"@type":"Recipe","name":"A","image":{"url":"a.jpg","width":"wide"}}`), "https://example.com/", ErrMalformedRecipe},
|
||||
{"empty prepTime", page("", `{"@type":"Recipe","name":"A","prepTime":""}`), "https://example.com/", ErrMalformedRecipe},
|
||||
{"blank cookTime", page("", `{"@type":"Recipe","name":"A","cookTime":" "}`), "https://example.com/", ErrMalformedRecipe},
|
||||
{"empty totalTime", page("", `{"@type":"Recipe","name":"A","totalTime":""}`), "https://example.com/", ErrMalformedRecipe},
|
||||
{"image @id not on page", page("", `{"@type":"Recipe","name":"A","image":{"@id":"#missing"}}`), "https://example.com/", ErrMalformedRecipe},
|
||||
{"image @id on two nodes", page("", `{"@type":"Recipe","name":"A","image":{"@id":"#img"}}`,
|
||||
`{"@id":"#img","url":"a.jpg"}`, `{"@id":"#img","url":"b.jpg"}`), "https://example.com/", ErrMalformedRecipe},
|
||||
{"non-text ingredient", page("", `{"@type":"Recipe","name":"A","recipeIngredient":[{"name":"salt"}]}`), "https://example.com/", ErrMalformedRecipe},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
_, err := Extract(tt.html, tt.url)
|
||||
if !errors.Is(err, tt.want) {
|
||||
t.Fatalf("err = %v, want %v", err, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
if _, err := Extract(page("", `{"@type":"Recipe","name":"A"}`), "/relative"); err == nil {
|
||||
t.Fatal("relative page URL accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestISOMinutes(t *testing.T) {
|
||||
good := map[string]int{
|
||||
"PT0S": 0, "PT45M": 45, "PT1H": 60, "PT2H42M": 162, "PT240M": 240, "P1D": 1440,
|
||||
"P1DT2H": 1560, "PT90S": 2, "PT1.5H": 90, "PT0,5H": 30, "P1W": 10080, " PT5M ": 5,
|
||||
}
|
||||
for in, want := range good {
|
||||
got, err := isoMinutes(in)
|
||||
if err != nil || got != want {
|
||||
t.Errorf("isoMinutes(%q) = %d, %v; want %d", in, got, err, want)
|
||||
}
|
||||
}
|
||||
for _, in := range []string{"", "P", "PT", "20 minutes", "PT5", "P1Y", "P1M", "PT1D", "P5H", "PT5M1H", "PT5M5M", "P1DT", "PTxM", "PT1..5M"} {
|
||||
if got, err := isoMinutes(in); err == nil {
|
||||
t.Errorf("isoMinutes(%q) = %d, want error", in, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
<!DOCTYPE html>
|
||||
<!-- Trimmed from https://www.bbcgoodfood.com/recipes/easy-chicken-curry (fetched 2026-10-05): head metadata and JSON-LD only. -->
|
||||
<html><head>
|
||||
<title>Easy chicken curry recipe | Good Food</title>
|
||||
<meta property="og:title" content="Easy chicken curry" data-testid="meta-og-title"/>
|
||||
<meta property="og:site_name" content="Good Food" data-testid="meta-og-site-name"/>
|
||||
<meta property="og:url" content="https://www.bbcgoodfood.com/recipes/easy-chicken-curry" data-testid="meta-og-url"/>
|
||||
<meta property="og:image" content="https://images.immediate.co.uk/production/volatile/sites/30/2020/08/easy-chicken-curry-aa22a0b.jpg?resize=440%2C230" data-testid="meta-og-image"/>
|
||||
<script data-testid="schema" type="application/ld+json">{"@context":"https://schema.org","@type":"VideoObject","name":"Easy Chicken Curry","contentUrl":"https://media.delight.video/0c18e53b610964fac175fe0e757adc30557e79b7/cb0e0397ffd102279027b8d892036a85e98afa8b/MEDIA/v0/HD/media.mp4","description":"This easy staple chicken curry is a fantastic recipe for family dinners. It's made with just a handful of ingredients and is enriched with creamy yogurt","embedUrl":"https://www.bbcgoodfood.com/recipes/easy-chicken-curry","thumbnailUrl":"https://media.delight.video/0c18e53b610964fac175fe0e757adc30557e79b7/cb0e0397ffd102279027b8d892036a85e98afa8b/MEDIA/v0/THUMBS/media.0000005.jpg","uploadDate":"2022-07-29T10:34:28.279Z","duration":"PT0M57S","customPosterUrl":"https://media.delight.video/0c18e53b610964fac175fe0e757adc30557e79b7/cb0e0397ffd102279027b8d892036a85e98afa8b/POSTER_USER/v0/cb0e0397ffd102279027b8d892036a85e98afa8b.jpeg"}</script>
|
||||
<script data-testid="page-schema" type="application/ld+json">{"@context":"https://schema.org","@id":"https://www.bbcgoodfood.com/recipes/easy-chicken-curry#Recipe","@type":"Recipe","description":"This easy staple chicken curry is a fantastic recipe for family dinners. It's made with just a handful of ingredients and is enriched with creamy yogurt","image":[{"@type":"ImageObject","url":"https://images.immediate.co.uk/production/volatile/sites/30/2020/08/easy-chicken-curry-aa22a0b.jpg?resize=440,400","width":440,"height":400}],"mainEntityOfPage":{"@type":"WebPage","@id":"https://www.bbcgoodfood.com/recipes/easy-chicken-curry"},"name":"Easy chicken curry","url":"https://www.bbcgoodfood.com/recipes/easy-chicken-curry","author":[{"@type":"Person","name":"Esther Clark","url":"https://www.bbcgoodfood.com/author/estherclark"}],"dateModified":"2026-08-13T14:40:21+01:00","datePublished":"2019-01-25T15:26:05+00:00","headline":"Easy chicken curry","keywords":"1 of 5-a-day, Chicken, Curry, Esther Clark, Family, Family friendly, Gluten free","publisher":{"@type":"Organization","name":"Good Food","url":"https://www.bbcgoodfood.com","logo":{"@type":"ImageObject","height":325,"url":"https://images.immediate.co.uk/production/volatile/sites/30/2026/02/goodfood.logo-f37096c.png?resize=1312,325","width":1312}},"cookTime":"PT45M","nutrition":{"@type":"NutritionInformation","calories":"354 calories","fatContent":"23 grams fat","saturatedFatContent":"4 grams saturated fat","carbohydrateContent":"10 grams carbohydrates","sugarContent":"8 grams sugar","fiberContent":"3 grams fiber","proteinContent":"24 grams protein","sodiumContent":"0.6 milligram of sodium"},"prepTime":"PT5M","recipeCategory":"Dinner, Main course","recipeIngredient":["2 tbsp sunflower oil","1 onion thinly sliced","2 garlic cloves crushed","thumb-sized piece of ginger grated","6 chicken thighs boneless and skinless","3 tbsp medium spice paste (tikka works well)","400g can chopped tomatoes","100g Greek yogurt","1 small bunch of coriander leaves chopped","50g ground almonds","naan breads or cooked basmati rice, to serve"],"recipeInstructions":[{"@type":"HowToStep","text":"Heat the oil in a flameproof casserole dish or large frying pan over a medium heat. Add the onion and a generous pinch of salt and fry for 8-10 mins, or until the onion has turned golden brown and sticky."},{"@type":"HowToStep","text":"Add the garlic and ginger, cooking for a further minute."},{"@type":"HowToStep","text":"Chop the chicken into chunky 3cm pieces, add to the pan and fry for 5 mins before stirring through the spice paste and tomatoes, along with 250ml water. Bring to the boil, lower to a simmer and cook on a gentle heat uncovered for 25-30 mins or until rich and slightly reduced."},{"@type":"HowToStep","text":"Stir though the yogurt, coriander and ground almonds, season and serve with warm naan or fluffy basmati rice."}],"recipeYield":4,"suitableForDiet":"https://schema.org/GlutenFreeDiet","totalTime":"PT50M"}</script>
|
||||
<script type="application/ld+json">{"@context":"https://schema.org/","@type":"BreadcrumbList","itemListElement":[{"@type":"ListItem","position":1,"name":"Home","item":"https://www.bbcgoodfood.com/"},{"@type":"ListItem","position":2,"name":"Recipes","item":"https://www.bbcgoodfood.com/recipes"},{"@type":"ListItem","position":3,"name":"Easy chicken curry"}]}</script>
|
||||
<script type="application/ld+json">{"@context":"https://schema.org/","@id":"https://www.bbcgoodfood.com/recipes/easy-chicken-curry#Recipe","aggregateRating":{"@type":"AggregateRating","ratingValue":5,"reviewCount":363,"bestRating":5,"worstRating":1}}</script>
|
||||
</head><body><main>(page body trimmed)</main></body></html>
|
||||
+10
File diff suppressed because one or more lines are too long
@@ -0,0 +1,10 @@
|
||||
<!DOCTYPE html>
|
||||
<!-- Trimmed from https://www.budgetbytes.com/about/ (fetched 2026-10-05): head metadata and JSON-LD only. -->
|
||||
<html><head>
|
||||
<title>The Budget Bytes Story - BudgetBytes.com</title>
|
||||
<meta property="og:title" content="The Budget Bytes Story - BudgetBytes.com" />
|
||||
<meta property="og:url" content="https://www.budgetbytes.com/about/" />
|
||||
<meta property="og:site_name" content="Budget Bytes" />
|
||||
<meta property="og:image" content="https://www.budgetbytes.com/wp-content/uploads/2025/04/Budget-Bytes-Group-Shot_Web.jpg" />
|
||||
<script type="application/ld+json" class="yoast-schema-graph">{"@context":"https:\/\/schema.org","@graph":[{"@type":"WebPage","@id":"https:\/\/www.budgetbytes.com\/about\/","url":"https:\/\/www.budgetbytes.com\/about\/","name":"The Budget Bytes Story - BudgetBytes.com","isPartOf":{"@id":"https:\/\/www.budgetbytes.com\/#website"},"primaryImageOfPage":{"@id":"https:\/\/www.budgetbytes.com\/about\/#primaryimage"},"image":{"@id":"https:\/\/www.budgetbytes.com\/about\/#primaryimage"},"thumbnailUrl":"https:\/\/www.budgetbytes.com\/wp-content\/uploads\/2025\/04\/Budget-Bytes-Group-Shot_Web.jpg","datePublished":"2013-05-02T18:19:55+00:00","dateModified":"2026-07-20T15:36:17+00:00","description":"Beth Moncel created Budget Bytes in 2009 to help people learn how to cook and create delicious recipes designed for small budgets.","inLanguage":"en-US","potentialAction":[{"@type":"ReadAction","target":["https:\/\/www.budgetbytes.com\/about\/"]}]},{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/www.budgetbytes.com\/about\/#primaryimage","url":"https:\/\/www.budgetbytes.com\/wp-content\/uploads\/2025\/04\/Budget-Bytes-Group-Shot_Web.jpg","contentUrl":"https:\/\/www.budgetbytes.com\/wp-content\/uploads\/2025\/04\/Budget-Bytes-Group-Shot_Web.jpg","width":1200,"height":800,"caption":"The Budget Bytes team that develops recipes out of our Nashville studio. Left to right: Rebecca Denton (Photographer), Jess Rice (Recipe Development Manager), Jennie Alley (Senior Recipe Developer), and Melissa Nolan (Senior Recipe Developer). We develop, test, and photograph each and every recipe on Budget Bytes from Nashville Tennessee."},{"@type":"WebSite","@id":"https:\/\/www.budgetbytes.com\/#website","url":"https:\/\/www.budgetbytes.com\/","name":"Budget Bytes","description":"Delicious Recipes Designed for Small Budgets","publisher":{"@id":"https:\/\/www.budgetbytes.com\/#organization"},"potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/www.budgetbytes.com\/?s={search_term_string}"},"query-input":{"@type":"PropertyValueSpecification","valueRequired":true,"valueName":"search_term_string"}}],"inLanguage":"en-US"},{"@type":"Organization","@id":"https:\/\/www.budgetbytes.com\/#organization","name":"Budget Bytes","url":"https:\/\/www.budgetbytes.com\/","logo":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/www.budgetbytes.com\/#\/schema\/logo\/image\/","url":"https:\/\/www.budgetbytes.com\/wp-content\/uploads\/2024\/11\/budgetbytes_square_icon_transparent-300x300-1.png","contentUrl":"https:\/\/www.budgetbytes.com\/wp-content\/uploads\/2024\/11\/budgetbytes_square_icon_transparent-300x300-1.png","width":300,"height":300,"caption":"Budget Bytes"},"image":{"@id":"https:\/\/www.budgetbytes.com\/#\/schema\/logo\/image\/"},"sameAs":["https:\/\/www.facebook.com\/budgetbytes1\/?ref=hl","https:\/\/x.com\/Budget_Bytes","https:\/\/instagram.com\/budgetbytes\/","https:\/\/www.pinterest.com\/budgetbytes\/","https:\/\/www.youtube.com\/channel\/UC17vdVOZBVxTSDcldUBBlRg"]}]}</script>
|
||||
</head><body><main>(page body trimmed)</main></body></html>
|
||||
+10
File diff suppressed because one or more lines are too long
+10
File diff suppressed because one or more lines are too long
+9
@@ -0,0 +1,9 @@
|
||||
<!DOCTYPE html>
|
||||
<!-- Trimmed from https://www.food.com/recipe/best-banana-bread-2886 (fetched 2026-10-05): head metadata and JSON-LD only; the 80 KB review array removed from the Recipe node. -->
|
||||
<html><head>
|
||||
<title>Best Banana Bread Recipe - Food.com</title>
|
||||
<meta name="og:title" content="Best Banana Bread Recipe - Food.com"/>
|
||||
<meta name="og:image" content="https://img.sndimg.com/food/image/upload/w_555,h_416,c_fit,fl_progressive,q_95/v1/img/recipes/28/86/wBZxNua1T8yfDwbfo9Fz_0S9A9315.jpg"/>
|
||||
<meta name="og:url" content=""/>
|
||||
<script type="application/ld+json">{"@context":"http://schema.org","@type":"Recipe","mainEntityOfPage":"true","name":"Best Banana Bread","author":"lkadlec","cookTime":"PT1H","prepTime":"PT10M","totalTime":"PT1H10M","datePublished":"1999-09-26T20:49Z","description":"Make and share this Best Banana Bread recipe from Food.com.","image":"https://img.sndimg.com/food/image/upload/q_92,fl_progressive,w_1200,c_scale/v1/img/recipes/28/86/wBZxNua1T8yfDwbfo9Fz_0S9A9315.jpg","recipeCategory":"Quick Breads","keywords":"Breads,Fruit,Oven,< 4 Hours","recipeIngredient":["1/2 cup butter, softened ","1 cup granulated sugar","2 eggs, beaten ","3 bananas, finely crushed (for serious and extreme moist and delicious, try 4 bananas)","1 1/2 cups all-purpose flour","1 teaspoon baking soda","1/2 teaspoon salt","1/2 teaspoon vanilla (optional)"],"aggregateRating":{"@type":"AggregateRating","ratingValue":"5.0","reviewCount":"2355"},"nutrition":{"@type":"NutritionInformation","calories":"272.8","fatContent":"10.5","saturatedFatContent":"6.2","cholesterolContent":"61.6","sodiumContent":"338.3","carbohydrateContent":"42.5","fiberContent":"1.4","sugarContent":"24.4","proteinContent":"3.7"},"recipeInstructions":[{"@type":"HowToStep","text":"Remove odd pots and pans from oven."},{"@type":"HowToStep","text":"Preheat oven to 350º / 180º."},{"@type":"HowToStep","text":"Cream together butter and sugar."},{"@type":"HowToStep","text":"Add eggs and crushed bananas."},{"@type":"HowToStep","text":"Combine well."},{"@type":"HowToStep","text":"Sift together flour, soda and salt. Add to creamed mixture. Add vanilla."},{"@type":"HowToStep","text":"Mix just until combined. Do not overmix."},{"@type":"HowToStep","text":"Pour into greased and floured loaf pan."},{"@type":"HowToStep","text":"Bake at 350º / 180º for 55 minutes."},{"@type":"HowToStep","text":"Keeps well, refrigerated."}],"recipeYield":"1 loaf, 10 serving(s)","publisher":{"@type":"Organization","name":"Food.com","logo":{"@type":"ImageObject","url":"https://geniuskitchen.sndimg.com/fdc-new/img/FDC-Logo.png"},"url":"https://www.food.com"},"video":[{"@type":"VideoObject","name":"Best Banana Bread","duration":"42S","thumbnailUrl":"https://img.sndimg.com/food/image/upload/w_1024,h_576,c_fit/v1/img/mpx/5458761_16x9.jpg","contentUrl":"https://www.food.com/recipe/best-banana-bread-2886"}]}</script>
|
||||
</head><body><main>(page body trimmed)</main></body></html>
|
||||
+79
@@ -0,0 +1,79 @@
|
||||
<!DOCTYPE html>
|
||||
<!-- Trimmed from https://www.kingarthurbaking.com/recipes/classic-sandwich-bread-recipe (fetched 2026-10-05): head metadata and JSON-LD only. -->
|
||||
<html><head>
|
||||
<title>Classic Sandwich Bread Recipe | King Arthur Baking</title>
|
||||
<meta property="og:site_name" content="King Arthur Baking" />
|
||||
<meta property="og:url" content="https://www.kingarthurbaking.com/recipes/classic-sandwich-bread-recipe" />
|
||||
<meta property="og:title" content="Classic Sandwich Bread" />
|
||||
<meta property="og:image" content="https://www.kingarthurbaking.com/sites/default/files/2026-02/Classic-Sandwich-Bread-12.jpg" />
|
||||
<script type="application/ld+json">{
|
||||
"@context": "https://schema.org",
|
||||
"@graph": [
|
||||
{
|
||||
"@type": "Recipe",
|
||||
"keywords": "Sandwich bread;;Tangzhong",
|
||||
"name": "Classic Sandwich Bread",
|
||||
"description": "Made with all of the ingredients that make homemade white bread superior to storebought — butter, milk, and high-quality flour — this sandwich bread recipe is our version of a classic, perfected. The soft and tender interior makes this loaf ideal for sandwiches, toast, or even French toast. It\u0027s a timeless recipe you\u0027ll come back to time and time again.",
|
||||
"recipeYield": [
|
||||
"16",
|
||||
"1 loaf"
|
||||
],
|
||||
"datePublished": "December 31, 2007 at 7:00pm",
|
||||
"prepTime": "PT12M",
|
||||
"cookTime": "PT40M",
|
||||
"totalTime": "PT2H42M",
|
||||
"recipeInstructions": [
|
||||
"Weigh your flour; or measure it by gently spooning it into a cup, then sweeping off any excess.",
|
||||
"In a large bowl, combine all of the ingredients. Mix and knead everything together — by hand, mixer, or bread machine set on the dough cycle — until you\u0027ve made a smooth dough. If you\u0027re kneading in a stand mixer, it should take 5 to 7 minutes at medium-low speed, and the dough should barely clean the sides of the bowl, perhaps sticking a bit at the bottom. In a bread machine (or by hand), knead the dough for about 6 to 8 minutes, until it forms a smooth ball. ",
|
||||
"Transfer the dough to a lightly greased bowl, cover the bowl, and allow the dough to rise until puffy though not necessarily doubled in bulk, about 1 to 2 hours, depending on the warmth of your kitchen. If you\u0027re using a bread machine, allow the machine to complete its cycle, then leave the dough in the machine until it\u0027s doubled in bulk, perhaps an additional 30 minutes or so.",
|
||||
"Gently deflate the dough and transfer it to a lightly oiled work surface. Shape the dough into an 8\u0022 log.",
|
||||
"Place the log in a lightly greased 8 1/2\u0022 x 4 1/2\u0022 loaf pan, cover the pan loosely with lightly greased plastic wrap, and allow the bread to rise for about 60 minutes, until it\u0027s domed about 1\u0022 above the edge of the pan. A finger pressed into the dough should leave a mark that rebounds slowly. Towards the end of the rise, preheat your oven to 350°F.",
|
||||
"Bake the bread for 30 to 35 minutes, until it\u0027s light golden brown. Test it for doneness by removing it from the pan and thumping it on the bottom (it should sound hollow), or by measuring its interior temperature with a digital thermometer (it should register 190°F at the center of the loaf).",
|
||||
"Remove the bread from the oven, and cool it on a rack before slicing. Store the bread in a plastic bag at room temperature for several days; freeze for longer storage."
|
||||
],
|
||||
"recipeIngredient": [
|
||||
"3 cups (360g) King Arthur Unbleached All-Purpose Flour",
|
||||
"1/2 cup (113g) milk, (skim, 1%, 2% or whole, your choice)*",
|
||||
"1/2 to 2/3 cup (113g to 152g) hot water, enough to make a soft, smooth dough* (see \u0022tips,\u0022 below)",
|
||||
"4 tablespoons (57g) melted butter or 1/4 cup (50g) vegetable oil",
|
||||
"2 tablespoons (25g) granulated sugar",
|
||||
"1 1/4 teaspoons (8g) table salt",
|
||||
"2 teaspoons instant yeast"
|
||||
],
|
||||
"publisher": {
|
||||
"@type": "Organization",
|
||||
"name": "King Arthur Baking",
|
||||
"url": "https://www.kingarthurbaking.com/"
|
||||
},
|
||||
"author": {
|
||||
"@type": "Person",
|
||||
"name": "PJ Hamel",
|
||||
"url": "https://www.kingarthurbaking.com/author/pj-hamel"
|
||||
},
|
||||
"image": {
|
||||
"@type": "ImageObject",
|
||||
"url": "https://www.kingarthurbaking.com/sites/default/files/2026-02/Classic-Sandwich-Bread-12.jpg"
|
||||
},
|
||||
"nutrition": {
|
||||
"@type": "NutritionInformation",
|
||||
"servingSize": "43g",
|
||||
"calories": "120 calories",
|
||||
"carbohydrateContent": "19g",
|
||||
"cholesterolContent": "10mg",
|
||||
"fiberContent": "1g",
|
||||
"proteinContent": "3g",
|
||||
"sodiumContent": "190mg",
|
||||
"sugarContent": "2g",
|
||||
"fatContent": "3g",
|
||||
"saturatedFatContent": "2g",
|
||||
"transFatContent": "0g"
|
||||
},
|
||||
"aggregateRating": {
|
||||
"@type": "AggregateRating",
|
||||
"ratingValue": "4.60",
|
||||
"ratingCount": "681"
|
||||
}
|
||||
}
|
||||
]
|
||||
}</script>
|
||||
</head><body><main>(page body trimmed)</main></body></html>
|
||||
+11
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user