package recipes import ( "encoding/json" "errors" "fmt" "html" "io" "math" "net/url" "strconv" "strings" ) var ( // ErrNoRecipe is returned for a page with no schema.org Recipe node in // its JSON-LD. ErrNoRecipe = errors.New("recipes: page has no schema.org Recipe") // ErrManyRecipes is returned for a page with more than one Recipe node: // which one the page is about is not knowable. ErrManyRecipes = errors.New("recipes: page has more than one schema.org Recipe") // ErrMalformedRecipe is returned when the page's JSON-LD, or a field of // its Recipe node, is not in a shape schema.org allows. ErrMalformedRecipe = errors.New("recipes: malformed schema.org Recipe") ) // Recipe is the schema.org Recipe extracted from a recipe page. type Recipe struct { // Title is the recipe's name. Title string // SiteName is the publisher's name, else the page's og:site_name, else // HostSiteName of the page's URL. SiteName string // ImageURL is the largest image, absolute; empty when the recipe has // none. ImageURL string Ingredients []string // Steps are the instructions in order, HowToSection groups flattened. Steps []string // PrepMinutes, CookMinutes and TotalMinutes are nil when not given. // TotalMinutes is prep plus cook when the page gives both but no total. PrepMinutes *int CookMinutes *int TotalMinutes *int // Yield is recipeYield as text; empty when not given. Yield string // JSONLD is the Recipe node as found. JSONLD json.RawMessage } // Extract finds the schema.org Recipe in a page's application/ld+json // scripts. Recipe nodes are found at the top level of a script, in a // top-level array, or in an @graph. An image or publisher given as an // {"@id": ...} reference resolves to the node with that @id found in the // same places, in any of the page's scripts. pageURL is the page's own // absolute URL; relative image URLs resolve against it. func Extract(page []byte, pageURL string) (Recipe, error) { base, err := url.Parse(pageURL) if err != nil || !base.IsAbs() || base.Host == "" { return Recipe{}, fmt.Errorf("recipes: page URL %q is not absolute", pageURL) } doc := scanHTML(string(page)) g := graph{ids: map[string][]map[string]any{}} for i, script := range doc.jsonLD { v, err := decodeJSON(script) if err != nil { return Recipe{}, fmt.Errorf("%w: JSON-LD script %d: %v", ErrMalformedRecipe, i+1, err) } g.collect(v) } nodes := g.recipes switch len(nodes) { case 0: return Recipe{}, ErrNoRecipe case 1: default: return Recipe{}, fmt.Errorf("%w: found %d", ErrManyRecipes, len(nodes)) } r, err := recipeFrom(nodes[0], g, base, doc.ogSiteName) if err != nil { return Recipe{}, fmt.Errorf("%w: %v", ErrMalformedRecipe, err) } return r, nil } func decodeJSON(s string) (any, error) { dec := json.NewDecoder(strings.NewReader(s)) dec.UseNumber() var v any if err := dec.Decode(&v); err != nil { return nil, err } if _, err := dec.Token(); err != io.EOF { return nil, errors.New("trailing data after JSON value") } return v, nil } // graph holds the page's JSON-LD nodes: the Recipe nodes, and every node // with an @id by that @id. type graph struct { recipes []map[string]any ids map[string][]map[string]any } // collect gathers the nodes in v: v itself, the elements of an array, and // the members of an @graph. A bare {"@id": ...} reference is not a node. func (g *graph) collect(v any) { switch n := v.(type) { case []any: for _, e := range n { g.collect(e) } case map[string]any: if hasType(n, "Recipe") { g.recipes = append(g.recipes, n) } if id, ok := n["@id"].(string); ok && len(n) > 1 { g.ids[id] = append(g.ids[id], n) } if members, ok := n["@graph"]; ok { g.collect(members) } } } // resolve returns the node a bare {"@id": ...} reference names, and v // itself when v is not a reference. found is false for a reference to no // node; a reference to more than one node is an error. func (g graph) resolve(v any) (node any, found bool, err error) { m, ok := v.(map[string]any) if !ok || len(m) != 1 { return v, true, nil } id, ok := m["@id"].(string) if !ok { return v, true, nil } switch nodes := g.ids[id]; len(nodes) { case 0: return v, false, nil case 1: return nodes[0], true, nil default: return nil, false, fmt.Errorf("@id %q names %d nodes", id, len(nodes)) } } // hasType reports whether node's @type, a string or an array of strings, // names the schema.org type typ, as a term or an absolute schema.org IRI. func hasType(node map[string]any, typ string) bool { switch t := node["@type"].(type) { case string: return isSchemaType(t, typ) case []any: for _, e := range t { if s, ok := e.(string); ok && isSchemaType(s, typ) { return true } } } return false } func isSchemaType(s, typ string) bool { return s == typ || s == "https://schema.org/"+typ || s == "http://schema.org/"+typ } func recipeFrom(n map[string]any, g graph, base *url.URL, ogSiteName string) (Recipe, error) { var r Recipe var err error name, ok := n["name"].(string) if r.Title = cleanText(name); !ok || r.Title == "" { return Recipe{}, errors.New("name is missing") } if r.SiteName, err = siteName(n["publisher"], g, ogSiteName, base); err != nil { return Recipe{}, err } if r.ImageURL, err = imageURL(n["image"], g, base); err != nil { return Recipe{}, err } if r.Ingredients, err = ingredients(n["recipeIngredient"]); err != nil { return Recipe{}, err } if r.Steps, err = steps(n["recipeInstructions"]); err != nil { return Recipe{}, err } if r.PrepMinutes, err = minutes(n, "prepTime"); err != nil { return Recipe{}, err } if r.CookMinutes, err = minutes(n, "cookTime"); err != nil { return Recipe{}, err } if r.TotalMinutes, err = minutes(n, "totalTime"); err != nil { return Recipe{}, err } if r.TotalMinutes == nil && r.PrepMinutes != nil && r.CookMinutes != nil { total := *r.PrepMinutes + *r.CookMinutes r.TotalMinutes = &total } if r.Yield, err = yield(n["recipeYield"]); err != nil { return Recipe{}, err } if r.JSONLD, err = json.Marshal(n); err != nil { return Recipe{}, err } return r, nil } // cleanText decodes HTML entities, which recipe plugins often leave in // JSON-LD text, and trims surrounding space. func cleanText(s string) string { return strings.TrimSpace(html.UnescapeString(s)) } func siteName(publisher any, g graph, ogSiteName string, base *url.URL) (string, error) { pubs, ok := publisher.([]any) if !ok { pubs = []any{publisher} } for _, p := range pubs { p, _, err := g.resolve(p) if err != nil { return "", fmt.Errorf("publisher: %w", err) } if m, ok := p.(map[string]any); ok { if name, ok := m["name"].(string); ok && cleanText(name) != "" { return cleanText(name), nil } } } if ogSiteName != "" { return ogSiteName, nil } return HostSiteName(base), nil } // HostSiteName is the site name of a page that names none: its host // without a leading "www.". func HostSiteName(u *url.URL) string { return strings.TrimPrefix(u.Hostname(), "www.") } // imageURL picks the largest of the recipe's images by width × height; // images without both dimensions count as size zero, and the first wins a // tie, so a list of bare URLs gives its first. func imageURL(v any, g graph, base *url.URL) (string, error) { items, ok := v.([]any) if !ok { if v == nil { return "", nil } items = []any{v} } best, bestArea := "", int64(-1) for _, item := range items { item, found, err := g.resolve(item) if err != nil { return "", fmt.Errorf("image: %w", err) } if !found { return "", fmt.Errorf("image references @id %v, which no node on the page has", item.(map[string]any)["@id"]) } var u string var area int64 switch img := item.(type) { case string: u = img case map[string]any: if s, ok := img["url"].(string); ok { u = s } else if s, ok := img["contentUrl"].(string); ok { u = s } w, err := dimension(img["width"]) if err != nil { return "", fmt.Errorf("image width: %w", err) } h, err := dimension(img["height"]) if err != nil { return "", fmt.Errorf("image height: %w", err) } area = w * h default: return "", fmt.Errorf("image is a %T, not a URL or ImageObject", item) } u = strings.TrimSpace(u) if u == "" { return "", errors.New("image has no URL") } if area > bestArea { best, bestArea = u, area } } if best == "" { return "", nil } ref, err := url.Parse(best) if err != nil { return "", fmt.Errorf("image URL %q: %w", best, err) } return base.ResolveReference(ref).String(), nil } // dimension reads an ImageObject width or height: a number or a string of // digits; zero when absent. func dimension(v any) (int64, error) { switch d := v.(type) { case nil: return 0, nil case json.Number: f, err := d.Float64() if err != nil || f < 0 { return 0, fmt.Errorf("%q is not a size", d) } return int64(f), nil case string: n, err := strconv.ParseInt(strings.TrimSpace(d), 10, 64) if err != nil || n < 0 { return 0, fmt.Errorf("%q is not a size", d) } return n, nil } return 0, fmt.Errorf("a %T is not a size", v) } func ingredients(v any) ([]string, error) { items, ok := v.([]any) if !ok { if v == nil { return nil, nil } items = []any{v} } var out []string for _, item := range items { s, ok := item.(string) if !ok { return nil, fmt.Errorf("recipeIngredient holds a %T, not text", item) } if s = cleanText(s); s != "" { out = append(out, s) } } return out, nil } // steps flattens recipeInstructions: one string of lines, or a list of // strings, HowToSteps and HowToSections of those, in order. func steps(v any) ([]string, error) { switch in := v.(type) { case nil: return nil, nil case string: var out []string for _, line := range strings.Split(in, "\n") { if line = cleanText(line); line != "" { out = append(out, line) } } return out, nil case map[string]any: return stepList([]any{in}) case []any: return stepList(in) } return nil, fmt.Errorf("recipeInstructions is a %T", v) } func stepList(items []any) ([]string, error) { var out []string for _, item := range items { switch s := item.(type) { case string: if text := cleanText(s); text != "" { out = append(out, text) } case map[string]any: switch { case hasType(s, "HowToStep"): text, _ := s["text"].(string) if text = cleanText(text); text == "" { return nil, errors.New("HowToStep has no text") } out = append(out, text) case hasType(s, "HowToSection"): list, ok := s["itemListElement"].([]any) if !ok { return nil, errors.New("HowToSection has no itemListElement list") } section, err := stepList(list) if err != nil { return nil, err } out = append(out, section...) default: return nil, fmt.Errorf("recipeInstructions holds a %v, not a HowToStep or HowToSection", s["@type"]) } default: return nil, fmt.Errorf("recipeInstructions holds a %T", item) } } return out, nil } // minutes reads a duration field: nil when absent or null; any string // present, empty included, must be an ISO-8601 duration. func minutes(n map[string]any, field string) (*int, error) { v, ok := n[field] if !ok || v == nil { return nil, nil } s, ok := v.(string) if !ok { return nil, fmt.Errorf("%s is a %T, not an ISO-8601 duration", field, v) } m, err := isoMinutes(s) if err != nil { return nil, fmt.Errorf("%s: %w", field, err) } return &m, nil } // isoMinutes converts an ISO-8601 duration of weeks, days, hours, minutes // and seconds to whole minutes, rounded. Years and months are refused: // their length in minutes is not fixed. func isoMinutes(s string) (int, error) { bad := fmt.Errorf("%q is not an ISO-8601 duration of days, hours, minutes or seconds", s) rest, ok := strings.CutPrefix(strings.TrimSpace(s), "P") if !ok || rest == "" { return 0, bad } const units = "WDHMS" seconds := map[byte]float64{'W': 7 * 86400, 'D': 86400, 'H': 3600, 'M': 60, 'S': 1} inTime, last, total := false, -1, 0.0 for rest != "" { if rest[0] == 'T' { if inTime || len(rest) == 1 { return 0, bad } inTime, rest = true, rest[1:] continue } i := 0 for i < len(rest) && (rest[i] >= '0' && rest[i] <= '9' || rest[i] == '.' || rest[i] == ',') { i++ } if i == 0 || i == len(rest) { return 0, bad } num, err := strconv.ParseFloat(strings.Replace(rest[:i], ",", ".", 1), 64) if err != nil { return 0, bad } unit := rest[i] pos := strings.IndexByte(units, unit) if pos < 0 || pos <= last || inTime != (pos >= 2) { return 0, bad } last, total, rest = pos, total+num*seconds[unit], rest[i+1:] } if last < 0 { return 0, bad } return int(math.Round(total / 60)), nil } func yield(v any) (string, error) { if list, ok := v.([]any); ok { if len(list) == 0 { return "", nil } v = list[0] } switch y := v.(type) { case nil: return "", nil case string: return cleanText(y), nil case json.Number: return y.String(), nil } return "", fmt.Errorf("recipeYield is a %T", v) } // htmlDoc is what Extract needs from a page's HTML. type htmlDoc struct { jsonLD []string ogSiteName string } // scanHTML is a small targeted scan, not a full HTML parser: it walks the // tags, skipping comments and the bodies of scripts and styles, and keeps // the bodies of application/ld+json scripts and the first og:site_name. func scanHTML(doc string) htmlDoc { var out htmlDoc lower := asciiLower(doc) for pos := 0; ; { lt := strings.IndexByte(doc[pos:], '<') if lt < 0 { return out } pos += lt if strings.HasPrefix(doc[pos:], "") if end < 0 { return out } pos += 4 + end + 3 continue } name := tagName(lower[pos+1:]) if name == "" { pos++ continue } end := tagEnd(doc, pos+1+len(name)) if end < 0 { return out } attrs := parseAttrs(doc[pos+1+len(name) : end]) pos = end + 1 switch name { case "meta": if out.ogSiteName == "" && attrs["property"] == "og:site_name" { out.ogSiteName = strings.TrimSpace(attrs["content"]) } case "script", "style": closeAt := strings.Index(lower[pos:], " 0 && '0' <= s[i] && s[i] <= '9') { i++ } if i == 0 || i < len(s) && !isSpace(s[i]) && s[i] != '>' && s[i] != '/' { return "" } return s[:i] } // tagEnd finds the '>' closing a tag whose attributes start at i, skipping // quoted attribute values. func tagEnd(doc string, i int) int { var quote byte for ; i < len(doc); i++ { switch c := doc[i]; { case quote != 0: if c == quote { quote = 0 } case c == '"' || c == '\'': quote = c case c == '>': return i } } return -1 } // parseAttrs reads a tag's attributes; names are lower-cased, values are // entity-decoded, and the first of a repeated name wins. func parseAttrs(s string) map[string]string { attrs := map[string]string{} i := 0 for i < len(s) { for i < len(s) && (isSpace(s[i]) || s[i] == '/') { i++ } start := i for i < len(s) && !isSpace(s[i]) && s[i] != '=' && s[i] != '/' { i++ } if i == start { i++ continue } name := asciiLower(s[start:i]) for i < len(s) && isSpace(s[i]) { i++ } value := "" if i < len(s) && s[i] == '=' { i++ for i < len(s) && isSpace(s[i]) { i++ } if i < len(s) && (s[i] == '"' || s[i] == '\'') { q := s[i] end := strings.IndexByte(s[i+1:], q) if end < 0 { end = len(s) - i - 1 } value = s[i+1 : i+1+end] i += end + 2 } else { start := i for i < len(s) && !isSpace(s[i]) { i++ } value = s[start:i] } } if _, seen := attrs[name]; !seen { attrs[name] = html.UnescapeString(value) } } return attrs } func isSpace(c byte) bool { return c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' }