diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 0d77b3d..632ecc9 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -208,7 +208,17 @@ automatic** (D-021). - Dinners are grounded in real recipe pages found through recipe search (`searxng` or `brave`, per `recipe_search_settings`). -- Fetching is **SSRF-safe**. +- Fetching is **SSRF-safe** (`recipes.Fetcher`): https:// only, at every + redirect hop. Its dialer resolves the host once and refuses unless every + answer is public — loopback, RFC 1918, link-local, carrier-grade NAT + (100.64/10), multicast, unspecified, reserved, IPv6 unique-local, + link-local and multicast, and IPv4-mapped or IPv4-embedding (NAT64, 6to4, + Teredo) forms are refused — then dials the address it checked, so a + rebinding DNS answer cannot swap in a private one. Keep-alives and + environment proxies are off, so every request dials and checks afresh. At + most 3 redirects; a body over 2 MB is an error, never truncated; 10 s for + the whole fetch; a descriptive `User-Agent`; requests to one host at least + 1 s apart; 2xx pages cached by URL for an hour (64 pages, oldest evicted). - Recipe data is parsed from schema.org `Recipe` JSON-LD. - Photos are cached locally (`photos`) and processed with `golang.org/x/image`. diff --git a/internal/recipes/fetcher.go b/internal/recipes/fetcher.go new file mode 100644 index 0000000..58e18dd --- /dev/null +++ b/internal/recipes/fetcher.go @@ -0,0 +1,373 @@ +// Package recipes fetches and holds real recipe pages that ground dinners. +package recipes + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/netip" + "net/url" + "strings" + "sync" + "time" +) + +const ( + // UserAgent identifies MealQuest to the recipe sites it fetches. + UserAgent = "MealQuest/1.0 (household meal planner; recipe fetcher)" + // MaxBodyBytes caps a fetched body. A larger page is an error, never + // silently truncated: a cut-off recipe page would parse as a wrong one. + MaxBodyBytes = 2 << 20 + // MaxRedirects is how many redirects a fetch follows; each hop is + // re-checked for scheme and address. + MaxRedirects = 3 + // Timeout bounds a whole fetch: rate-limit waits, every redirect hop + // and reading the body. + Timeout = 10 * time.Second + // HostInterval is the least time between two requests to one host. + HostInterval = time.Second + // CacheTTL is how long a fetched page is served from the cache. + CacheTTL = time.Hour + // CacheSize is how many pages the cache holds; the oldest is evicted + // first. + CacheSize = 64 +) + +var ( + // ErrNotHTTPS is returned for a URL, or a redirect, that is not + // https://. + ErrNotHTTPS = errors.New("recipes: only https:// URLs are fetched") + // ErrBlockedAddress is returned when a host resolves to an address + // that is not public, such as loopback or a private network. + ErrBlockedAddress = errors.New("recipes: address is not public") + // ErrTooManyRedirects is returned after MaxRedirects redirects. + ErrTooManyRedirects = errors.New("recipes: too many redirects") + // ErrTooLarge is returned for a body over MaxBodyBytes. + ErrTooLarge = errors.New("recipes: body exceeds size cap") + // ErrTimeout is returned when a fetch runs past Timeout. + ErrTimeout = errors.New("recipes: fetch timed out") +) + +// StatusError is returned for a final response whose status is not 2xx. +type StatusError struct { + URL string + Status int +} + +func (e *StatusError) Error() string { + return fmt.Sprintf("recipes: GET %s: HTTP %d", e.URL, e.Status) +} + +// Page is a fetched page. +type Page struct { + // URL is the final URL, after redirects. + URL string + ContentType string + Body []byte +} + +// Resolver looks up a host's addresses; *net.Resolver satisfies it. +type Resolver interface { + LookupIPAddr(ctx context.Context, host string) ([]net.IPAddr, error) +} + +// dialFunc dials an address that has already passed the public-address +// check. +type dialFunc func(ctx context.Context, network, address string) (net.Conn, error) + +// Fetcher retrieves recipe pages without reaching private networks. Its +// dialer resolves each host itself, refuses the connection unless every +// answer is a public address, and dials the checked address, so a second DNS +// answer cannot swap in a private one. Keep-alives are off: every request, +// redirect hops included, dials and checks afresh. Proxies from the +// environment are not used, since a proxy would dial on the Fetcher's +// behalf. A Fetcher is safe for concurrent use. +type Fetcher struct { + client *http.Client + resolver Resolver + dial dialFunc + // timeout bounds a fetch; tests shorten it. + timeout time.Duration + // now is the clock; tests replace it. + now func() time.Time + // sleep waits d or until ctx is done; tests replace it. + sleep func(ctx context.Context, d time.Duration) error + + mu sync.Mutex + nextSlot map[string]time.Time + cache map[string]cacheEntry +} + +type cacheEntry struct { + page Page + fetched time.Time +} + +// NewFetcher returns a Fetcher that resolves through the system resolver. +func NewFetcher() *Fetcher { + return newFetcher(net.DefaultResolver, (&net.Dialer{}).DialContext, nil) +} + +// newFetcher returns a Fetcher over resolver and dial; tlsConfig, when not +// nil, replaces the default TLS settings so tests can trust their servers. +func newFetcher(resolver Resolver, dial dialFunc, tlsConfig *tls.Config) *Fetcher { + f := &Fetcher{ + resolver: resolver, + dial: dial, + timeout: Timeout, + now: time.Now, + sleep: sleep, + nextSlot: map[string]time.Time{}, + cache: map[string]cacheEntry{}, + } + transport := &http.Transport{ + Proxy: nil, + DialContext: f.dialPublic, + TLSClientConfig: tlsConfig, + DisableKeepAlives: true, + ForceAttemptHTTP2: true, + TLSHandshakeTimeout: Timeout, + } + f.client = &http.Client{ + Transport: &limitedTransport{f: f, next: transport}, + CheckRedirect: checkRedirect, + } + return f +} + +// Fetch returns the page at rawURL, from the cache when it was fetched +// within CacheTTL. Only 2xx responses are returned and cached. +func (f *Fetcher) Fetch(ctx context.Context, rawURL string) (Page, error) { + u, err := url.Parse(rawURL) + if err != nil { + return Page{}, fmt.Errorf("recipes: parse URL: %w", err) + } + if err := checkURL(u); err != nil { + return Page{}, err + } + key := u.String() + if page, ok := f.cached(key); ok { + return page, nil + } + + fetchCtx, cancel := context.WithTimeout(ctx, f.timeout) + defer cancel() + page, err := f.get(fetchCtx, key) + if err != nil { + if ctx.Err() == nil && errors.Is(fetchCtx.Err(), context.DeadlineExceeded) { + return Page{}, fmt.Errorf("%w after %s: %s", ErrTimeout, f.timeout, key) + } + return Page{}, err + } + f.store(key, page) + return page, nil +} + +func (f *Fetcher) get(ctx context.Context, rawURL string) (Page, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return Page{}, fmt.Errorf("recipes: build request: %w", err) + } + req.Header.Set("User-Agent", UserAgent) + req.Header.Set("Accept", "text/html,application/xhtml+xml") + resp, err := f.client.Do(req) + if err != nil { + return Page{}, err + } + defer resp.Body.Close() + final := resp.Request.URL.String() + if resp.StatusCode < 200 || resp.StatusCode > 299 { + return Page{}, &StatusError{URL: final, Status: resp.StatusCode} + } + if resp.ContentLength > MaxBodyBytes { + return Page{}, fmt.Errorf("%w: %s declares %d bytes", ErrTooLarge, final, resp.ContentLength) + } + body, err := io.ReadAll(io.LimitReader(resp.Body, MaxBodyBytes+1)) + if err != nil { + return Page{}, fmt.Errorf("recipes: read %s: %w", final, err) + } + if len(body) > MaxBodyBytes { + return Page{}, fmt.Errorf("%w: %s is over %d bytes", ErrTooLarge, final, MaxBodyBytes) + } + return Page{URL: final, ContentType: resp.Header.Get("Content-Type"), Body: body}, nil +} + +// cached returns the page cached under key if it is younger than CacheTTL. +func (f *Fetcher) cached(key string) (Page, bool) { + f.mu.Lock() + defer f.mu.Unlock() + e, ok := f.cache[key] + if !ok { + return Page{}, false + } + if f.now().Sub(e.fetched) >= CacheTTL { + delete(f.cache, key) + return Page{}, false + } + return e.page, true +} + +// store caches page under key, evicting the oldest entry when full. +func (f *Fetcher) store(key string, page Page) { + f.mu.Lock() + defer f.mu.Unlock() + if _, ok := f.cache[key]; !ok && len(f.cache) >= CacheSize { + var oldest string + var oldestAt time.Time + for k, e := range f.cache { + if oldest == "" || e.fetched.Before(oldestAt) { + oldest, oldestAt = k, e.fetched + } + } + delete(f.cache, oldest) + } + f.cache[key] = cacheEntry{page: page, fetched: f.now()} +} + +// waitTurn blocks until host's next request slot, keeping requests to one +// host at least HostInterval apart. +func (f *Fetcher) waitTurn(ctx context.Context, host string) error { + f.mu.Lock() + now := f.now() + slot := f.nextSlot[host] + if slot.Before(now) { + slot = now + } + f.nextSlot[host] = slot.Add(HostInterval) + f.mu.Unlock() + if wait := slot.Sub(now); wait > 0 { + return f.sleep(ctx, wait) + } + return nil +} + +// dialPublic resolves address's host once, refuses unless every answer is +// public, then dials the checked addresses in turn. +func (f *Fetcher) dialPublic(ctx context.Context, network, address string) (net.Conn, error) { + host, port, err := net.SplitHostPort(address) + if err != nil { + return nil, fmt.Errorf("recipes: dial address %q: %w", address, err) + } + var addrs []netip.Addr + if ip, err := netip.ParseAddr(host); err == nil { + addrs = []netip.Addr{ip} + } else { + answers, err := f.resolver.LookupIPAddr(ctx, host) + if err != nil { + return nil, fmt.Errorf("recipes: resolve %s: %w", host, err) + } + for _, a := range answers { + ip, ok := netip.AddrFromSlice(a.IP) + if !ok { + return nil, fmt.Errorf("recipes: resolve %s: malformed address %v", host, a.IP) + } + addrs = append(addrs, ip) + } + } + if len(addrs) == 0 { + return nil, fmt.Errorf("recipes: resolve %s: no addresses", host) + } + for _, ip := range addrs { + if !IsPublic(ip) { + return nil, fmt.Errorf("%w: %s resolves to %s", ErrBlockedAddress, host, ip) + } + } + var errs []error + for _, ip := range addrs { + conn, err := f.dial(ctx, network, net.JoinHostPort(ip.Unmap().WithZone("").String(), port)) + if err == nil { + return conn, nil + } + errs = append(errs, err) + } + return nil, errors.Join(errs...) +} + +// limitedTransport waits for each request's per-host rate-limit slot, +// redirect hops included, before sending it. +type limitedTransport struct { + f *Fetcher + next http.RoundTripper +} + +func (t *limitedTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if err := t.f.waitTurn(req.Context(), strings.ToLower(req.URL.Hostname())); err != nil { + return nil, err + } + return t.next.RoundTrip(req) +} + +// checkURL refuses anything but an absolute https:// URL with a host. +func checkURL(u *url.URL) error { + if u.Scheme != "https" { + return fmt.Errorf("%w: %q", ErrNotHTTPS, u.Redacted()) + } + if u.Hostname() == "" { + return fmt.Errorf("recipes: URL %q has no host", u.Redacted()) + } + return nil +} + +// checkRedirect applies checkURL to every hop and stops after +// MaxRedirects. The hop's address is checked when it is dialed. +func checkRedirect(req *http.Request, via []*http.Request) error { + if len(via) > MaxRedirects { + return fmt.Errorf("%w: more than %d", ErrTooManyRedirects, MaxRedirects) + } + return checkURL(req.URL) +} + +// blockedPrefixes are the non-public ranges. IPv4-mapped IPv6 addresses +// are unmapped before the check, so the IPv4 entries cover them too. +var blockedPrefixes = []netip.Prefix{ + netip.MustParsePrefix("0.0.0.0/8"), // unspecified, "this network" + netip.MustParsePrefix("10.0.0.0/8"), // RFC 1918 + netip.MustParsePrefix("100.64.0.0/10"), // carrier-grade NAT + netip.MustParsePrefix("127.0.0.0/8"), // loopback + netip.MustParsePrefix("169.254.0.0/16"), // link-local + netip.MustParsePrefix("172.16.0.0/12"), // RFC 1918 + netip.MustParsePrefix("192.168.0.0/16"), // RFC 1918 + netip.MustParsePrefix("224.0.0.0/4"), // multicast + netip.MustParsePrefix("240.0.0.0/4"), // reserved, broadcast + netip.MustParsePrefix("::/96"), // unspecified, loopback, IPv4-compatible + netip.MustParsePrefix("fc00::/7"), // unique local + netip.MustParsePrefix("fe80::/10"), // link-local + netip.MustParsePrefix("fec0::/10"), // site-local (deprecated) + netip.MustParsePrefix("ff00::/8"), // multicast + netip.MustParsePrefix("64:ff9b::/96"), // NAT64, embeds any IPv4 + netip.MustParsePrefix("64:ff9b:1::/48"), // local-use NAT64 + netip.MustParsePrefix("2002::/16"), // 6to4, embeds any IPv4 + netip.MustParsePrefix("2001::/32"), // Teredo, embeds any IPv4 + netip.MustParsePrefix("::ffff:0:0:0/96"), // IPv4-translated +} + +// IsPublic reports whether ip is a public unicast address the Fetcher may +// dial. +func IsPublic(ip netip.Addr) bool { + if !ip.IsValid() { + return false + } + ip = ip.Unmap().WithZone("") + for _, p := range blockedPrefixes { + if p.Contains(ip) { + return false + } + } + return true +} + +// sleep waits d or until ctx is done, returning ctx's error if it is. +func sleep(ctx context.Context, d time.Duration) error { + t := time.NewTimer(d) + defer t.Stop() + select { + case <-t.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} diff --git a/internal/recipes/fetcher_test.go b/internal/recipes/fetcher_test.go new file mode 100644 index 0000000..f1c2f53 --- /dev/null +++ b/internal/recipes/fetcher_test.go @@ -0,0 +1,515 @@ +package recipes + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "net" + "net/http" + "net/http/httptest" + "net/netip" + "strings" + "sync" + "testing" + "time" +) + +// stubResolver answers each host from a list of answers, one per lookup; +// the last answer repeats. +type stubResolver struct { + mu sync.Mutex + answers map[string][][]string + calls map[string]int +} + +func (r *stubResolver) LookupIPAddr(_ context.Context, host string) ([]net.IPAddr, error) { + r.mu.Lock() + defer r.mu.Unlock() + seq, ok := r.answers[host] + if !ok { + return nil, fmt.Errorf("stub: no such host %s", host) + } + n := r.calls[host] + r.calls[host]++ + if n >= len(seq) { + n = len(seq) - 1 + } + var out []net.IPAddr + for _, s := range seq[n] { + out = append(out, net.IPAddr{IP: net.ParseIP(s)}) + } + return out, nil +} + +func (r *stubResolver) lookups(host string) int { + r.mu.Lock() + defer r.mu.Unlock() + return r.calls[host] +} + +// testEnv is a Fetcher whose public test addresses all reach one httptest +// TLS server, and which records every address it dials. +type testEnv struct { + f *Fetcher + resolver *stubResolver + server *httptest.Server + + mu sync.Mutex + dialed []string +} + +// publicIP is a documentation address the tests treat as a public host. +const publicIP = "203.0.113.10" + +func newTestEnv(t *testing.T, handler http.Handler, answers map[string][][]string) *testEnv { + t.Helper() + server := httptest.NewTLSServer(handler) + t.Cleanup(server.Close) + pool := x509.NewCertPool() + pool.AddCert(server.Certificate()) + env := &testEnv{ + resolver: &stubResolver{answers: answers, calls: map[string]int{}}, + server: server, + } + dial := func(ctx context.Context, network, address string) (net.Conn, error) { + env.mu.Lock() + env.dialed = append(env.dialed, address) + env.mu.Unlock() + host, _, err := net.SplitHostPort(address) + if err != nil { + return nil, err + } + if strings.HasPrefix(host, "203.0.113.") { + var d net.Dialer + return d.DialContext(ctx, network, server.Listener.Addr().String()) + } + return nil, fmt.Errorf("test dial: no route to %s", address) + } + env.f = newFetcher(env.resolver, dial, &tls.Config{RootCAs: pool}) + // Real requests in these tests are not rate-limited unless a test + // installs a fake clock. + env.f.sleep = func(context.Context, time.Duration) error { return nil } + return env +} + +func (e *testEnv) dials() []string { + e.mu.Lock() + defer e.mu.Unlock() + return append([]string(nil), e.dialed...) +} + +func publicAnswers() map[string][][]string { + return map[string][][]string{"example.com": {{publicIP}}} +} + +func TestFetchReturnsPage(t *testing.T) { + t.Parallel() + var gotUA string + env := newTestEnv(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotUA = r.Header.Get("User-Agent") + w.Header().Set("Content-Type", "text/html") + fmt.Fprint(w, "tacos") + }), publicAnswers()) + + page, err := env.f.Fetch(t.Context(), "https://example.com/tacos") + if err != nil { + t.Fatalf("Fetch: %v", err) + } + if string(page.Body) != "tacos" || page.ContentType != "text/html" || page.URL != "https://example.com/tacos" { + t.Errorf("page = %+v", page) + } + if gotUA != UserAgent { + t.Errorf("User-Agent = %q, want %q", gotUA, UserAgent) + } + if got := env.dials(); len(got) != 1 || got[0] != publicIP+":443" { + t.Errorf("dialed %v, want [%s:443]", got, publicIP) + } +} + +func TestFetchRejectsNonHTTPS(t *testing.T) { + t.Parallel() + env := newTestEnv(t, http.NotFoundHandler(), publicAnswers()) + for _, raw := range []string{"http://example.com/tacos", "ftp://example.com/", "example.com/tacos", "https:///tacos"} { + _, err := env.f.Fetch(t.Context(), raw) + if err == nil { + t.Errorf("Fetch(%q) succeeded", raw) + } + if strings.HasPrefix(raw, "http://") && !errors.Is(err, ErrNotHTTPS) { + t.Errorf("Fetch(%q) error = %v, want ErrNotHTTPS", raw, err) + } + } + if got := env.dials(); len(got) != 0 { + t.Errorf("dialed %v, want nothing", got) + } + if n := env.resolver.lookups("example.com"); n != 0 { + t.Errorf("resolved example.com %d times, want 0", n) + } +} + +// blockedIPs has an address in every refused range, including IPv4-mapped +// forms of the IPv4 ranges. +var blockedIPs = []string{ + "127.0.0.1", "127.255.255.254", "::1", "::ffff:127.0.0.1", // loopback + "10.0.0.1", "172.16.0.1", "172.31.255.255", "192.168.1.1", // RFC 1918 + "::ffff:10.1.2.3", "::ffff:172.20.0.1", "::ffff:192.168.0.10", + "169.254.169.254", "::ffff:169.254.169.254", "fe80::1", // link-local + "100.64.0.1", "100.127.255.254", "::ffff:100.100.0.1", // carrier-grade NAT + "224.0.0.1", "239.255.255.250", "ff02::1", "::ffff:224.0.0.251", // multicast + "0.0.0.0", "0.1.2.3", "::", "::ffff:0.0.0.0", // unspecified + "fc00::1", "fd12:3456:789a::1", // unique local + "255.255.255.255", "240.0.0.1", "fec0::1", + "64:ff9b::a00:1", "2002:a00:1::1", "2001:0:a00:1::1", "::a00:1", +} + +func TestIsPublic(t *testing.T) { + t.Parallel() + for _, s := range blockedIPs { + if IsPublic(netip.MustParseAddr(s)) { + t.Errorf("IsPublic(%s) = true, want false", s) + } + } + for _, s := range []string{"8.8.8.8", "203.0.113.10", "100.63.255.255", "100.128.0.1", "172.32.0.1", "2606:4700::1111", "::ffff:8.8.8.8"} { + if !IsPublic(netip.MustParseAddr(s)) { + t.Errorf("IsPublic(%s) = false, want true", s) + } + } + if IsPublic(netip.Addr{}) { + t.Error("IsPublic(zero Addr) = true, want false") + } +} + +func TestFetchRefusesEachBlockedRange(t *testing.T) { + t.Parallel() + for _, ip := range blockedIPs { + t.Run(ip, func(t *testing.T) { + t.Parallel() + env := newTestEnv(t, http.NotFoundHandler(), map[string][][]string{"example.com": {{ip}}}) + _, err := env.f.Fetch(t.Context(), "https://example.com/") + if !errors.Is(err, ErrBlockedAddress) { + t.Fatalf("Fetch error = %v, want ErrBlockedAddress", err) + } + if got := env.dials(); len(got) != 0 { + t.Errorf("dialed %v, want nothing", got) + } + }) + } +} + +func TestFetchRefusesBlockedLiteralHost(t *testing.T) { + t.Parallel() + env := newTestEnv(t, http.NotFoundHandler(), nil) + for _, raw := range []string{"https://127.0.0.1/", "https://[::1]/", "https://[::ffff:10.0.0.1]/", "https://169.254.169.254/latest/meta-data/", "https://[fe80::1%25eth0]/"} { + if _, err := env.f.Fetch(t.Context(), raw); !errors.Is(err, ErrBlockedAddress) { + t.Errorf("Fetch(%q) error = %v, want ErrBlockedAddress", raw, err) + } + } + if got := env.dials(); len(got) != 0 { + t.Errorf("dialed %v, want nothing", got) + } +} + +func TestFetchRefusesMixedAnswer(t *testing.T) { + t.Parallel() + env := newTestEnv(t, http.NotFoundHandler(), map[string][][]string{"example.com": {{publicIP, "10.0.0.1"}}}) + if _, err := env.f.Fetch(t.Context(), "https://example.com/"); !errors.Is(err, ErrBlockedAddress) { + t.Fatalf("Fetch error = %v, want ErrBlockedAddress", err) + } + if got := env.dials(); len(got) != 0 { + t.Errorf("dialed %v, want nothing", got) + } +} + +func TestFetchRefusesRedirectToPrivateHost(t *testing.T) { + t.Parallel() + answers := map[string][][]string{ + "example.com": {{publicIP}}, + "evil.example.com": {{"10.0.0.5"}}, + } + for _, target := range []string{"https://evil.example.com/admin", "https://127.0.0.1/admin", "https://[::ffff:192.168.0.1]/"} { + t.Run(target, func(t *testing.T) { + t.Parallel() + env := newTestEnv(t, http.RedirectHandler(target, http.StatusFound), answers) + _, err := env.f.Fetch(t.Context(), "https://example.com/tacos") + if !errors.Is(err, ErrBlockedAddress) { + t.Fatalf("Fetch error = %v, want ErrBlockedAddress", err) + } + if got := env.dials(); len(got) != 1 || got[0] != publicIP+":443" { + t.Errorf("dialed %v, want only the first hop", got) + } + }) + } +} + +func TestFetchRefusesRedirectToHTTP(t *testing.T) { + t.Parallel() + env := newTestEnv(t, http.RedirectHandler("http://example.com/plain", http.StatusMovedPermanently), publicAnswers()) + if _, err := env.f.Fetch(t.Context(), "https://example.com/"); !errors.Is(err, ErrNotHTTPS) { + t.Fatalf("Fetch error = %v, want ErrNotHTTPS", err) + } +} + +// chainHandler redirects /hop/N to /hop/N-1 until /hop/0, which serves a +// page. +var chainHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var n int + if _, err := fmt.Sscanf(r.URL.Path, "/hop/%d", &n); err != nil { + http.NotFound(w, r) + return + } + if n == 0 { + fmt.Fprint(w, "arrived") + return + } + http.Redirect(w, r, fmt.Sprintf("https://recipes.example.com/hop/%d", n-1), http.StatusFound) +}) + +func TestFetchFollowsAtMostThreeRedirects(t *testing.T) { + t.Parallel() + answers := map[string][][]string{ + "example.com": {{publicIP}}, + "recipes.example.com": {{"203.0.113.20"}}, + } + env := newTestEnv(t, chainHandler, answers) + page, err := env.f.Fetch(t.Context(), "https://example.com/hop/3") + if err != nil { + t.Fatalf("Fetch with 3 redirects: %v", err) + } + if page.URL != "https://recipes.example.com/hop/0" || string(page.Body) != "arrived" { + t.Errorf("page = %+v", page) + } + // Every hop is dialed, so every hop's address is checked. + if got := env.dials(); len(got) != 4 || got[0] != publicIP+":443" || got[3] != "203.0.113.20:443" { + t.Errorf("dialed %v, want 4 checked hops", got) + } + + if _, err := env.f.Fetch(t.Context(), "https://example.com/hop/4"); !errors.Is(err, ErrTooManyRedirects) { + t.Fatalf("Fetch with 4 redirects error = %v, want ErrTooManyRedirects", err) + } +} + +// TestFetchRefusesDNSSwitchToPrivate covers DNS rebinding: the host's first +// answer is public and every later one private. The first fetch dials the +// address it checked; the next lookup's private answer is refused, and the +// private address is never dialed. +func TestFetchRefusesDNSSwitchToPrivate(t *testing.T) { + t.Parallel() + answers := map[string][][]string{"example.com": {{publicIP}, {"127.0.0.1"}}} + env := newTestEnv(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "ok") + }), answers) + + if _, err := env.f.Fetch(t.Context(), "https://example.com/one"); err != nil { + t.Fatalf("first Fetch: %v", err) + } + if n := env.resolver.lookups("example.com"); n != 1 { + t.Fatalf("first Fetch resolved %d times, want 1 (checked IP must be dialed IP)", n) + } + _, err := env.f.Fetch(t.Context(), "https://example.com/two") + if !errors.Is(err, ErrBlockedAddress) { + t.Fatalf("second Fetch error = %v, want ErrBlockedAddress", err) + } + if got := env.dials(); len(got) != 1 || got[0] != publicIP+":443" { + t.Errorf("dialed %v, want only %s:443", got, publicIP) + } +} + +func TestFetchSizeCap(t *testing.T) { + t.Parallel() + env := newTestEnv(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + size := MaxBodyBytes + if r.URL.Path != "/exact" { + size++ + } + if r.URL.Path == "/declared" { + w.Header().Set("Content-Length", fmt.Sprint(size)) + } + w.Write([]byte(strings.Repeat("a", size))) + }), publicAnswers()) + + page, err := env.f.Fetch(t.Context(), "https://example.com/exact") + if err != nil { + t.Fatalf("Fetch at the cap: %v", err) + } + if len(page.Body) != MaxBodyBytes { + t.Errorf("body is %d bytes, want %d", len(page.Body), MaxBodyBytes) + } + for _, path := range []string{"/declared", "/chunked"} { + if _, err := env.f.Fetch(t.Context(), "https://example.com"+path); !errors.Is(err, ErrTooLarge) { + t.Errorf("Fetch %s error = %v, want ErrTooLarge", path, err) + } + } +} + +func TestFetchTimeout(t *testing.T) { + t.Parallel() + env := newTestEnv(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/slow-body" { + w.Write([]byte("start")) + w.(http.Flusher).Flush() + } + <-r.Context().Done() + }), publicAnswers()) + env.f.timeout = 100 * time.Millisecond + + for _, path := range []string{"/slow-headers", "/slow-body"} { + start := time.Now() + _, err := env.f.Fetch(t.Context(), "https://example.com"+path) + if !errors.Is(err, ErrTimeout) { + t.Errorf("Fetch %s error = %v, want ErrTimeout", path, err) + } + if elapsed := time.Since(start); elapsed > 5*time.Second { + t.Errorf("Fetch %s took %s", path, elapsed) + } + } + if Timeout != 10*time.Second { + t.Errorf("Timeout = %s, want 10s", Timeout) + } +} + +func TestFetchCallerCancelIsNotTimeout(t *testing.T) { + t.Parallel() + env := newTestEnv(t, http.NotFoundHandler(), publicAnswers()) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + _, err := env.f.Fetch(ctx, "https://example.com/") + if !errors.Is(err, context.Canceled) || errors.Is(err, ErrTimeout) { + t.Fatalf("Fetch error = %v, want context.Canceled", err) + } +} + +// fakeClock is a clock whose sleep advances it and records the wait. +type fakeClock struct { + mu sync.Mutex + t time.Time + sleeps []time.Duration +} + +func (c *fakeClock) now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.t +} + +func (c *fakeClock) sleep(_ context.Context, d time.Duration) error { + c.mu.Lock() + defer c.mu.Unlock() + c.sleeps = append(c.sleeps, d) + c.t = c.t.Add(d) + return nil +} + +func (c *fakeClock) advance(d time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + c.t = c.t.Add(d) +} + +func (c *fakeClock) waits() []time.Duration { + c.mu.Lock() + defer c.mu.Unlock() + return append([]time.Duration(nil), c.sleeps...) +} + +func installClock(f *Fetcher) *fakeClock { + c := &fakeClock{t: time.Date(2026, 10, 5, 18, 0, 0, 0, time.UTC)} + f.now = c.now + f.sleep = c.sleep + return c +} + +func TestFetchCachesByURL(t *testing.T) { + t.Parallel() + var mu sync.Mutex + hits := map[string]int{} + env := newTestEnv(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + hits[r.URL.Path]++ + n := hits[r.URL.Path] + mu.Unlock() + if r.URL.Path == "/missing" { + http.NotFound(w, r) + return + } + fmt.Fprintf(w, "%s #%d", r.URL.Path, n) + }), publicAnswers()) + clock := installClock(env.f) + + fetch := func(path string) string { + t.Helper() + page, err := env.f.Fetch(t.Context(), "https://example.com"+path) + if err != nil { + t.Fatalf("Fetch %s: %v", path, err) + } + return string(page.Body) + } + if got := fetch("/a"); got != "/a #1" { + t.Errorf("first /a = %q", got) + } + if got := fetch("/a"); got != "/a #1" { + t.Errorf("cached /a = %q, want the first response", got) + } + if got := fetch("/b"); got != "/b #1" { + t.Errorf("/b = %q, want its own response", got) + } + clock.advance(CacheTTL) + if got := fetch("/a"); got != "/a #2" { + t.Errorf("/a after CacheTTL = %q, want a refetch", got) + } + + for range 2 { + var se *StatusError + if _, err := env.f.Fetch(t.Context(), "https://example.com/missing"); !errors.As(err, &se) || se.Status != http.StatusNotFound { + t.Fatalf("Fetch /missing error = %v, want StatusError 404", err) + } + } + mu.Lock() + defer mu.Unlock() + if hits["/missing"] != 2 { + t.Errorf("/missing was requested %d times, want 2 (errors are not cached)", hits["/missing"]) + } +} + +func TestCacheEvictsOldest(t *testing.T) { + t.Parallel() + f := newFetcher(&stubResolver{}, nil, nil) + clock := installClock(f) + for i := range CacheSize + 1 { + f.store(fmt.Sprintf("https://example.com/%d", i), Page{}) + clock.advance(time.Second) + } + if len(f.cache) != CacheSize { + t.Errorf("cache holds %d pages, want %d", len(f.cache), CacheSize) + } + if _, ok := f.cached("https://example.com/0"); ok { + t.Error("oldest page was not evicted") + } + if _, ok := f.cached(fmt.Sprintf("https://example.com/%d", CacheSize)); !ok { + t.Error("newest page is missing") + } +} + +func TestFetchRateLimitsPerHost(t *testing.T) { + t.Parallel() + answers := map[string][][]string{ + "example.com": {{publicIP}}, + "recipes.example.com": {{"203.0.113.20"}}, + } + env := newTestEnv(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, "ok") + }), answers) + clock := installClock(env.f) + + for _, raw := range []string{"https://example.com/1", "https://recipes.example.com/1", "https://example.com/2", "https://example.com/3"} { + if _, err := env.f.Fetch(t.Context(), raw); err != nil { + t.Fatalf("Fetch %s: %v", raw, err) + } + } + // example.com/2 waits a full interval; example.com/3 waits another. + // recipes.example.com has its own slot and never waits. + got := clock.waits() + if len(got) != 2 || got[0] != HostInterval || got[1] != HostInterval { + t.Errorf("waits = %v, want [%s %s]", got, HostInterval, HostInterval) + } +}