Add photo cache and /media/photos endpoint
recipes.PhotoCache.CachePhoto downloads an image through the safe
Fetcher's new FetchImage (JPEG/PNG/WebP/GIF content types only, 8 MB
cap, uncached), decodes it (GIF first frame), refuses images under
300 px on the short side, scales to 1200 px and 400 px on the long side
with golang.org/x/image/draw, encodes JPEG quality 82 under
$MEALQUEST_DATA_DIR/media/photos/{id}-{size}.jpg, deduplicates by the
SHA-256 of the original bytes and records a photos row.
GET /media/photos/{id}-{400|1200}.jpg serves a stored size behind the
session with Cache-Control "private, max-age=31536000, immutable";
anything unknown is 404.
Adds golang.org/x/image v0.45.0 (approved; v0.46.0 needs Go 1.26).
Maestro task: 8a432108-d3ba-4013-ae8c-11963981d641
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+66
-14
@@ -8,10 +8,12 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -23,6 +25,8 @@ const (
|
||||
// MaxBodyBytes caps a fetched body. A larger page is an error, never
|
||||
// silently truncated: a cut-off recipe page would parse as a wrong one.
|
||||
MaxBodyBytes = 2 << 20
|
||||
// MaxImageBytes caps a fetched image body, as MaxBodyBytes caps a page.
|
||||
MaxImageBytes = 8 << 20
|
||||
// MaxRedirects is how many redirects a fetch follows; each hop is
|
||||
// re-checked for scheme and address.
|
||||
MaxRedirects = 3
|
||||
@@ -51,6 +55,25 @@ var (
|
||||
ErrTooLarge = errors.New("recipes: body exceeds size cap")
|
||||
// ErrTimeout is returned when a fetch runs past Timeout.
|
||||
ErrTimeout = errors.New("recipes: fetch timed out")
|
||||
// ErrNotImage is returned by FetchImage for a response whose
|
||||
// Content-Type is not one of ImageContentTypes.
|
||||
ErrNotImage = errors.New("recipes: response is not a supported image")
|
||||
)
|
||||
|
||||
// ImageContentTypes are the media types FetchImage accepts.
|
||||
var ImageContentTypes = []string{"image/jpeg", "image/png", "image/webp", "image/gif"}
|
||||
|
||||
// fetchKind is what a request asks for: its Accept header, the body cap and
|
||||
// the media types allowed, where nil allows any.
|
||||
type fetchKind struct {
|
||||
accept string
|
||||
maxBytes int64
|
||||
contentTypes []string
|
||||
}
|
||||
|
||||
var (
|
||||
pageKind = fetchKind{accept: "text/html,application/xhtml+xml", maxBytes: MaxBodyBytes}
|
||||
imageKind = fetchKind{accept: strings.Join(ImageContentTypes, ","), maxBytes: MaxImageBytes, contentTypes: ImageContentTypes}
|
||||
)
|
||||
|
||||
// StatusError is returned for a final response whose status is not 2xx.
|
||||
@@ -163,27 +186,49 @@ func (f *Fetcher) Fetch(ctx context.Context, rawURL string) (Page, error) {
|
||||
if page, ok := f.cached(key); ok {
|
||||
return page, nil
|
||||
}
|
||||
|
||||
fetchCtx, cancel := context.WithTimeout(ctx, f.timeout)
|
||||
defer cancel()
|
||||
page, err := f.get(fetchCtx, key)
|
||||
page, err := f.fetch(ctx, key, pageKind)
|
||||
if err != nil {
|
||||
if ctx.Err() == nil && errors.Is(fetchCtx.Err(), context.DeadlineExceeded) {
|
||||
return Page{}, fmt.Errorf("%w after %s: %s", ErrTimeout, f.timeout, key)
|
||||
}
|
||||
return Page{}, err
|
||||
}
|
||||
f.store(key, page)
|
||||
return page, nil
|
||||
}
|
||||
|
||||
func (f *Fetcher) get(ctx context.Context, rawURL string) (Page, error) {
|
||||
// FetchImage returns the image at rawURL under the same safeguards as
|
||||
// Fetch, but with a body cap of MaxImageBytes, and only for a response
|
||||
// whose Content-Type is one of ImageContentTypes. Images are not cached.
|
||||
func (f *Fetcher) FetchImage(ctx context.Context, rawURL string) (Page, error) {
|
||||
u, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
return Page{}, fmt.Errorf("recipes: parse URL: %w", err)
|
||||
}
|
||||
if err := checkURL(u); err != nil {
|
||||
return Page{}, err
|
||||
}
|
||||
return f.fetch(ctx, u.String(), imageKind)
|
||||
}
|
||||
|
||||
// fetch gets rawURL as kind within the Fetcher's timeout.
|
||||
func (f *Fetcher) fetch(ctx context.Context, rawURL string, kind fetchKind) (Page, error) {
|
||||
fetchCtx, cancel := context.WithTimeout(ctx, f.timeout)
|
||||
defer cancel()
|
||||
page, err := f.get(fetchCtx, rawURL, kind)
|
||||
if err != nil {
|
||||
if ctx.Err() == nil && errors.Is(fetchCtx.Err(), context.DeadlineExceeded) {
|
||||
return Page{}, fmt.Errorf("%w after %s: %s", ErrTimeout, f.timeout, rawURL)
|
||||
}
|
||||
return Page{}, err
|
||||
}
|
||||
return page, nil
|
||||
}
|
||||
|
||||
func (f *Fetcher) get(ctx context.Context, rawURL string, kind fetchKind) (Page, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
|
||||
if err != nil {
|
||||
return Page{}, fmt.Errorf("recipes: build request: %w", err)
|
||||
}
|
||||
req.Header.Set("User-Agent", UserAgent)
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml")
|
||||
req.Header.Set("Accept", kind.accept)
|
||||
resp, err := f.client.Do(req)
|
||||
if err != nil {
|
||||
return Page{}, err
|
||||
@@ -193,17 +238,24 @@ func (f *Fetcher) get(ctx context.Context, rawURL string) (Page, error) {
|
||||
if resp.StatusCode < 200 || resp.StatusCode > 299 {
|
||||
return Page{}, &StatusError{URL: final, Status: resp.StatusCode}
|
||||
}
|
||||
if resp.ContentLength > MaxBodyBytes {
|
||||
contentType := resp.Header.Get("Content-Type")
|
||||
if kind.contentTypes != nil {
|
||||
mediaType, _, err := mime.ParseMediaType(contentType)
|
||||
if err != nil || !slices.Contains(kind.contentTypes, mediaType) {
|
||||
return Page{}, fmt.Errorf("%w: %s has Content-Type %q", ErrNotImage, final, contentType)
|
||||
}
|
||||
}
|
||||
if resp.ContentLength > kind.maxBytes {
|
||||
return Page{}, fmt.Errorf("%w: %s declares %d bytes", ErrTooLarge, final, resp.ContentLength)
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, MaxBodyBytes+1))
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, kind.maxBytes+1))
|
||||
if err != nil {
|
||||
return Page{}, fmt.Errorf("recipes: read %s: %w", final, err)
|
||||
}
|
||||
if len(body) > MaxBodyBytes {
|
||||
return Page{}, fmt.Errorf("%w: %s is over %d bytes", ErrTooLarge, final, MaxBodyBytes)
|
||||
if int64(len(body)) > kind.maxBytes {
|
||||
return Page{}, fmt.Errorf("%w: %s is over %d bytes", ErrTooLarge, final, kind.maxBytes)
|
||||
}
|
||||
return Page{URL: final, ContentType: resp.Header.Get("Content-Type"), Body: body}, nil
|
||||
return Page{URL: final, ContentType: contentType, Body: body}, nil
|
||||
}
|
||||
|
||||
// cached returns the page cached under key if it is younger than CacheTTL.
|
||||
|
||||
@@ -0,0 +1,248 @@
|
||||
package recipes
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/gif"
|
||||
"image/jpeg"
|
||||
"image/png"
|
||||
"io/fs"
|
||||
"math"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strconv"
|
||||
"sync"
|
||||
|
||||
"github.com/oklog/ulid/v2"
|
||||
"golang.org/x/image/draw"
|
||||
"golang.org/x/image/webp"
|
||||
|
||||
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/db"
|
||||
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/store"
|
||||
)
|
||||
|
||||
const (
|
||||
// PhotoDir is where cached photos are stored, relative to the data
|
||||
// directory.
|
||||
PhotoDir = "media/photos"
|
||||
// PhotoMinShortSide is the least a photo's shorter side may measure, in
|
||||
// pixels.
|
||||
PhotoMinShortSide = 300
|
||||
// PhotoMaxPixels bounds the pixels a photo may decode to, so a small
|
||||
// compressed file cannot claim gigabytes of memory.
|
||||
PhotoMaxPixels = 40_000_000
|
||||
// PhotoQuality is the JPEG quality of every stored size.
|
||||
PhotoQuality = 82
|
||||
)
|
||||
|
||||
// PhotoSizes are the long-side lengths, in pixels, each photo is stored at.
|
||||
var PhotoSizes = []int{1200, 400}
|
||||
|
||||
var (
|
||||
// ErrPhotoTooSmall is returned for an image whose shorter side is under
|
||||
// PhotoMinShortSide.
|
||||
ErrPhotoTooSmall = errors.New("recipes: photo is too small")
|
||||
// ErrPhotoTooLarge is returned for an image over PhotoMaxPixels.
|
||||
ErrPhotoTooLarge = errors.New("recipes: photo has too many pixels")
|
||||
// ErrPhotoFormat is returned for bytes that are not a JPEG, PNG, WebP
|
||||
// or GIF image.
|
||||
ErrPhotoFormat = errors.New("recipes: photo is not a JPEG, PNG, WebP or GIF image")
|
||||
)
|
||||
|
||||
// photoDecoders are the formats a photo may be in. A GIF decodes to its
|
||||
// first frame.
|
||||
var photoDecoders = map[string]func([]byte) (image.Image, error){
|
||||
"jpeg": func(b []byte) (image.Image, error) { return jpeg.Decode(bytes.NewReader(b)) },
|
||||
"png": func(b []byte) (image.Image, error) { return png.Decode(bytes.NewReader(b)) },
|
||||
"gif": func(b []byte) (image.Image, error) { return gif.Decode(bytes.NewReader(b)) },
|
||||
"webp": func(b []byte) (image.Image, error) { return webp.Decode(bytes.NewReader(b)) },
|
||||
}
|
||||
|
||||
// PhotoCache downloads photos through a Fetcher and stores each at every
|
||||
// PhotoSizes length as {id}-{size}.jpg under PhotoDir, with a photos row.
|
||||
// Photos are deduplicated by the SHA-256 of the downloaded bytes. A
|
||||
// PhotoCache is safe for concurrent use; photos are processed one at a time.
|
||||
type PhotoCache struct {
|
||||
fetcher *Fetcher
|
||||
store *store.Store
|
||||
// dir is the absolute photo directory.
|
||||
dir string
|
||||
|
||||
// mu serialises the dedupe lookup, processing and insert, so one image
|
||||
// is never stored twice.
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
// NewPhotoCache returns a PhotoCache storing photos under dataDir, creating
|
||||
// the photo directory if it is absent.
|
||||
func NewPhotoCache(fetcher *Fetcher, st *store.Store, dataDir string) (*PhotoCache, error) {
|
||||
if fetcher == nil {
|
||||
return nil, errors.New("recipes: nil fetcher")
|
||||
}
|
||||
if st == nil {
|
||||
return nil, errors.New("recipes: nil store")
|
||||
}
|
||||
dir, err := filepath.Abs(filepath.Join(dataDir, filepath.FromSlash(PhotoDir)))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("recipes: resolve photo directory: %w", err)
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0o750); err != nil {
|
||||
return nil, fmt.Errorf("recipes: create photo directory: %w", err)
|
||||
}
|
||||
return &PhotoCache{fetcher: fetcher, store: st, dir: dir}, nil
|
||||
}
|
||||
|
||||
// CachePhoto downloads the image at rawURL and returns its photos row. An
|
||||
// image whose bytes are already cached returns the existing row. Otherwise
|
||||
// it is decoded, refused if its shorter side is under PhotoMinShortSide,
|
||||
// scaled so its long side is each of PhotoSizes, encoded as JPEG and
|
||||
// recorded. The row's width and height are the original image's, and its
|
||||
// local_path is the largest size's file relative to the data directory.
|
||||
func (c *PhotoCache) CachePhoto(ctx context.Context, rawURL string) (db.Photo, error) {
|
||||
page, err := c.fetcher.FetchImage(ctx, rawURL)
|
||||
if err != nil {
|
||||
return db.Photo{}, err
|
||||
}
|
||||
digest := sha256.Sum256(page.Body)
|
||||
sum := hex.EncodeToString(digest[:])
|
||||
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
existing, err := db.New(c.store.Reader).GetPhotoBySHA256(ctx, sum)
|
||||
if err == nil {
|
||||
return existing, nil
|
||||
}
|
||||
if !errors.Is(err, sql.ErrNoRows) {
|
||||
return db.Photo{}, fmt.Errorf("recipes: look up photo: %w", err)
|
||||
}
|
||||
|
||||
img, err := decodePhoto(page.Body)
|
||||
if err != nil {
|
||||
return db.Photo{}, fmt.Errorf("%w: %s", err, page.URL)
|
||||
}
|
||||
id := store.NewID()
|
||||
var written []string
|
||||
removeWritten := func() error {
|
||||
var errs []error
|
||||
for _, p := range written {
|
||||
errs = append(errs, os.Remove(p))
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
for _, size := range PhotoSizes {
|
||||
p := c.path(id, size)
|
||||
if err := writeJPEG(p, scaleLongSide(img, size)); err != nil {
|
||||
return db.Photo{}, errors.Join(err, removeWritten())
|
||||
}
|
||||
written = append(written, p)
|
||||
}
|
||||
|
||||
now := store.Now().String()
|
||||
bounds := img.Bounds()
|
||||
photo, err := db.New(c.store.Writer).CreatePhoto(ctx, db.CreatePhotoParams{
|
||||
ID: id,
|
||||
SourceUrl: page.URL,
|
||||
LocalPath: path.Join(PhotoDir, photoFileName(id, PhotoSizes[0])),
|
||||
Width: int64(bounds.Dx()),
|
||||
Height: int64(bounds.Dy()),
|
||||
Sha256: sum,
|
||||
FetchedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
})
|
||||
if err != nil {
|
||||
return db.Photo{}, errors.Join(fmt.Errorf("recipes: record photo: %w", err), removeWritten())
|
||||
}
|
||||
return photo, nil
|
||||
}
|
||||
|
||||
// OpenPhoto opens the stored file of photo id at size. An id that is not a
|
||||
// canonical ULID, a size not in PhotoSizes, or a photo that is not stored
|
||||
// is an error matching fs.ErrNotExist.
|
||||
func (c *PhotoCache) OpenPhoto(id string, size int) (*os.File, error) {
|
||||
parsed, err := ulid.ParseStrict(id)
|
||||
if err != nil || parsed.String() != id || !slices.Contains(PhotoSizes, size) {
|
||||
return nil, fmt.Errorf("recipes: photo %q at size %d: %w", id, size, fs.ErrNotExist)
|
||||
}
|
||||
return os.Open(c.path(id, size))
|
||||
}
|
||||
|
||||
func (c *PhotoCache) path(id string, size int) string {
|
||||
return filepath.Join(c.dir, photoFileName(id, size))
|
||||
}
|
||||
|
||||
func photoFileName(id string, size int) string {
|
||||
return id + "-" + strconv.Itoa(size) + ".jpg"
|
||||
}
|
||||
|
||||
// decodePhoto decodes a JPEG, PNG, WebP or GIF image, checking its size
|
||||
// from the header before decoding the pixels.
|
||||
func decodePhoto(b []byte) (image.Image, error) {
|
||||
cfg, format, err := image.DecodeConfig(bytes.NewReader(b))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrPhotoFormat, err)
|
||||
}
|
||||
decode, ok := photoDecoders[format]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%w: format %q", ErrPhotoFormat, format)
|
||||
}
|
||||
if int64(cfg.Width)*int64(cfg.Height) > PhotoMaxPixels {
|
||||
return nil, fmt.Errorf("%w: %dx%d", ErrPhotoTooLarge, cfg.Width, cfg.Height)
|
||||
}
|
||||
if min(cfg.Width, cfg.Height) < PhotoMinShortSide {
|
||||
return nil, fmt.Errorf("%w: %dx%d, shorter side under %d px", ErrPhotoTooSmall, cfg.Width, cfg.Height, PhotoMinShortSide)
|
||||
}
|
||||
img, err := decode(b)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrPhotoFormat, err)
|
||||
}
|
||||
return img, nil
|
||||
}
|
||||
|
||||
// scaleLongSide returns img scaled so its longer side is long pixels,
|
||||
// keeping its aspect ratio. Transparent areas become white, since JPEG has
|
||||
// no alpha.
|
||||
func scaleLongSide(img image.Image, long int) *image.RGBA {
|
||||
b := img.Bounds()
|
||||
w, h := b.Dx(), b.Dy()
|
||||
dw, dh := long, long
|
||||
if w >= h {
|
||||
dh = max(1, int(math.Round(float64(h)*float64(long)/float64(w))))
|
||||
} else {
|
||||
dw = max(1, int(math.Round(float64(w)*float64(long)/float64(h))))
|
||||
}
|
||||
dst := image.NewRGBA(image.Rect(0, 0, dw, dh))
|
||||
draw.Draw(dst, dst.Bounds(), image.White, image.Point{}, draw.Src)
|
||||
draw.CatmullRom.Scale(dst, dst.Bounds(), img, b, draw.Over, nil)
|
||||
return dst
|
||||
}
|
||||
|
||||
// writeJPEG encodes img at PhotoQuality into a temporary file beside path
|
||||
// and renames it into place, so a reader never sees a partial file.
|
||||
func writeJPEG(path string, img image.Image) error {
|
||||
f, err := os.CreateTemp(filepath.Dir(path), ".photo-*.tmp")
|
||||
if err != nil {
|
||||
return fmt.Errorf("recipes: create photo file: %w", err)
|
||||
}
|
||||
tmp := f.Name()
|
||||
err = jpeg.Encode(f, img, &jpeg.Options{Quality: PhotoQuality})
|
||||
if err == nil {
|
||||
err = f.Sync()
|
||||
}
|
||||
err = errors.Join(err, f.Close())
|
||||
if err == nil {
|
||||
err = os.Rename(tmp, path)
|
||||
}
|
||||
if err != nil {
|
||||
return errors.Join(fmt.Errorf("recipes: write photo %s: %w", filepath.Base(path), err), os.Remove(tmp))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,405 @@
|
||||
package recipes
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/gif"
|
||||
"image/jpeg"
|
||||
"image/png"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/store"
|
||||
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/migrations"
|
||||
)
|
||||
|
||||
// photoServer serves each path's body with its Content-Type and counts
|
||||
// requests.
|
||||
type photoServer struct {
|
||||
files map[string]photoFile
|
||||
requests atomic.Int32
|
||||
}
|
||||
|
||||
type photoFile struct {
|
||||
contentType string
|
||||
body []byte
|
||||
}
|
||||
|
||||
func (s *photoServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
s.requests.Add(1)
|
||||
f, ok := s.files[r.URL.Path]
|
||||
if !ok {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", f.contentType)
|
||||
w.Write(f.body)
|
||||
}
|
||||
|
||||
// newTestPhotoCache returns a PhotoCache over files, a migrated temp
|
||||
// database and a temp data directory.
|
||||
func newTestPhotoCache(t *testing.T, files map[string]photoFile) (*PhotoCache, *store.Store, string, *photoServer) {
|
||||
t.Helper()
|
||||
srv := &photoServer{files: files}
|
||||
env := newTestEnv(t, srv, publicAnswers())
|
||||
installClock(env.f)
|
||||
dataDir := t.TempDir()
|
||||
st, err := store.Open(t.Context(), dataDir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
if err := st.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
})
|
||||
if err := store.Migrate(t.Context(), st.Writer, migrations.FS); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cache, err := NewPhotoCache(env.f, st, dataDir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return cache, st, dataDir, srv
|
||||
}
|
||||
|
||||
// gradient is a w×h opaque image.
|
||||
func gradient(w, h int) *image.RGBA {
|
||||
img := image.NewRGBA(image.Rect(0, 0, w, h))
|
||||
for y := range h {
|
||||
for x := range w {
|
||||
img.Set(x, y, color.RGBA{uint8(x * 255 / w), uint8(y * 255 / h), 128, 255})
|
||||
}
|
||||
}
|
||||
return img
|
||||
}
|
||||
|
||||
func encodeJPEG(t *testing.T, img image.Image) []byte {
|
||||
t.Helper()
|
||||
var b bytes.Buffer
|
||||
if err := jpeg.Encode(&b, img, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
func encodePNG(t *testing.T, img image.Image) []byte {
|
||||
t.Helper()
|
||||
var b bytes.Buffer
|
||||
if err := png.Encode(&b, img); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
// twoFrameGIF is a w×h animated GIF whose first frame is red and second
|
||||
// blue.
|
||||
func twoFrameGIF(t *testing.T, w, h int) []byte {
|
||||
t.Helper()
|
||||
frame := func(c color.Color) *image.Paletted {
|
||||
p := image.NewPaletted(image.Rect(0, 0, w, h), color.Palette{c})
|
||||
return p
|
||||
}
|
||||
var b bytes.Buffer
|
||||
anim := &gif.GIF{
|
||||
Image: []*image.Paletted{frame(color.RGBA{255, 0, 0, 255}), frame(color.RGBA{0, 0, 255, 255})},
|
||||
Delay: []int{10, 10},
|
||||
}
|
||||
if err := gif.EncodeAll(&b, anim); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
func readTestdata(t *testing.T, name string) []byte {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile(filepath.Join("testdata", name))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// storedSize decodes the stored JPEG of id at size and returns it.
|
||||
func storedSize(t *testing.T, dataDir, id string, size int) image.Image {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile(filepath.Join(dataDir, "media", "photos", photoFileName(id, size)))
|
||||
if err != nil {
|
||||
t.Fatalf("read stored photo: %v", err)
|
||||
}
|
||||
img, format, err := image.Decode(bytes.NewReader(b))
|
||||
if err != nil || format != "jpeg" {
|
||||
t.Fatalf("stored photo is %q, %v; want a JPEG", format, err)
|
||||
}
|
||||
return img
|
||||
}
|
||||
|
||||
func TestCachePhotoFormatsAndSizes(t *testing.T) {
|
||||
t.Parallel()
|
||||
files := map[string]photoFile{
|
||||
"/landscape.jpg": {"image/jpeg", encodeJPEG(t, gradient(800, 600))},
|
||||
"/portrait.png": {"image/png", encodePNG(t, gradient(600, 900))},
|
||||
"/photo.webp": {"image/webp", readTestdata(t, "photo.webp")},
|
||||
"/anim.gif": {"image/gif; charset=binary", twoFrameGIF(t, 300, 450)},
|
||||
"/square.png": {"image/png", encodePNG(t, gradient(1500, 1500))},
|
||||
}
|
||||
cache, _, dataDir, _ := newTestPhotoCache(t, files)
|
||||
|
||||
tests := []struct {
|
||||
path string
|
||||
width, height int64
|
||||
// sizes maps each stored size to its expected dimensions.
|
||||
sizes map[int]image.Point
|
||||
}{
|
||||
{"/landscape.jpg", 800, 600, map[int]image.Point{1200: {1200, 900}, 400: {400, 300}}},
|
||||
{"/portrait.png", 600, 900, map[int]image.Point{1200: {800, 1200}, 400: {267, 400}}},
|
||||
{"/photo.webp", 640, 480, map[int]image.Point{1200: {1200, 900}, 400: {400, 300}}},
|
||||
{"/anim.gif", 300, 450, map[int]image.Point{1200: {800, 1200}, 400: {267, 400}}},
|
||||
{"/square.png", 1500, 1500, map[int]image.Point{1200: {1200, 1200}, 400: {400, 400}}},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
photo, err := cache.CachePhoto(t.Context(), "https://example.com"+tc.path)
|
||||
if err != nil {
|
||||
t.Fatalf("CachePhoto %s: %v", tc.path, err)
|
||||
}
|
||||
if photo.Width != tc.width || photo.Height != tc.height {
|
||||
t.Errorf("%s: row is %dx%d, want %dx%d", tc.path, photo.Width, photo.Height, tc.width, tc.height)
|
||||
}
|
||||
if photo.SourceUrl != "https://example.com"+tc.path || len(photo.Sha256) != 64 {
|
||||
t.Errorf("%s: row = %+v", tc.path, photo)
|
||||
}
|
||||
if want := "media/photos/" + photo.ID + "-1200.jpg"; photo.LocalPath != want {
|
||||
t.Errorf("%s: local_path = %q, want %q", tc.path, photo.LocalPath, want)
|
||||
}
|
||||
for size, want := range tc.sizes {
|
||||
img := storedSize(t, dataDir, photo.ID, size)
|
||||
if got := img.Bounds().Size(); got != want {
|
||||
t.Errorf("%s at %d: stored %v, want %v", tc.path, size, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCachePhotoGIFUsesFirstFrame(t *testing.T) {
|
||||
t.Parallel()
|
||||
cache, _, dataDir, _ := newTestPhotoCache(t, map[string]photoFile{
|
||||
"/anim.gif": {"image/gif", twoFrameGIF(t, 400, 400)},
|
||||
})
|
||||
photo, err := cache.CachePhoto(t.Context(), "https://example.com/anim.gif")
|
||||
if err != nil {
|
||||
t.Fatalf("CachePhoto: %v", err)
|
||||
}
|
||||
r, g, b, _ := storedSize(t, dataDir, photo.ID, 400).At(200, 200).RGBA()
|
||||
if r>>8 < 200 || g>>8 > 50 || b>>8 > 50 {
|
||||
t.Errorf("centre pixel = (%d,%d,%d), want the first frame's red", r>>8, g>>8, b>>8)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCachePhotoFillsTransparencyWhite(t *testing.T) {
|
||||
t.Parallel()
|
||||
cache, _, dataDir, _ := newTestPhotoCache(t, map[string]photoFile{
|
||||
"/clear.png": {"image/png", encodePNG(t, image.NewNRGBA(image.Rect(0, 0, 400, 400)))},
|
||||
})
|
||||
photo, err := cache.CachePhoto(t.Context(), "https://example.com/clear.png")
|
||||
if err != nil {
|
||||
t.Fatalf("CachePhoto: %v", err)
|
||||
}
|
||||
r, g, b, _ := storedSize(t, dataDir, photo.ID, 400).At(200, 200).RGBA()
|
||||
if r>>8 < 250 || g>>8 < 250 || b>>8 < 250 {
|
||||
t.Errorf("transparent pixel stored as (%d,%d,%d), want white", r>>8, g>>8, b>>8)
|
||||
}
|
||||
}
|
||||
|
||||
func countPhotos(t *testing.T, st *store.Store) int {
|
||||
t.Helper()
|
||||
var n int
|
||||
if err := st.Reader.QueryRowContext(context.Background(), "SELECT count(*) FROM photos").Scan(&n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func photoFiles(t *testing.T, dataDir string) []string {
|
||||
t.Helper()
|
||||
entries, err := os.ReadDir(filepath.Join(dataDir, "media", "photos"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var names []string
|
||||
for _, e := range entries {
|
||||
names = append(names, e.Name())
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func TestCachePhotoRejectsTooSmall(t *testing.T) {
|
||||
t.Parallel()
|
||||
cache, st, dataDir, _ := newTestPhotoCache(t, map[string]photoFile{
|
||||
"/narrow.png": {"image/png", encodePNG(t, gradient(299, 1000))},
|
||||
"/short.jpg": {"image/jpeg", encodeJPEG(t, gradient(1000, 299))},
|
||||
"/edge.png": {"image/png", encodePNG(t, gradient(300, 1000))},
|
||||
})
|
||||
for _, p := range []string{"/narrow.png", "/short.jpg"} {
|
||||
if _, err := cache.CachePhoto(t.Context(), "https://example.com"+p); !errors.Is(err, ErrPhotoTooSmall) {
|
||||
t.Errorf("CachePhoto %s error = %v, want ErrPhotoTooSmall", p, err)
|
||||
}
|
||||
}
|
||||
if n := countPhotos(t, st); n != 0 {
|
||||
t.Errorf("%d photos recorded for refused images", n)
|
||||
}
|
||||
if names := photoFiles(t, dataDir); len(names) != 0 {
|
||||
t.Errorf("files stored for refused images: %v", names)
|
||||
}
|
||||
if _, err := cache.CachePhoto(t.Context(), "https://example.com/edge.png"); err != nil {
|
||||
t.Errorf("CachePhoto with a 300 px short side: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCachePhotoRejectsNonImages(t *testing.T) {
|
||||
t.Parallel()
|
||||
cache, st, _, _ := newTestPhotoCache(t, map[string]photoFile{
|
||||
"/page.html": {"text/html", encodePNG(t, gradient(400, 400))},
|
||||
"/fake.jpg": {"image/jpeg", []byte("<html>not a picture</html>")},
|
||||
})
|
||||
if _, err := cache.CachePhoto(t.Context(), "https://example.com/page.html"); !errors.Is(err, ErrNotImage) {
|
||||
t.Errorf("text/html error = %v, want ErrNotImage", err)
|
||||
}
|
||||
if _, err := cache.CachePhoto(t.Context(), "https://example.com/fake.jpg"); !errors.Is(err, ErrPhotoFormat) {
|
||||
t.Errorf("undecodable image error = %v, want ErrPhotoFormat", err)
|
||||
}
|
||||
if n := countPhotos(t, st); n != 0 {
|
||||
t.Errorf("%d photos recorded", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCachePhotoDeduplicatesBySHA256(t *testing.T) {
|
||||
t.Parallel()
|
||||
body := encodeJPEG(t, gradient(640, 480))
|
||||
cache, st, dataDir, srv := newTestPhotoCache(t, map[string]photoFile{
|
||||
"/a.jpg": {"image/jpeg", body},
|
||||
"/b.jpg": {"image/jpeg", body},
|
||||
"/other.jpg": {"image/jpeg", encodeJPEG(t, gradient(480, 640))},
|
||||
})
|
||||
first, err := cache.CachePhoto(t.Context(), "https://example.com/a.jpg")
|
||||
if err != nil {
|
||||
t.Fatalf("CachePhoto a: %v", err)
|
||||
}
|
||||
again, err := cache.CachePhoto(t.Context(), "https://example.com/b.jpg")
|
||||
if err != nil {
|
||||
t.Fatalf("CachePhoto b: %v", err)
|
||||
}
|
||||
if again != first {
|
||||
t.Errorf("same bytes at another URL = %+v, want the first row %+v", again, first)
|
||||
}
|
||||
if n := srv.requests.Load(); n != 2 {
|
||||
t.Errorf("%d downloads, want 2: each URL is fetched to hash it", n)
|
||||
}
|
||||
other, err := cache.CachePhoto(t.Context(), "https://example.com/other.jpg")
|
||||
if err != nil {
|
||||
t.Fatalf("CachePhoto other: %v", err)
|
||||
}
|
||||
if other.ID == first.ID || other.Sha256 == first.Sha256 {
|
||||
t.Errorf("different bytes share a photo: %+v and %+v", first, other)
|
||||
}
|
||||
if n := countPhotos(t, st); n != 2 {
|
||||
t.Errorf("%d photos recorded, want 2", n)
|
||||
}
|
||||
if names := photoFiles(t, dataDir); len(names) != 4 {
|
||||
t.Errorf("stored files %v, want two sizes of two photos", names)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenPhoto(t *testing.T) {
|
||||
t.Parallel()
|
||||
cache, _, _, _ := newTestPhotoCache(t, map[string]photoFile{
|
||||
"/a.png": {"image/png", encodePNG(t, gradient(400, 300))},
|
||||
})
|
||||
photo, err := cache.CachePhoto(t.Context(), "https://example.com/a.png")
|
||||
if err != nil {
|
||||
t.Fatalf("CachePhoto: %v", err)
|
||||
}
|
||||
for _, size := range PhotoSizes {
|
||||
f, err := cache.OpenPhoto(photo.ID, size)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenPhoto %d: %v", size, err)
|
||||
}
|
||||
f.Close()
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
id string
|
||||
size int
|
||||
}{
|
||||
{photo.ID, 800},
|
||||
{store.NewID(), 400},
|
||||
{strings.ToLower(photo.ID), 400},
|
||||
{"../" + photo.ID, 400},
|
||||
{"", 400},
|
||||
} {
|
||||
if _, err := cache.OpenPhoto(tc.id, tc.size); !errors.Is(err, fs.ErrNotExist) {
|
||||
t.Errorf("OpenPhoto(%q, %d) error = %v, want fs.ErrNotExist", tc.id, tc.size, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchImageSizeCap(t *testing.T) {
|
||||
t.Parallel()
|
||||
env := newTestEnv(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
size := MaxImageBytes
|
||||
if r.URL.Path != "/exact" {
|
||||
size++
|
||||
}
|
||||
w.Header().Set("Content-Type", "image/jpeg")
|
||||
w.Write(bytes.Repeat([]byte("a"), size))
|
||||
}), publicAnswers())
|
||||
installClock(env.f)
|
||||
|
||||
page, err := env.f.FetchImage(t.Context(), "https://example.com/exact")
|
||||
if err != nil {
|
||||
t.Fatalf("FetchImage at the cap: %v", err)
|
||||
}
|
||||
if len(page.Body) != MaxImageBytes {
|
||||
t.Errorf("body is %d bytes, want %d", len(page.Body), MaxImageBytes)
|
||||
}
|
||||
if _, err := env.f.FetchImage(t.Context(), "https://example.com/over"); !errors.Is(err, ErrTooLarge) {
|
||||
t.Errorf("FetchImage over the cap error = %v, want ErrTooLarge", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchImageContentTypes(t *testing.T) {
|
||||
t.Parallel()
|
||||
var accept atomic.Value
|
||||
env := newTestEnv(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
accept.Store(r.Header.Get("Accept"))
|
||||
w.Header().Set("Content-Type", r.URL.Query().Get("type"))
|
||||
w.Write([]byte("x"))
|
||||
}), publicAnswers())
|
||||
installClock(env.f)
|
||||
typeURL := func(ct string) string { return "https://example.com/?type=" + url.QueryEscape(ct) }
|
||||
|
||||
for _, ct := range []string{"image/jpeg", "image/png", "image/webp", "image/gif", "IMAGE/PNG; q=1"} {
|
||||
if _, err := env.f.FetchImage(t.Context(), typeURL(ct)); err != nil {
|
||||
t.Errorf("FetchImage %q: %v", ct, err)
|
||||
}
|
||||
}
|
||||
if got := accept.Load(); got != "image/jpeg,image/png,image/webp,image/gif" {
|
||||
t.Errorf("Accept = %q", got)
|
||||
}
|
||||
for _, ct := range []string{"text/html", "image/svg+xml", "image/avif", "application/octet-stream", ""} {
|
||||
if _, err := env.f.FetchImage(t.Context(), typeURL(ct)); !errors.Is(err, ErrNotImage) {
|
||||
t.Errorf("FetchImage %q error = %v, want ErrNotImage", ct, err)
|
||||
}
|
||||
}
|
||||
// Pages still fetch whatever the Content-Type.
|
||||
if _, err := env.f.Fetch(t.Context(), typeURL("text/plain")); err != nil {
|
||||
t.Errorf("Fetch: %v", err)
|
||||
}
|
||||
}
|
||||
Vendored
BIN
Binary file not shown.
|
After Width: | Height: | Size: 2.3 KiB |
Reference in New Issue
Block a user