MealQuest/internal/ai/llm/retry.go

package llm

import (
	"context"
	"errors"
	"math/rand/v2"
	"time"
)

const (
	// DefaultTimeout bounds each Generate attempt whose Request sets no
	// Timeout.
	DefaultTimeout = 60 * time.Second
	// MaxRetries is how many times a failed Generate is retried, so a call
	// makes at most MaxRetries+1 attempts.
	MaxRetries = 2
	// MaxRetryAfter caps the wait a rate-limited vendor may ask for.
	MaxRetryAfter = 20 * time.Second
	// backoffBase is the first retry's backoff before jitter; each later
	// retry doubles it.
	backoffBase = time.Second
)

// Retrying is a Provider that retries transient Generate failures of the
// Provider it wraps. It retries ErrRateLimited, ErrTimeout and a 5xx
// ErrProvider at most MaxRetries times, and nothing else. A rate-limited
// retry waits for the vendor's RetryAfter, capped at MaxRetryAfter; every
// other retry, and a rate limit that names no wait, waits a jittered
// exponential backoff. A retry whose wait would not end before the
// caller's context deadline is not made: the last error is returned at
// once. ListModels and Verify pass straight through.
type Retrying struct {
	Provider
	// now is the clock; tests replace it.
	now func() time.Time
	// sleep waits d or until ctx is done; tests replace it.
	sleep func(ctx context.Context, d time.Duration) error
	// jitter returns a random duration in [0, n) for n > 0; tests replace
	// it.
	jitter func(n time.Duration) time.Duration
}

// NewRetrying returns inner wrapped with retries and the default Generate
// timeout.
func NewRetrying(inner Provider) *Retrying {
	return &Retrying{
		Provider: inner,
		now:      time.Now,
		sleep:    sleep,
		jitter:   rand.N[time.Duration],
	}
}

// Generate sends req to the wrapped Provider, retrying transient failures.
// A req.Timeout of 0 becomes DefaultTimeout; either bounds each attempt,
// not the whole call, which ctx bounds.
func (r *Retrying) Generate(ctx context.Context, req Request) (Response, error) {
	if req.Timeout == 0 {
		req.Timeout = DefaultTimeout
	}
	for retry := 0; ; retry++ {
		resp, err := r.Provider.Generate(ctx, req)
		if err == nil || retry == MaxRetries || ctx.Err() != nil {
			return resp, err
		}
		wait, ok := r.wait(err, retry)
		if !ok {
			return resp, err
		}
		if deadline, has := ctx.Deadline(); has && !r.now().Add(wait).Before(deadline) {
			return resp, err
		}
		if serr := r.sleep(ctx, wait); serr != nil {
			return resp, err
		}
	}
}

// wait returns how long to wait before retrying after err, the retry-th
// retry counting from 0, and false when err is not retried.
func (r *Retrying) wait(err error, retry int) (time.Duration, bool) {
	var rl *RateLimitedError
	if errors.As(err, &rl) && rl.RetryAfter > 0 {
		return min(rl.RetryAfter, MaxRetryAfter), true
	}
	var pe *ProviderError
	switch {
	case errors.Is(err, ErrRateLimited), errors.Is(err, ErrTimeout):
	case errors.As(err, &pe) && pe.Status() >= 500 && pe.Status() <= 599:
	default:
		return 0, false
	}
	half := (backoffBase << retry) / 2
	return half + r.jitter(half), true
}

// sleep waits d or until ctx is done, returning ctx's error in that case.
func sleep(ctx context.Context, d time.Duration) error {
	t := time.NewTimer(d)
	defer t.Stop()
	select {
	case <-t.C:
		return nil
	case <-ctx.Done():
		return ctx.Err()
	}
}