MealQuest/internal/ai/llm/retry_test.go

package llm

import (
	"context"
	"errors"
	"fmt"
	"slices"
	"testing"
	"time"
)

// fakeClock is a clock whose sleeps return at once, advance it and are
// recorded.
type fakeClock struct {
	t      time.Time
	sleeps []time.Duration
}

func (c *fakeClock) now() time.Time { return c.t }

func (c *fakeClock) sleep(ctx context.Context, d time.Duration) error {
	if err := ctx.Err(); err != nil {
		return err
	}
	c.sleeps = append(c.sleeps, d)
	c.t = c.t.Add(d)
	return nil
}

// newTestRetrying returns a Retrying over a fresh Fake with a fake clock
// at the real current time, so real context deadlines compare sensibly,
// and jitter that always returns its maximum.
func newTestRetrying() (*Retrying, *Fake, *fakeClock) {
	f := &Fake{}
	c := &fakeClock{t: time.Now()}
	r := NewRetrying(f)
	r.now = c.now
	r.sleep = c.sleep
	r.jitter = func(n time.Duration) time.Duration { return n - 1 }
	return r, f, c
}

var retryReq = Request{Messages: []Message{{Role: RoleUser, Content: "plan dinners"}}}

func ok(text string) GenerateResult { return GenerateResult{Response: Response{Text: text}} }

func fail(err error) GenerateResult { return GenerateResult{Err: err} }

func assertConsumed(t *testing.T, f *Fake, attempts int) {
	t.Helper()
	if g, _, _ := f.Remaining(); g != 0 {
		t.Errorf("%d scripted Generate results unused", g)
	}
	if n := len(f.Requests()); n != attempts {
		t.Errorf("made %d attempts, want %d", n, attempts)
	}
}

func TestRetryingRateLimitedThenSuccess(t *testing.T) {
	r, f, c := newTestRetrying()
	f.QueueGenerate(fail(NewRateLimitedError(3*time.Second, "")), ok("dinner"))

	resp, err := r.Generate(context.Background(), retryReq)
	if err != nil || resp.Text != "dinner" {
		t.Fatalf("Generate = %+v, %v", resp, err)
	}
	if want := []time.Duration{3 * time.Second}; !slices.Equal(c.sleeps, want) {
		t.Errorf("sleeps = %v, want %v", c.sleeps, want)
	}
	assertConsumed(t, f, 2)
}

func TestRetryingRateLimitedWaitIsCapped(t *testing.T) {
	r, f, c := newTestRetrying()
	f.QueueGenerate(fail(NewRateLimitedError(time.Hour, "")), ok("dinner"))

	if _, err := r.Generate(context.Background(), retryReq); err != nil {
		t.Fatal(err)
	}
	if want := []time.Duration{20 * time.Second}; !slices.Equal(c.sleeps, want) {
		t.Errorf("sleeps = %v, want %v", c.sleeps, want)
	}
	assertConsumed(t, f, 2)
}

func TestRetryingRateLimitedWithoutRetryAfterBacksOff(t *testing.T) {
	r, f, c := newTestRetrying()
	f.QueueGenerate(fail(NewRateLimitedError(0, "")), ok("dinner"))

	if _, err := r.Generate(context.Background(), retryReq); err != nil {
		t.Fatal(err)
	}
	if want := []time.Duration{time.Second - 1}; !slices.Equal(c.sleeps, want) {
		t.Errorf("sleeps = %v, want %v", c.sleeps, want)
	}
}

func TestRetrying5xxTwiceThenSuccess(t *testing.T) {
	r, f, c := newTestRetrying()
	f.QueueGenerate(
		fail(NewProviderError(503, []byte("busy"), "")),
		// Adapters wrap the error they return.
		fail(fmt.Errorf("openai: generate: %w", NewProviderError(503, []byte("busy"), ""))),
		ok("dinner"),
	)

	resp, err := r.Generate(context.Background(), retryReq)
	if err != nil || resp.Text != "dinner" {
		t.Fatalf("Generate = %+v, %v", resp, err)
	}
	// Max jitter: [0.5 s, 1 s) then [1 s, 2 s).
	if want := []time.Duration{time.Second - 1, 2*time.Second - 1}; !slices.Equal(c.sleeps, want) {
		t.Errorf("sleeps = %v, want %v", c.sleeps, want)
	}
	assertConsumed(t, f, 3)
}

func TestRetryingBackoffJitterLowerBound(t *testing.T) {
	r, f, c := newTestRetrying()
	r.jitter = func(time.Duration) time.Duration { return 0 }
	f.QueueGenerate(fail(ErrTimeout), fail(ErrTimeout), ok("dinner"))

	if _, err := r.Generate(context.Background(), retryReq); err != nil {
		t.Fatal(err)
	}
	if want := []time.Duration{500 * time.Millisecond, time.Second}; !slices.Equal(c.sleeps, want) {
		t.Errorf("sleeps = %v, want %v", c.sleeps, want)
	}
}

func TestRetrying5xxThreeTimesFails(t *testing.T) {
	r, f, c := newTestRetrying()
	last := NewProviderError(503, []byte("third"), "")
	f.QueueGenerate(
		fail(NewProviderError(503, []byte("first"), "")),
		fail(NewProviderError(503, []byte("second"), "")),
		fail(last),
	)

	_, err := r.Generate(context.Background(), retryReq)
	if err != last {
		t.Fatalf("err = %v, want the last attempt's error", err)
	}
	if len(c.sleeps) != MaxRetries {
		t.Errorf("slept %d times, want %d", len(c.sleeps), MaxRetries)
	}
	assertConsumed(t, f, 3)
}

func TestRetryingDoesNotRetryPermanentErrors(t *testing.T) {
	for name, err := range map[string]error{
		"401 auth":    ErrAuth,
		"400":         NewProviderError(400, []byte("bad request"), ""),
		"404":         NewProviderError(404, []byte("no such model"), ""),
		"408":         NewTimeoutStatusError(408, ""),
		"wrapped 408": fmt.Errorf("openai: generate: %w", NewTimeoutStatusError(408, "")),
		"bad output":  ErrBadOutput,
		"other":       errors.New("llm: request has no messages"),
	} {
		t.Run(name, func(t *testing.T) {
			r, f, c := newTestRetrying()
			f.QueueGenerate(fail(err))

			if _, got := r.Generate(context.Background(), retryReq); got != err {
				t.Fatalf("err = %v, want %v", got, err)
			}
			if len(c.sleeps) != 0 {
				t.Errorf("slept %v", c.sleeps)
			}
			assertConsumed(t, f, 1)
		})
	}
}

// A transport timeout matches ErrTimeout like an HTTP 408, but carries no
// status, so it is retried.
func TestRetryingTransportTimeoutRetried(t *testing.T) {
	r, f, _ := newTestRetrying()
	f.QueueGenerate(fail(fmt.Errorf("openai: generate: %w", ErrTimeout)), ok("dinner"))

	if _, err := r.Generate(context.Background(), retryReq); err != nil {
		t.Fatal(err)
	}
	assertConsumed(t, f, 2)
}

func TestRetryingDeadlineShorterThanRetryAfter(t *testing.T) {
	r, f, c := newTestRetrying()
	rl := NewRateLimitedError(10*time.Second, "")
	f.QueueGenerate(fail(rl))
	ctx, cancel := context.WithDeadline(context.Background(), c.t.Add(5*time.Second))
	defer cancel()

	if _, err := r.Generate(ctx, retryReq); err != rl {
		t.Fatalf("err = %v, want the rate-limited error", err)
	}
	if len(c.sleeps) != 0 {
		t.Errorf("slept %v", c.sleeps)
	}
	assertConsumed(t, f, 1)
}

func TestRetryingDeadlineShorterThanBackoff(t *testing.T) {
	r, f, c := newTestRetrying()
	f.QueueGenerate(fail(ErrTimeout))
	// The first backoff with maximum jitter is just under 1 s.
	ctx, cancel := context.WithDeadline(context.Background(), c.t.Add(time.Second-1))
	defer cancel()

	if _, err := r.Generate(ctx, retryReq); !errors.Is(err, ErrTimeout) {
		t.Fatalf("err = %v, want ErrTimeout", err)
	}
	if len(c.sleeps) != 0 {
		t.Errorf("slept %v", c.sleeps)
	}
	assertConsumed(t, f, 1)
}

func TestRetryingRetriesWithinDeadline(t *testing.T) {
	r, f, c := newTestRetrying()
	f.QueueGenerate(fail(NewRateLimitedError(3*time.Second, "")), ok("dinner"))
	ctx, cancel := context.WithDeadline(context.Background(), c.t.Add(time.Minute))
	defer cancel()

	if _, err := r.Generate(ctx, retryReq); err != nil {
		t.Fatal(err)
	}
	assertConsumed(t, f, 2)
}

func TestRetryingStopsWhenContextDone(t *testing.T) {
	r, f, _ := newTestRetrying()
	ctx, cancel := context.WithCancel(context.Background())
	cancel()
	f.QueueGenerate(fail(ErrTimeout))

	if _, err := r.Generate(ctx, retryReq); !errors.Is(err, ErrTimeout) {
		t.Fatalf("err = %v, want ErrTimeout", err)
	}
	assertConsumed(t, f, 1)
}

func TestRetryingWaitInterruptedReturnsContextError(t *testing.T) {
	t.Run("cancelled", func(t *testing.T) {
		r, f, _ := newTestRetrying()
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()
		r.sleep = func(ctx context.Context, d time.Duration) error {
			cancel()
			return sleep(ctx, d)
		}
		f.QueueGenerate(fail(NewRateLimitedError(3*time.Second, "")))

		_, err := r.Generate(ctx, retryReq)
		if !errors.Is(err, context.Canceled) || errors.Is(err, ErrRateLimited) {
			t.Fatalf("err = %v, want context.Canceled", err)
		}
		assertConsumed(t, f, 1)
	})
	t.Run("deadline", func(t *testing.T) {
		r, f, _ := newTestRetrying()
		// The fake clock stands still, so the 500ms backoff fits before
		// the caller's deadline; the real wait outlasts it.
		r.jitter = func(time.Duration) time.Duration { return 0 }
		r.sleep = func(ctx context.Context, _ time.Duration) error {
			return sleep(ctx, time.Hour)
		}
		ctx, cancel := context.WithTimeout(context.Background(), 600*time.Millisecond)
		defer cancel()
		f.QueueGenerate(fail(NewRateLimitedError(0, "")))

		_, err := r.Generate(ctx, retryReq)
		if !errors.Is(err, context.DeadlineExceeded) || errors.Is(err, ErrRateLimited) {
			t.Fatalf("err = %v, want context.DeadlineExceeded", err)
		}
		assertConsumed(t, f, 1)
	})
}

// slowProvider is a Provider whose every attempt takes d on clock c,
// records the deadline of the context it received, and times out.
type slowProvider struct {
	Provider
	c         *fakeClock
	d         time.Duration
	reqs      []Request
	deadlines []time.Time
}

func (p *slowProvider) Generate(ctx context.Context, req Request) (Response, error) {
	p.reqs = append(p.reqs, req)
	deadline, _ := ctx.Deadline()
	p.deadlines = append(p.deadlines, deadline)
	p.c.t = p.c.t.Add(p.d)
	return Response{}, ErrTimeout
}

func TestRetryingTimeoutBoundsWholeCall(t *testing.T) {
	for _, tc := range []struct {
		name    string
		timeout time.Duration
		want    time.Duration
	}{
		{"default", 0, 60 * time.Second},
		{"custom", 10 * time.Second, 10 * time.Second},
	} {
		t.Run(tc.name, func(t *testing.T) {
			r, _, c := newTestRetrying()
			// Each attempt uses most of the one deadline, so a fresh full
			// timeout per attempt would allow every retry; one deadline
			// leaves room for exactly one.
			p := &slowProvider{c: c, d: tc.want * 5 / 6}
			r.Provider = p
			start := c.t
			req := retryReq
			req.Timeout = tc.timeout

			if _, err := r.Generate(context.Background(), req); !errors.Is(err, ErrTimeout) {
				t.Fatalf("err = %v, want ErrTimeout", err)
			}
			if len(p.deadlines) != 2 {
				t.Fatalf("made %d attempts, want 2", len(p.deadlines))
			}
			for i, d := range p.deadlines {
				if !d.Equal(start.Add(tc.want)) {
					t.Errorf("attempt %d deadline = start+%v, want start+%v", i, d.Sub(start), tc.want)
				}
				if p.reqs[i].Timeout != 0 {
					t.Errorf("attempt %d Timeout = %v, want 0", i, p.reqs[i].Timeout)
				}
			}
			if want := []time.Duration{time.Second - 1}; !slices.Equal(c.sleeps, want) {
				t.Errorf("sleeps = %v, want %v", c.sleeps, want)
			}
		})
	}
}

func TestRetryingCallerDeadlineShorterThanTimeout(t *testing.T) {
	r, _, c := newTestRetrying()
	p := &slowProvider{c: c, d: time.Second}
	r.Provider = p
	ctx, cancel := context.WithDeadline(context.Background(), c.t.Add(5*time.Second))
	defer cancel()

	if _, err := r.Generate(ctx, retryReq); !errors.Is(err, ErrTimeout) {
		t.Fatalf("err = %v, want ErrTimeout", err)
	}
	for i, d := range p.deadlines {
		if want, _ := ctx.Deadline(); !d.Equal(want) {
			t.Errorf("attempt %d deadline = %v, want caller's %v", i, d, want)
		}
	}
}

func TestRetryingRejectsNegativeTimeout(t *testing.T) {
	r, f, _ := newTestRetrying()
	f.QueueGenerate(fail(errors.New("llm: timeout is negative")))
	req := retryReq
	req.Timeout = -time.Second

	if _, err := r.Generate(context.Background(), req); err == nil {
		t.Fatal("Generate accepted a negative timeout")
	}
	assertConsumed(t, f, 1)
	if got := f.Requests()[0].Timeout; got != -time.Second {
		t.Errorf("wrapped Provider got Timeout %v, want -1s to reject", got)
	}
}

func TestRetryingPassesThroughListModelsAndVerify(t *testing.T) {
	r, f, _ := newTestRetrying()
	f.QueueListModels(ListModelsResult{Err: ErrTimeout})
	f.QueueVerify(ErrTimeout)

	if _, err := r.ListModels(context.Background()); !errors.Is(err, ErrTimeout) {
		t.Fatalf("ListModels err = %v", err)
	}
	if err := r.Verify(context.Background()); !errors.Is(err, ErrTimeout) {
		t.Fatalf("Verify err = %v", err)
	}
	if _, l, v := f.Remaining(); l != 0 || v != 0 {
		t.Fatalf("Remaining = %d, %d", l, v)
	}
}

func TestSleepStopsWhenContextDone(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	cancel()
	if err := sleep(ctx, time.Hour); !errors.Is(err, context.Canceled) {
		t.Fatalf("sleep err = %v", err)
	}
	if err := sleep(context.Background(), time.Millisecond); err != nil {
		t.Fatalf("sleep err = %v", err)
	}
}