Wire retrying Provider into presets.FromSettings

FromSettings now returns llm.NewRetrying around the configured adapter, so every Provider it builds retries transient Generate failures and gets the default timeout; ListModels and Verify pass through. The test recording transport now answers 400 instead of failing the round trip: the ollama adapter reports a transport failure as ErrTimeout, which is retried, so the helper would see three requests. Tests: a 429 then 200 succeeds on the second request; exhausted retries return ErrProvider after MaxRetries+1 requests with no key in any error. Maestro task: 1a882923-fdb4-4576-ab82-9529e9974b26 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Add retrying Provider wrapper with backoff and default timeout
2026-10-04 10:49:46 -04:00 · 2026-10-04 10:48:07 -04:00 · 2026-10-04 09:36:10 -04:00 · 2026-10-04 09:18:34 -04:00
8 changed files with 691 additions and 15 deletions
@@ -145,6 +145,15 @@ var all = map[string]Preset{
 		StructuredOutputs:        []openai.StructuredOutput{openai.JSONSchema, openai.JSONObject},
 		OmitTemperature:          []*regexp.Regexp{openRouterReasoningModels},
 	},
+	"xai": {
+		Name:                     "xai",
+		Adapter:                  AdapterOpenAI,
+		BaseURL:                  "https://api.x.ai/v1",
+		Key:                      KeyRequired,
+		StructuredOutput:         openai.JSONSchema,
+		FallbackStructuredOutput: openai.JSONObject,
+		StructuredOutputs:        []openai.StructuredOutput{openai.JSONSchema, openai.JSONObject},
+	},
 	"ollama": {
 		Name:    "ollama",
 		Adapter: AdapterOllama,
@@ -169,13 +178,14 @@ var all = map[string]Preset{
 //
 // The Provider sends s.Temperature with every request whose Temperature is
 // nil; a request that sets one keeps it. Neither is sent to a model the
-// preset's OmitTemperature matches.
+// preset's OmitTemperature matches. It is wrapped in llm.NewRetrying, so
+// Generate retries transient vendor failures and has the default timeout.
 func FromSettings(s llm.Settings, key string) (llm.Provider, error) {
 	cp, err := fromSettings(s, key)
 	if err != nil {
 		return nil, redact(err, key)
 	}
-	return cp, nil
+	return llm.NewRetrying(cp), nil
 }

 func fromSettings(s llm.Settings, key string) (*configured, error) {
@@ -22,10 +22,8 @@ type captured struct {
 	body   map[string]json.RawMessage
 }

-var errStop = errors.New("recorded")
-
-// recordingTransport records each request and fails it, so no request
-// leaves the process.
+// recordingTransport records each request and answers it 400, so no
+// request leaves the process and the Provider never retries it.
 type recordingTransport struct{ reqs []captured }

 func (rt *recordingTransport) RoundTrip(r *http.Request) (*http.Response, error) {
@@ -40,7 +38,12 @@ func (rt *recordingTransport) RoundTrip(r *http.Request) (*http.Response, error)
 		}
 	}
 	rt.reqs = append(rt.reqs, c)
-	return nil, errStop
+	return &http.Response{
+		StatusCode: http.StatusBadRequest,
+		Header:     http.Header{"Content-Type": {"application/json"}},
+		Body:       io.NopCloser(strings.NewReader(`{}`)),
+		Request:    r,
+	}, nil
 }

 // generate builds the provider s describes and returns the request its
@@ -180,6 +183,43 @@ func TestOpenRouterFallbackModeIsJSONObject(t *testing.T) {
 	}
 }

+func TestXAIPreset(t *testing.T) {
+	c := generate(t, llm.Settings{Provider: "xai", Model: "grok-4", Temperature: 0.6}, testKey, schemaRequest())
+	if c.url != "https://api.x.ai/v1/chat/completions" {
+		t.Fatalf("URL = %q", c.url)
+	}
+	if got := c.header.Get("Authorization"); got != "Bearer "+testKey {
+		t.Fatalf("Authorization = %q", got)
+	}
+	for _, h := range []string{"X-Api-Key", "HTTP-Referer", "X-Title"} {
+		if got := c.header.Get(h); got != "" {
+			t.Fatalf("%s = %q, want none", h, got)
+		}
+	}
+	if got := responseFormatType(t, c); got != "json_schema" {
+		t.Fatalf("default response_format type = %q, want json_schema", got)
+	}
+	if got, ok := temperature(t, c); !ok || got != 0.6 {
+		t.Fatalf("temperature = %v, %v; want 0.6", got, ok)
+	}
+}
+
+func TestXAIFallbackModeIsJSONObject(t *testing.T) {
+	p, _ := Lookup("xai")
+	s := llm.Settings{Provider: "xai", Model: "grok-4", StructuredOutput: string(p.FallbackStructuredOutput)}
+	c := generate(t, s, testKey, schemaRequest())
+	if got := responseFormatType(t, c); got != "json_object" {
+		t.Fatalf("response_format type = %q, want json_object", got)
+	}
+}
+
+func TestXAIPresetAcceptsItsOwnBaseURL(t *testing.T) {
+	c := generate(t, llm.Settings{Provider: "xai", BaseURL: "https://api.x.ai/v1", Model: "grok-4"}, testKey, plainRequest())
+	if c.url != "https://api.x.ai/v1/chat/completions" {
+		t.Fatalf("URL = %q", c.url)
+	}
+}
+
 func TestOllamaPreset(t *testing.T) {
 	s := llm.Settings{Provider: "ollama", BaseURL: "http://ollama.lan:11434", Model: "llama3.2", Temperature: 0.5}
 	c := generate(t, s, "", plainRequest())
@@ -290,7 +330,7 @@ func TestNegativeTemperatureRejectedForRejectingModels(t *testing.T) {
 		}
 		req := plainRequest()
 		req.Temperature = ptr(-0.5)
-		if _, err := p.Generate(context.Background(), req); err == nil || errors.Is(err, errStop) || !strings.Contains(err.Error(), "temperature") {
+		if _, err := p.Generate(context.Background(), req); err == nil || errors.Is(err, llm.ErrProvider) || !strings.Contains(err.Error(), "temperature") {
 			t.Errorf("%s: Generate = %v, want a temperature validation error", s.Model, err)
 		}
 	}
@@ -318,6 +358,7 @@ func TestPresetTable(t *testing.T) {
 		{"anthropic", AdapterAnthropic, "https://api.anthropic.com", KeyRequired},
 		{"openai", AdapterOpenAI, "https://api.openai.com/v1", KeyRequired},
 		{"openrouter", AdapterOpenAI, "https://openrouter.ai/api/v1", KeyRequired},
+		{"xai", AdapterOpenAI, "https://api.x.ai/v1", KeyRequired},
 		{"ollama", AdapterOllama, "", KeyNone},
 		{"openai_compatible", AdapterOpenAI, "", KeyOptional},
 	}
@@ -350,10 +391,12 @@ func TestConfigErrors(t *testing.T) {
 		{"anthropic no key", llm.Settings{Provider: "anthropic", Model: "m"}, "", llm.ErrMissingKey},
 		{"openai no key", llm.Settings{Provider: "openai", Model: "m"}, "", llm.ErrMissingKey},
 		{"openrouter no key", llm.Settings{Provider: "openrouter", Model: "m", PublicURL: "https://x.example"}, "", llm.ErrMissingKey},
+		{"xai no key", llm.Settings{Provider: "xai", Model: "grok-4"}, "", llm.ErrMissingKey},
 		{"ollama with key", llm.Settings{Provider: "ollama", BaseURL: "http://o:11434", Model: "m"}, testKey, llm.ErrUnexpectedKey},
 		{"anthropic key with newline", llm.Settings{Provider: "anthropic", Model: "m"}, testKey + "\n", llm.ErrInvalidKey},
 		{"openai key with newline", llm.Settings{Provider: "openai", Model: "m"}, testKey + "\n", llm.ErrInvalidKey},
 		{"openrouter key with CR LF", llm.Settings{Provider: "openrouter", Model: "m", PublicURL: "https://x.example"}, testKey + "\r\nX-Injected: 1", llm.ErrInvalidKey},
+		{"xai key with newline", llm.Settings{Provider: "xai", Model: "grok-4"}, testKey + "\n", llm.ErrInvalidKey},
 		{"openai_compatible key with NUL", llm.Settings{Provider: "openai_compatible", BaseURL: "http://l/v1", Model: "m"}, "\x00" + testKey, llm.ErrInvalidKey},
 		{"ollama no base URL", llm.Settings{Provider: "ollama", Model: "m"}, "", llm.ErrMissingBaseURL},
 		{"openai_compatible no base URL", llm.Settings{Provider: "openai_compatible", Model: "m"}, "", llm.ErrMissingBaseURL},
@@ -362,8 +405,11 @@ func TestConfigErrors(t *testing.T) {
 		{"openrouter no model", llm.Settings{Provider: "openrouter", PublicURL: "https://x.example"}, testKey, llm.ErrMissingModel},
 		{"ollama no model", llm.Settings{Provider: "ollama", BaseURL: "http://o:11434"}, "", llm.ErrMissingModel},
 		{"openai_compatible no model", llm.Settings{Provider: "openai_compatible", BaseURL: "http://l/v1"}, "", llm.ErrMissingModel},
+		{"xai no model", llm.Settings{Provider: "xai"}, testKey, llm.ErrMissingModel},
 		{"openrouter no public URL", llm.Settings{Provider: "openrouter", Model: "m"}, testKey, llm.ErrMissingPublicURL},
 		{"fixed base URL overridden", llm.Settings{Provider: "openai", BaseURL: "https://evil.example/v1", Model: "m"}, testKey, llm.ErrInvalidSetting},
+		{"xai base URL overridden", llm.Settings{Provider: "xai", BaseURL: "https://api.openai.com/v1", Model: "grok-4"}, testKey, llm.ErrInvalidSetting},
+		{"xai prompt_only", llm.Settings{Provider: "xai", Model: "grok-4", StructuredOutput: "prompt_only"}, testKey, llm.ErrInvalidSetting},
 		{"openai json_object", llm.Settings{Provider: "openai", Model: "m", StructuredOutput: "json_object"}, testKey, llm.ErrInvalidSetting},
 		{"anthropic mode", llm.Settings{Provider: "anthropic", Model: "m", StructuredOutput: "json_schema"}, testKey, llm.ErrInvalidSetting},
 		{"unknown mode", llm.Settings{Provider: "openai_compatible", BaseURL: "http://l/v1", Model: "m", StructuredOutput: "xml"}, "", llm.ErrInvalidSetting},
@@ -396,6 +442,7 @@ func TestReadyProviderForEveryPreset(t *testing.T) {
 		{llm.Settings{Provider: "anthropic", Model: "m"}, testKey},
 		{llm.Settings{Provider: "openai", Model: "m"}, testKey},
 		{llm.Settings{Provider: "openrouter", Model: "m", PublicURL: "https://x.example"}, testKey},
+		{llm.Settings{Provider: "xai", Model: "m"}, testKey},
 		{llm.Settings{Provider: "ollama", BaseURL: "http://o:11434", Model: "m"}, ""},
 		{llm.Settings{Provider: "openai_compatible", BaseURL: "http://l/v1", Model: "m"}, ""},
 	} {
@@ -421,7 +468,7 @@ func TestGenerateValidationErrorsNeverLeakKey(t *testing.T) {
 		req := plainRequest()
 		req.Temperature = ptr(-0.5)
 		_, err = p.Generate(context.Background(), req)
-		if err == nil || errors.Is(err, errStop) {
+		if err == nil || errors.Is(err, llm.ErrProvider) {
 			t.Fatalf("%s: Generate = %v, want a validation error", key, err)
 		}
 		for e := error(err); e != nil; e = errors.Unwrap(e) {
@@ -454,6 +501,10 @@ func TestConfigErrorsNeverLeakKey(t *testing.T) {
 		{"key in bad base URL", llm.Settings{Provider: "openai_compatible", BaseURL: "ftp://k3y", Model: "m"}, "k3y", llm.ErrInvalidSetting},
 		{"ollama with key", llm.Settings{Provider: "ollama", BaseURL: "http://o:11434", Model: "m"}, "key", llm.ErrUnexpectedKey},
 		{"key with newline", llm.Settings{Provider: "openai", Model: "m"}, "sk-test\n", llm.ErrInvalidKey},
+		{"xai key as preset name", llm.Settings{Provider: "xai"}, "xai", llm.ErrMissingModel},
+		{"xai key as missing-model text", llm.Settings{Provider: "xai"}, "model is required", llm.ErrMissingModel},
+		{"xai key as fixed-URL text", llm.Settings{Provider: "xai", BaseURL: "https://x.example/v1", Model: "m"}, "fixed", llm.ErrInvalidSetting},
+		{"xai key as structured-output text", llm.Settings{Provider: "xai", Model: "m", StructuredOutput: "prompt_only"}, "structured output", llm.ErrInvalidSetting},
 		{"key with newline as text", llm.Settings{Provider: "openai", Model: "m"}, "header\n", llm.ErrInvalidKey},
 	}
 	for _, c := range cases {
@@ -470,3 +521,86 @@ func TestConfigErrorsNeverLeakKey(t *testing.T) {
 		})
 	}
 }
+
+// scriptedTransport answers each request with the next scripted status and
+// body, echoing the key in every failure body, and counts the requests.
+type scriptedTransport struct {
+	statuses []int
+	key      string
+	sent     int
+}
+
+func (st *scriptedTransport) RoundTrip(r *http.Request) (*http.Response, error) {
+	if st.sent == len(st.statuses) {
+		return nil, fmt.Errorf("unscripted request %d", st.sent+1)
+	}
+	status := st.statuses[st.sent]
+	st.sent++
+	header := http.Header{"Content-Type": {"application/json"}}
+	body := `{"error":{"message":"bad key ` + st.key + `"}}`
+	switch status {
+	case http.StatusOK:
+		body = `{"model":"m","choices":[{"message":{"content":"tacos"},"finish_reason":"stop"}]}`
+	case http.StatusTooManyRequests:
+		header.Set("Retry-After", "1")
+	}
+	return &http.Response{
+		StatusCode: status,
+		Header:     header,
+		Body:       io.NopCloser(strings.NewReader(body)),
+		Request:    r,
+	}, nil
+}
+
+// useTransport makes the adapters send through rt for the rest of the test.
+func useTransport(t *testing.T, rt http.RoundTripper) {
+	t.Helper()
+	orig := http.DefaultTransport
+	http.DefaultTransport = rt
+	t.Cleanup(func() { http.DefaultTransport = orig })
+}
+
+// The Provider FromSettings returns retries a rate-limited Generate.
+func TestFromSettingsRetriesRateLimited(t *testing.T) {
+	st := &scriptedTransport{statuses: []int{http.StatusTooManyRequests, http.StatusOK}, key: testKey}
+	useTransport(t, st)
+
+	p, err := FromSettings(llm.Settings{Provider: "openai", Model: "m"}, testKey)
+	if err != nil {
+		t.Fatalf("FromSettings: %v", err)
+	}
+	resp, err := p.Generate(context.Background(), plainRequest())
+	if err != nil {
+		t.Fatalf("Generate: %v", err)
+	}
+	if resp.Text != "tacos" {
+		t.Errorf("Text = %q, want %q", resp.Text, "tacos")
+	}
+	if st.sent != 2 {
+		t.Errorf("sent %d requests, want 2", st.sent)
+	}
+}
+
+// When every retry fails, the error FromSettings's Provider returns keeps
+// its sentinel and never holds the key the vendor echoed.
+func TestFromSettingsRetryErrorNeverLeaksKey(t *testing.T) {
+	st := &scriptedTransport{statuses: []int{http.StatusTooManyRequests, http.StatusTooManyRequests, http.StatusServiceUnavailable}, key: testKey}
+	useTransport(t, st)
+
+	p, err := FromSettings(llm.Settings{Provider: "openai", Model: "m"}, testKey)
+	if err != nil {
+		t.Fatalf("FromSettings: %v", err)
+	}
+	_, err = p.Generate(context.Background(), plainRequest())
+	if !errors.Is(err, llm.ErrProvider) {
+		t.Fatalf("Generate = %v, want %v", err, llm.ErrProvider)
+	}
+	if st.sent != llm.MaxRetries+1 {
+		t.Errorf("sent %d requests, want %d", st.sent, llm.MaxRetries+1)
+	}
+	for e := error(err); e != nil; e = errors.Unwrap(e) {
+		if s := e.Error() + fmt.Sprintf("%#v", e); strings.Contains(s, testKey) {
+			t.Fatalf("error leaks the key: %s", s)
+		}
+	}
+}
@@ -0,0 +1,107 @@
+package llm
+
+import (
+	"context"
+	"errors"
+	"math/rand/v2"
+	"time"
+)
+
+const (
+	// DefaultTimeout bounds each Generate attempt whose Request sets no
+	// Timeout.
+	DefaultTimeout = 60 * time.Second
+	// MaxRetries is how many times a failed Generate is retried, so a call
+	// makes at most MaxRetries+1 attempts.
+	MaxRetries = 2
+	// MaxRetryAfter caps the wait a rate-limited vendor may ask for.
+	MaxRetryAfter = 20 * time.Second
+	// backoffBase is the first retry's backoff before jitter; each later
+	// retry doubles it.
+	backoffBase = time.Second
+)
+
+// Retrying is a Provider that retries transient Generate failures of the
+// Provider it wraps. It retries ErrRateLimited, ErrTimeout and a 5xx
+// ErrProvider at most MaxRetries times, and nothing else. A rate-limited
+// retry waits for the vendor's RetryAfter, capped at MaxRetryAfter; every
+// other retry, and a rate limit that names no wait, waits a jittered
+// exponential backoff. A retry whose wait would not end before the
+// caller's context deadline is not made: the last error is returned at
+// once. ListModels and Verify pass straight through.
+type Retrying struct {
+	Provider
+	// now is the clock; tests replace it.
+	now func() time.Time
+	// sleep waits d or until ctx is done; tests replace it.
+	sleep func(ctx context.Context, d time.Duration) error
+	// jitter returns a random duration in [0, n) for n > 0; tests replace
+	// it.
+	jitter func(n time.Duration) time.Duration
+}
+
+// NewRetrying returns inner wrapped with retries and the default Generate
+// timeout.
+func NewRetrying(inner Provider) *Retrying {
+	return &Retrying{
+		Provider: inner,
+		now:      time.Now,
+		sleep:    sleep,
+		jitter:   rand.N[time.Duration],
+	}
+}
+
+// Generate sends req to the wrapped Provider, retrying transient failures.
+// A req.Timeout of 0 becomes DefaultTimeout; either bounds each attempt,
+// not the whole call, which ctx bounds.
+func (r *Retrying) Generate(ctx context.Context, req Request) (Response, error) {
+	if req.Timeout == 0 {
+		req.Timeout = DefaultTimeout
+	}
+	for retry := 0; ; retry++ {
+		resp, err := r.Provider.Generate(ctx, req)
+		if err == nil || retry == MaxRetries || ctx.Err() != nil {
+			return resp, err
+		}
+		wait, ok := r.wait(err, retry)
+		if !ok {
+			return resp, err
+		}
+		if deadline, has := ctx.Deadline(); has && !r.now().Add(wait).Before(deadline) {
+			return resp, err
+		}
+		if serr := r.sleep(ctx, wait); serr != nil {
+			return resp, err
+		}
+	}
+}
+
+// wait returns how long to wait before retrying after err, the retry-th
+// retry counting from 0, and false when err is not retried.
+func (r *Retrying) wait(err error, retry int) (time.Duration, bool) {
+	var rl *RateLimitedError
+	if errors.As(err, &rl) && rl.RetryAfter > 0 {
+		return min(rl.RetryAfter, MaxRetryAfter), true
+	}
+	var pe *ProviderError
+	switch {
+	case errors.Is(err, ErrRateLimited), errors.Is(err, ErrTimeout):
+	case errors.As(err, &pe) && pe.Status() >= 500 && pe.Status() <= 599:
+	default:
+		return 0, false
+	}
+	half := (backoffBase << retry) / 2
+	return half + r.jitter(half), true
+}
+
+// sleep waits d or until ctx is done, returning ctx's error in that case.
+func sleep(ctx context.Context, d time.Duration) error {
+	t := time.NewTimer(d)
+	defer t.Stop()
+	select {
+	case <-t.C:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
@@ -0,0 +1,271 @@
+package llm
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"slices"
+	"testing"
+	"time"
+)
+
+// fakeClock is a clock whose sleeps return at once, advance it and are
+// recorded.
+type fakeClock struct {
+	t      time.Time
+	sleeps []time.Duration
+}
+
+func (c *fakeClock) now() time.Time { return c.t }
+
+func (c *fakeClock) sleep(ctx context.Context, d time.Duration) error {
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	c.sleeps = append(c.sleeps, d)
+	c.t = c.t.Add(d)
+	return nil
+}
+
+// newTestRetrying returns a Retrying over a fresh Fake with a fake clock
+// at the real current time, so real context deadlines compare sensibly,
+// and jitter that always returns its maximum.
+func newTestRetrying() (*Retrying, *Fake, *fakeClock) {
+	f := &Fake{}
+	c := &fakeClock{t: time.Now()}
+	r := NewRetrying(f)
+	r.now = c.now
+	r.sleep = c.sleep
+	r.jitter = func(n time.Duration) time.Duration { return n - 1 }
+	return r, f, c
+}
+
+var retryReq = Request{Messages: []Message{{Role: RoleUser, Content: "plan dinners"}}}
+
+func ok(text string) GenerateResult { return GenerateResult{Response: Response{Text: text}} }
+
+func fail(err error) GenerateResult { return GenerateResult{Err: err} }
+
+func assertConsumed(t *testing.T, f *Fake, attempts int) {
+	t.Helper()
+	if g, _, _ := f.Remaining(); g != 0 {
+		t.Errorf("%d scripted Generate results unused", g)
+	}
+	if n := len(f.Requests()); n != attempts {
+		t.Errorf("made %d attempts, want %d", n, attempts)
+	}
+}
+
+func TestRetryingRateLimitedThenSuccess(t *testing.T) {
+	r, f, c := newTestRetrying()
+	f.QueueGenerate(fail(NewRateLimitedError(3*time.Second, "")), ok("dinner"))
+
+	resp, err := r.Generate(context.Background(), retryReq)
+	if err != nil || resp.Text != "dinner" {
+		t.Fatalf("Generate = %+v, %v", resp, err)
+	}
+	if want := []time.Duration{3 * time.Second}; !slices.Equal(c.sleeps, want) {
+		t.Errorf("sleeps = %v, want %v", c.sleeps, want)
+	}
+	assertConsumed(t, f, 2)
+}
+
+func TestRetryingRateLimitedWaitIsCapped(t *testing.T) {
+	r, f, c := newTestRetrying()
+	f.QueueGenerate(fail(NewRateLimitedError(time.Hour, "")), ok("dinner"))
+
+	if _, err := r.Generate(context.Background(), retryReq); err != nil {
+		t.Fatal(err)
+	}
+	if want := []time.Duration{MaxRetryAfter}; !slices.Equal(c.sleeps, want) {
+		t.Errorf("sleeps = %v, want %v", c.sleeps, want)
+	}
+	assertConsumed(t, f, 2)
+}
+
+func TestRetryingRateLimitedWithoutRetryAfterBacksOff(t *testing.T) {
+	r, f, c := newTestRetrying()
+	f.QueueGenerate(fail(NewRateLimitedError(0, "")), ok("dinner"))
+
+	if _, err := r.Generate(context.Background(), retryReq); err != nil {
+		t.Fatal(err)
+	}
+	if want := []time.Duration{time.Second - 1}; !slices.Equal(c.sleeps, want) {
+		t.Errorf("sleeps = %v, want %v", c.sleeps, want)
+	}
+}
+
+func TestRetrying5xxTwiceThenSuccess(t *testing.T) {
+	r, f, c := newTestRetrying()
+	f.QueueGenerate(
+		fail(NewProviderError(503, []byte("busy"), "")),
+		// Adapters wrap the error they return.
+		fail(fmt.Errorf("openai: generate: %w", NewProviderError(503, []byte("busy"), ""))),
+		ok("dinner"),
+	)
+
+	resp, err := r.Generate(context.Background(), retryReq)
+	if err != nil || resp.Text != "dinner" {
+		t.Fatalf("Generate = %+v, %v", resp, err)
+	}
+	// Max jitter: [0.5 s, 1 s) then [1 s, 2 s).
+	if want := []time.Duration{time.Second - 1, 2*time.Second - 1}; !slices.Equal(c.sleeps, want) {
+		t.Errorf("sleeps = %v, want %v", c.sleeps, want)
+	}
+	assertConsumed(t, f, 3)
+}
+
+func TestRetryingBackoffJitterLowerBound(t *testing.T) {
+	r, f, c := newTestRetrying()
+	r.jitter = func(time.Duration) time.Duration { return 0 }
+	f.QueueGenerate(fail(ErrTimeout), fail(ErrTimeout), ok("dinner"))
+
+	if _, err := r.Generate(context.Background(), retryReq); err != nil {
+		t.Fatal(err)
+	}
+	if want := []time.Duration{500 * time.Millisecond, time.Second}; !slices.Equal(c.sleeps, want) {
+		t.Errorf("sleeps = %v, want %v", c.sleeps, want)
+	}
+}
+
+func TestRetrying5xxThreeTimesFails(t *testing.T) {
+	r, f, c := newTestRetrying()
+	last := NewProviderError(503, []byte("third"), "")
+	f.QueueGenerate(
+		fail(NewProviderError(503, []byte("first"), "")),
+		fail(NewProviderError(503, []byte("second"), "")),
+		fail(last),
+	)
+
+	_, err := r.Generate(context.Background(), retryReq)
+	if err != last {
+		t.Fatalf("err = %v, want the last attempt's error", err)
+	}
+	if len(c.sleeps) != MaxRetries {
+		t.Errorf("slept %d times, want %d", len(c.sleeps), MaxRetries)
+	}
+	assertConsumed(t, f, 3)
+}
+
+func TestRetryingDoesNotRetryPermanentErrors(t *testing.T) {
+	for name, err := range map[string]error{
+		"401 auth":   ErrAuth,
+		"400":        NewProviderError(400, []byte("bad request"), ""),
+		"404":        NewProviderError(404, []byte("no such model"), ""),
+		"bad output": ErrBadOutput,
+		"other":      errors.New("llm: request has no messages"),
+	} {
+		t.Run(name, func(t *testing.T) {
+			r, f, c := newTestRetrying()
+			f.QueueGenerate(fail(err))
+
+			if _, got := r.Generate(context.Background(), retryReq); got != err {
+				t.Fatalf("err = %v, want %v", got, err)
+			}
+			if len(c.sleeps) != 0 {
+				t.Errorf("slept %v", c.sleeps)
+			}
+			assertConsumed(t, f, 1)
+		})
+	}
+}
+
+func TestRetryingDeadlineShorterThanRetryAfter(t *testing.T) {
+	r, f, c := newTestRetrying()
+	rl := NewRateLimitedError(10*time.Second, "")
+	f.QueueGenerate(fail(rl))
+	ctx, cancel := context.WithDeadline(context.Background(), c.t.Add(5*time.Second))
+	defer cancel()
+
+	if _, err := r.Generate(ctx, retryReq); err != rl {
+		t.Fatalf("err = %v, want the rate-limited error", err)
+	}
+	if len(c.sleeps) != 0 {
+		t.Errorf("slept %v", c.sleeps)
+	}
+	assertConsumed(t, f, 1)
+}
+
+func TestRetryingDeadlineShorterThanBackoff(t *testing.T) {
+	r, f, c := newTestRetrying()
+	f.QueueGenerate(fail(ErrTimeout))
+	// The first backoff with maximum jitter is just under 1 s.
+	ctx, cancel := context.WithDeadline(context.Background(), c.t.Add(time.Second-1))
+	defer cancel()
+
+	if _, err := r.Generate(ctx, retryReq); !errors.Is(err, ErrTimeout) {
+		t.Fatalf("err = %v, want ErrTimeout", err)
+	}
+	if len(c.sleeps) != 0 {
+		t.Errorf("slept %v", c.sleeps)
+	}
+	assertConsumed(t, f, 1)
+}
+
+func TestRetryingRetriesWithinDeadline(t *testing.T) {
+	r, f, c := newTestRetrying()
+	f.QueueGenerate(fail(NewRateLimitedError(3*time.Second, "")), ok("dinner"))
+	ctx, cancel := context.WithDeadline(context.Background(), c.t.Add(time.Minute))
+	defer cancel()
+
+	if _, err := r.Generate(ctx, retryReq); err != nil {
+		t.Fatal(err)
+	}
+	assertConsumed(t, f, 2)
+}
+
+func TestRetryingStopsWhenContextDone(t *testing.T) {
+	r, f, _ := newTestRetrying()
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	f.QueueGenerate(fail(ErrTimeout))
+
+	if _, err := r.Generate(ctx, retryReq); !errors.Is(err, ErrTimeout) {
+		t.Fatalf("err = %v, want ErrTimeout", err)
+	}
+	assertConsumed(t, f, 1)
+}
+
+func TestRetryingDefaultTimeout(t *testing.T) {
+	r, f, _ := newTestRetrying()
+	f.QueueGenerate(ok("a"), ok("b"))
+	custom := retryReq
+	custom.Timeout = 5 * time.Second
+
+	for _, req := range []Request{retryReq, custom} {
+		if _, err := r.Generate(context.Background(), req); err != nil {
+			t.Fatal(err)
+		}
+	}
+	got := f.Requests()
+	if got[0].Timeout != DefaultTimeout || got[1].Timeout != 5*time.Second {
+		t.Fatalf("timeouts = %v, %v; want %v, 5s", got[0].Timeout, got[1].Timeout, DefaultTimeout)
+	}
+}
+
+func TestRetryingPassesThroughListModelsAndVerify(t *testing.T) {
+	r, f, _ := newTestRetrying()
+	f.QueueListModels(ListModelsResult{Err: ErrTimeout})
+	f.QueueVerify(ErrTimeout)
+
+	if _, err := r.ListModels(context.Background()); !errors.Is(err, ErrTimeout) {
+		t.Fatalf("ListModels err = %v", err)
+	}
+	if err := r.Verify(context.Background()); !errors.Is(err, ErrTimeout) {
+		t.Fatalf("Verify err = %v", err)
+	}
+	if _, l, v := f.Remaining(); l != 0 || v != 0 {
+		t.Fatalf("Remaining = %d, %d", l, v)
+	}
+}
+
+func TestSleepStopsWhenContextDone(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	if err := sleep(ctx, time.Hour); !errors.Is(err, context.Canceled) {
+		t.Fatalf("sleep err = %v", err)
+	}
+	if err := sleep(context.Background(), time.Millisecond); err != nil {
+		t.Fatalf("sleep err = %v", err)
+	}
+}
@@ -18,7 +18,7 @@ import "errors"
 // to presets.FromSettings separately.
 type Settings struct {
 	// Provider names the preset: "anthropic", "openai", "openrouter",
-	// "ollama" or "openai_compatible".
+	// "xai", "ollama" or "openai_compatible".
 	Provider string
 	// BaseURL is the API root. It is required for ollama and
 	// openai_compatible; the other presets fix their own, and a non-empty
@@ -175,8 +175,8 @@ func TestMigrateEmbeddedMigrations(t *testing.T) {
 	if err := Migrate(context.Background(), s.Writer, migrations.FS); err != nil {
 		t.Fatal(err)
 	}
-	if got := versions(migrationRows(t, s)); got != "0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets" {
-		t.Errorf("versions = %q, want 0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets", got)
+	if got := versions(migrationRows(t, s)); got != "0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai" {
+		t.Errorf("versions = %q, want 0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai", got)
 	}
 }

@@ -196,8 +196,8 @@ func TestMigrateBootstrapThenEmbeddedInitApplies(t *testing.T) {
 	if err := Migrate(ctx, s.Writer, migrations.FS); err != nil {
 		t.Fatal(err)
 	}
-	if got := versions(migrationRows(t, s)); got != "0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets" {
-		t.Errorf("versions = %q, want 0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets", got)
+	if got := versions(migrationRows(t, s)); got != "0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai" {
+		t.Errorf("versions = %q, want 0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai", got)
 	}
 	if !tableExists(t, s, "dinners") {
 		t.Error("table from 0001_init was not created")
@@ -396,3 +396,93 @@ func TestMigrateAIProviderPresetsKeepsRowsAndReferences(t *testing.T) {
 		}
 	}
 }
+
+// TestMigrateAIProviderXAIKeepsRowsAndReferences upgrades a database at 0004
+// holding ai_settings and ai_runs rows with 0004's providers, with
+// suggestion_batches and week_plans referencing the run, and checks 0005
+// carries every row over, keeps the references enforced, accepts xai and
+// still rejects an unknown provider.
+func TestMigrateAIProviderXAIKeepsRowsAndReferences(t *testing.T) {
+	ctx := context.Background()
+	s := openTestStore(t)
+	before := fstest.MapFS{}
+	for _, name := range []string{"0000_bootstrap.sql", "0001_init.sql", "0002_household_settings_updated_by.sql", "0003_jobs_status.sql", "0004_ai_provider_presets.sql"} {
+		body, err := fs.ReadFile(migrations.FS, name)
+		if err != nil {
+			t.Fatal(err)
+		}
+		before[name] = &fstest.MapFile{Data: body}
+	}
+	if err := Migrate(ctx, s.Writer, before); err != nil {
+		t.Fatal(err)
+	}
+	for _, stmt := range []string{
+		`INSERT INTO ai_settings (id, provider, base_url, model, temperature, system_prompt, api_key_ct, api_key_last4, key_verified_at, created_at, updated_at)
+			VALUES (1, 'openrouter', 'https://openrouter.ai/api/v1', 'openai/gpt-4o', 0.7, 'sp', x'0102', 'abcd', 'v', 'c', 'u')`,
+		`INSERT INTO ai_runs (id, skill, skill_version, provider, model, temperature, status, latency_ms, tokens_in, tokens_out, error, request_redacted, response_redacted, created_at, updated_at)
+			VALUES ('run-1', 'suggest', '1', 'openai', 'gpt-4o', 0.5, 'ok', 10, 20, 30, 'e', 'req', 'resp', 'c', 'u')`,
+		`INSERT INTO suggestion_batches (id, trigger, status, ai_run_id, created_at, updated_at)
+			VALUES ('batch-1', 'manual', 'done', 'run-1', 'c', 'u')`,
+		`INSERT INTO week_plans (id, week_start, status, ai_run_id, created_at, updated_at)
+			VALUES ('plan-1', '2026-10-04', 'draft', 'run-1', 'c', 'u')`,
+	} {
+		if _, err := s.Writer.Exec(stmt); err != nil {
+			t.Fatalf("seed: %v", err)
+		}
+	}
+
+	if err := Migrate(ctx, s.Writer, migrations.FS); err != nil {
+		t.Fatal(err)
+	}
+
+	var provider, baseURL, model, systemPrompt, last4, verifiedAt, createdAt, updatedAt string
+	var temperature float64
+	var keyCT []byte
+	if err := s.Reader.QueryRow(`SELECT provider, base_url, model, temperature, system_prompt, api_key_ct, api_key_last4, key_verified_at, created_at, updated_at FROM ai_settings WHERE id = 1`).
+		Scan(&provider, &baseURL, &model, &temperature, &systemPrompt, &keyCT, &last4, &verifiedAt, &createdAt, &updatedAt); err != nil {
+		t.Fatalf("read ai_settings: %v", err)
+	}
+	settings := []any{provider, baseURL, model, temperature, systemPrompt, string(keyCT), last4, verifiedAt, createdAt, updatedAt}
+	if want := []any{"openrouter", "https://openrouter.ai/api/v1", "openai/gpt-4o", 0.7, "sp", "\x01\x02", "abcd", "v", "c", "u"}; !slices.Equal(settings, want) {
+		t.Errorf("ai_settings = %v, want %v", settings, want)
+	}
+
+	var skill, skillVersion, status, errText, request, response string
+	var latency, tokensIn, tokensOut int
+	if err := s.Reader.QueryRow(`SELECT skill, skill_version, provider, model, temperature, status, latency_ms, tokens_in, tokens_out, error, request_redacted, response_redacted, created_at, updated_at FROM ai_runs WHERE id = 'run-1'`).
+		Scan(&skill, &skillVersion, &provider, &model, &temperature, &status, &latency, &tokensIn, &tokensOut, &errText, &request, &response, &createdAt, &updatedAt); err != nil {
+		t.Fatalf("read ai_runs: %v", err)
+	}
+	run := []any{skill, skillVersion, provider, model, temperature, status, latency, tokensIn, tokensOut, errText, request, response, createdAt, updatedAt}
+	if want := []any{"suggest", "1", "openai", "gpt-4o", 0.5, "ok", 10, 20, 30, "e", "req", "resp", "c", "u"}; !slices.Equal(run, want) {
+		t.Errorf("ai_runs = %v, want %v", run, want)
+	}
+
+	rows, err := s.Reader.Query("PRAGMA foreign_key_check")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if rows.Next() {
+		t.Error("foreign_key_check reported violations after 0005")
+	}
+	if err := rows.Close(); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := s.Writer.Exec(`DELETE FROM ai_runs WHERE id = 'run-1'`); err == nil || !strings.Contains(err.Error(), "FOREIGN KEY constraint failed") {
+		t.Errorf("deleting a referenced ai_run: error %v, want FOREIGN KEY constraint failure", err)
+	}
+
+	for _, table := range []string{"ai_settings", "ai_runs"} {
+		if _, err := s.Writer.Exec(`UPDATE ` + table + ` SET provider = 'xai'`); err != nil {
+			t.Errorf("%s.provider xai rejected: %v", table, err)
+		}
+		if _, err := s.Writer.Exec(`UPDATE ` + table + ` SET provider = 'grok'`); err == nil || !strings.Contains(err.Error(), "CHECK constraint failed") {
+			t.Errorf("%s.provider grok: error %v, want CHECK constraint failure", table, err)
+		}
+	}
+	for _, table := range []string{"ai_settings_copy", "ai_runs_copy"} {
+		if tableExists(t, s, table) {
+			t.Errorf("%s left behind", table)
+		}
+	}
+}
@@ -364,7 +364,7 @@ func TestSchemaRotationReviewEventAllowed(t *testing.T) {
 // both ai_settings and ai_runs.
 func TestSchemaAIProvidersAllowed(t *testing.T) {
 	s := seededStore(t)
-	for _, provider := range []string{"openai_compatible", "anthropic", "ollama", "openai", "openrouter"} {
+	for _, provider := range []string{"openai_compatible", "anthropic", "ollama", "openai", "openrouter", "xai"} {
 		for _, table := range []string{"ai_settings", "ai_runs"} {
 			inRolledBackTx(t, s, func(tx *sql.Tx) {
 				row := testRow(table, 1)
@@ -0,0 +1,64 @@
+-- 0005_ai_provider_xai: ai_settings.provider and ai_runs.provider also
+-- accept xai, the xAI Grok preset in internal/ai/llm/presets. Every 0004
+-- value is kept, so existing rows copy unchanged. The rebuild is 0004's:
+-- copy the rows aside, drop the table, create it again with the wider CHECK,
+-- copy the rows back and drop the copy. Neither table has indexes or
+-- triggers.
+--
+-- suggestion_batches.ai_run_id and week_plans.ai_run_id reference ai_runs,
+-- so foreign keys are deferred until commit, after the ai_runs rows are
+-- back. The pragma lasts only for this migration's transaction.
+
+PRAGMA defer_foreign_keys = ON;
+
+CREATE TABLE ai_settings_copy AS SELECT * FROM ai_settings;
+
+DROP TABLE ai_settings;
+
+CREATE TABLE ai_settings (
+	id              INTEGER PRIMARY KEY CHECK (id = 1),
+	provider        TEXT    NOT NULL CHECK (provider IN ('openai_compatible', 'anthropic', 'ollama', 'openai', 'openrouter', 'xai')),
+	base_url        TEXT    NOT NULL,
+	model           TEXT    NOT NULL,
+	temperature     REAL    NOT NULL,
+	system_prompt   TEXT,
+	api_key_ct      BLOB,
+	api_key_last4   TEXT,
+	key_verified_at TEXT,
+	created_at      TEXT    NOT NULL,
+	updated_at      TEXT    NOT NULL
+);
+
+INSERT INTO ai_settings (id, provider, base_url, model, temperature, system_prompt, api_key_ct, api_key_last4, key_verified_at, created_at, updated_at)
+SELECT id, provider, base_url, model, temperature, system_prompt, api_key_ct, api_key_last4, key_verified_at, created_at, updated_at
+FROM ai_settings_copy;
+
+DROP TABLE ai_settings_copy;
+
+CREATE TABLE ai_runs_copy AS SELECT * FROM ai_runs;
+
+DROP TABLE ai_runs;
+
+CREATE TABLE ai_runs (
+	id                 TEXT    NOT NULL PRIMARY KEY,
+	skill              TEXT    NOT NULL,
+	skill_version      TEXT    NOT NULL,
+	provider           TEXT    NOT NULL CHECK (provider IN ('openai_compatible', 'anthropic', 'ollama', 'openai', 'openrouter', 'xai')),
+	model              TEXT    NOT NULL,
+	temperature        REAL    NOT NULL,
+	status             TEXT    NOT NULL CHECK (status IN ('running', 'ok', 'error')),
+	latency_ms         INTEGER,
+	tokens_in          INTEGER,
+	tokens_out         INTEGER,
+	error              TEXT,
+	request_redacted   TEXT    NOT NULL,
+	response_redacted  TEXT,
+	created_at         TEXT    NOT NULL,
+	updated_at         TEXT    NOT NULL
+);
+
+INSERT INTO ai_runs (id, skill, skill_version, provider, model, temperature, status, latency_ms, tokens_in, tokens_out, error, request_redacted, response_redacted, created_at, updated_at)
+SELECT id, skill, skill_version, provider, model, temperature, status, latency_ms, tokens_in, tokens_out, error, request_redacted, response_redacted, created_at, updated_at
+FROM ai_runs_copy;
+
+DROP TABLE ai_runs_copy;
Author	SHA1	Message	Date
dev_agentandClaude Opus 5.5	bcdea85a2d	Wire retrying Provider into presets.FromSettings CI / ci (pull_request) Successful in 1m31s Details CI / ci (push) Successful in 1m32s Details FromSettings now returns llm.NewRetrying around the configured adapter, so every Provider it builds retries transient Generate failures and gets the default timeout; ListModels and Verify pass through. The test recording transport now answers 400 instead of failing the round trip: the ollama adapter reports a transport failure as ErrTimeout, which is retried, so the helper would see three requests. Tests: a 429 then 200 succeeds on the second request; exhausted retries return ErrProvider after MaxRetries+1 requests with no key in any error. Maestro task: 1a882923-fdb4-4576-ab82-9529e9974b26 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>	2026-10-04 10:49:46 -04:00
dev_agentandClaude Opus 5.5	cbf24e63e1	Add retrying Provider wrapper with backoff and default timeout llm.NewRetrying wraps a Provider and retries ErrRateLimited, ErrTimeout and 5xx ErrProvider at most twice. A rate limit waits RetryAfter, capped at 20 s; other retries use jittered exponential backoff. ErrAuth, 4xx and other errors are never retried, and no retry is made whose wait would not end before the caller's context deadline. Generate gets a 60 s per-attempt timeout when Request.Timeout is 0. Maestro task: 1a882923-fdb4-4576-ab82-9529e9974b26 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>	2026-10-04 10:48:07 -04:00
dev_agent	e105cdb9c5	Add xAI Grok as a provider (#46 ) CI / ci (push) Successful in 1m28s Details Co-authored-by: dev_agent <dev_agent@cubecraftlabs.com> Co-committed-by: dev_agent <dev_agent@cubecraftlabs.com>	2026-10-04 09:36:10 -04:00
dev_agent	9ace3aba9f	Provider presets and factory (#44 ) CI / ci (push) Successful in 1m28s Details Co-authored-by: dev_agent <dev_agent@cubecraftlabs.com> Co-committed-by: dev_agent <dev_agent@cubecraftlabs.com>	2026-10-04 09:18:34 -04:00