MealQuest/internal/ai/llm/presets/presets_test.go

package presets

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"strings"
	"testing"

	"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm"
)

const testKey = "sk-test-0123456789"

// captured is the HTTP request an adapter sent.
type captured struct {
	url    string
	header http.Header
	body   map[string]json.RawMessage
}

// recordingTransport records each request and answers it 400, so no
// request leaves the process and the Provider never retries it.
type recordingTransport struct{ reqs []captured }

func (rt *recordingTransport) RoundTrip(r *http.Request) (*http.Response, error) {
	c := captured{url: r.URL.String(), header: r.Header.Clone()}
	if r.Body != nil {
		raw, err := io.ReadAll(r.Body)
		if err != nil {
			return nil, err
		}
		if err := json.Unmarshal(raw, &c.body); err != nil {
			return nil, err
		}
	}
	rt.reqs = append(rt.reqs, c)
	return &http.Response{
		StatusCode: http.StatusBadRequest,
		Header:     http.Header{"Content-Type": {"application/json"}},
		Body:       io.NopCloser(strings.NewReader(`{}`)),
		Request:    r,
	}, nil
}

// generate builds the provider s describes and returns the request its
// Generate sends for req. The adapters use http.DefaultTransport, which the
// test replaces, so the real preset base URLs are never contacted.
func generate(t *testing.T, s llm.Settings, key string, req llm.Request) captured {
	t.Helper()
	rt := &recordingTransport{}
	orig := http.DefaultTransport
	http.DefaultTransport = rt
	t.Cleanup(func() { http.DefaultTransport = orig })

	p, err := FromSettings(s, key)
	if err != nil {
		t.Fatalf("FromSettings: %v", err)
	}
	if _, err := p.Generate(context.Background(), req); err == nil {
		t.Fatal("Generate succeeded through the recording transport")
	}
	if len(rt.reqs) != 1 {
		t.Fatalf("sent %d requests, want 1", len(rt.reqs))
	}
	return rt.reqs[0]
}

func plainRequest() llm.Request {
	return llm.Request{Messages: []llm.Message{{Role: llm.RoleUser, Content: "dinner?"}}}
}

func schemaRequest() llm.Request {
	r := plainRequest()
	r.Schema = &llm.JSONSchema{Name: "side_quests", Schema: json.RawMessage(`{"type":"object"}`)}
	return r
}

func ptr(f float64) *float64 { return &f }

func responseFormatType(t *testing.T, c captured) string {
	t.Helper()
	raw, ok := c.body["response_format"]
	if !ok {
		return ""
	}
	var rf struct{ Type string }
	if err := json.Unmarshal(raw, &rf); err != nil {
		t.Fatalf("response_format: %v", err)
	}
	return rf.Type
}

func temperature(t *testing.T, c captured) (float64, bool) {
	t.Helper()
	raw, ok := c.body["temperature"]
	if !ok {
		return 0, false
	}
	var f float64
	if err := json.Unmarshal(raw, &f); err != nil {
		t.Fatalf("temperature: %v", err)
	}
	return f, true
}

func TestAnthropicPreset(t *testing.T) {
	c := generate(t, llm.Settings{Provider: "anthropic", Model: "claude-sonnet-5", Temperature: 0.7}, testKey, plainRequest())
	if c.url != "https://api.anthropic.com/v1/messages" {
		t.Fatalf("URL = %q", c.url)
	}
	if got := c.header.Get("X-Api-Key"); got != testKey {
		t.Fatalf("x-api-key = %q", got)
	}
	if got := c.header.Get("Authorization"); got != "" {
		t.Fatalf("Authorization = %q, want none", got)
	}
	if got, ok := temperature(t, c); !ok || got != 0.7 {
		t.Fatalf("temperature = %v, %v; want 0.7", got, ok)
	}
	if string(c.body["max_tokens"]) != "4096" {
		t.Fatalf("max_tokens = %s", c.body["max_tokens"])
	}
}

func TestAnthropicPresetAcceptsItsOwnBaseURL(t *testing.T) {
	c := generate(t, llm.Settings{Provider: "anthropic", BaseURL: "https://api.anthropic.com", Model: "claude-sonnet-5"}, testKey, plainRequest())
	if c.url != "https://api.anthropic.com/v1/messages" {
		t.Fatalf("URL = %q", c.url)
	}
}

func TestOpenAIPreset(t *testing.T) {
	c := generate(t, llm.Settings{Provider: "openai", Model: "gpt-4o", Temperature: 0.4}, testKey, schemaRequest())
	if c.url != "https://api.openai.com/v1/chat/completions" {
		t.Fatalf("URL = %q", c.url)
	}
	if got := c.header.Get("Authorization"); got != "Bearer "+testKey {
		t.Fatalf("Authorization = %q", got)
	}
	if got := responseFormatType(t, c); got != "json_schema" {
		t.Fatalf("response_format type = %q, want json_schema", got)
	}
	if c.header.Get("HTTP-Referer") != "" || c.header.Get("X-Title") != "" {
		t.Fatal("openai sends OpenRouter headers")
	}
}

func TestOpenRouterPreset(t *testing.T) {
	s := llm.Settings{Provider: "openrouter", Model: "openai/gpt-4o", PublicURL: "https://meals.example.com/"}
	c := generate(t, s, testKey, schemaRequest())
	if c.url != "https://openrouter.ai/api/v1/chat/completions" {
		t.Fatalf("URL = %q", c.url)
	}
	if got := c.header.Get("Authorization"); got != "Bearer "+testKey {
		t.Fatalf("Authorization = %q", got)
	}
	if got := c.header.Get("HTTP-Referer"); got != "https://meals.example.com/" {
		t.Fatalf("HTTP-Referer = %q", got)
	}
	if got := c.header.Get("X-Title"); got != "MealQuest" {
		t.Fatalf("X-Title = %q", got)
	}
	if got := responseFormatType(t, c); got != "json_schema" {
		t.Fatalf("default response_format type = %q, want json_schema", got)
	}
}

func TestOpenRouterFallbackModeIsJSONObject(t *testing.T) {
	p, _ := Lookup("openrouter")
	s := llm.Settings{
		Provider:         "openrouter",
		Model:            "meta-llama/llama-3.3-70b-instruct",
		PublicURL:        "https://meals.example.com",
		StructuredOutput: string(p.FallbackStructuredOutput),
	}
	c := generate(t, s, testKey, schemaRequest())
	if got := responseFormatType(t, c); got != "json_object" {
		t.Fatalf("response_format type = %q, want json_object", got)
	}
}

func TestXAIPreset(t *testing.T) {
	c := generate(t, llm.Settings{Provider: "xai", Model: "grok-4", Temperature: 0.6}, testKey, schemaRequest())
	if c.url != "https://api.x.ai/v1/chat/completions" {
		t.Fatalf("URL = %q", c.url)
	}
	if got := c.header.Get("Authorization"); got != "Bearer "+testKey {
		t.Fatalf("Authorization = %q", got)
	}
	for _, h := range []string{"X-Api-Key", "HTTP-Referer", "X-Title"} {
		if got := c.header.Get(h); got != "" {
			t.Fatalf("%s = %q, want none", h, got)
		}
	}
	if got := responseFormatType(t, c); got != "json_schema" {
		t.Fatalf("default response_format type = %q, want json_schema", got)
	}
	if got, ok := temperature(t, c); !ok || got != 0.6 {
		t.Fatalf("temperature = %v, %v; want 0.6", got, ok)
	}
}

func TestXAIFallbackModeIsJSONObject(t *testing.T) {
	p, _ := Lookup("xai")
	s := llm.Settings{Provider: "xai", Model: "grok-4", StructuredOutput: string(p.FallbackStructuredOutput)}
	c := generate(t, s, testKey, schemaRequest())
	if got := responseFormatType(t, c); got != "json_object" {
		t.Fatalf("response_format type = %q, want json_object", got)
	}
}

func TestXAIPresetAcceptsItsOwnBaseURL(t *testing.T) {
	c := generate(t, llm.Settings{Provider: "xai", BaseURL: "https://api.x.ai/v1", Model: "grok-4"}, testKey, plainRequest())
	if c.url != "https://api.x.ai/v1/chat/completions" {
		t.Fatalf("URL = %q", c.url)
	}
}

func TestOllamaPreset(t *testing.T) {
	s := llm.Settings{Provider: "ollama", BaseURL: "http://ollama.lan:11434", Model: "llama3.2", Temperature: 0.5}
	c := generate(t, s, "", plainRequest())
	if c.url != "http://ollama.lan:11434/api/chat" {
		t.Fatalf("URL = %q", c.url)
	}
	if got := c.header.Get("Authorization"); got != "" {
		t.Fatalf("Authorization = %q, want none", got)
	}
	var opts struct{ Temperature *float64 }
	if err := json.Unmarshal(c.body["options"], &opts); err != nil || opts.Temperature == nil || *opts.Temperature != 0.5 {
		t.Fatalf("options = %s", c.body["options"])
	}
}

func TestOpenAICompatiblePreset(t *testing.T) {
	base := llm.Settings{Provider: "openai_compatible", BaseURL: "http://llm.lan:8000/v1", Model: "qwen2.5"}

	c := generate(t, base, "", schemaRequest())
	if c.url != "http://llm.lan:8000/v1/chat/completions" {
		t.Fatalf("URL = %q", c.url)
	}
	if got := c.header.Get("Authorization"); got != "" {
		t.Fatalf("Authorization without key = %q, want none", got)
	}
	if got := responseFormatType(t, c); got != "json_schema" {
		t.Fatalf("default response_format type = %q, want json_schema", got)
	}

	c = generate(t, base, testKey, plainRequest())
	if got := c.header.Get("Authorization"); got != "Bearer "+testKey {
		t.Fatalf("Authorization with key = %q", got)
	}

	for mode, want := range map[string]string{"json_schema": "json_schema", "json_object": "json_object", "prompt_only": ""} {
		s := base
		s.StructuredOutput = mode
		if got := responseFormatType(t, generate(t, s, "", schemaRequest())); got != want {
			t.Errorf("mode %s: response_format type = %q, want %q", mode, got, want)
		}
	}
}

func TestTemperatureOmittedForRejectingModels(t *testing.T) {
	cases := map[string][]string{
		"openai":            {"o1", "o1-mini", "o1-2024-12-17", "o3", "o3-mini"},
		"openai_compatible": {"o1", "o1-mini", "o3-mini", "openai/o1"},
		"openrouter":        {"openai/o1", "openai/o1-mini", "openai/o3", "openai/o3-mini", "openai/o1:online"},
	}
	for provider, models := range cases {
		for _, model := range models {
			s := llm.Settings{Provider: provider, Model: model, Temperature: 0.8}
			switch provider {
			case "openai_compatible":
				s.BaseURL = "http://llm.lan/v1"
			case "openrouter":
				s.PublicURL = "https://mealquest.example"
			}
			for _, req := range []llm.Request{plainRequest(), func() llm.Request { r := plainRequest(); r.Temperature = ptr(0.2); return r }()} {
				if got, ok := temperature(t, generate(t, s, testKey, req)); ok {
					t.Errorf("%s %s: temperature %v sent, want omitted", provider, model, got)
				}
			}
		}
	}
}

func TestTemperatureSentForOtherModels(t *testing.T) {
	cases := map[string][]string{
		"openai":     {"gpt-4o", "gpt-4o-mini", "chatgpt-o1-like", "o123", "o3custom", "o1mini", "openai/o1"},
		"openrouter": {"openai/gpt-4o", "openai/o123", "openai/o3custom", "acme/o1", "o1-mini"},
	}
	for provider, models := range cases {
		for _, model := range models {
			s := llm.Settings{Provider: provider, Model: model, Temperature: 0.8}
			if provider == "openrouter" {
				s.PublicURL = "https://mealquest.example"
			}
			if got, ok := temperature(t, generate(t, s, testKey, plainRequest())); !ok || got != 0.8 {
				t.Errorf("%s %s: temperature = %v, %v; want 0.8", provider, model, got, ok)
			}
		}
	}
	// A request's own temperature wins over the household's.
	req := plainRequest()
	req.Temperature = ptr(0.1)
	c := generate(t, llm.Settings{Provider: "openai", Model: "gpt-4o", Temperature: 0.8}, testKey, req)
	if got, ok := temperature(t, c); !ok || got != 0.1 {
		t.Fatalf("temperature = %v, %v; want 0.1", got, ok)
	}
}

// A request is validated before the temperature quirk clears its
// temperature, so a negative one is an error and nothing is sent.
func TestNegativeTemperatureRejectedForRejectingModels(t *testing.T) {
	rt := &recordingTransport{}
	orig := http.DefaultTransport
	http.DefaultTransport = rt
	t.Cleanup(func() { http.DefaultTransport = orig })

	for _, s := range []llm.Settings{
		{Provider: "openai", Model: "o1"},
		{Provider: "openrouter", Model: "openai/o1", PublicURL: "https://mealquest.example"},
	} {
		p, err := FromSettings(s, testKey)
		if err != nil {
			t.Fatalf("%s: FromSettings: %v", s.Model, err)
		}
		req := plainRequest()
		req.Temperature = ptr(-0.5)
		if _, err := p.Generate(context.Background(), req); err == nil || errors.Is(err, llm.ErrProvider) || !strings.Contains(err.Error(), "temperature") {
			t.Errorf("%s: Generate = %v, want a temperature validation error", s.Model, err)
		}
	}
	if len(rt.reqs) != 0 {
		t.Fatalf("sent %d requests, want 0", len(rt.reqs))
	}
}

func TestTemperatureQuirkPresets(t *testing.T) {
	want := map[string]bool{"openai": true, "openrouter": true, "openai_compatible": true}
	for name, p := range all {
		if got := len(p.OmitTemperature) > 0; got != want[name] {
			t.Errorf("%s: lists temperature quirk = %v, want %v", name, got, want[name])
		}
	}
}

func TestPresetTable(t *testing.T) {
	cases := []struct {
		name    string
		adapter Adapter
		baseURL string
		key     KeyPolicy
	}{
		{"anthropic", AdapterAnthropic, "https://api.anthropic.com", KeyRequired},
		{"openai", AdapterOpenAI, "https://api.openai.com/v1", KeyRequired},
		{"openrouter", AdapterOpenAI, "https://openrouter.ai/api/v1", KeyRequired},
		{"xai", AdapterOpenAI, "https://api.x.ai/v1", KeyRequired},
		{"ollama", AdapterOllama, "", KeyNone},
		{"openai_compatible", AdapterOpenAI, "", KeyOptional},
	}
	if len(all) != len(cases) {
		t.Fatalf("%d presets, want %d", len(all), len(cases))
	}
	for _, c := range cases {
		p, ok := Lookup(c.name)
		if !ok {
			t.Fatalf("no preset %q", c.name)
		}
		if p.Name != c.name || p.Adapter != c.adapter || p.BaseURL != c.baseURL || p.Key != c.key {
			t.Errorf("%s = {%s %s %q %d}", c.name, p.Name, p.Adapter, p.BaseURL, p.Key)
		}
	}
}

func TestConfigErrors(t *testing.T) {
	sentinels := []error{
		llm.ErrUnknownProvider, llm.ErrMissingKey, llm.ErrUnexpectedKey, llm.ErrInvalidKey, llm.ErrMissingBaseURL,
		llm.ErrMissingModel, llm.ErrMissingPublicURL, llm.ErrInvalidSetting,
	}
	cases := []struct {
		name string
		s    llm.Settings
		key  string
		want error
	}{
		{"unknown provider", llm.Settings{Provider: "gemini", Model: "m"}, testKey, llm.ErrUnknownProvider},
		{"anthropic no key", llm.Settings{Provider: "anthropic", Model: "m"}, "", llm.ErrMissingKey},
		{"openai no key", llm.Settings{Provider: "openai", Model: "m"}, "", llm.ErrMissingKey},
		{"openrouter no key", llm.Settings{Provider: "openrouter", Model: "m", PublicURL: "https://x.example"}, "", llm.ErrMissingKey},
		{"xai no key", llm.Settings{Provider: "xai", Model: "grok-4"}, "", llm.ErrMissingKey},
		{"ollama with key", llm.Settings{Provider: "ollama", BaseURL: "http://o:11434", Model: "m"}, testKey, llm.ErrUnexpectedKey},
		{"anthropic key with newline", llm.Settings{Provider: "anthropic", Model: "m"}, testKey + "\n", llm.ErrInvalidKey},
		{"openai key with newline", llm.Settings{Provider: "openai", Model: "m"}, testKey + "\n", llm.ErrInvalidKey},
		{"openrouter key with CR LF", llm.Settings{Provider: "openrouter", Model: "m", PublicURL: "https://x.example"}, testKey + "\r\nX-Injected: 1", llm.ErrInvalidKey},
		{"xai key with newline", llm.Settings{Provider: "xai", Model: "grok-4"}, testKey + "\n", llm.ErrInvalidKey},
		{"openai_compatible key with NUL", llm.Settings{Provider: "openai_compatible", BaseURL: "http://l/v1", Model: "m"}, "\x00" + testKey, llm.ErrInvalidKey},
		{"ollama no base URL", llm.Settings{Provider: "ollama", Model: "m"}, "", llm.ErrMissingBaseURL},
		{"openai_compatible no base URL", llm.Settings{Provider: "openai_compatible", Model: "m"}, "", llm.ErrMissingBaseURL},
		{"anthropic no model", llm.Settings{Provider: "anthropic"}, testKey, llm.ErrMissingModel},
		{"openai no model", llm.Settings{Provider: "openai"}, testKey, llm.ErrMissingModel},
		{"openrouter no model", llm.Settings{Provider: "openrouter", PublicURL: "https://x.example"}, testKey, llm.ErrMissingModel},
		{"ollama no model", llm.Settings{Provider: "ollama", BaseURL: "http://o:11434"}, "", llm.ErrMissingModel},
		{"openai_compatible no model", llm.Settings{Provider: "openai_compatible", BaseURL: "http://l/v1"}, "", llm.ErrMissingModel},
		{"xai no model", llm.Settings{Provider: "xai"}, testKey, llm.ErrMissingModel},
		{"openrouter no public URL", llm.Settings{Provider: "openrouter", Model: "m"}, testKey, llm.ErrMissingPublicURL},
		{"fixed base URL overridden", llm.Settings{Provider: "openai", BaseURL: "https://evil.example/v1", Model: "m"}, testKey, llm.ErrInvalidSetting},
		{"xai base URL overridden", llm.Settings{Provider: "xai", BaseURL: "https://api.openai.com/v1", Model: "grok-4"}, testKey, llm.ErrInvalidSetting},
		{"xai prompt_only", llm.Settings{Provider: "xai", Model: "grok-4", StructuredOutput: "prompt_only"}, testKey, llm.ErrInvalidSetting},
		{"openai json_object", llm.Settings{Provider: "openai", Model: "m", StructuredOutput: "json_object"}, testKey, llm.ErrInvalidSetting},
		{"anthropic mode", llm.Settings{Provider: "anthropic", Model: "m", StructuredOutput: "json_schema"}, testKey, llm.ErrInvalidSetting},
		{"unknown mode", llm.Settings{Provider: "openai_compatible", BaseURL: "http://l/v1", Model: "m", StructuredOutput: "xml"}, "", llm.ErrInvalidSetting},
		{"negative temperature", llm.Settings{Provider: "openai", Model: "m", Temperature: -0.1}, testKey, llm.ErrInvalidSetting},
		{"bad base URL", llm.Settings{Provider: "ollama", BaseURL: "ftp://o", Model: "m"}, "", llm.ErrInvalidSetting},
	}
	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			p, err := FromSettings(c.s, c.key)
			if p != nil || !errors.Is(err, c.want) {
				t.Fatalf("FromSettings = %v, %v; want %v", p, err, c.want)
			}
			for _, other := range sentinels {
				if other != c.want && errors.Is(err, other) {
					t.Fatalf("error %v also matches %v", err, other)
				}
			}
			if c.key != "" && strings.Contains(err.Error(), c.key) {
				t.Fatalf("error leaks the key: %v", err)
			}
		})
	}
}

func TestReadyProviderForEveryPreset(t *testing.T) {
	for _, s := range []struct {
		s   llm.Settings
		key string
	}{
		{llm.Settings{Provider: "anthropic", Model: "m"}, testKey},
		{llm.Settings{Provider: "openai", Model: "m"}, testKey},
		{llm.Settings{Provider: "openrouter", Model: "m", PublicURL: "https://x.example"}, testKey},
		{llm.Settings{Provider: "xai", Model: "m"}, testKey},
		{llm.Settings{Provider: "ollama", BaseURL: "http://o:11434", Model: "m"}, ""},
		{llm.Settings{Provider: "openai_compatible", BaseURL: "http://l/v1", Model: "m"}, ""},
	} {
		if p, err := FromSettings(s.s, s.key); err != nil || p == nil {
			t.Errorf("%s: FromSettings = %v, %v", s.s.Provider, p, err)
		}
	}
}

// A key that matches a value the request carries, or text the error spells
// anyway, never appears in a validation error, and nothing is sent.
func TestGenerateValidationErrorsNeverLeakKey(t *testing.T) {
	rt := &recordingTransport{}
	orig := http.DefaultTransport
	http.DefaultTransport = rt
	t.Cleanup(func() { http.DefaultTransport = orig })

	for _, key := range []string{"-0.5", "temperature", "llm", "negative"} {
		p, err := FromSettings(llm.Settings{Provider: "openai_compatible", BaseURL: "http://l/v1", Model: "m"}, key)
		if err != nil {
			t.Fatalf("%s: FromSettings: %v", key, err)
		}
		req := plainRequest()
		req.Temperature = ptr(-0.5)
		_, err = p.Generate(context.Background(), req)
		if err == nil || errors.Is(err, llm.ErrProvider) {
			t.Fatalf("%s: Generate = %v, want a validation error", key, err)
		}
		for e := error(err); e != nil; e = errors.Unwrap(e) {
			if s := e.Error() + fmt.Sprintf("%#v", e); strings.Contains(s, key) {
				t.Fatalf("%s: error leaks the key: %s", key, s)
			}
		}
	}
	if len(rt.reqs) != 0 {
		t.Fatalf("sent %d requests, want 0", len(rt.reqs))
	}
}

// A configuration error keeps its sentinel but never holds the key, even
// when the key is also a setting's value or text the error spells anyway.
func TestConfigErrorsNeverLeakKey(t *testing.T) {
	cases := []struct {
		name string
		s    llm.Settings
		key  string
		want error
	}{
		{"unknown provider named by key", llm.Settings{Provider: "llm", Model: "m"}, "llm", llm.ErrUnknownProvider},
		{"no base URL", llm.Settings{Provider: "openai_compatible", Model: "m"}, "base URL", llm.ErrMissingBaseURL},
		{"no model", llm.Settings{Provider: "openai_compatible", BaseURL: "http://l/v1"}, "model", llm.ErrMissingModel},
		{"key as model, no public URL", llm.Settings{Provider: "openrouter", Model: "missing"}, "missing", llm.ErrMissingPublicURL},
		{"key as base URL", llm.Settings{Provider: "openai", BaseURL: "llm", Model: "m"}, "llm", llm.ErrInvalidSetting},
		{"key as mode", llm.Settings{Provider: "openai_compatible", BaseURL: "http://l/v1", Model: "m", StructuredOutput: "xml"}, "xml", llm.ErrInvalidSetting},
		{"key as temperature", llm.Settings{Provider: "openai", Model: "m", Temperature: -0.1}, "-0.1", llm.ErrInvalidSetting},
		{"key in bad base URL", llm.Settings{Provider: "openai_compatible", BaseURL: "ftp://k3y", Model: "m"}, "k3y", llm.ErrInvalidSetting},
		{"ollama with key", llm.Settings{Provider: "ollama", BaseURL: "http://o:11434", Model: "m"}, "key", llm.ErrUnexpectedKey},
		{"key with newline", llm.Settings{Provider: "openai", Model: "m"}, "sk-test\n", llm.ErrInvalidKey},
		{"xai key as preset name", llm.Settings{Provider: "xai"}, "xai", llm.ErrMissingModel},
		{"xai key as missing-model text", llm.Settings{Provider: "xai"}, "model is required", llm.ErrMissingModel},
		{"xai key as fixed-URL text", llm.Settings{Provider: "xai", BaseURL: "https://x.example/v1", Model: "m"}, "fixed", llm.ErrInvalidSetting},
		{"xai key as structured-output text", llm.Settings{Provider: "xai", Model: "m", StructuredOutput: "prompt_only"}, "structured output", llm.ErrInvalidSetting},
		{"key with newline as text", llm.Settings{Provider: "openai", Model: "m"}, "header\n", llm.ErrInvalidKey},
	}
	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			p, err := FromSettings(c.s, c.key)
			if p != nil || !errors.Is(err, c.want) {
				t.Fatalf("FromSettings = %v, %v; want %v", p, err, c.want)
			}
			for e := error(err); e != nil; e = errors.Unwrap(e) {
				if s := e.Error() + fmt.Sprintf("%#v", e); strings.Contains(s, c.key) {
					t.Fatalf("error leaks the key: %s", s)
				}
			}
		})
	}
}

// scriptedTransport answers each request with the next scripted status and
// body, echoing the key in every failure body, and counts the requests. A
// scripted status of 0 is a transport timeout instead of a response.
type scriptedTransport struct {
	statuses []int
	key      string
	sent     int
}

func (st *scriptedTransport) RoundTrip(r *http.Request) (*http.Response, error) {
	if st.sent == len(st.statuses) {
		return nil, fmt.Errorf("unscripted request %d", st.sent+1)
	}
	status := st.statuses[st.sent]
	st.sent++
	if status == 0 {
		return nil, transportTimeout{}
	}
	header := http.Header{"Content-Type": {"application/json"}}
	body := `{"error":{"message":"bad key ` + st.key + `"}}`
	switch status {
	case http.StatusOK:
		body = `{"model":"m","choices":[{"message":{"content":"tacos"},"finish_reason":"stop"}]}`
	case http.StatusTooManyRequests:
		header.Set("Retry-After", "1")
	}
	return &http.Response{
		StatusCode: status,
		Header:     header,
		Body:       io.NopCloser(strings.NewReader(body)),
		Request:    r,
	}, nil
}

// transportTimeout is a net.Error that timed out.
type transportTimeout struct{}

func (transportTimeout) Error() string   { return "i/o timeout" }
func (transportTimeout) Timeout() bool   { return true }
func (transportTimeout) Temporary() bool { return true }

// useTransport makes the adapters send through rt for the rest of the test.
func useTransport(t *testing.T, rt http.RoundTripper) {
	t.Helper()
	orig := http.DefaultTransport
	http.DefaultTransport = rt
	t.Cleanup(func() { http.DefaultTransport = orig })
}

// The Provider FromSettings returns retries a rate-limited Generate.
func TestFromSettingsRetriesRateLimited(t *testing.T) {
	st := &scriptedTransport{statuses: []int{http.StatusTooManyRequests, http.StatusOK}, key: testKey}
	useTransport(t, st)

	p, err := FromSettings(llm.Settings{Provider: "openai", Model: "m"}, testKey)
	if err != nil {
		t.Fatalf("FromSettings: %v", err)
	}
	resp, err := p.Generate(context.Background(), plainRequest())
	if err != nil {
		t.Fatalf("Generate: %v", err)
	}
	if resp.Text != "tacos" {
		t.Errorf("Text = %q, want %q", resp.Text, "tacos")
	}
	if st.sent != 2 {
		t.Errorf("sent %d requests, want 2", st.sent)
	}
}

// When every retry fails, the error FromSettings's Provider returns keeps
// its sentinel and never holds the key the vendor echoed.
func TestFromSettingsRetryErrorNeverLeaksKey(t *testing.T) {
	st := &scriptedTransport{statuses: []int{http.StatusTooManyRequests, http.StatusTooManyRequests, http.StatusServiceUnavailable}, key: testKey}
	useTransport(t, st)

	p, err := FromSettings(llm.Settings{Provider: "openai", Model: "m"}, testKey)
	if err != nil {
		t.Fatalf("FromSettings: %v", err)
	}
	_, err = p.Generate(context.Background(), plainRequest())
	if !errors.Is(err, llm.ErrProvider) {
		t.Fatalf("Generate = %v, want %v", err, llm.ErrProvider)
	}
	if st.sent != llm.MaxRetries+1 {
		t.Errorf("sent %d requests, want %d", st.sent, llm.MaxRetries+1)
	}
	for e := error(err); e != nil; e = errors.Unwrap(e) {
		if s := e.Error() + fmt.Sprintf("%#v", e); strings.Contains(s, testKey) {
			t.Fatalf("error leaks the key: %s", s)
		}
	}
}

// An HTTP 408 from the vendor is a 4xx answer and is not retried, though it
// matches llm.ErrTimeout.
func TestFromSettingsDoesNotRetryHTTP408(t *testing.T) {
	for _, provider := range []string{"openai", "anthropic"} {
		t.Run(provider, func(t *testing.T) {
			st := &scriptedTransport{statuses: []int{http.StatusRequestTimeout}, key: testKey}
			useTransport(t, st)

			p, err := FromSettings(llm.Settings{Provider: provider, Model: "m"}, testKey)
			if err != nil {
				t.Fatalf("FromSettings: %v", err)
			}
			_, err = p.Generate(context.Background(), plainRequest())
			var ts *llm.TimeoutStatusError
			if !errors.Is(err, llm.ErrTimeout) || !errors.As(err, &ts) || ts.Status() != http.StatusRequestTimeout {
				t.Fatalf("Generate = %v, want an HTTP 408 ErrTimeout", err)
			}
			if st.sent != 1 {
				t.Errorf("sent %d requests, want 1", st.sent)
			}
		})
	}
}

// A transport timeout carries no status and is retried.
func TestFromSettingsRetriesTransportTimeout(t *testing.T) {
	st := &scriptedTransport{statuses: []int{0, http.StatusOK}, key: testKey}
	useTransport(t, st)

	p, err := FromSettings(llm.Settings{Provider: "openai", Model: "m"}, testKey)
	if err != nil {
		t.Fatalf("FromSettings: %v", err)
	}
	resp, err := p.Generate(context.Background(), plainRequest())
	if err != nil {
		t.Fatalf("Generate: %v", err)
	}
	if resp.Text != "tacos" {
		t.Errorf("Text = %q, want %q", resp.Text, "tacos")
	}
	if st.sent != 2 {
		t.Errorf("sent %d requests, want 2", st.sent)
	}
}