From 72042516e2d7530bbfe449cadfba7a58be531ae6 Mon Sep 17 00:00:00 2001 From: dev_agent Date: Sat, 3 Oct 2026 16:14:12 -0400 Subject: [PATCH 1/7] Feature 08 task 1: Provider interface, typed errors and Fake provider (#33) Co-authored-by: dev_agent Co-committed-by: dev_agent --- internal/ai/llm/errors.go | 103 ++++++++++++++++++++++++ internal/ai/llm/errors_test.go | 121 +++++++++++++++++++++++++++++ internal/ai/llm/fake.go | 127 ++++++++++++++++++++++++++++++ internal/ai/llm/fake_test.go | 138 +++++++++++++++++++++++++++++++++ internal/ai/llm/llm.go | 132 +++++++++++++++++++++++++++++++ internal/ai/llm/llm_test.go | 44 +++++++++++ 6 files changed, 665 insertions(+) create mode 100644 internal/ai/llm/errors.go create mode 100644 internal/ai/llm/errors_test.go create mode 100644 internal/ai/llm/fake.go create mode 100644 internal/ai/llm/fake_test.go create mode 100644 internal/ai/llm/llm.go create mode 100644 internal/ai/llm/llm_test.go diff --git a/internal/ai/llm/errors.go b/internal/ai/llm/errors.go new file mode 100644 index 0000000..be8e922 --- /dev/null +++ b/internal/ai/llm/errors.go @@ -0,0 +1,103 @@ +package llm + +import ( + "errors" + "fmt" + "strings" + "time" + "unicode/utf8" +) + +var ( + // ErrAuth means the vendor rejected the credentials. + ErrAuth = errors.New("llm: authentication failed") + // ErrRateLimited means the vendor throttled the request. The concrete + // error is a *RateLimitedError carrying the requested wait. + ErrRateLimited = errors.New("llm: rate limited") + // ErrTimeout means the request did not finish within its timeout. + ErrTimeout = errors.New("llm: request timed out") + // ErrBadOutput means the model replied but the reply is unusable, e.g. + // structured output that does not parse. + ErrBadOutput = errors.New("llm: unusable model output") + // ErrProvider means the vendor returned an unexpected HTTP status. The + // concrete error is a *ProviderError carrying the status and a redacted + // body excerpt. + ErrProvider = errors.New("llm: provider error") +) + +// RateLimitedError is a throttled request. It matches ErrRateLimited. +type RateLimitedError struct { + // RetryAfter is how long the vendor asked callers to wait; 0 means it + // did not say. + RetryAfter time.Duration +} + +func (e *RateLimitedError) Error() string { + if e.RetryAfter == 0 { + return ErrRateLimited.Error() + } + return fmt.Sprintf("%s: retry after %v", ErrRateLimited, e.RetryAfter) +} + +// Is reports whether target is ErrRateLimited. +func (e *RateLimitedError) Is(target error) bool { return target == ErrRateLimited } + +// MaxBodyExcerpt is the longest response-body excerpt, in bytes, a +// ProviderError keeps. +const MaxBodyExcerpt = 512 + +// redacted replaces each secret in a ProviderError body. +const redacted = "[REDACTED]" + +// ProviderError is an unexpected HTTP status from the vendor. It matches +// ErrProvider. Build it with NewProviderError, which redacts the body. +type ProviderError struct { + status int + body string +} + +// NewProviderError returns the error for an unexpected HTTP status. +// secrets must hold the API key (and any other credential sent with the +// request). Every occurrence of each non-empty secret in body is replaced +// before the body is cut to MaxBodyExcerpt bytes, so no part of a secret +// survives. If secrets holds no non-empty value the body cannot be +// redacted, so no excerpt is kept at all. +func NewProviderError(status int, body []byte, secrets []string) *ProviderError { + s := string(body) + redactedAny := false + for _, secret := range secrets { + if secret != "" { + s = strings.ReplaceAll(s, secret, redacted) + redactedAny = true + } + } + if !redactedAny { + return &ProviderError{status: status} + } + s = strings.TrimSpace(s) + if len(s) > MaxBodyExcerpt { + cut := MaxBodyExcerpt + for cut > 0 && !utf8.RuneStart(s[cut]) { + cut-- + } + s = s[:cut] + "…" + } + return &ProviderError{status: status, body: s} +} + +// Status returns the HTTP status code. +func (e *ProviderError) Status() int { return e.status } + +// Body returns the redacted, truncated response-body excerpt; it is empty +// when the vendor sent no body or no secret was given to redact it with. +func (e *ProviderError) Body() string { return e.body } + +func (e *ProviderError) Error() string { + if e.body == "" { + return fmt.Sprintf("%s: HTTP %d", ErrProvider, e.status) + } + return fmt.Sprintf("%s: HTTP %d: %s", ErrProvider, e.status, e.body) +} + +// Is reports whether target is ErrProvider. +func (e *ProviderError) Is(target error) bool { return target == ErrProvider } diff --git a/internal/ai/llm/errors_test.go b/internal/ai/llm/errors_test.go new file mode 100644 index 0000000..321f887 --- /dev/null +++ b/internal/ai/llm/errors_test.go @@ -0,0 +1,121 @@ +package llm + +import ( + "errors" + "fmt" + "strings" + "testing" + "time" +) + +func TestSentinelErrorsMatchWhenWrapped(t *testing.T) { + for _, sentinel := range []error{ErrAuth, ErrTimeout, ErrBadOutput} { + wrapped := fmt.Errorf("openai: generate: %w", sentinel) + if !errors.Is(wrapped, sentinel) { + t.Errorf("errors.Is(%v, %v) = false", wrapped, sentinel) + } + for _, other := range []error{ErrAuth, ErrTimeout, ErrBadOutput, ErrRateLimited, ErrProvider} { + if other != sentinel && errors.Is(wrapped, other) { + t.Errorf("errors.Is(%v, %v) = true", wrapped, other) + } + } + } +} + +func TestRateLimitedError(t *testing.T) { + err := fmt.Errorf("anthropic: %w", &RateLimitedError{RetryAfter: 30 * time.Second}) + if !errors.Is(err, ErrRateLimited) { + t.Fatal("errors.Is(err, ErrRateLimited) = false") + } + if errors.Is(err, ErrProvider) || errors.Is(err, ErrAuth) { + t.Fatal("rate-limited error matches an unrelated sentinel") + } + var rl *RateLimitedError + if !errors.As(err, &rl) { + t.Fatal("errors.As(err, *RateLimitedError) = false") + } + if rl.RetryAfter != 30*time.Second { + t.Fatalf("RetryAfter = %v, want 30s", rl.RetryAfter) + } + if got := err.Error(); got != "anthropic: llm: rate limited: retry after 30s" { + t.Fatalf("Error() = %q", got) + } + if got := (&RateLimitedError{}).Error(); got != "llm: rate limited" { + t.Fatalf("Error() without RetryAfter = %q", got) + } +} + +func TestProviderError(t *testing.T) { + err := fmt.Errorf("gemini: %w", NewProviderError(503, []byte(` {"error":"overloaded"} `), []string{"sk-test"})) + if !errors.Is(err, ErrProvider) { + t.Fatal("errors.Is(err, ErrProvider) = false") + } + if errors.Is(err, ErrRateLimited) || errors.Is(err, ErrAuth) { + t.Fatal("provider error matches an unrelated sentinel") + } + var pe *ProviderError + if !errors.As(err, &pe) { + t.Fatal("errors.As(err, *ProviderError) = false") + } + if pe.Status() != 503 || pe.Body() != `{"error":"overloaded"}` { + t.Fatalf("Status, Body = %d, %q", pe.Status(), pe.Body()) + } + if got := err.Error(); got != `gemini: llm: provider error: HTTP 503: {"error":"overloaded"}` { + t.Fatalf("Error() = %q", got) + } + if got := NewProviderError(500, nil, []string{"sk-test"}).Error(); got != "llm: provider error: HTTP 500" { + t.Fatalf("Error() without body = %q", got) + } +} + +func TestProviderErrorRedactsSecrets(t *testing.T) { + const key = "sk-live-0123456789abcdef" + body := []byte(`{"error":"invalid key sk-live-0123456789abcdef","echo":"Bearer sk-live-0123456789abcdef"}`) + pe := NewProviderError(400, body, []string{key, ""}) + for _, s := range []string{pe.Error(), pe.Body()} { + if strings.Contains(s, key) { + t.Fatalf("secret leaked: %q", s) + } + } + want := `{"error":"invalid key [REDACTED]","echo":"Bearer [REDACTED]"}` + if pe.Body() != want { + t.Fatalf("Body() = %q, want %q", pe.Body(), want) + } +} + +func TestProviderErrorRedactsBeforeTruncating(t *testing.T) { + const key = "sk-secret-straddling-the-cut" + // The key starts just before the excerpt limit, so truncating first + // would leave its prefix in the excerpt. + body := strings.Repeat("x", MaxBodyExcerpt-5) + key + pe := NewProviderError(500, []byte(body), []string{key}) + if strings.Contains(pe.Body(), "sk-se") { + t.Fatalf("secret prefix leaked: %q", pe.Body()) + } +} + +func TestProviderErrorTruncatesOnRuneBoundary(t *testing.T) { + body := strings.Repeat("a", MaxBodyExcerpt-1) + "é" + strings.Repeat("b", 100) + pe := NewProviderError(500, []byte(body), []string{"sk-test"}) + want := strings.Repeat("a", MaxBodyExcerpt-1) + "…" + if pe.Body() != want { + t.Fatalf("Body() has %d bytes, want %d", len(pe.Body()), len(want)) + } +} + +func TestProviderErrorOmitsBodyWithoutSecret(t *testing.T) { + const key = "sk-live-0123456789abcdef" + body := []byte(`{"error":"invalid key sk-live-0123456789abcdef"}`) + for _, secrets := range [][]string{nil, {}, {""}} { + pe := NewProviderError(401, body, secrets) + if pe.Body() != "" { + t.Fatalf("secrets %q: Body() = %q, want empty", secrets, pe.Body()) + } + if got := pe.Error(); got != "llm: provider error: HTTP 401" { + t.Fatalf("secrets %q: Error() = %q", secrets, got) + } + if strings.Contains(pe.Error(), key) { + t.Fatalf("secrets %q: secret leaked: %q", secrets, pe.Error()) + } + } +} diff --git a/internal/ai/llm/fake.go b/internal/ai/llm/fake.go new file mode 100644 index 0000000..b418d82 --- /dev/null +++ b/internal/ai/llm/fake.go @@ -0,0 +1,127 @@ +package llm + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "sync" +) + +// Fake is a scripted Provider for tests. Each method returns its queued +// results in order, one per call, and records every Request Generate +// receives. A call with nothing left in its queue fails, so an unexpected +// call is never silently answered. It is safe for concurrent use. +type Fake struct { + mu sync.Mutex + generate []GenerateResult + models []ListModelsResult + verify []error + requests []Request +} + +var _ Provider = (*Fake)(nil) + +// GenerateResult is one scripted Generate outcome. +type GenerateResult struct { + Response Response + Err error +} + +// ListModelsResult is one scripted ListModels outcome. +type ListModelsResult struct { + Models []Model + Err error +} + +// QueueGenerate appends results to the Generate script. +func (f *Fake) QueueGenerate(results ...GenerateResult) { + f.mu.Lock() + defer f.mu.Unlock() + f.generate = append(f.generate, results...) +} + +// QueueListModels appends results to the ListModels script. +func (f *Fake) QueueListModels(results ...ListModelsResult) { + f.mu.Lock() + defer f.mu.Unlock() + f.models = append(f.models, results...) +} + +// QueueVerify appends results to the Verify script; a nil entry is success. +func (f *Fake) QueueVerify(results ...error) { + f.mu.Lock() + defer f.mu.Unlock() + f.verify = append(f.verify, results...) +} + +// Generate records req and returns the next scripted result. +func (f *Fake) Generate(_ context.Context, req Request) (Response, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.requests = append(f.requests, cloneRequest(req)) + if len(f.generate) == 0 { + return Response{}, fmt.Errorf("llm: fake: unscripted Generate call %d", len(f.requests)) + } + r := f.generate[0] + f.generate = f.generate[1:] + return r.Response, r.Err +} + +// ListModels returns the next scripted result. +func (f *Fake) ListModels(context.Context) ([]Model, error) { + f.mu.Lock() + defer f.mu.Unlock() + if len(f.models) == 0 { + return nil, fmt.Errorf("llm: fake: unscripted ListModels call") + } + r := f.models[0] + f.models = f.models[1:] + return slices.Clone(r.Models), r.Err +} + +// Verify returns the next scripted result. +func (f *Fake) Verify(context.Context) error { + f.mu.Lock() + defer f.mu.Unlock() + if len(f.verify) == 0 { + return fmt.Errorf("llm: fake: unscripted Verify call") + } + err := f.verify[0] + f.verify = f.verify[1:] + return err +} + +// Requests returns a copy of every Request Generate received, in order. +func (f *Fake) Requests() []Request { + f.mu.Lock() + defer f.mu.Unlock() + out := make([]Request, len(f.requests)) + for i, r := range f.requests { + out[i] = cloneRequest(r) + } + return out +} + +// Remaining reports how many scripted results of each kind are unused, so a +// test can assert its script was consumed. +func (f *Fake) Remaining() (generate, listModels, verify int) { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.generate), len(f.models), len(f.verify) +} + +// cloneRequest deep-copies r so a recorded Request is unaffected by later +// changes the caller makes to its slices and pointers. +func cloneRequest(r Request) Request { + r.Messages = slices.Clone(r.Messages) + if r.Temperature != nil { + t := *r.Temperature + r.Temperature = &t + } + if r.Schema != nil { + s := JSONSchema{Name: r.Schema.Name, Schema: json.RawMessage(slices.Clone([]byte(r.Schema.Schema)))} + r.Schema = &s + } + return r +} diff --git a/internal/ai/llm/fake_test.go b/internal/ai/llm/fake_test.go new file mode 100644 index 0000000..bba1b3c --- /dev/null +++ b/internal/ai/llm/fake_test.go @@ -0,0 +1,138 @@ +package llm + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sync" + "testing" + "time" +) + +func TestFakeGenerateReturnsScriptInOrder(t *testing.T) { + ctx := context.Background() + var f Fake + f.QueueGenerate( + GenerateResult{Response: Response{Text: "first", Model: "m1"}}, + GenerateResult{Err: &RateLimitedError{RetryAfter: time.Second}}, + ) + f.QueueGenerate(GenerateResult{Response: Response{JSON: json.RawMessage(`{"ok":true}`), FinishReason: "stop"}}) + + req := Request{Messages: []Message{{Role: RoleUser, Content: "one"}}} + resp, err := f.Generate(ctx, req) + if err != nil || resp.Text != "first" || resp.Model != "m1" { + t.Fatalf("call 1 = %+v, %v", resp, err) + } + if _, err := f.Generate(ctx, req); !errors.Is(err, ErrRateLimited) { + t.Fatalf("call 2 err = %v, want ErrRateLimited", err) + } + resp, err = f.Generate(ctx, req) + if err != nil || string(resp.JSON) != `{"ok":true}` || resp.FinishReason != "stop" { + t.Fatalf("call 3 = %+v, %v", resp, err) + } + if _, err := f.Generate(ctx, req); err == nil { + t.Fatal("unscripted call 4 succeeded") + } + if g, l, v := f.Remaining(); g != 0 || l != 0 || v != 0 { + t.Fatalf("Remaining() = %d, %d, %d", g, l, v) + } + if n := len(f.Requests()); n != 4 { + t.Fatalf("recorded %d requests, want 4 (unscripted calls are recorded too)", n) + } +} + +func TestFakeRecordsRequests(t *testing.T) { + var f Fake + f.QueueGenerate(GenerateResult{}, GenerateResult{}) + temp := 0.2 + first := Request{ + System: "be brief", + Messages: []Message{{Role: RoleUser, Content: "plan dinners"}}, + Temperature: &temp, + MaxTokens: 100, + Schema: &JSONSchema{Name: "plan", Schema: json.RawMessage(`{"type":"object"}`)}, + Timeout: 5 * time.Second, + } + if _, err := f.Generate(context.Background(), first); err != nil { + t.Fatal(err) + } + if _, err := f.Generate(context.Background(), Request{System: "second"}); err != nil { + t.Fatal(err) + } + + // Later changes by the caller must not alter what was recorded. + first.Messages[0].Content = "changed" + temp = 1 + first.Schema.Schema[2] = 'X' + + got := f.Requests() + if len(got) != 2 || got[1].System != "second" { + t.Fatalf("Requests() = %+v", got) + } + r := got[0] + if r.System != "be brief" || r.Messages[0].Content != "plan dinners" || *r.Temperature != 0.2 || + r.MaxTokens != 100 || r.Schema.Name != "plan" || string(r.Schema.Schema) != `{"type":"object"}` || + r.Timeout != 5*time.Second { + t.Fatalf("recorded request = %+v", r) + } + + // Changing a returned copy must not alter the record either. + got[0].Messages[0].Content = "mutated" + if f.Requests()[0].Messages[0].Content != "plan dinners" { + t.Fatal("Requests() returned shared state") + } +} + +func TestFakeListModelsAndVerify(t *testing.T) { + ctx := context.Background() + var f Fake + f.QueueListModels( + ListModelsResult{Models: []Model{{ID: "a", DisplayName: "A"}, {ID: "b", DisplayName: "b"}}}, + ListModelsResult{Err: ErrAuth}, + ) + f.QueueVerify(nil, fmt.Errorf("wrapped: %w", ErrAuth)) + + models, err := f.ListModels(ctx) + if err != nil || len(models) != 2 || models[0].ID != "a" { + t.Fatalf("ListModels 1 = %+v, %v", models, err) + } + if _, err := f.ListModels(ctx); !errors.Is(err, ErrAuth) { + t.Fatalf("ListModels 2 err = %v, want ErrAuth", err) + } + if _, err := f.ListModels(ctx); err == nil { + t.Fatal("unscripted ListModels succeeded") + } + if err := f.Verify(ctx); err != nil { + t.Fatalf("Verify 1 = %v", err) + } + if err := f.Verify(ctx); !errors.Is(err, ErrAuth) { + t.Fatalf("Verify 2 = %v, want ErrAuth", err) + } + if err := f.Verify(ctx); err == nil { + t.Fatal("unscripted Verify succeeded") + } +} + +func TestFakeConcurrentUse(t *testing.T) { + const n = 50 + var f Fake + for i := range n { + f.QueueGenerate(GenerateResult{Response: Response{Text: fmt.Sprint(i)}}) + } + var wg sync.WaitGroup + for range n { + wg.Go(func() { + if _, err := f.Generate(context.Background(), Request{}); err != nil { + t.Error(err) + } + }) + } + wg.Wait() + if len(f.Requests()) != n { + t.Fatalf("recorded %d requests, want %d", len(f.Requests()), n) + } + if g, _, _ := f.Remaining(); g != 0 { + t.Fatalf("%d Generate results unused", g) + } +} diff --git a/internal/ai/llm/llm.go b/internal/ai/llm/llm.go new file mode 100644 index 0000000..ce2aaad --- /dev/null +++ b/internal/ai/llm/llm.go @@ -0,0 +1,132 @@ +// Package llm is the single interface MealQuest uses to talk to a language +// model. Every vendor adapter implements Provider over plain net/http (no +// vendor SDKs) and reports failures with this package's typed errors, so +// callers decide what to do from errors.Is and errors.As alone. +// +// Errors from this package and its adapters never contain an API key. +package llm + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" +) + +// Provider is a language-model vendor. +type Provider interface { + // Generate sends one request and returns the model's reply. + Generate(ctx context.Context, req Request) (Response, error) + // ListModels returns the models the configured credentials can use. + ListModels(ctx context.Context) ([]Model, error) + // Verify checks that the configured credentials are accepted. + Verify(ctx context.Context) error +} + +// Role is the author of a Message. +type Role string + +// The roles a Message may have. The system prompt is Request.System, not a +// message. +const ( + RoleUser Role = "user" + RoleAssistant Role = "assistant" +) + +// Message is one turn of a conversation. +type Message struct { + Role Role + Content string +} + +// JSONSchema is a named structured-output contract: when a Request carries +// one, the model must reply with JSON matching Schema. +type JSONSchema struct { + // Name identifies the contract to the vendor, e.g. "side_quests". + Name string + // Schema is a JSON Schema object. + Schema json.RawMessage +} + +// Request is one generation request. +type Request struct { + // System is the system prompt; empty means none. + System string + // Messages is the conversation, oldest first. It must not be empty. + Messages []Message + // Temperature is the sampling temperature; nil means the vendor default. + Temperature *float64 + // MaxTokens caps the reply length; 0 means the vendor default. + MaxTokens int + // Schema, when set, requests structured output; the reply is then in + // Response.JSON. + Schema *JSONSchema + // Timeout bounds the request; 0 means only ctx bounds it. + Timeout time.Duration +} + +// Validate reports whether r is well formed. Adapters call it before +// contacting the vendor. +func (r Request) Validate() error { + if len(r.Messages) == 0 { + return errors.New("llm: request has no messages") + } + for i, m := range r.Messages { + if m.Role != RoleUser && m.Role != RoleAssistant { + return fmt.Errorf("llm: message %d has unknown role %q", i, m.Role) + } + } + if r.Temperature != nil && *r.Temperature < 0 { + return fmt.Errorf("llm: temperature %v is negative", *r.Temperature) + } + if r.MaxTokens < 0 { + return fmt.Errorf("llm: max tokens %d is negative", r.MaxTokens) + } + if r.Timeout < 0 { + return fmt.Errorf("llm: timeout %v is negative", r.Timeout) + } + if r.Schema != nil { + if r.Schema.Name == "" { + return errors.New("llm: schema has no name") + } + var obj map[string]json.RawMessage + if err := json.Unmarshal(r.Schema.Schema, &obj); err != nil || obj == nil { + return fmt.Errorf("llm: schema %q is not a JSON object", r.Schema.Name) + } + } + return nil +} + +// Usage is the token count of one Generate call. +type Usage struct { + InputTokens int + OutputTokens int +} + +// Total returns InputTokens + OutputTokens. +func (u Usage) Total() int { return u.InputTokens + u.OutputTokens } + +// Response is the model's reply to a Request. +type Response struct { + // Text is the reply text. + Text string + // JSON is the structured reply when the Request carried a Schema, and + // nil otherwise. + JSON json.RawMessage + // Usage is the tokens the call consumed. + Usage Usage + // Model is the model that answered, as the vendor reports it. + Model string + // FinishReason is why generation stopped, as the vendor reports it. + FinishReason string +} + +// Model is a model a Provider offers. +type Model struct { + // ID is the vendor's model identifier, used in requests. + ID string + // DisplayName is a human-readable name; it equals ID when the vendor + // gives none. + DisplayName string +} diff --git a/internal/ai/llm/llm_test.go b/internal/ai/llm/llm_test.go new file mode 100644 index 0000000..1b2774a --- /dev/null +++ b/internal/ai/llm/llm_test.go @@ -0,0 +1,44 @@ +package llm + +import ( + "encoding/json" + "testing" +) + +func TestRequestValidate(t *testing.T) { + neg := -0.1 + valid := func() Request { + return Request{ + Messages: []Message{{Role: RoleUser, Content: "hi"}, {Role: RoleAssistant, Content: "hello"}}, + Schema: &JSONSchema{Name: "plan", Schema: json.RawMessage(`{"type":"object"}`)}, + } + } + if err := valid().Validate(); err != nil { + t.Fatalf("valid request: %v", err) + } + cases := map[string]func(*Request){ + "no messages": func(r *Request) { r.Messages = nil }, + "unknown role": func(r *Request) { r.Messages[0].Role = "system" }, + "negative temp": func(r *Request) { r.Temperature = &neg }, + "negative max": func(r *Request) { r.MaxTokens = -1 }, + "negative time": func(r *Request) { r.Timeout = -1 }, + "unnamed schema": func(r *Request) { r.Schema.Name = "" }, + "schema array": func(r *Request) { r.Schema.Schema = json.RawMessage(`[]`) }, + "schema null": func(r *Request) { r.Schema.Schema = json.RawMessage(`null`) }, + "schema invalid": func(r *Request) { r.Schema.Schema = json.RawMessage(`{`) }, + "schema empty": func(r *Request) { r.Schema.Schema = nil }, + } + for name, mutate := range cases { + r := valid() + mutate(&r) + if err := r.Validate(); err == nil { + t.Errorf("%s: Validate() = nil", name) + } + } +} + +func TestUsageTotal(t *testing.T) { + if got := (Usage{InputTokens: 12, OutputTokens: 30}).Total(); got != 42 { + t.Fatalf("Total() = %d, want 42", got) + } +} -- 2.55.0 From acd29c82212dc6f7c21f87f75697e0c7620d8373 Mon Sep 17 00:00:00 2001 From: dev_agent Date: Sun, 4 Oct 2026 07:41:45 -0400 Subject: [PATCH 2/7] Feature 08 task 2: OpenAI-compatible adapter (#35) Co-authored-by: dev_agent Co-committed-by: dev_agent --- internal/ai/llm/errors.go | 89 +- internal/ai/llm/errors_test.go | 109 +- internal/ai/llm/llm.go | 7 +- internal/ai/llm/openai/openai.go | 599 +++++++++ internal/ai/llm/openai/openai_test.go | 1121 +++++++++++++++++ internal/ai/llm/openai/testdata/README.md | 71 ++ .../testdata/ollama_chat_prompt_only.json | 18 + .../llm/openai/testdata/ollama_chat_text.json | 11 + .../testdata/ollama_chat_truncated.json | 15 + .../testdata/ollama_error_not_found.json | 1 + .../ai/llm/openai/testdata/ollama_models.json | 7 + .../testdata/openai_chat_json_schema.json | 28 + .../openai/testdata/openai_chat_refusal.json | 16 + .../testdata/openai_error_invalid_key.json | 8 + .../openai/testdata/openai_error_server.json | 1 + .../ai/llm/openai/testdata/openai_models.json | 7 + .../testdata/openrouter_chat_json_object.json | 22 + .../openrouter_chat_upstream_error.json | 5 + .../openrouter_error_rate_limited.json | 1 + .../openai/testdata/openrouter_models.json | 22 + 20 files changed, 2109 insertions(+), 49 deletions(-) create mode 100644 internal/ai/llm/openai/openai.go create mode 100644 internal/ai/llm/openai/openai_test.go create mode 100644 internal/ai/llm/openai/testdata/README.md create mode 100644 internal/ai/llm/openai/testdata/ollama_chat_prompt_only.json create mode 100644 internal/ai/llm/openai/testdata/ollama_chat_text.json create mode 100644 internal/ai/llm/openai/testdata/ollama_chat_truncated.json create mode 100644 internal/ai/llm/openai/testdata/ollama_error_not_found.json create mode 100644 internal/ai/llm/openai/testdata/ollama_models.json create mode 100644 internal/ai/llm/openai/testdata/openai_chat_json_schema.json create mode 100644 internal/ai/llm/openai/testdata/openai_chat_refusal.json create mode 100644 internal/ai/llm/openai/testdata/openai_error_invalid_key.json create mode 100644 internal/ai/llm/openai/testdata/openai_error_server.json create mode 100644 internal/ai/llm/openai/testdata/openai_models.json create mode 100644 internal/ai/llm/openai/testdata/openrouter_chat_json_object.json create mode 100644 internal/ai/llm/openai/testdata/openrouter_chat_upstream_error.json create mode 100644 internal/ai/llm/openai/testdata/openrouter_error_rate_limited.json create mode 100644 internal/ai/llm/openai/testdata/openrouter_models.json diff --git a/internal/ai/llm/errors.go b/internal/ai/llm/errors.go index be8e922..07fddef 100644 --- a/internal/ai/llm/errors.go +++ b/internal/ai/llm/errors.go @@ -26,13 +26,32 @@ var ( ) // RateLimitedError is a throttled request. It matches ErrRateLimited. +// Adapters build it with NewRateLimitedError so its text never holds their +// credential. type RateLimitedError struct { // RetryAfter is how long the vendor asked callers to wait; 0 means it // did not say. RetryAfter time.Duration + // text is the redacted Error text set by NewRateLimitedError; "" means + // the error was built without a credential. + text string +} + +// NewRateLimitedError returns the error for a throttled request. secret is +// the credential sent with the request, or "" when the request sent none. +// Every occurrence of a non-empty secret is removed from the error text +// when it is built, even where the fixed text happens to spell it, and the +// secret itself is not kept. +func NewRateLimitedError(retryAfter time.Duration, secret string) *RateLimitedError { + e := &RateLimitedError{RetryAfter: retryAfter} + e.text = Redact(e.Error(), secret) + return e } func (e *RateLimitedError) Error() string { + if e.text != "" { + return e.text + } if e.RetryAfter == 0 { return ErrRateLimited.Error() } @@ -46,58 +65,68 @@ func (e *RateLimitedError) Is(target error) bool { return target == ErrRateLimit // ProviderError keeps. const MaxBodyExcerpt = 512 -// redacted replaces each secret in a ProviderError body. -const redacted = "[REDACTED]" +// redactionMarker replaces each secret in redacted text, unless the marker +// would itself hold the secret. +const redactionMarker = "[REDACTED]" + +// Redact returns s with every occurrence of a non-empty secret removed. Each +// occurrence becomes "[REDACTED]" when the result then holds no secret; +// otherwise, because the marker contains the secret or joins with its +// neighbours to form it, occurrences are deleted until none remains. The +// result never contains secret. +func Redact(s, secret string) string { + if secret == "" { + return s + } + if out := strings.ReplaceAll(s, secret, redactionMarker); !strings.Contains(out, secret) { + return out + } + for strings.Contains(s, secret) { + s = strings.ReplaceAll(s, secret, "") + } + return s +} // ProviderError is an unexpected HTTP status from the vendor. It matches // ErrProvider. Build it with NewProviderError, which redacts the body. type ProviderError struct { status int body string + text string } -// NewProviderError returns the error for an unexpected HTTP status. -// secrets must hold the API key (and any other credential sent with the -// request). Every occurrence of each non-empty secret in body is replaced -// before the body is cut to MaxBodyExcerpt bytes, so no part of a secret -// survives. If secrets holds no non-empty value the body cannot be -// redacted, so no excerpt is kept at all. -func NewProviderError(status int, body []byte, secrets []string) *ProviderError { - s := string(body) - redactedAny := false - for _, secret := range secrets { - if secret != "" { - s = strings.ReplaceAll(s, secret, redacted) - redactedAny = true - } - } - if !redactedAny { - return &ProviderError{status: status} - } - s = strings.TrimSpace(s) +// NewProviderError returns the error for an unexpected HTTP status. It +// always keeps a body excerpt of at most MaxBodyExcerpt bytes. secret is the +// credential sent with the request, or "" when the request sent none; every +// occurrence of a non-empty secret in body is removed by Redact before the +// body is cut, so no part of it survives, and again after the cut, so the +// truncation mark cannot complete it. It is removed from the complete error +// text too, even where the fixed text happens to spell it, and the secret +// itself is not kept. +func NewProviderError(status int, body []byte, secret string) *ProviderError { + s := strings.TrimSpace(Redact(string(body), secret)) if len(s) > MaxBodyExcerpt { cut := MaxBodyExcerpt for cut > 0 && !utf8.RuneStart(s[cut]) { cut-- } - s = s[:cut] + "…" + s = Redact(s[:cut]+"…", secret) } - return &ProviderError{status: status, body: s} + text := fmt.Sprintf("%s: HTTP %d", ErrProvider, status) + if s != "" { + text += ": " + s + } + return &ProviderError{status: status, body: s, text: Redact(text, secret)} } // Status returns the HTTP status code. func (e *ProviderError) Status() int { return e.status } // Body returns the redacted, truncated response-body excerpt; it is empty -// when the vendor sent no body or no secret was given to redact it with. +// when the vendor sent no body. func (e *ProviderError) Body() string { return e.body } -func (e *ProviderError) Error() string { - if e.body == "" { - return fmt.Sprintf("%s: HTTP %d", ErrProvider, e.status) - } - return fmt.Sprintf("%s: HTTP %d: %s", ErrProvider, e.status, e.body) -} +func (e *ProviderError) Error() string { return e.text } // Is reports whether target is ErrProvider. func (e *ProviderError) Is(target error) bool { return target == ErrProvider } diff --git a/internal/ai/llm/errors_test.go b/internal/ai/llm/errors_test.go index 321f887..5ac5fa7 100644 --- a/internal/ai/llm/errors_test.go +++ b/internal/ai/llm/errors_test.go @@ -46,7 +46,7 @@ func TestRateLimitedError(t *testing.T) { } func TestProviderError(t *testing.T) { - err := fmt.Errorf("gemini: %w", NewProviderError(503, []byte(` {"error":"overloaded"} `), []string{"sk-test"})) + err := fmt.Errorf("gemini: %w", NewProviderError(503, []byte(` {"error":"overloaded"} `), "sk-test")) if !errors.Is(err, ErrProvider) { t.Fatal("errors.Is(err, ErrProvider) = false") } @@ -63,7 +63,7 @@ func TestProviderError(t *testing.T) { if got := err.Error(); got != `gemini: llm: provider error: HTTP 503: {"error":"overloaded"}` { t.Fatalf("Error() = %q", got) } - if got := NewProviderError(500, nil, []string{"sk-test"}).Error(); got != "llm: provider error: HTTP 500" { + if got := NewProviderError(500, nil, "sk-test").Error(); got != "llm: provider error: HTTP 500" { t.Fatalf("Error() without body = %q", got) } } @@ -71,7 +71,7 @@ func TestProviderError(t *testing.T) { func TestProviderErrorRedactsSecrets(t *testing.T) { const key = "sk-live-0123456789abcdef" body := []byte(`{"error":"invalid key sk-live-0123456789abcdef","echo":"Bearer sk-live-0123456789abcdef"}`) - pe := NewProviderError(400, body, []string{key, ""}) + pe := NewProviderError(400, body, key) for _, s := range []string{pe.Error(), pe.Body()} { if strings.Contains(s, key) { t.Fatalf("secret leaked: %q", s) @@ -88,7 +88,7 @@ func TestProviderErrorRedactsBeforeTruncating(t *testing.T) { // The key starts just before the excerpt limit, so truncating first // would leave its prefix in the excerpt. body := strings.Repeat("x", MaxBodyExcerpt-5) + key - pe := NewProviderError(500, []byte(body), []string{key}) + pe := NewProviderError(500, []byte(body), key) if strings.Contains(pe.Body(), "sk-se") { t.Fatalf("secret prefix leaked: %q", pe.Body()) } @@ -96,26 +96,103 @@ func TestProviderErrorRedactsBeforeTruncating(t *testing.T) { func TestProviderErrorTruncatesOnRuneBoundary(t *testing.T) { body := strings.Repeat("a", MaxBodyExcerpt-1) + "é" + strings.Repeat("b", 100) - pe := NewProviderError(500, []byte(body), []string{"sk-test"}) + pe := NewProviderError(500, []byte(body), "sk-test") want := strings.Repeat("a", MaxBodyExcerpt-1) + "…" if pe.Body() != want { t.Fatalf("Body() has %d bytes, want %d", len(pe.Body()), len(want)) } } -func TestProviderErrorOmitsBodyWithoutSecret(t *testing.T) { - const key = "sk-live-0123456789abcdef" - body := []byte(`{"error":"invalid key sk-live-0123456789abcdef"}`) - for _, secrets := range [][]string{nil, {}, {""}} { - pe := NewProviderError(401, body, secrets) - if pe.Body() != "" { - t.Fatalf("secrets %q: Body() = %q, want empty", secrets, pe.Body()) +func TestProviderErrorKeepsBoundedBodyWithoutSecret(t *testing.T) { + body := []byte(` {"error":"model \"llama9\" not found"} `) + pe := NewProviderError(404, body, "") + if pe.Body() != `{"error":"model \"llama9\" not found"}` { + t.Fatalf("Body() = %q", pe.Body()) + } + if got := pe.Error(); got != `llm: provider error: HTTP 404: {"error":"model \"llama9\" not found"}` { + t.Fatalf("Error() = %q", got) + } + long := NewProviderError(500, []byte(strings.Repeat("x", 4*MaxBodyExcerpt)), "") + if want := strings.Repeat("x", MaxBodyExcerpt) + "…"; long.Body() != want { + t.Fatalf("Body() has %d bytes, want %d", len(long.Body()), len(want)) + } +} + +func TestRedactNeverLeavesSecret(t *testing.T) { + cases := []struct{ s, secret, want string }{ + {"key sk-1 and Bearer sk-1", "sk-1", "key [REDACTED] and Bearer [REDACTED]"}, + {"no secret here", "sk-1", "no secret here"}, + {"x", "", "x"}, + // The marker holds the secret, so occurrences are deleted. + {"rejected REDACTED", "REDACTED", "rejected "}, + {"A then A", "A", " then "}, + // The marker joins with a neighbour to form the secret. + {"]]aa", "]a", ""}, + // The marker holds the secret and deleting once leaves a new one. + {"DDAA", "DA", ""}, + {"aabb", "ab", "a[REDACTED]b"}, + } + for _, c := range cases { + if got := Redact(c.s, c.secret); got != c.want { + t.Errorf("Redact(%q, %q) = %q, want %q", c.s, c.secret, got, c.want) } - if got := pe.Error(); got != "llm: provider error: HTTP 401" { - t.Fatalf("secrets %q: Error() = %q", secrets, got) + } +} + +func TestProviderErrorWithMarkerSecretNeverLeaksSecret(t *testing.T) { + const key = "REDACTED" + pe := NewProviderError(500, []byte(`{"error":"bad key REDACTED","echo":"Bearer REDACTED"}`), key) + for _, s := range []string{pe.Error(), pe.Body()} { + if strings.Contains(s, key) { + t.Fatalf("secret leaked: %q", s) } - if strings.Contains(pe.Error(), key) { - t.Fatalf("secrets %q: secret leaked: %q", secrets, pe.Error()) + } + if want := `{"error":"bad key ","echo":"Bearer "}`; pe.Body() != want { + t.Fatalf("Body() = %q, want %q", pe.Body(), want) + } +} + +func TestProviderErrorTruncationMarkCannotCompleteSecret(t *testing.T) { + const key = "z…" + body := strings.Repeat("a", MaxBodyExcerpt-1) + "z" + strings.Repeat("b", 10) + if pe := NewProviderError(500, []byte(body), key); strings.Contains(pe.Body(), key) { + t.Fatalf("secret leaked: %q", pe.Body()) + } +} + +func TestProviderErrorWithFixedWordSecretNeverLeaksSecret(t *testing.T) { + const secret = "HTTP" + pe := NewProviderError(500, []byte(`{"error":"HTTP down"}`), secret) + if strings.Contains(pe.Error(), secret) || strings.Contains(pe.Body(), secret) { + t.Fatalf("secret leaked: Error %q, Body %q", pe.Error(), pe.Body()) + } + if got := NewProviderError(500, nil, secret).Error(); got != "llm: provider error: [REDACTED] 500" { + t.Fatalf("Error() without body = %q", got) + } +} + +func TestRateLimitedErrorWithFixedWordSecretNeverLeaksSecret(t *testing.T) { + for _, secret := range []string{"rate", "retry after"} { + for _, wait := range []time.Duration{0, 30 * time.Second} { + rl := NewRateLimitedError(wait, secret) + if strings.Contains(rl.Error(), secret) { + t.Fatalf("secret %q leaked: %q", secret, rl.Error()) + } + if rl.RetryAfter != wait || !errors.Is(rl, ErrRateLimited) { + t.Fatalf("NewRateLimitedError(%v) = %+v", wait, rl) + } + } + } +} + +func TestTypedErrorsDoNotKeepSecret(t *testing.T) { + const secret = "sk-live-0123456789abcdef" + for _, err := range []error{ + NewProviderError(500, []byte("down "+secret), secret), + NewRateLimitedError(30*time.Second, secret), + } { + if got := fmt.Sprintf("%#v %+v", err, err); strings.Contains(got, secret) { + t.Fatalf("formatting %T leaks the secret: %q", err, got) } } } diff --git a/internal/ai/llm/llm.go b/internal/ai/llm/llm.go index ce2aaad..23e7b9a 100644 --- a/internal/ai/llm/llm.go +++ b/internal/ai/llm/llm.go @@ -67,14 +67,15 @@ type Request struct { } // Validate reports whether r is well formed. Adapters call it before -// contacting the vendor. +// contacting the vendor. Its errors never quote the request's text, such as +// a role or schema name: any of it may equal the API key. func (r Request) Validate() error { if len(r.Messages) == 0 { return errors.New("llm: request has no messages") } for i, m := range r.Messages { if m.Role != RoleUser && m.Role != RoleAssistant { - return fmt.Errorf("llm: message %d has unknown role %q", i, m.Role) + return fmt.Errorf("llm: message %d has an unknown role", i) } } if r.Temperature != nil && *r.Temperature < 0 { @@ -92,7 +93,7 @@ func (r Request) Validate() error { } var obj map[string]json.RawMessage if err := json.Unmarshal(r.Schema.Schema, &obj); err != nil || obj == nil { - return fmt.Errorf("llm: schema %q is not a JSON object", r.Schema.Name) + return errors.New("llm: schema is not a JSON object") } } return nil diff --git a/internal/ai/llm/openai/openai.go b/internal/ai/llm/openai/openai.go new file mode 100644 index 0000000..e6493de --- /dev/null +++ b/internal/ai/llm/openai/openai.go @@ -0,0 +1,599 @@ +// Package openai is the llm.Provider adapter for OpenAI-compatible chat +// APIs: OpenAI itself, OpenRouter and Ollama's /v1 endpoint. It calls +// POST {base}/chat/completions and GET {base}/models over plain net/http. +// +// The adapter never logs, and no error it returns contains the API key or +// the Authorization header value: errors do not quote caller- or +// vendor-supplied text, and every exported method passes its error through +// redact as a backstop. +package openai + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/textproto" + "net/url" + "reflect" + "strconv" + "strings" + "time" + + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm" +) + +// StructuredOutput is how the adapter asks the vendor for JSON when a +// Request carries a Schema. Which mode a vendor and model support comes +// from its preset. +type StructuredOutput string + +const ( + // JSONSchema sends response_format {type: json_schema, strict: true} + // with the Request's schema. + JSONSchema StructuredOutput = "json_schema" + // JSONObject sends response_format {type: json_object} and puts the + // schema in the system prompt. + JSONObject StructuredOutput = "json_object" + // PromptOnly sends no response_format, puts the schema in the system + // prompt and takes the first JSON object found in the reply. + PromptOnly StructuredOutput = "prompt_only" +) + +// Config configures a Provider. +type Config struct { + // BaseURL is the API root the endpoint paths are appended to, e.g. + // "https://api.openai.com/v1", "https://openrouter.ai/api/v1" or + // "http://ollama:11434/v1". + BaseURL string + // APIKey is sent as "Authorization: Bearer ". Empty sends no + // Authorization header, for endpoints such as Ollama that need none. + APIKey string + // Model is the model Generate requests. ListModels and Verify do not + // need it. + Model string + // Headers are extra request headers, e.g. OpenRouter's HTTP-Referer and + // X-Title. An Accept header replaces the default "application/json". + // Headers the adapter or net/http set themselves (see managedHeaders) + // are rejected. + Headers map[string]string + // StructuredOutput is how JSON is requested when a Request has a Schema. + StructuredOutput StructuredOutput +} + +// Provider is an OpenAI-compatible llm.Provider. +type Provider struct { + base string + apiKey string + model string + headers http.Header + mode StructuredOutput + client *http.Client +} + +var _ llm.Provider = (*Provider)(nil) + +// New validates cfg and returns a Provider. +func New(cfg Config) (*Provider, error) { + p, err := newFromConfig(cfg) + if err != nil { + return nil, redact(err, cfg.APIKey) + } + return p, nil +} + +func newFromConfig(cfg Config) (*Provider, error) { + // Base URL errors never echo the URL: a rejected one may carry a + // credential. + u, err := url.Parse(cfg.BaseURL) + if err != nil { + return nil, errors.New("openai: base URL does not parse") + } + if (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + return nil, errors.New("openai: base URL must be an absolute http or https URL") + } + // A trailing "?" or "#" parses to an empty query or fragment, yet the + // endpoint path appended to it would land in the query or fragment. + if u.User != nil || strings.ContainsAny(cfg.BaseURL, "?#") { + return nil, errors.New("openai: base URL must not carry credentials, a query or a fragment") + } + // No error from New quotes caller-supplied text or a name derived from + // it: any such text may equal or contain the API key, and redaction + // cannot catch a fragment or a case-changed copy of it. + switch cfg.StructuredOutput { + case JSONSchema, JSONObject, PromptOnly: + default: + return nil, errors.New("openai: unknown structured output mode") + } + headers := make(http.Header, len(cfg.Headers)+1) + for name, value := range cfg.Headers { + if !validHeaderName(name) { + return nil, errors.New("openai: an extra header name is not a valid HTTP token") + } + canonical := textproto.CanonicalMIMEHeaderKey(name) + if managedHeaders[canonical] { + return nil, errors.New("openai: an extra header is set by the adapter") + } + if !validHeaderValue(value) { + return nil, errors.New("openai: an extra header value has a control character") + } + headers.Set(canonical, value) + } + if headers.Get("Accept") == "" { + headers.Set("Accept", "application/json") + } + return &Provider{ + base: strings.TrimRight(cfg.BaseURL, "/"), + apiKey: cfg.APIKey, + model: cfg.Model, + headers: headers, + mode: cfg.StructuredOutput, + client: &http.Client{}, + }, nil +} + +// managedHeaders are the request headers the adapter or net/http set; a +// configured value for one would be overwritten or ignored. +var managedHeaders = map[string]bool{ + "Authorization": true, + "Content-Type": true, + "Content-Length": true, + "Transfer-Encoding": true, + "Host": true, + "Connection": true, +} + +// validHeaderName reports whether name is an RFC 9110 field-name: a +// non-empty token of letters, digits and !#$%&'*+-.^_`|~. +func validHeaderName(name string) bool { + if name == "" { + return false + } + for i := 0; i < len(name); i++ { + c := name[i] + switch { + case 'a' <= c && c <= 'z', 'A' <= c && c <= 'Z', '0' <= c && c <= '9': + case strings.IndexByte("!#$%&'*+-.^_`|~", c) >= 0: + default: + return false + } + } + return true +} + +// validHeaderValue reports whether value is an RFC 9110 field-value: no +// control characters other than horizontal tab, so no CR, LF or DEL. +func validHeaderValue(value string) bool { + for i := 0; i < len(value); i++ { + if c := value[i]; (c < 0x20 && c != '\t') || c == 0x7f { + return false + } + } + return true +} + +// maxResponseBody caps a successful response body; a larger one is an error. +const maxResponseBody = 16 << 20 + +// maxErrorBody caps how much of an error response body is read. It is far +// above llm.MaxBodyExcerpt, so redaction still sees every secret that could +// reach the excerpt. +const maxErrorBody = 64 << 10 + +type chatMessage struct { + Role string `json:"role"` + Content string `json:"content"` +} + +type jsonSchemaFormat struct { + Name string `json:"name"` + Schema json.RawMessage `json:"schema"` + Strict bool `json:"strict"` +} + +type responseFormat struct { + Type string `json:"type"` + JSONSchema *jsonSchemaFormat `json:"json_schema,omitempty"` +} + +type chatRequest struct { + Model string `json:"model"` + Messages []chatMessage `json:"messages"` + Temperature *float64 `json:"temperature,omitempty"` + MaxTokens int `json:"max_tokens,omitempty"` + ResponseFormat *responseFormat `json:"response_format,omitempty"` +} + +type chatResponse struct { + Model string `json:"model"` + Choices []struct { + Message struct { + Content *string `json:"content"` + Refusal *string `json:"refusal"` + } `json:"message"` + FinishReason string `json:"finish_reason"` + } `json:"choices"` + Usage struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + } `json:"usage"` + Error json.RawMessage `json:"error"` +} + +// schemaInstruction is appended to the system prompt in the json_object and +// prompt_only modes, where the schema is not sent in response_format. +const schemaInstruction = "Reply with only a JSON object, and no other text, that matches this JSON Schema:\n" + +// Generate sends req to {base}/chat/completions. +func (p *Provider) Generate(ctx context.Context, req llm.Request) (llm.Response, error) { + resp, err := p.generate(ctx, req) + return resp, redact(err, p.apiKey) +} + +func (p *Provider) generate(ctx context.Context, req llm.Request) (llm.Response, error) { + if err := req.Validate(); err != nil { + return llm.Response{}, fmt.Errorf("openai: generate: %w", err) + } + if p.model == "" { + return llm.Response{}, errors.New("openai: generate: no model configured") + } + body := chatRequest{ + Model: p.model, + Temperature: req.Temperature, + MaxTokens: req.MaxTokens, + } + system := req.System + if req.Schema != nil { + switch p.mode { + case JSONSchema: + body.ResponseFormat = &responseFormat{ + Type: "json_schema", + JSONSchema: &jsonSchemaFormat{Name: req.Schema.Name, Schema: req.Schema.Schema, Strict: true}, + } + case JSONObject: + body.ResponseFormat = &responseFormat{Type: "json_object"} + system = appendInstruction(system, req.Schema.Schema) + case PromptOnly: + system = appendInstruction(system, req.Schema.Schema) + } + } + if system != "" { + body.Messages = append(body.Messages, chatMessage{Role: "system", Content: system}) + } + for _, m := range req.Messages { + body.Messages = append(body.Messages, chatMessage{Role: string(m.Role), Content: m.Content}) + } + payload, err := json.Marshal(body) + if err != nil { + return llm.Response{}, fmt.Errorf("openai: generate: encode request: %w", err) + } + + if req.Timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, req.Timeout) + defer cancel() + } + raw, status, err := p.do(ctx, http.MethodPost, "/chat/completions", payload) + if err != nil { + return llm.Response{}, fmt.Errorf("openai: generate: %w", err) + } + + var cr chatResponse + if err := json.Unmarshal(raw, &cr); err != nil { + return llm.Response{}, fmt.Errorf("openai: generate: decode response: %w", err) + } + if len(cr.Error) > 0 && string(cr.Error) != "null" { + // OpenRouter can report an upstream failure in a 2xx body. + return llm.Response{}, fmt.Errorf("openai: generate: %w", llm.NewProviderError(status, raw, p.apiKey)) + } + if len(cr.Choices) == 0 { + return llm.Response{}, errors.New("openai: generate: response has no choices") + } + choice := cr.Choices[0] + resp := llm.Response{ + Usage: llm.Usage{InputTokens: cr.Usage.PromptTokens, OutputTokens: cr.Usage.CompletionTokens}, + Model: cr.Model, + FinishReason: choice.FinishReason, + } + if choice.Message.Content != nil { + resp.Text = *choice.Message.Content + } + if req.Schema == nil { + return resp, nil + } + if choice.Message.Refusal != nil && *choice.Message.Refusal != "" { + return resp, fmt.Errorf("openai: generate: model refused the structured request: %w", llm.ErrBadOutput) + } + if p.mode == PromptOnly { + resp.JSON = firstJSONObject(resp.Text) + } else if text := strings.TrimSpace(resp.Text); isJSONObject(text) { + resp.JSON = json.RawMessage(text) + } + if resp.JSON == nil { + return resp, fmt.Errorf("openai: generate: reply holds no JSON object: %w", llm.ErrBadOutput) + } + return resp, nil +} + +func appendInstruction(system string, schema json.RawMessage) string { + instruction := schemaInstruction + string(schema) + if system == "" { + return instruction + } + return system + "\n\n" + instruction +} + +// isJSONObject reports whether s is exactly one JSON object. +func isJSONObject(s string) bool { + return strings.HasPrefix(s, "{") && json.Valid([]byte(s)) +} + +// firstJSONObject returns the first complete JSON object in s, trying each +// "{" in order, or nil when there is none. Text around it, such as prose or +// a Markdown code fence, is ignored. +func firstJSONObject(s string) json.RawMessage { + for i := strings.IndexByte(s, '{'); i >= 0; { + var obj json.RawMessage + if err := json.NewDecoder(strings.NewReader(s[i:])).Decode(&obj); err == nil { + return obj + } + next := strings.IndexByte(s[i+1:], '{') + if next < 0 { + break + } + i += 1 + next + } + return nil +} + +type modelsResponse struct { + Data []struct { + ID string `json:"id"` + Name string `json:"name"` + } `json:"data"` +} + +// ListModels returns the models {base}/models lists. +func (p *Provider) ListModels(ctx context.Context) ([]llm.Model, error) { + models, err := p.listModels(ctx) + return models, redact(err, p.apiKey) +} + +func (p *Provider) listModels(ctx context.Context) ([]llm.Model, error) { + raw, _, err := p.do(ctx, http.MethodGet, "/models", nil) + if err != nil { + return nil, fmt.Errorf("openai: list models: %w", err) + } + var mr modelsResponse + if err := json.Unmarshal(raw, &mr); err != nil { + return nil, fmt.Errorf("openai: list models: decode response: %w", err) + } + if mr.Data == nil { + return nil, errors.New("openai: list models: response has no data list") + } + models := make([]llm.Model, 0, len(mr.Data)) + for i, m := range mr.Data { + if m.ID == "" { + return nil, fmt.Errorf("openai: list models: model %d has no id", i) + } + name := m.Name + if name == "" { + name = m.ID + } + models = append(models, llm.Model{ID: m.ID, DisplayName: name}) + } + return models, nil +} + +// Verify checks the credentials with an authenticated ListModels call. +func (p *Provider) Verify(ctx context.Context) error { + _, err := p.listModels(ctx) + return redact(err, p.apiKey) +} + +// do sends one request and returns the body and status of a 2xx response. +// Any other status becomes the matching llm typed error. +func (p *Provider) do(ctx context.Context, method, path string, payload []byte) ([]byte, int, error) { + var reqBody io.Reader + if payload != nil { + reqBody = bytes.NewReader(payload) + } + httpReq, err := http.NewRequestWithContext(ctx, method, p.base+path, reqBody) + if err != nil { + // The parse error names the URL, whose path may hold the API key. + return nil, 0, errors.New("build request: request URL is invalid") + } + for name, values := range p.headers { + httpReq.Header[name] = values + } + if payload != nil { + httpReq.Header.Set("Content-Type", "application/json") + } + if p.apiKey != "" { + httpReq.Header.Set("Authorization", "Bearer "+p.apiKey) + } + + httpResp, err := p.client.Do(httpReq) + if err != nil { + return nil, 0, p.transportError(err) + } + defer httpResp.Body.Close() + + if httpResp.StatusCode < 200 || httpResp.StatusCode > 299 { + body, err := io.ReadAll(io.LimitReader(httpResp.Body, maxErrorBody)) + if err != nil { + return nil, httpResp.StatusCode, p.transportError(err) + } + return nil, httpResp.StatusCode, p.statusError(httpResp, body) + } + body, err := io.ReadAll(io.LimitReader(httpResp.Body, maxResponseBody+1)) + if err != nil { + return nil, httpResp.StatusCode, p.transportError(err) + } + if len(body) > maxResponseBody { + return nil, httpResp.StatusCode, fmt.Errorf("response body exceeds %d bytes", maxResponseBody) + } + return body, httpResp.StatusCode, nil +} + +// statusError maps a non-2xx response to an llm typed error. +func (p *Provider) statusError(resp *http.Response, body []byte) error { + switch resp.StatusCode { + case http.StatusUnauthorized, http.StatusForbidden: + return fmt.Errorf("HTTP %d: %w", resp.StatusCode, llm.ErrAuth) + case http.StatusTooManyRequests: + return llm.NewRateLimitedError(retryAfter(resp.Header.Get("Retry-After"), time.Now()), p.apiKey) + case http.StatusRequestTimeout: + return fmt.Errorf("HTTP %d: %w", resp.StatusCode, llm.ErrTimeout) + default: + return llm.NewProviderError(resp.StatusCode, body, p.apiKey) + } +} + +// retryAfter parses a Retry-After header, in delay-seconds or HTTP-date +// form. A missing, malformed or past value means the vendor gave no usable +// wait, reported as 0. +func retryAfter(value string, now time.Time) time.Duration { + value = strings.TrimSpace(value) + if value == "" { + return 0 + } + if secs, err := strconv.Atoi(value); err == nil { + if secs < 0 { + return 0 + } + return time.Duration(secs) * time.Second + } + if at, err := http.ParseTime(value); err == nil && at.After(now) { + return at.Sub(now).Round(time.Second) + } + return 0 +} + +func isTimeout(err error) bool { + var ne net.Error + return errors.Is(err, context.DeadlineExceeded) || (errors.As(err, &ne) && ne.Timeout()) +} + +// transportFailure is a failed HTTP exchange. It carries the cause's text +// with the request URL removed and the API key redacted, and unwraps only to +// the cause's category: llm.ErrTimeout, context.Canceled or nothing. +type transportFailure struct { + detail string + kind error +} + +func (e *transportFailure) Error() string { + if e.kind == llm.ErrTimeout { + return fmt.Sprintf("%s: transport: %s", llm.ErrTimeout, e.detail) + } + return "transport: " + e.detail +} + +func (e *transportFailure) Unwrap() error { return e.kind } + +// transportError reports a failure to complete the HTTP exchange. A +// *url.Error names the request URL, and a base URL path may hold the API +// key, so only the wrapped cause is kept, rendered with the key redacted. +func (p *Provider) transportError(err error) error { + var kind error + switch { + case isTimeout(err): + kind = llm.ErrTimeout + case errors.Is(err, context.Canceled): + kind = context.Canceled + } + var ue *url.Error + if errors.As(err, &ue) { + err = ue.Err + } + return &transportFailure{detail: llm.Redact(err.Error(), p.apiKey), kind: kind} +} + +// redactedError stands in for one layer of an error chain that holds the +// API key. Its text is the layer's text with the key redacted, and Unwrap +// returns the layer's causes passed through redact, so no error reached by +// unwrapping has text holding the key. Is consults the original layer and +// its chain, so sentinels still match. As does too. The llm typed errors +// remove the key from their own complete text, fixed words included, so +// they always match; As refuses only a match from outside llm whose text +// holds the key, so it never hands back an unredacted error. +type redactedError struct { + text string + err error + key string +} + +func (e *redactedError) Error() string { return e.text } + +func (e *redactedError) Unwrap() error { return redact(errors.Unwrap(e.err), e.key) } + +func (e *redactedError) Is(target error) bool { return errors.Is(e.err, target) } + +func (e *redactedError) As(target any) bool { + if !errors.As(e.err, target) { + return false + } + found := reflect.ValueOf(target).Elem() + if matched, ok := found.Interface().(error); ok && holdsKey(matched, e.key) { + found.SetZero() + return false + } + return true +} + +// redactedJoin is redactedError for a layer with several causes, such as +// one built by errors.Join or by fmt.Errorf with more than one %w. +type redactedJoin struct { + redactedError +} + +func (e *redactedJoin) Unwrap() []error { + causes := e.err.(interface{ Unwrap() []error }).Unwrap() + safe := make([]error, len(causes)) + for i, c := range causes { + safe[i] = redact(c, e.key) + } + return safe +} + +// redact is the backstop every exported method applies to its error: when +// the text of err or of any error in its chain holds a non-empty key, it +// returns a stand-in whose text, and the text of every error it unwraps +// to, has the key redacted. Any other err is returned unchanged. +func redact(err error, key string) error { + if err == nil || key == "" || !holdsKey(err, key) { + return err + } + re := redactedError{text: llm.Redact(err.Error(), key), err: err, key: key} + if _, ok := err.(interface{ Unwrap() []error }); ok { + return &redactedJoin{re} + } + return &re +} + +// holdsKey reports whether the text of err or of any error in its chain +// contains key. +func holdsKey(err error, key string) bool { + if err == nil { + return false + } + if strings.Contains(err.Error(), key) { + return true + } + switch u := err.(type) { + case interface{ Unwrap() error }: + return holdsKey(u.Unwrap(), key) + case interface{ Unwrap() []error }: + for _, c := range u.Unwrap() { + if holdsKey(c, key) { + return true + } + } + } + return false +} diff --git a/internal/ai/llm/openai/openai_test.go b/internal/ai/llm/openai/openai_test.go new file mode 100644 index 0000000..8f0a248 --- /dev/null +++ b/internal/ai/llm/openai/openai_test.go @@ -0,0 +1,1121 @@ +package openai + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm" +) + +const testKey = "sk-test-9f8e7d6c5b4a39281706f5e4d3c2b1a0" + +// recorded is one request the test server received. +type recorded struct { + method string + path string + header http.Header + body []byte +} + +// reply is the response the test server replays. +type reply struct { + status int + header map[string]string + fixture string // testdata file; {{KEY}} is replaced with testKey + delay time.Duration +} + +// vendorServer replays one recorded payload and records the request. +type vendorServer struct { + *httptest.Server + mu sync.Mutex + got []recorded +} + +func newVendorServer(t *testing.T, r reply) *vendorServer { + t.Helper() + var payload []byte + if r.fixture != "" { + raw, err := os.ReadFile(filepath.Join("testdata", r.fixture)) + if err != nil { + t.Fatal(err) + } + payload = bytes.ReplaceAll(raw, []byte("{{KEY}}"), []byte(testKey)) + } + vs := &vendorServer{} + vs.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + body, _ := io.ReadAll(req.Body) + vs.mu.Lock() + vs.got = append(vs.got, recorded{req.Method, req.URL.Path, req.Header.Clone(), body}) + vs.mu.Unlock() + if r.delay > 0 { + select { + case <-time.After(r.delay): + case <-req.Context().Done(): + return + } + } + for k, v := range r.header { + w.Header().Set(k, v) + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(r.status) + w.Write(payload) + })) + t.Cleanup(vs.Close) + return vs +} + +func (vs *vendorServer) only(t *testing.T) recorded { + t.Helper() + vs.mu.Lock() + defer vs.mu.Unlock() + if len(vs.got) != 1 { + t.Fatalf("server got %d requests, want 1", len(vs.got)) + } + return vs.got[0] +} + +func newProvider(t *testing.T, base string, mode StructuredOutput, headers map[string]string) *Provider { + t.Helper() + p, err := New(Config{BaseURL: base, APIKey: testKey, Model: "test-model", Headers: headers, StructuredOutput: mode}) + if err != nil { + t.Fatal(err) + } + return p +} + +var dinnersSchema = json.RawMessage(`{"type":"object","properties":{"dinners":{"type":"array"}},"required":["dinners"],"additionalProperties":false}`) + +func schemaRequest() llm.Request { + temp := 0.0 + return llm.Request{ + System: "You plan dinners for Our kitchen.", + Messages: []llm.Message{{Role: llm.RoleUser, Content: "Suggest a Side quest."}}, + Temperature: &temp, + MaxTokens: 400, + Schema: &llm.JSONSchema{Name: "side_quests", Schema: dinnersSchema}, + } +} + +// sentBody decodes the chat request body into a generic map. +func sentBody(t *testing.T, r recorded) map[string]any { + t.Helper() + var m map[string]any + if err := json.Unmarshal(r.body, &m); err != nil { + t.Fatalf("request body %q: %v", r.body, err) + } + return m +} + +func assertJSON(t *testing.T, got json.RawMessage, want string) { + t.Helper() + var g, w any + if err := json.Unmarshal(got, &g); err != nil { + t.Fatalf("Response.JSON %q: %v", got, err) + } + if err := json.Unmarshal([]byte(want), &w); err != nil { + t.Fatal(err) + } + gb, _ := json.Marshal(g) + wb, _ := json.Marshal(w) + if !bytes.Equal(gb, wb) { + t.Fatalf("Response.JSON = %s, want %s", gb, wb) + } +} + +func TestGenerateJSONSchemaOpenAI(t *testing.T) { + vs := newVendorServer(t, reply{status: 200, fixture: "openai_chat_json_schema.json"}) + p := newProvider(t, vs.URL+"/v1/", JSONSchema, nil) + + resp, err := p.Generate(context.Background(), schemaRequest()) + if err != nil { + t.Fatal(err) + } + assertJSON(t, resp.JSON, `{"dinners":[{"name":"Sheet-pan gnocchi","why":"Like your Regular tray bake, but quicker."}]}`) + if resp.Text != string(resp.JSON) { + t.Errorf("Text = %q, want the raw reply", resp.Text) + } + if resp.Usage != (llm.Usage{InputTokens: 1117, OutputTokens: 46}) || resp.Model != "gpt-4o-2024-08-06" || resp.FinishReason != "stop" { + t.Errorf("Usage, Model, FinishReason = %+v, %q, %q", resp.Usage, resp.Model, resp.FinishReason) + } + + got := vs.only(t) + if got.method != http.MethodPost || got.path != "/v1/chat/completions" { + t.Errorf("request = %s %s", got.method, got.path) + } + if h := got.header.Get("Authorization"); h != "Bearer "+testKey { + t.Errorf("Authorization = %q", h) + } + if h := got.header.Get("Content-Type"); h != "application/json" { + t.Errorf("Content-Type = %q", h) + } + body := sentBody(t, got) + if body["model"] != "test-model" || body["temperature"] != 0.0 || body["max_tokens"] != 400.0 { + t.Errorf("model, temperature, max_tokens = %v, %v, %v", body["model"], body["temperature"], body["max_tokens"]) + } + msgs := body["messages"].([]any) + if len(msgs) != 2 { + t.Fatalf("messages = %v", msgs) + } + system := msgs[0].(map[string]any) + if system["role"] != "system" || system["content"] != "You plan dinners for Our kitchen." { + t.Errorf("system message = %v (the schema must not be in the prompt in json_schema mode)", system) + } + if user := msgs[1].(map[string]any); user["role"] != "user" || user["content"] != "Suggest a Side quest." { + t.Errorf("user message = %v", user) + } + rf := body["response_format"].(map[string]any) + js := rf["json_schema"].(map[string]any) + if rf["type"] != "json_schema" || js["name"] != "side_quests" || js["strict"] != true { + t.Errorf("response_format = %v", rf) + } + schema, _ := json.Marshal(js["schema"]) + var want any + json.Unmarshal(dinnersSchema, &want) + wantB, _ := json.Marshal(want) + if !bytes.Equal(schema, wantB) { + t.Errorf("response_format schema = %s", schema) + } +} + +func TestGenerateJSONObjectOpenRouter(t *testing.T) { + vs := newVendorServer(t, reply{status: 200, fixture: "openrouter_chat_json_object.json"}) + p := newProvider(t, vs.URL+"/api/v1", JSONObject, map[string]string{"http-referer": "https://mealquest.example", "X-Title": "MealQuest"}) + + req := schemaRequest() + req.Temperature = nil + req.MaxTokens = 0 + resp, err := p.Generate(context.Background(), req) + if err != nil { + t.Fatal(err) + } + assertJSON(t, resp.JSON, `{"dinners":[{"name":"Chicken tinga tacos","why":"Close to your taco-night Regular."}]}`) + if resp.Usage != (llm.Usage{InputTokens: 842, OutputTokens: 31}) || resp.Model != "openai/gpt-4o-mini" || resp.FinishReason != "stop" { + t.Errorf("Usage, Model, FinishReason = %+v, %q, %q", resp.Usage, resp.Model, resp.FinishReason) + } + + got := vs.only(t) + if got.path != "/api/v1/chat/completions" { + t.Errorf("path = %s", got.path) + } + if got.header.Get("HTTP-Referer") != "https://mealquest.example" || got.header.Get("X-Title") != "MealQuest" { + t.Errorf("extra headers = %v", got.header) + } + body := sentBody(t, got) + if _, ok := body["temperature"]; ok { + t.Error("temperature sent although Request.Temperature is nil") + } + if _, ok := body["max_tokens"]; ok { + t.Error("max_tokens sent although Request.MaxTokens is 0") + } + if rf := body["response_format"].(map[string]any); rf["type"] != "json_object" || rf["json_schema"] != nil { + t.Errorf("response_format = %v", rf) + } + system := body["messages"].([]any)[0].(map[string]any)["content"].(string) + if !strings.HasPrefix(system, "You plan dinners for Our kitchen.\n\n"+schemaInstruction) || !strings.Contains(system, `"required":["dinners"]`) { + t.Errorf("system prompt lacks the schema instruction: %q", system) + } +} + +func TestGeneratePromptOnlyOllama(t *testing.T) { + vs := newVendorServer(t, reply{status: 200, fixture: "ollama_chat_prompt_only.json"}) + p, err := New(Config{BaseURL: vs.URL + "/v1", Model: "llama3.2", StructuredOutput: PromptOnly}) + if err != nil { + t.Fatal(err) + } + req := schemaRequest() + req.System = "" + resp, err := p.Generate(context.Background(), req) + if err != nil { + t.Fatal(err) + } + // The reply has a non-JSON "{idea}" first and a second object after the + // fenced one; only the first complete object is taken. + assertJSON(t, resp.JSON, `{"dinners":[{"name":"Crock pot chili","why":"A Quick dinner like your {Regular} stew."}]}`) + if !strings.HasPrefix(resp.Text, "Sure! Here is") { + t.Errorf("Text = %q, want the whole reply", resp.Text) + } + if resp.Usage != (llm.Usage{InputTokens: 512, OutputTokens: 58}) || resp.Model != "llama3.2" || resp.FinishReason != "stop" { + t.Errorf("Usage, Model, FinishReason = %+v, %q, %q", resp.Usage, resp.Model, resp.FinishReason) + } + + got := vs.only(t) + if _, ok := got.header["Authorization"]; ok { + t.Error("Authorization sent without an API key") + } + body := sentBody(t, got) + if _, ok := body["response_format"]; ok { + t.Error("response_format sent in prompt_only mode") + } + system := body["messages"].([]any)[0].(map[string]any) + if system["role"] != "system" || !strings.HasPrefix(system["content"].(string), schemaInstruction) { + t.Errorf("system message = %v", system) + } +} + +func TestGenerateWithoutSchemaReturnsText(t *testing.T) { + vs := newVendorServer(t, reply{status: 200, fixture: "ollama_chat_text.json"}) + p := newProvider(t, vs.URL+"/v1", PromptOnly, nil) + resp, err := p.Generate(context.Background(), llm.Request{Messages: []llm.Message{{Role: llm.RoleUser, Content: "Dinner?"}}}) + if err != nil { + t.Fatal(err) + } + if resp.Text != "Try a lemon orzo bake tonight." || resp.JSON != nil { + t.Errorf("Text, JSON = %q, %q", resp.Text, resp.JSON) + } + body := sentBody(t, vs.only(t)) + if msgs := body["messages"].([]any); len(msgs) != 1 { + t.Errorf("messages = %v, want only the user message", msgs) + } + if _, ok := body["response_format"]; ok { + t.Error("response_format sent without a schema") + } +} + +func TestGenerateBadOutput(t *testing.T) { + cases := []struct { + name string + fixture string + mode StructuredOutput + }{ + {"refusal", "openai_chat_refusal.json", JSONSchema}, + {"truncated prompt_only", "ollama_chat_truncated.json", PromptOnly}, + {"truncated json_object", "ollama_chat_truncated.json", JSONObject}, + {"prose in json_object", "ollama_chat_text.json", JSONObject}, + {"prose in prompt_only", "ollama_chat_text.json", PromptOnly}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + vs := newVendorServer(t, reply{status: 200, fixture: c.fixture}) + p := newProvider(t, vs.URL, c.mode, nil) + _, err := p.Generate(context.Background(), schemaRequest()) + if !errors.Is(err, llm.ErrBadOutput) { + t.Fatalf("err = %v, want ErrBadOutput", err) + } + }) + } +} + +func TestListModels(t *testing.T) { + cases := []struct { + fixture string + base string + want []llm.Model + }{ + {"openai_models.json", "/v1", []llm.Model{ + {ID: "gpt-4o-2024-08-06", DisplayName: "gpt-4o-2024-08-06"}, + {ID: "gpt-4o-mini", DisplayName: "gpt-4o-mini"}, + }}, + {"openrouter_models.json", "/api/v1", []llm.Model{ + {ID: "openai/gpt-4o-mini", DisplayName: "OpenAI: GPT-4o-mini"}, + {ID: "anthropic/claude-sonnet-5", DisplayName: "Anthropic: Claude Sonnet 5"}, + }}, + {"ollama_models.json", "/v1", []llm.Model{ + {ID: "llama3.2:latest", DisplayName: "llama3.2:latest"}, + {ID: "qwen2.5:7b", DisplayName: "qwen2.5:7b"}, + }}, + } + for _, c := range cases { + t.Run(c.fixture, func(t *testing.T) { + vs := newVendorServer(t, reply{status: 200, fixture: c.fixture}) + p := newProvider(t, vs.URL+c.base, JSONSchema, map[string]string{"X-Title": "MealQuest"}) + models, err := p.ListModels(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(models) != len(c.want) { + t.Fatalf("models = %+v", models) + } + for i := range c.want { + if models[i] != c.want[i] { + t.Errorf("model %d = %+v, want %+v", i, models[i], c.want[i]) + } + } + got := vs.only(t) + if got.method != http.MethodGet || got.path != c.base+"/models" { + t.Errorf("request = %s %s", got.method, got.path) + } + if got.header.Get("Authorization") != "Bearer "+testKey || got.header.Get("X-Title") != "MealQuest" { + t.Errorf("headers = %v", got.header) + } + }) + } +} + +func TestVerify(t *testing.T) { + ok := newVendorServer(t, reply{status: 200, fixture: "openai_models.json"}) + if err := newProvider(t, ok.URL, JSONSchema, nil).Verify(context.Background()); err != nil { + t.Fatalf("Verify() = %v", err) + } + if got := ok.only(t); got.path != "/models" { + t.Errorf("Verify called %s", got.path) + } + bad := newVendorServer(t, reply{status: 401, fixture: "openai_error_invalid_key.json"}) + if err := newProvider(t, bad.URL, JSONSchema, nil).Verify(context.Background()); !errors.Is(err, llm.ErrAuth) { + t.Fatalf("Verify() = %v, want ErrAuth", err) + } +} + +// assertNoSecret fails if err's text holds the key or Authorization value. +func assertNoSecret(t *testing.T, err error) { + t.Helper() + for _, secret := range []string{testKey, "Bearer " + testKey, testKey[:12]} { + if strings.Contains(strings.ToLower(err.Error()), strings.ToLower(secret)) { + t.Fatalf("error leaks %q: %v", secret, err) + } + } +} + +func TestErrorMapping(t *testing.T) { + cases := []struct { + name string + reply reply + want error + check func(t *testing.T, err error) + }{ + {name: "401 OpenAI", reply: reply{status: 401, fixture: "openai_error_invalid_key.json"}, want: llm.ErrAuth}, + {name: "403", reply: reply{status: 403, fixture: "openai_error_invalid_key.json"}, want: llm.ErrAuth}, + {name: "429 seconds OpenRouter", reply: reply{status: 429, header: map[string]string{"Retry-After": "17"}, fixture: "openrouter_error_rate_limited.json"}, want: llm.ErrRateLimited, + check: func(t *testing.T, err error) { + var rl *llm.RateLimitedError + if !errors.As(err, &rl) || rl.RetryAfter != 17*time.Second { + t.Fatalf("RetryAfter from %v", err) + } + }}, + {name: "429 HTTP-date", reply: reply{status: 429, header: map[string]string{"Retry-After": time.Now().Add(90 * time.Second).UTC().Format(http.TimeFormat)}, fixture: "openrouter_error_rate_limited.json"}, want: llm.ErrRateLimited, + check: func(t *testing.T, err error) { + var rl *llm.RateLimitedError + if !errors.As(err, &rl) || rl.RetryAfter < 85*time.Second || rl.RetryAfter > 91*time.Second { + t.Fatalf("RetryAfter from %v", err) + } + }}, + {name: "429 no Retry-After", reply: reply{status: 429, fixture: "openrouter_error_rate_limited.json"}, want: llm.ErrRateLimited, + check: func(t *testing.T, err error) { + var rl *llm.RateLimitedError + if !errors.As(err, &rl) || rl.RetryAfter != 0 { + t.Fatalf("RetryAfter from %v", err) + } + }}, + {name: "408", reply: reply{status: 408}, want: llm.ErrTimeout}, + {name: "404 Ollama", reply: reply{status: 404, fixture: "ollama_error_not_found.json"}, want: llm.ErrProvider, + check: func(t *testing.T, err error) { + var pe *llm.ProviderError + if !errors.As(err, &pe) || pe.Status() != 404 || !strings.Contains(pe.Body(), `model \"llama9\" not found`) { + t.Fatalf("ProviderError from %v", err) + } + }}, + {name: "500 echoing the key", reply: reply{status: 500, fixture: "openai_error_server.json"}, want: llm.ErrProvider, + check: func(t *testing.T, err error) { + var pe *llm.ProviderError + if !errors.As(err, &pe) || pe.Status() != 500 || !strings.Contains(pe.Body(), "Authorization: Bearer [REDACTED]") { + t.Fatalf("ProviderError from %v", err) + } + }}, + {name: "400 echoing the key", reply: reply{status: 400, fixture: "openai_error_invalid_key.json"}, want: llm.ErrProvider}, + {name: "200 OpenRouter upstream error", reply: reply{status: 200, fixture: "openrouter_chat_upstream_error.json"}, want: llm.ErrProvider, + check: func(t *testing.T, err error) { + var pe *llm.ProviderError + if !errors.As(err, &pe) || !strings.Contains(pe.Body(), "Upstream error from provider") { + t.Fatalf("ProviderError from %v", err) + } + }}, + } + all := []error{llm.ErrAuth, llm.ErrRateLimited, llm.ErrTimeout, llm.ErrBadOutput, llm.ErrProvider} + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + vs := newVendorServer(t, c.reply) + p := newProvider(t, vs.URL+"/v1", JSONSchema, nil) + _, genErr := p.Generate(context.Background(), schemaRequest()) + errs := []error{genErr} + if c.reply.status != 200 { + _, listErr := p.ListModels(context.Background()) + errs = append(errs, listErr) + } + for _, err := range errs { + for _, sentinel := range all { + if errors.Is(err, sentinel) != (sentinel == c.want) { + t.Fatalf("errors.Is(%v, %v) = %v", err, sentinel, !(sentinel == c.want)) + } + } + assertNoSecret(t, err) + if c.check != nil { + c.check(t, err) + } + } + }) + } +} + +func TestTimeouts(t *testing.T) { + vs := newVendorServer(t, reply{status: 200, fixture: "openai_chat_json_schema.json", delay: 5 * time.Second}) + p := newProvider(t, vs.URL, JSONSchema, nil) + + req := schemaRequest() + req.Timeout = 50 * time.Millisecond + start := time.Now() + _, err := p.Generate(context.Background(), req) + if !errors.Is(err, llm.ErrTimeout) { + t.Fatalf("Request.Timeout: err = %v, want ErrTimeout", err) + } + if elapsed := time.Since(start); elapsed > 2*time.Second { + t.Fatalf("Request.Timeout not honoured: took %v", elapsed) + } + assertNoSecret(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + if _, err := p.ListModels(ctx); !errors.Is(err, llm.ErrTimeout) { + t.Fatalf("context deadline: err = %v, want ErrTimeout", err) + } + + ctx, cancel = context.WithCancel(context.Background()) + cancel() + if _, err := p.Generate(ctx, schemaRequest()); !errors.Is(err, context.Canceled) || errors.Is(err, llm.ErrTimeout) { + t.Fatalf("cancelled context: err = %v, want context.Canceled only", err) + } +} + +func TestTransportErrorHasNoSecret(t *testing.T) { + vs := newVendorServer(t, reply{status: 200}) + base := vs.URL + vs.Close() + _, err := newProvider(t, base, JSONSchema, nil).Generate(context.Background(), schemaRequest()) + if err == nil { + t.Fatal("Generate against a closed server succeeded") + } + assertNoSecret(t, err) +} + +// TestTransportErrorWithKeyInBaseURLPath fails the exchange against a base +// URL whose path holds the key, which New accepts; net/http names the full +// request URL in its transport errors. +func TestTransportErrorWithKeyInBaseURLPath(t *testing.T) { + refused := newVendorServer(t, reply{status: 200}) + refusedBase := refused.URL + refused.Close() + + dropped := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + conn, _, err := w.(http.Hijacker).Hijack() + if err != nil { + t.Error(err) + return + } + conn.Close() + })) + t.Cleanup(dropped.Close) + + for name, base := range map[string]string{ + "connection refused": refusedBase + "/v1/" + testKey, + "connection dropped": dropped.URL + "/v1/" + testKey, + } { + p := newProvider(t, base, JSONSchema, nil) + _, genErr := p.Generate(context.Background(), schemaRequest()) + _, listErr := p.ListModels(context.Background()) + verifyErr := p.Verify(context.Background()) + for _, err := range []error{genErr, listErr, verifyErr} { + if err == nil { + t.Fatalf("%s: nil error", name) + } + assertNoSecret(t, err) + var ue *url.Error + if errors.As(err, &ue) { + t.Fatalf("%s: error chain exposes the request URL: %v", name, ue) + } + if errors.Is(err, llm.ErrTimeout) || errors.Is(err, llm.ErrProvider) { + t.Fatalf("%s: err = %v, want an untyped transport failure", name, err) + } + } + } +} + +// TestKeyedErrorsNeverLeakKey drives every error path of the keyed OpenAI +// and OpenRouter shapes with vendor bodies that echo the key. +func TestKeyedErrorsNeverLeakKey(t *testing.T) { + shapes := map[string]struct { + path string + headers map[string]string + }{ + "OpenAI": {path: "/v1"}, + "OpenRouter": {path: "/api/v1", headers: map[string]string{"HTTP-Referer": "https://mealquest.example", "X-Title": "MealQuest"}}, + } + fixtures := []string{"openai_error_invalid_key.json", "openai_error_server.json", "openrouter_error_rate_limited.json"} + statuses := []int{400, 401, 403, 404, 408, 429, 500, 502, 503} + for name, shape := range shapes { + for _, fixture := range fixtures { + for _, status := range statuses { + vs := newVendorServer(t, reply{status: status, fixture: fixture}) + p := newProvider(t, vs.URL+shape.path, JSONSchema, shape.headers) + _, genErr := p.Generate(context.Background(), schemaRequest()) + _, listErr := p.ListModels(context.Background()) + verifyErr := p.Verify(context.Background()) + for _, err := range []error{genErr, listErr, verifyErr} { + if err == nil { + t.Fatalf("%s %s HTTP %d: nil error", name, fixture, status) + } + assertNoSecret(t, err) + var pe *llm.ProviderError + if errors.As(err, &pe) { + assertNoSecret(t, errors.New(pe.Body())) + } + } + } + } + } +} + +func TestKeylessOllamaErrorKeepsExcerpt(t *testing.T) { + vs := newVendorServer(t, reply{status: 404, fixture: "ollama_error_not_found.json"}) + p, err := New(Config{BaseURL: vs.URL + "/v1", Model: "llama9", StructuredOutput: PromptOnly}) + if err != nil { + t.Fatal(err) + } + _, err = p.Generate(context.Background(), schemaRequest()) + var pe *llm.ProviderError + if !errors.As(err, &pe) || pe.Status() != 404 { + t.Fatalf("err = %v, want a 404 ProviderError", err) + } + if !strings.Contains(pe.Body(), `model \"llama9\" not found`) || len(pe.Body()) > llm.MaxBodyExcerpt { + t.Fatalf("Body() = %q, want the bounded not-found explanation", pe.Body()) + } + if !strings.Contains(err.Error(), pe.Body()) { + t.Fatalf("Error() = %q, want it to carry the excerpt", err) + } + assertNoSecret(t, err) + if got := vs.only(t).header.Get("Authorization"); got != "" { + t.Fatalf("keyless request sent Authorization %q", got) + } +} + +func TestGenerateRejectsInvalidRequests(t *testing.T) { + vs := newVendorServer(t, reply{status: 200, fixture: "ollama_chat_text.json"}) + p := newProvider(t, vs.URL, JSONSchema, nil) + if _, err := p.Generate(context.Background(), llm.Request{}); err == nil { + t.Error("Generate accepted a request with no messages") + } + noModel, err := New(Config{BaseURL: vs.URL, APIKey: testKey, StructuredOutput: JSONSchema}) + if err != nil { + t.Fatal(err) + } + if _, err := noModel.Generate(context.Background(), schemaRequest()); err == nil { + t.Error("Generate succeeded with no model configured") + } + vs.mu.Lock() + defer vs.mu.Unlock() + if len(vs.got) != 0 { + t.Errorf("invalid requests reached the vendor %d times", len(vs.got)) + } +} + +func TestNewRejectsBadHeaderWithoutEchoingValue(t *testing.T) { + _, err := New(Config{BaseURL: "https://example.com/v1", StructuredOutput: JSONSchema, Headers: map[string]string{"X-Title": testKey + "\u0019"}}) + if err == nil { + t.Fatal("New() = nil error") + } + assertNoSecret(t, err) +} + +// TestNewConfigErrorsNeverLeakKey covers every New error that names +// caller-supplied text, each with the API key embedded in that text. +func TestNewConfigErrorsNeverLeakKey(t *testing.T) { + cases := map[string]Config{ + "key in invalid header name": {Headers: map[string]string{"Bearer " + testKey: "a"}}, + "key as invalid header name": {Headers: map[string]string{testKey + ":": "a"}}, + "key in name, bad value": {Headers: map[string]string{"x-" + testKey: "a\nb"}}, + "key in structured output": {StructuredOutput: StructuredOutput("Bearer " + testKey)}, + "key with quote in mode": {StructuredOutput: StructuredOutput(testKey + "\"\x00")}, + } + for name, cfg := range cases { + cfg.BaseURL = "https://example.com/v1" + cfg.APIKey = testKey + if cfg.StructuredOutput == "" { + cfg.StructuredOutput = JSONSchema + } + _, err := New(cfg) + if err == nil { + t.Errorf("%s: New() = nil error", name) + continue + } + assertNoSecret(t, err) + } +} + +// TestNewManagedHeaderErrorNeverLeaksKey uses an API key that is itself a +// managed header name, so naming the rejected header would print the key. +func TestNewManagedHeaderErrorNeverLeaksKey(t *testing.T) { + for _, key := range []string{"Authorization", "Content-Type", "Host"} { + _, err := New(Config{ + BaseURL: "https://example.com/v1", + APIKey: key, + StructuredOutput: JSONSchema, + Headers: map[string]string{key: "x"}, + }) + if err == nil { + t.Errorf("key %q: New() = nil error", key) + continue + } + for _, secret := range []string{key, "Bearer " + key} { + if strings.Contains(strings.ToLower(err.Error()), strings.ToLower(secret)) { + t.Errorf("key %q: error leaks %q: %v", key, secret, err) + } + } + } +} + +// TestNewModeErrorNeverLeaksKeyFragment uses a structured output mode that is +// a fragment of the API key, which redacting the whole key would not catch. +func TestNewModeErrorNeverLeaksKeyFragment(t *testing.T) { + _, err := New(Config{BaseURL: "https://example.com/v1", APIKey: testKey, StructuredOutput: StructuredOutput(testKey[:20])}) + if err == nil { + t.Fatal("New() = nil error") + } + assertNoSecret(t, err) +} + +func TestAcceptHeader(t *testing.T) { + for name, tc := range map[string]struct { + headers map[string]string + want string + }{ + "default": {nil, "application/json"}, + "configured": {map[string]string{"accept": "application/json, text/event-stream"}, "application/json, text/event-stream"}, + } { + vs := newVendorServer(t, reply{status: 200, fixture: "openai_models.json"}) + p := newProvider(t, vs.URL+"/v1", JSONSchema, tc.headers) + if _, err := p.ListModels(context.Background()); err != nil { + t.Fatalf("%s: %v", name, err) + } + if got := vs.only(t).header.Values("Accept"); len(got) != 1 || got[0] != tc.want { + t.Errorf("%s: Accept = %q, want %q", name, got, tc.want) + } + } +} + +func TestNewAcceptsValidHeaders(t *testing.T) { + headers := map[string]string{"X-Trace_ID.v2!#$%&'*+^`|~": "tab\tand obs-text \xe2\x9c\x93", "X-Empty": ""} + if _, err := New(Config{BaseURL: "https://example.com/v1", StructuredOutput: JSONSchema, Headers: headers}); err != nil { + t.Fatal(err) + } +} + +func TestNewRejectsBadConfig(t *testing.T) { + cases := map[string]Config{ + "empty base": {StructuredOutput: JSONSchema}, + "relative base": {BaseURL: "api.openai.com/v1", StructuredOutput: JSONSchema}, + "ftp base": {BaseURL: "ftp://example.com", StructuredOutput: JSONSchema}, + "credentials in URL": {BaseURL: "https://user:" + testKey + "@example.com/v1", StructuredOutput: JSONSchema}, + "key in query": {BaseURL: "https://example.com/v1?key=" + testKey, StructuredOutput: JSONSchema}, + "key in fragment": {BaseURL: "https://example.com/v1#" + testKey, StructuredOutput: JSONSchema}, + "key, relative base": {BaseURL: "example.com/v1?key=" + testKey, StructuredOutput: JSONSchema}, + "key, ftp base": {BaseURL: "ftp://example.com/v1?key=" + testKey, StructuredOutput: JSONSchema}, + "key, unparseable": {BaseURL: "https://example.com/%zz?key=" + testKey, StructuredOutput: JSONSchema}, + "empty query": {BaseURL: "https://example.com/v1?", StructuredOutput: JSONSchema}, + "empty fragment": {BaseURL: "https://example.com/v1#", StructuredOutput: JSONSchema}, + "empty both": {BaseURL: "https://example.com/v1?#", StructuredOutput: JSONSchema}, + "unknown mode": {BaseURL: "https://example.com/v1", StructuredOutput: "json"}, + "no mode": {BaseURL: "https://example.com/v1"}, + "authorization hdr": {BaseURL: "https://example.com/v1", StructuredOutput: JSONSchema, Headers: map[string]string{"authorization": "Bearer x"}}, + "content-type hdr": {BaseURL: "https://example.com/v1", StructuredOutput: JSONSchema, Headers: map[string]string{"Content-Type": "text/plain"}}, + "header newline": {BaseURL: "https://example.com/v1", StructuredOutput: JSONSchema, Headers: map[string]string{"X-Title": "a\r\nb"}}, + "header bad name": {BaseURL: "https://example.com/v1", StructuredOutput: JSONSchema, Headers: map[string]string{"X Title": "a"}}, + "header empty name": {BaseURL: "https://example.com/v1", StructuredOutput: JSONSchema, Headers: map[string]string{"": "a"}}, + "header bracket": {BaseURL: "https://example.com/v1", StructuredOutput: JSONSchema, Headers: map[string]string{"X[Trace]": "a"}}, + "header colon": {BaseURL: "https://example.com/v1", StructuredOutput: JSONSchema, Headers: map[string]string{"X-Title:": "a"}}, + "header non-ASCII": {BaseURL: "https://example.com/v1", StructuredOutput: JSONSchema, Headers: map[string]string{"X-Tïtle": "a"}}, + "header U+0019": {BaseURL: "https://example.com/v1", StructuredOutput: JSONSchema, Headers: map[string]string{"X-Title": "a\u0019b"}}, + "header NUL": {BaseURL: "https://example.com/v1", StructuredOutput: JSONSchema, Headers: map[string]string{"X-Title": "a\x00b"}}, + "header DEL": {BaseURL: "https://example.com/v1", StructuredOutput: JSONSchema, Headers: map[string]string{"X-Title": "a\x7fb"}}, + "host hdr": {BaseURL: "https://example.com/v1", StructuredOutput: JSONSchema, Headers: map[string]string{"host": "other.example"}}, + "content-length hdr": {BaseURL: "https://example.com/v1", StructuredOutput: JSONSchema, Headers: map[string]string{"Content-Length": "1"}}, + } + for name, cfg := range cases { + cfg.APIKey = testKey + _, err := New(cfg) + if err == nil { + t.Errorf("%s: New() = nil error", name) + continue + } + assertNoSecret(t, err) + } +} + +func TestFirstJSONObject(t *testing.T) { + cases := map[string]string{ + `{"a":1}`: `{"a":1}`, + "text {not json} then {\"b\":2}": `{"b":2}`, + "```json\n{\"c\":{\"d\":3}}\n```": `{"c":{"d":3}}`, + `{"a":1} {"b":2}`: `{"a":1}`, + "no object": ``, + `{"open":`: ``, + `[{"in":"array"}]`: `{"in":"array"}`, + } + for in, want := range cases { + if got := string(firstJSONObject(in)); got != want { + t.Errorf("firstJSONObject(%q) = %q, want %q", in, got, want) + } + } +} + +func TestRetryAfter(t *testing.T) { + now := time.Date(2026, 10, 3, 12, 0, 0, 0, time.UTC) + cases := map[string]time.Duration{ + "": 0, + "30": 30 * time.Second, + " 5 ": 5 * time.Second, + "-3": 0, + "soon": 0, + "Sat, 03 Oct 2026 12:02:00 GMT": 2 * time.Minute, + "Sat, 03 Oct 2026 11:59:00 GMT": 0, + } + for in, want := range cases { + if got := retryAfter(in, now); got != want { + t.Errorf("retryAfter(%q) = %v, want %v", in, got, want) + } + } +} + +// headerNameKey is an API key that is also a word callers and vendors put in +// text: an error that quotes such text prints the key. +const headerNameKey = "Authorization" + +// assertNoKey fails if err's text holds key or "Bearer "+key in any case. +func assertNoKey(t *testing.T, err error, key string) { + t.Helper() + for _, secret := range []string{key, "Bearer " + key} { + if strings.Contains(err.Error(), secret) || strings.Contains(strings.ToLower(err.Error()), strings.ToLower(secret)) { + t.Fatalf("error leaks %q: %v", secret, err) + } + } +} + +// keyServer replies to every request with status and body. +func keyServer(t *testing.T, status int, body string) *Provider { + t.Helper() + return keyServerWithKey(t, headerNameKey, status, body) +} + +// keyServerWithKey is keyServer for a provider configured with key. +func keyServerWithKey(t *testing.T, key string, status int, body string) *Provider { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + io.WriteString(w, body) + })) + t.Cleanup(srv.Close) + p, err := New(Config{BaseURL: srv.URL + "/v1", APIKey: key, Model: "test-model", StructuredOutput: PromptOnly}) + if err != nil { + t.Fatal(err) + } + return p +} + +func TestGenerateInvalidRoleNeverLeaksKey(t *testing.T) { + p := keyServer(t, 200, `{}`) + _, err := p.Generate(context.Background(), llm.Request{Messages: []llm.Message{{Role: llm.Role(headerNameKey), Content: "hi"}}}) + if err == nil { + t.Fatal("Generate accepted an unknown role") + } + assertNoKey(t, err, headerNameKey) +} + +func TestGenerateInvalidSchemaNeverLeaksKey(t *testing.T) { + p := keyServer(t, 200, `{}`) + req := schemaRequest() + req.Schema = &llm.JSONSchema{Name: headerNameKey, Schema: json.RawMessage(`[]`)} + _, err := p.Generate(context.Background(), req) + if err == nil { + t.Fatal("Generate accepted a schema that is not a JSON object") + } + assertNoKey(t, err, headerNameKey) +} + +func TestGenerateBadOutputNeverLeaksKey(t *testing.T) { + p := keyServer(t, 200, `{"model":"m","choices":[{"message":{"content":"no JSON here"},"finish_reason":"`+headerNameKey+`"}]}`) + resp, err := p.Generate(context.Background(), schemaRequest()) + if !errors.Is(err, llm.ErrBadOutput) { + t.Fatalf("err = %v, want ErrBadOutput", err) + } + assertNoKey(t, err, headerNameKey) + if resp.FinishReason != headerNameKey { + t.Fatalf("FinishReason = %q, want it kept on the response", resp.FinishReason) + } +} + +func TestProviderErrorBodyNeverLeaksKey(t *testing.T) { + p := keyServer(t, 500, `{"error":{"message":"rejected `+headerNameKey+`: Bearer `+headerNameKey+`"}}`) + _, genErr := p.Generate(context.Background(), schemaRequest()) + _, listErr := p.ListModels(context.Background()) + verifyErr := p.Verify(context.Background()) + for _, err := range []error{genErr, listErr, verifyErr} { + var pe *llm.ProviderError + if !errors.Is(err, llm.ErrProvider) || !errors.As(err, &pe) || pe.Status() != 500 { + t.Fatalf("err = %v, want a 500 ProviderError", err) + } + assertNoKey(t, err, headerNameKey) + } +} + +// TestRedactKeepsChainAndHidesKey checks the backstop every exported method +// applies: the key and its Bearer value leave the text, the chain stays. +func TestRedactKeepsChainAndHidesKey(t *testing.T) { + inner := fmt.Errorf("decode: body %q: %w", "Bearer "+testKey, llm.ErrBadOutput) + err := redact(inner, testKey) + assertChainNoKey(t, err, testKey) + if !errors.Is(err, llm.ErrBadOutput) { + t.Fatalf("redact broke the chain of %v", err) + } + if redact(inner, "") != inner { + t.Fatal("redact changed an error for the empty key") + } + clean := errors.New("openai: generate: no model configured") + if redact(clean, testKey) != clean { + t.Fatal("redact wrapped an error that holds no key") + } + // A key inside the redaction marker would survive replacement. + if got := redact(errors.New("x E y"), "E").Error(); strings.Contains(got, "E") { + t.Fatalf("redact kept the key: %q", got) + } +} + +// assertChainNoKey fails if err, or any error reached from it by unwrapping, +// has text holding key or "Bearer "+key. +func assertChainNoKey(t *testing.T, err error, key string) { + t.Helper() + if err == nil { + return + } + assertNoKey(t, err, key) + switch u := err.(type) { + case interface{ Unwrap() error }: + assertChainNoKey(t, u.Unwrap(), key) + case interface{ Unwrap() []error }: + for _, c := range u.Unwrap() { + assertChainNoKey(t, c, key) + } + } +} + +// TestGenerateInvalidJSONWithKeyNeverLeaksKeyByUnwrapping sends an invalid +// JSON body holding the key and walks the returned chain with errors.Unwrap. +func TestGenerateInvalidJSONWithKeyNeverLeaksKeyByUnwrapping(t *testing.T) { + p := keyServer(t, 200, `{"choices": Bearer `+headerNameKey+` `+headerNameKey+`}`) + _, err := p.Generate(context.Background(), schemaRequest()) + if err == nil { + t.Fatal("Generate accepted an invalid JSON body") + } + var syntax *json.SyntaxError + if !errors.As(err, &syntax) { + t.Fatalf("err = %v, want a *json.SyntaxError in the chain", err) + } + for layer := err; layer != nil; layer = errors.Unwrap(layer) { + assertNoKey(t, layer, headerNameKey) + } +} + +// TestRedactMakesEveryUnwrappedLayerSafe covers chains whose inner layers +// hold the key, including a layer whose own text does not show it and a +// layer with several causes, and checks Is and As still match. +func TestRedactMakesEveryUnwrappedLayerSafe(t *testing.T) { + pe := llm.NewProviderError(500, []byte(`{"error":"down"}`), testKey) + hidden := &opaqueError{err: fmt.Errorf("body Bearer %s: %w", testKey, llm.ErrAuth)} + inner := fmt.Errorf("decode %s: %w", testKey, llm.ErrBadOutput) + joined := errors.Join(inner, pe, context.Canceled, fmt.Errorf("limit: %w", llm.ErrRateLimited)) + cases := []error{ + fmt.Errorf("openai: generate: %w", inner), + fmt.Errorf("openai: generate: %w", hidden), + fmt.Errorf("openai: generate: %w", joined), + fmt.Errorf("openai: %w and %w", inner, &transportFailure{detail: "slow", kind: llm.ErrTimeout}), + } + for _, c := range cases { + err := redact(c, testKey) + if err == c { + t.Fatalf("redact left %v unchanged", c) + } + assertChainNoKey(t, err, testKey) + for layer := err; layer != nil; layer = errors.Unwrap(layer) { + assertNoKey(t, layer, testKey) + } + } + err := redact(fmt.Errorf("openai: %w", joined), testKey) + var got *llm.ProviderError + if !errors.As(err, &got) || got != pe { + t.Fatalf("errors.As lost the ProviderError in %v", err) + } + for _, target := range []error{llm.ErrBadOutput, llm.ErrProvider, llm.ErrRateLimited, context.Canceled} { + if !errors.Is(err, target) { + t.Fatalf("errors.Is(%v, %v) = false", err, target) + } + } + err = redact(fmt.Errorf("openai: %w", hidden), testKey) + if !errors.Is(err, llm.ErrAuth) { + t.Fatalf("errors.Is lost ErrAuth in %v", err) + } + err = redact(fmt.Errorf("openai %s: %w", testKey, &transportFailure{detail: "slow", kind: llm.ErrTimeout}), testKey) + if !errors.Is(err, llm.ErrTimeout) { + t.Fatalf("errors.Is lost ErrTimeout in %v", err) + } +} + +// opaqueError hides its cause's text but unwraps to it. +type opaqueError struct{ err error } + +func (e *opaqueError) Error() string { return "opaque" } + +func (e *opaqueError) Unwrap() error { return e.err } + +// assertChainNoExactKey fails if the text of err, of any error reached from +// it by unwrapping, or of a *llm.ProviderError found by errors.As holds key. +// It matches case-sensitively, so it suits keys too short for assertNoKey. +func assertChainNoExactKey(t *testing.T, err error, key string) { + t.Helper() + var walk func(error) + walk = func(e error) { + if e == nil { + return + } + if strings.Contains(e.Error(), key) { + t.Fatalf("error leaks %q: %q", key, e.Error()) + } + switch u := e.(type) { + case interface{ Unwrap() error }: + walk(u.Unwrap()) + case interface{ Unwrap() []error }: + for _, c := range u.Unwrap() { + walk(c) + } + } + } + walk(err) + var pe *llm.ProviderError + if errors.As(err, &pe) && (strings.Contains(pe.Body(), key) || strings.Contains(pe.Error(), key)) { + t.Fatalf("ProviderError leaks %q: body %q", key, pe.Body()) + } + var syntax *json.SyntaxError + if errors.As(err, &syntax) && strings.Contains(syntax.Error(), key) { + t.Fatalf("errors.As reached a *json.SyntaxError leaking %q: %q", key, syntax.Error()) + } +} + +// TestProviderErrorBodyWithMarkerKeyNeverLeaksKey uses the redaction marker +// text as the API key, so replacing the key with the marker would keep it. +func TestProviderErrorBodyWithMarkerKeyNeverLeaksKey(t *testing.T) { + const key = "REDACTED" + p := keyServerWithKey(t, key, 500, `{"error":{"message":"rejected `+key+`: Bearer `+key+`"}}`) + _, err := p.Generate(context.Background(), schemaRequest()) + var pe *llm.ProviderError + if !errors.Is(err, llm.ErrProvider) || !errors.As(err, &pe) || pe.Status() != 500 { + t.Fatalf("err = %v, want a 500 ProviderError", err) + } + if strings.Contains(pe.Body(), key) || strings.Contains(pe.Error(), key) { + t.Fatalf("ProviderError leaks the key: %q", pe.Error()) + } + assertChainNoExactKey(t, err, key) +} + +// TestGenerateMalformedJSONWithShortKeyNeverLeaksKey uses a one-letter key +// found in the redaction marker and in ordinary error prose. +func TestGenerateMalformedJSONWithShortKeyNeverLeaksKey(t *testing.T) { + const key = "A" + p := keyServerWithKey(t, key, 200, `{"choices": A Bearer A}`) + _, err := p.Generate(context.Background(), schemaRequest()) + if err == nil { + t.Fatal("Generate accepted a malformed JSON body") + } + assertChainNoExactKey(t, err, key) + for layer := err; layer != nil; layer = errors.Unwrap(layer) { + if strings.Contains(layer.Error(), key) { + t.Fatalf("layer leaks %q: %q", key, layer.Error()) + } + } +} + +// assertLayersNoExactKey fails if the text of err or of any error reached +// from it by errors.Unwrap holds key, matched case-sensitively. +func assertLayersNoExactKey(t *testing.T, err error, key string) { + t.Helper() + for layer := err; layer != nil; layer = errors.Unwrap(layer) { + if strings.Contains(layer.Error(), key) { + t.Fatalf("layer leaks %q: %q", key, layer.Error()) + } + } +} + +// TestProviderErrorWithFixedWordKeyKeepsTypedMatch uses the fixed word +// "HTTP" of ProviderError's text as the API key: the text must drop it and +// errors.As must still reach the ProviderError. +func TestProviderErrorWithFixedWordKeyKeepsTypedMatch(t *testing.T) { + const key = "HTTP" + p := keyServerWithKey(t, key, 500, `{"error":"down"}`) + _, err := p.Generate(context.Background(), schemaRequest()) + var pe *llm.ProviderError + if !errors.Is(err, llm.ErrProvider) || !errors.As(err, &pe) || pe.Status() != 500 { + t.Fatalf("err = %v, want a 500 ProviderError", err) + } + if strings.Contains(pe.Error(), key) || strings.Contains(pe.Body(), key) { + t.Fatalf("ProviderError leaks the key: Error %q, Body %q", pe.Error(), pe.Body()) + } + assertLayersNoExactKey(t, err, key) + assertChainNoExactKey(t, err, key) +} + +// TestRateLimitedErrorWithFixedWordKeyKeepsTypedMatch uses the fixed word +// "rate" of RateLimitedError's text as the API key. +func TestRateLimitedErrorWithFixedWordKeyKeepsTypedMatch(t *testing.T) { + const key = "rate" + srv := newVendorServer(t, reply{status: 429, header: map[string]string{"Retry-After": "17"}}) + p, err := New(Config{BaseURL: srv.URL + "/v1", APIKey: key, Model: "test-model", StructuredOutput: PromptOnly}) + if err != nil { + t.Fatal(err) + } + _, err = p.Generate(context.Background(), schemaRequest()) + var rl *llm.RateLimitedError + if !errors.Is(err, llm.ErrRateLimited) || !errors.As(err, &rl) || rl.RetryAfter != 17*time.Second { + t.Fatalf("err = %v, want a RateLimitedError with RetryAfter 17s", err) + } + if strings.Contains(rl.Error(), key) { + t.Fatalf("RateLimitedError leaks the key: %q", rl.Error()) + } + assertLayersNoExactKey(t, err, key) + assertChainNoExactKey(t, err, key) +} + +// TestGenerateWithAdapterNameKeyNeverLeaksKey uses the fixed prefix +// "openai" of the adapter's errors as the API key. +func TestGenerateWithAdapterNameKeyNeverLeaksKey(t *testing.T) { + const key = "openai" + t.Run("401", func(t *testing.T) { + p := keyServerWithKey(t, key, 401, `{"error":{"message":"bad key"}}`) + _, err := p.Generate(context.Background(), schemaRequest()) + if !errors.Is(err, llm.ErrAuth) { + t.Fatalf("err = %v, want ErrAuth", err) + } + assertLayersNoExactKey(t, err, key) + assertChainNoExactKey(t, err, key) + }) + t.Run("no model", func(t *testing.T) { + p, err := New(Config{BaseURL: "http://127.0.0.1:1/v1", APIKey: key, StructuredOutput: PromptOnly}) + if err != nil { + t.Fatal(err) + } + _, err = p.Generate(context.Background(), schemaRequest()) + if err == nil { + t.Fatal("Generate succeeded without a model") + } + assertLayersNoExactKey(t, err, key) + assertChainNoExactKey(t, err, key) + }) +} diff --git a/internal/ai/llm/openai/testdata/README.md b/internal/ai/llm/openai/testdata/README.md new file mode 100644 index 0000000..a058c82 --- /dev/null +++ b/internal/ai/llm/openai/testdata/README.md @@ -0,0 +1,71 @@ +# OpenAI-compatible adapter fixtures + +These are the vendor response bodies the contract tests in `openai_test.go` +replay. They are **not live captures**. Product AI is tested with fakes, and +live vendor calls are opt-in and never use credentials, so no request was +sent to a vendor to make them. Each fixture is adapted from the response +shape in that vendor's public API documentation or open source, then +sanitized: + +- IDs, timestamps, fingerprints and token counts are illustrative values in + the documented format, not values from a real account. +- Message content is replaced with MealQuest dinner text. +- No real credential appears anywhere. `{{KEY}}` is a placeholder the tests + replace with a fake key, to prove the adapter redacts a key echoed back in + a vendor body. + +The tests check the adapter against these bodies with independent +assertions on both the request sent (path, headers, `response_format`, +messages) and the response decoded. The request side does not depend on the +fixtures. + +Below, *not in the public example* marks fields or values that were added +for a test case and do not come from the vendor's documented example. + +## OpenAI + +Sources: API reference, "The chat completion object" and "List models" +(, +); the +Structured Outputs guide, for `refusal` +(); and the error +codes guide (). + +| Fixture | Shape | Not in the public example | +| --- | --- | --- | +| `openai_chat_json_schema.json` | Chat completion object with `usage` details, `service_tier`, `system_fingerprint`. | The JSON reply content. | +| `openai_chat_refusal.json` | Structured Outputs refusal: `content: null`, `refusal` set. | — | +| `openai_models.json` | `{object: list, data: [model]}`. | — | +| `openai_error_invalid_key.json` | Error object, `code: invalid_api_key`, message echoing the key. | The full key in the message; OpenAI masks most of it. Kept whole to test redaction. | +| `openai_error_server.json` | Error object, `type: server_error`. | The `Authorization: Bearer {{KEY}}` text; added to test redaction. | + +## OpenRouter + +Sources: API reference, chat completion response and errors +(, +), and "List available +models" (). + +| Fixture | Shape | Not in the public example | +| --- | --- | --- | +| `openrouter_chat_json_object.json` | Chat completion with `provider`, `native_finish_reason` and `message.reasoning`. | The JSON reply content. | +| `openrouter_chat_upstream_error.json` | An `error` object (`code`, `message`, `metadata.provider_name`, `metadata.raw`) in an HTTP 200 body, which OpenRouter documents for upstream failures. | `user_id`. | +| `openrouter_error_rate_limited.json` | Error object, `code: 429`, rate-limit `metadata`. | The `Key {{KEY}}` text; added to test redaction. | +| `openrouter_models.json` | `{data: [model]}` with `name`, `context_length`, `architecture`, `pricing`, `supported_parameters`. | The second model is a minimal entry, to test optional fields. | + +## Ollama + +Ollama's OpenAI compatibility page +() documents +requests but shows no response bodies. The shapes come from its `openai` +middleware source (): +`chatcmpl-` IDs, `system_fingerprint: fp_ollama`, `owned_by: library` on +models, and errors as `{error: {message, type: api_error, param, code}}`. + +| Fixture | Shape | Not in the public example | +| --- | --- | --- | +| `ollama_chat_text.json` | Plain chat completion. | — | +| `ollama_chat_prompt_only.json` | Chat completion. | The reply text: prose and stray braces around a fenced JSON object, to test prompt-only extraction. | +| `ollama_chat_truncated.json` | Chat completion with `finish_reason: length`. | The cut-off JSON reply. | +| `ollama_models.json` | `{object: list, data: [model]}`. | — | +| `ollama_error_not_found.json` | Middleware error object for a missing model. | — | diff --git a/internal/ai/llm/openai/testdata/ollama_chat_prompt_only.json b/internal/ai/llm/openai/testdata/ollama_chat_prompt_only.json new file mode 100644 index 0000000..f235cd4 --- /dev/null +++ b/internal/ai/llm/openai/testdata/ollama_chat_prompt_only.json @@ -0,0 +1,18 @@ +{ + "id": "chatcmpl-417", + "object": "chat.completion", + "created": 1741570322, + "model": "llama3.2", + "system_fingerprint": "fp_ollama", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Sure! Here is a dinner {idea} for you:\n```json\n{\"dinners\": [{\"name\": \"Crock pot chili\", \"why\": \"A Quick dinner like your {Regular} stew.\"}]}\n```\nEnjoy {\"not\": \"this one\"}!" + }, + "finish_reason": "stop" + } + ], + "usage": {"prompt_tokens": 512, "completion_tokens": 58, "total_tokens": 570} +} diff --git a/internal/ai/llm/openai/testdata/ollama_chat_text.json b/internal/ai/llm/openai/testdata/ollama_chat_text.json new file mode 100644 index 0000000..cebe0de --- /dev/null +++ b/internal/ai/llm/openai/testdata/ollama_chat_text.json @@ -0,0 +1,11 @@ +{ + "id": "chatcmpl-419", + "object": "chat.completion", + "created": 1741570340, + "model": "llama3.2", + "system_fingerprint": "fp_ollama", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "Try a lemon orzo bake tonight."}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 40, "completion_tokens": 9, "total_tokens": 49} +} diff --git a/internal/ai/llm/openai/testdata/ollama_chat_truncated.json b/internal/ai/llm/openai/testdata/ollama_chat_truncated.json new file mode 100644 index 0000000..ff036f8 --- /dev/null +++ b/internal/ai/llm/openai/testdata/ollama_chat_truncated.json @@ -0,0 +1,15 @@ +{ + "id": "chatcmpl-418", + "object": "chat.completion", + "created": 1741570330, + "model": "llama3.2", + "system_fingerprint": "fp_ollama", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "{\"dinners\": [{\"name\": \"Crock pot ch"}, + "finish_reason": "length" + } + ], + "usage": {"prompt_tokens": 512, "completion_tokens": 16, "total_tokens": 528} +} diff --git a/internal/ai/llm/openai/testdata/ollama_error_not_found.json b/internal/ai/llm/openai/testdata/ollama_error_not_found.json new file mode 100644 index 0000000..137e3ba --- /dev/null +++ b/internal/ai/llm/openai/testdata/ollama_error_not_found.json @@ -0,0 +1 @@ +{"error": {"message": "model \"llama9\" not found, try pulling it first", "type": "api_error", "param": null, "code": null}} diff --git a/internal/ai/llm/openai/testdata/ollama_models.json b/internal/ai/llm/openai/testdata/ollama_models.json new file mode 100644 index 0000000..2421a5e --- /dev/null +++ b/internal/ai/llm/openai/testdata/ollama_models.json @@ -0,0 +1,7 @@ +{ + "object": "list", + "data": [ + {"id": "llama3.2:latest", "object": "model", "created": 1741000000, "owned_by": "library"}, + {"id": "qwen2.5:7b", "object": "model", "created": 1740000000, "owned_by": "library"} + ] +} diff --git a/internal/ai/llm/openai/testdata/openai_chat_json_schema.json b/internal/ai/llm/openai/testdata/openai_chat_json_schema.json new file mode 100644 index 0000000..3d596d1 --- /dev/null +++ b/internal/ai/llm/openai/testdata/openai_chat_json_schema.json @@ -0,0 +1,28 @@ +{ + "id": "chatcmpl-B9MHDbslfkBeAs8l4bebGdFOJ6PeG", + "object": "chat.completion", + "created": 1741570283, + "model": "gpt-4o-2024-08-06", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "{\"dinners\":[{\"name\":\"Sheet-pan gnocchi\",\"why\":\"Like your Regular tray bake, but quicker.\"}]}", + "refusal": null, + "annotations": [] + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 1117, + "completion_tokens": 46, + "total_tokens": 1163, + "prompt_tokens_details": {"cached_tokens": 0, "audio_tokens": 0}, + "completion_tokens_details": {"reasoning_tokens": 0, "audio_tokens": 0, "accepted_prediction_tokens": 0, "rejected_prediction_tokens": 0} + }, + "service_tier": "default", + "system_fingerprint": "fp_fc9f1d7035" +} diff --git a/internal/ai/llm/openai/testdata/openai_chat_refusal.json b/internal/ai/llm/openai/testdata/openai_chat_refusal.json new file mode 100644 index 0000000..d00903b --- /dev/null +++ b/internal/ai/llm/openai/testdata/openai_chat_refusal.json @@ -0,0 +1,16 @@ +{ + "id": "chatcmpl-B9MHDbslfkBeAs8l4bebGdFOJ6PeH", + "object": "chat.completion", + "created": 1741570290, + "model": "gpt-4o-2024-08-06", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": null, "refusal": "I'm sorry, I can't help with that request.", "annotations": []}, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": {"prompt_tokens": 210, "completion_tokens": 11, "total_tokens": 221}, + "system_fingerprint": "fp_fc9f1d7035" +} diff --git a/internal/ai/llm/openai/testdata/openai_error_invalid_key.json b/internal/ai/llm/openai/testdata/openai_error_invalid_key.json new file mode 100644 index 0000000..f82a0ff --- /dev/null +++ b/internal/ai/llm/openai/testdata/openai_error_invalid_key.json @@ -0,0 +1,8 @@ +{ + "error": { + "message": "Incorrect API key provided: {{KEY}}. You can find your API key at https://platform.openai.com/account/api-keys.", + "type": "invalid_request_error", + "param": null, + "code": "invalid_api_key" + } +} diff --git a/internal/ai/llm/openai/testdata/openai_error_server.json b/internal/ai/llm/openai/testdata/openai_error_server.json new file mode 100644 index 0000000..006fc65 --- /dev/null +++ b/internal/ai/llm/openai/testdata/openai_error_server.json @@ -0,0 +1 @@ +{"error": {"message": "The server had an error while processing your request. Authorization: Bearer {{KEY}}", "type": "server_error", "param": null, "code": null}} diff --git a/internal/ai/llm/openai/testdata/openai_models.json b/internal/ai/llm/openai/testdata/openai_models.json new file mode 100644 index 0000000..c5dda95 --- /dev/null +++ b/internal/ai/llm/openai/testdata/openai_models.json @@ -0,0 +1,7 @@ +{ + "object": "list", + "data": [ + {"id": "gpt-4o-2024-08-06", "object": "model", "created": 1722814719, "owned_by": "system"}, + {"id": "gpt-4o-mini", "object": "model", "created": 1721172741, "owned_by": "system"} + ] +} diff --git a/internal/ai/llm/openai/testdata/openrouter_chat_json_object.json b/internal/ai/llm/openai/testdata/openrouter_chat_json_object.json new file mode 100644 index 0000000..a1fddac --- /dev/null +++ b/internal/ai/llm/openai/testdata/openrouter_chat_json_object.json @@ -0,0 +1,22 @@ +{ + "id": "gen-1741570300-xQmT9vWb4kR2nP7sL3aE", + "provider": "OpenAI", + "model": "openai/gpt-4o-mini", + "object": "chat.completion", + "created": 1741570300, + "choices": [ + { + "logprobs": null, + "finish_reason": "stop", + "native_finish_reason": "stop", + "index": 0, + "message": { + "role": "assistant", + "content": "{\n \"dinners\": [{\"name\": \"Chicken tinga tacos\", \"why\": \"Close to your taco-night Regular.\"}]\n}", + "refusal": null, + "reasoning": null + } + } + ], + "usage": {"prompt_tokens": 842, "completion_tokens": 31, "total_tokens": 873} +} diff --git a/internal/ai/llm/openai/testdata/openrouter_chat_upstream_error.json b/internal/ai/llm/openai/testdata/openrouter_chat_upstream_error.json new file mode 100644 index 0000000..6167272 --- /dev/null +++ b/internal/ai/llm/openai/testdata/openrouter_chat_upstream_error.json @@ -0,0 +1,5 @@ +{ + "id": "gen-1741570311-aB3dE5fG7hJ9kL1mN2pQ", + "error": {"code": 502, "message": "Upstream error from provider", "metadata": {"provider_name": "Lambda", "raw": "upstream returned 502"}}, + "user_id": "user_2abcDEF" +} diff --git a/internal/ai/llm/openai/testdata/openrouter_error_rate_limited.json b/internal/ai/llm/openai/testdata/openrouter_error_rate_limited.json new file mode 100644 index 0000000..bc12a1e --- /dev/null +++ b/internal/ai/llm/openai/testdata/openrouter_error_rate_limited.json @@ -0,0 +1 @@ +{"error": {"code": 429, "message": "Rate limit exceeded: free-models-per-min. Key {{KEY}}", "metadata": {"headers": {"X-RateLimit-Limit": "20", "X-RateLimit-Remaining": "0"}}}} diff --git a/internal/ai/llm/openai/testdata/openrouter_models.json b/internal/ai/llm/openai/testdata/openrouter_models.json new file mode 100644 index 0000000..741effd --- /dev/null +++ b/internal/ai/llm/openai/testdata/openrouter_models.json @@ -0,0 +1,22 @@ +{ + "data": [ + { + "id": "openai/gpt-4o-mini", + "canonical_slug": "openai/gpt-4o-mini", + "name": "OpenAI: GPT-4o-mini", + "created": 1721260800, + "description": "GPT-4o mini is OpenAI's newest model after GPT-4 Omni.", + "context_length": 128000, + "architecture": {"modality": "text+image->text", "input_modalities": ["text", "image"], "output_modalities": ["text"], "tokenizer": "GPT"}, + "pricing": {"prompt": "0.00000015", "completion": "0.0000006"}, + "supported_parameters": ["max_tokens", "temperature", "response_format", "structured_outputs"] + }, + { + "id": "anthropic/claude-sonnet-5", + "name": "Anthropic: Claude Sonnet 5", + "created": 1767225600, + "context_length": 200000, + "supported_parameters": ["max_tokens", "temperature"] + } + ] +} -- 2.55.0 From 15886f2c55e27f391744bfad1c5eba742cebb3da Mon Sep 17 00:00:00 2001 From: dev_agent Date: Sun, 4 Oct 2026 08:09:20 -0400 Subject: [PATCH 3/7] Anthropic Messages adapter (#40) Co-authored-by: dev_agent Co-committed-by: dev_agent --- internal/ai/llm/anthropic/anthropic.go | 560 +++++++++ internal/ai/llm/anthropic/anthropic_test.go | 1025 +++++++++++++++++ internal/ai/llm/anthropic/testdata/README.md | 41 + .../ai/llm/anthropic/testdata/error_api.json | 1 + .../testdata/error_authentication.json | 1 + .../testdata/error_invalid_request.json | 1 + .../anthropic/testdata/error_not_found.json | 1 + .../anthropic/testdata/error_overloaded.json | 1 + .../anthropic/testdata/error_permission.json | 1 + .../anthropic/testdata/error_rate_limit.json | 1 + .../testdata/messages_max_tokens.json | 12 + .../llm/anthropic/testdata/messages_text.json | 18 + .../anthropic/testdata/messages_tool_use.json | 22 + .../llm/anthropic/testdata/models_page1.json | 13 + .../llm/anthropic/testdata/models_page2.json | 18 + internal/ai/llm/errors.go | 7 + internal/ai/llm/openai/openai.go | 67 +- internal/ai/llm/openai/openai_test.go | 142 ++- 18 files changed, 1909 insertions(+), 23 deletions(-) create mode 100644 internal/ai/llm/anthropic/anthropic.go create mode 100644 internal/ai/llm/anthropic/anthropic_test.go create mode 100644 internal/ai/llm/anthropic/testdata/README.md create mode 100644 internal/ai/llm/anthropic/testdata/error_api.json create mode 100644 internal/ai/llm/anthropic/testdata/error_authentication.json create mode 100644 internal/ai/llm/anthropic/testdata/error_invalid_request.json create mode 100644 internal/ai/llm/anthropic/testdata/error_not_found.json create mode 100644 internal/ai/llm/anthropic/testdata/error_overloaded.json create mode 100644 internal/ai/llm/anthropic/testdata/error_permission.json create mode 100644 internal/ai/llm/anthropic/testdata/error_rate_limit.json create mode 100644 internal/ai/llm/anthropic/testdata/messages_max_tokens.json create mode 100644 internal/ai/llm/anthropic/testdata/messages_text.json create mode 100644 internal/ai/llm/anthropic/testdata/messages_tool_use.json create mode 100644 internal/ai/llm/anthropic/testdata/models_page1.json create mode 100644 internal/ai/llm/anthropic/testdata/models_page2.json diff --git a/internal/ai/llm/anthropic/anthropic.go b/internal/ai/llm/anthropic/anthropic.go new file mode 100644 index 0000000..37b1239 --- /dev/null +++ b/internal/ai/llm/anthropic/anthropic.go @@ -0,0 +1,560 @@ +// Package anthropic is the llm.Provider adapter for Anthropic's Messages +// API. It calls POST {base}/v1/messages and GET {base}/v1/models over plain +// net/http. +// +// The adapter never logs, and no error it returns contains the API key: +// errors do not quote caller- or vendor-supplied text, and every exported +// method passes its error through redact as a backstop. +package anthropic + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "net" + "net/http" + "net/url" + "reflect" + "regexp" + "strconv" + "strings" + "time" + + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm" +) + +// APIVersion is the Messages API version sent as the anthropic-version +// header. +const APIVersion = "2023-06-01" + +// Config configures a Provider. +type Config struct { + // BaseURL is the API root the /v1 endpoint paths are appended to, e.g. + // "https://api.anthropic.com". + BaseURL string + // APIKey is sent as the x-api-key header. It must not be empty. + APIKey string + // Model is the model Generate requests. ListModels and Verify do not + // need it. + Model string + // MaxTokens is the max_tokens Generate sends when a Request sets none: + // the Messages API requires one. It must be positive. + MaxTokens int +} + +// Provider is an Anthropic Messages API llm.Provider. +type Provider struct { + base string + apiKey string + model string + maxTokens int + client *http.Client +} + +var _ llm.Provider = (*Provider)(nil) + +// New validates cfg and returns a Provider. +func New(cfg Config) (*Provider, error) { + p, err := newFromConfig(cfg) + if err != nil { + return nil, redact(err, cfg.APIKey) + } + return p, nil +} + +func newFromConfig(cfg Config) (*Provider, error) { + // No error from New quotes caller-supplied text: any of it may equal or + // contain the API key. Base URL errors never echo the URL. + u, err := url.Parse(cfg.BaseURL) + if err != nil { + return nil, errors.New("anthropic: base URL does not parse") + } + if (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + return nil, errors.New("anthropic: base URL must be an absolute http or https URL") + } + // A trailing "?" or "#" parses to an empty query or fragment, yet the + // endpoint path appended to it would land in the query or fragment. + if u.User != nil || strings.ContainsAny(cfg.BaseURL, "?#") { + return nil, errors.New("anthropic: base URL must not carry credentials, a query or a fragment") + } + if cfg.APIKey == "" { + return nil, errors.New("anthropic: no API key configured") + } + if !validHeaderValue(cfg.APIKey) { + return nil, errors.New("anthropic: API key has a control character") + } + if cfg.MaxTokens <= 0 { + return nil, errors.New("anthropic: max tokens must be positive") + } + return &Provider{ + base: strings.TrimRight(cfg.BaseURL, "/"), + apiKey: cfg.APIKey, + model: cfg.Model, + maxTokens: cfg.MaxTokens, + client: noRedirectClient(), + }, nil +} + +// validHeaderValue reports whether value is an RFC 9110 field-value: no +// control characters other than horizontal tab, so no CR, LF or DEL. +func validHeaderValue(value string) bool { + for i := 0; i < len(value); i++ { + if c := value[i]; (c < 0x20 && c != '\t') || c == 0x7f { + return false + } + } + return true +} + +// toolName is the pattern the Messages API requires of a tool name. +var toolName = regexp.MustCompile(`^[a-zA-Z0-9_-]{1,64}$`) + +// maxResponseBody caps a successful response body; a larger one is an error. +const maxResponseBody = 16 << 20 + +// maxErrorBody caps how much of an error response body is read. It is far +// above llm.MaxBodyExcerpt, so redaction still sees every secret that could +// reach the excerpt. +const maxErrorBody = 64 << 10 + +type message struct { + Role string `json:"role"` + Content string `json:"content"` +} + +type tool struct { + Name string `json:"name"` + InputSchema json.RawMessage `json:"input_schema"` +} + +type toolChoice struct { + Type string `json:"type"` + Name string `json:"name"` +} + +type messagesRequest struct { + Model string `json:"model"` + MaxTokens int `json:"max_tokens"` + System string `json:"system,omitempty"` + Messages []message `json:"messages"` + Temperature *float64 `json:"temperature,omitempty"` + Tools []tool `json:"tools,omitempty"` + ToolChoice *toolChoice `json:"tool_choice,omitempty"` +} + +type messagesResponse struct { + Type string `json:"type"` + Model string `json:"model"` + Content []struct { + Type string `json:"type"` + Text string `json:"text"` + Name string `json:"name"` + Input json.RawMessage `json:"input"` + } `json:"content"` + StopReason string `json:"stop_reason"` + Usage struct { + InputTokens int `json:"input_tokens"` + OutputTokens int `json:"output_tokens"` + } `json:"usage"` +} + +// Generate sends req to {base}/v1/messages. A Request with a Schema is sent +// as one tool whose input_schema is the schema, with tool_choice forcing +// it; Response.JSON is then that tool's input. +func (p *Provider) Generate(ctx context.Context, req llm.Request) (llm.Response, error) { + resp, err := p.generate(ctx, req) + return resp, redact(err, p.apiKey) +} + +func (p *Provider) generate(ctx context.Context, req llm.Request) (llm.Response, error) { + if err := req.Validate(); err != nil { + return llm.Response{}, fmt.Errorf("anthropic: generate: %w", err) + } + if p.model == "" { + return llm.Response{}, errors.New("anthropic: generate: no model configured") + } + body := messagesRequest{ + Model: p.model, + MaxTokens: p.maxTokens, + System: req.System, + Temperature: req.Temperature, + } + if req.MaxTokens > 0 { + body.MaxTokens = req.MaxTokens + } + if req.Schema != nil { + if !toolName.MatchString(req.Schema.Name) { + return llm.Response{}, errors.New("anthropic: generate: schema name is not a valid tool name") + } + body.Tools = []tool{{Name: req.Schema.Name, InputSchema: req.Schema.Schema}} + body.ToolChoice = &toolChoice{Type: "tool", Name: req.Schema.Name} + } + for _, m := range req.Messages { + body.Messages = append(body.Messages, message{Role: string(m.Role), Content: m.Content}) + } + payload, err := json.Marshal(body) + if err != nil { + return llm.Response{}, fmt.Errorf("anthropic: generate: encode request: %w", err) + } + + if req.Timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, req.Timeout) + defer cancel() + } + raw, err := p.do(ctx, http.MethodPost, "/v1/messages", payload) + if err != nil { + return llm.Response{}, fmt.Errorf("anthropic: generate: %w", err) + } + + var mr messagesResponse + if err := json.Unmarshal(raw, &mr); err != nil { + return llm.Response{}, fmt.Errorf("anthropic: generate: decode response: %w", err) + } + if mr.Type != "message" { + return llm.Response{}, errors.New("anthropic: generate: response is not a message") + } + resp := llm.Response{ + Usage: llm.Usage{InputTokens: mr.Usage.InputTokens, OutputTokens: mr.Usage.OutputTokens}, + Model: mr.Model, + FinishReason: mr.StopReason, + } + var text strings.Builder + for _, block := range mr.Content { + switch block.Type { + case "text": + text.WriteString(block.Text) + case "tool_use": + if req.Schema != nil && block.Name == req.Schema.Name && resp.JSON == nil { + resp.JSON = block.Input + } + } + } + resp.Text = text.String() + if req.Schema == nil { + return resp, nil + } + if !isJSONObject(resp.JSON) { + resp.JSON = nil + return resp, fmt.Errorf("anthropic: generate: reply holds no input for the forced tool: %w", llm.ErrBadOutput) + } + return resp, nil +} + +// isJSONObject reports whether raw is exactly one JSON object. +func isJSONObject(raw json.RawMessage) bool { + s := bytes.TrimSpace(raw) + return bytes.HasPrefix(s, []byte("{")) && json.Valid(s) +} + +type modelsResponse struct { + Data []struct { + ID string `json:"id"` + DisplayName string `json:"display_name"` + } `json:"data"` + HasMore bool `json:"has_more"` + LastID string `json:"last_id"` +} + +// modelsPageLimit is the largest page GET /v1/models serves. +const modelsPageLimit = 1000 + +// ListModels returns every model {base}/v1/models lists, following pages. +func (p *Provider) ListModels(ctx context.Context) ([]llm.Model, error) { + models, err := p.listModels(ctx) + return models, redact(err, p.apiKey) +} + +func (p *Provider) listModels(ctx context.Context) ([]llm.Model, error) { + models := []llm.Model{} + after := "" + for { + page, err := p.modelsPage(ctx, modelsPageLimit, after) + if err != nil { + return nil, fmt.Errorf("anthropic: list models: %w", err) + } + for _, m := range page.Data { + if m.ID == "" { + return nil, fmt.Errorf("anthropic: list models: model %d has no id", len(models)) + } + name := m.DisplayName + if name == "" { + name = m.ID + } + models = append(models, llm.Model{ID: m.ID, DisplayName: name}) + } + if !page.HasMore { + return models, nil + } + if page.LastID == "" || page.LastID == after { + return nil, errors.New("anthropic: list models: response has more pages but no new cursor") + } + after = page.LastID + } +} + +// modelsPage fetches one page of GET {base}/v1/models. +func (p *Provider) modelsPage(ctx context.Context, limit int, after string) (modelsResponse, error) { + q := url.Values{"limit": {strconv.Itoa(limit)}} + if after != "" { + q.Set("after_id", after) + } + raw, err := p.do(ctx, http.MethodGet, "/v1/models?"+q.Encode(), nil) + if err != nil { + return modelsResponse{}, err + } + var mr modelsResponse + if err := json.Unmarshal(raw, &mr); err != nil { + return modelsResponse{}, fmt.Errorf("decode response: %w", err) + } + if mr.Data == nil { + return modelsResponse{}, errors.New("response has no data list") + } + return mr, nil +} + +// Verify checks the credentials with the cheapest authenticated call: one +// page of GET {base}/v1/models holding at most one model. +func (p *Provider) Verify(ctx context.Context) error { + _, err := p.modelsPage(ctx, 1, "") + if err != nil { + err = fmt.Errorf("anthropic: verify: %w", err) + } + return redact(err, p.apiKey) +} + +// do sends one request and returns the body of a 2xx response. Any other +// status becomes the matching llm typed error. +func (p *Provider) do(ctx context.Context, method, path string, payload []byte) ([]byte, error) { + var reqBody io.Reader + if payload != nil { + reqBody = bytes.NewReader(payload) + } + httpReq, err := http.NewRequestWithContext(ctx, method, p.base+path, reqBody) + if err != nil { + // The parse error names the URL, whose path may hold the API key. + return nil, errors.New("build request: request URL is invalid") + } + httpReq.Header.Set("x-api-key", p.apiKey) + httpReq.Header.Set("anthropic-version", APIVersion) + httpReq.Header.Set("Accept", "application/json") + if payload != nil { + httpReq.Header.Set("Content-Type", "application/json") + } + + httpResp, err := p.client.Do(httpReq) + if err != nil { + return nil, p.transportError(err) + } + defer httpResp.Body.Close() + + if httpResp.StatusCode < 200 || httpResp.StatusCode > 299 { + body, err := io.ReadAll(io.LimitReader(httpResp.Body, maxErrorBody)) + if err != nil { + return nil, p.transportError(err) + } + return nil, p.statusError(httpResp, body) + } + body, err := io.ReadAll(io.LimitReader(httpResp.Body, maxResponseBody+1)) + if err != nil { + return nil, p.transportError(err) + } + if len(body) > maxResponseBody { + return nil, fmt.Errorf("response body exceeds %d bytes", maxResponseBody) + } + return body, nil +} + +// statusOverloaded is the status the Messages API returns when it is +// temporarily overloaded. +const statusOverloaded = 529 + +// statusError maps a non-2xx response to an llm typed error. +func (p *Provider) statusError(resp *http.Response, body []byte) error { + switch resp.StatusCode { + case http.StatusUnauthorized, http.StatusForbidden: + return fmt.Errorf("HTTP %d: %w", resp.StatusCode, llm.ErrAuth) + case http.StatusTooManyRequests, statusOverloaded: + return llm.NewRateLimitedError(retryAfter(resp.Header.Get("Retry-After"), time.Now()), p.apiKey) + case http.StatusRequestTimeout: + return fmt.Errorf("HTTP %d: %w", resp.StatusCode, llm.ErrTimeout) + default: + return llm.NewProviderError(resp.StatusCode, body, p.apiKey) + } +} + +// retryAfter parses a Retry-After header, in delay-seconds or HTTP-date +// form. A missing, malformed or past value means the vendor gave no usable +// wait, reported as 0. +func retryAfter(value string, now time.Time) time.Duration { + value = strings.TrimSpace(value) + if value == "" { + return 0 + } + if secs, err := strconv.ParseInt(value, 10, 64); err == nil || errors.Is(err, strconv.ErrRange) { + if secs < 0 { + return 0 + } + // time.Duration counts nanoseconds in an int64; a larger wait is + // capped to the most whole seconds it holds instead of overflowing. + if secs > maxRetryAfterSeconds { + secs = maxRetryAfterSeconds + } + return time.Duration(secs) * time.Second + } + if at, err := http.ParseTime(value); err == nil && at.After(now) { + return at.Sub(now).Round(time.Second) + } + return 0 +} + +// maxRetryAfterSeconds is the largest delay-seconds value retryAfter +// returns unchanged: the most whole seconds a time.Duration holds. +const maxRetryAfterSeconds = math.MaxInt64 / int64(time.Second) + +// noRedirectClient returns an HTTP client that never follows a redirect. +// A followed redirect would resend the credential headers, which net/http +// strips only for its own list of sensitive headers and only across hosts; +// instead the 3xx response is returned and becomes a provider error. +func noRedirectClient() *http.Client { + return &http.Client{ + CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }, + } +} + +func isTimeout(err error) bool { + var ne net.Error + return errors.Is(err, context.DeadlineExceeded) || (errors.As(err, &ne) && ne.Timeout()) +} + +// transportFailure is a failed HTTP exchange. It carries the cause's text +// with the request URL removed and the API key redacted, and unwraps only to +// the cause's category: llm.ErrTimeout, context.Canceled or nothing. +type transportFailure struct { + text string + kind error +} + +func (e *transportFailure) Error() string { return e.text } + +// GoString keeps %#v to the redacted Error text instead of the fields. +func (e *transportFailure) GoString() string { return e.text } + +func (e *transportFailure) Unwrap() error { return e.kind } + +// transportError reports a failure to complete the HTTP exchange. A +// *url.Error names the request URL, and a base URL path may hold the API +// key, so only the wrapped cause is kept, rendered with the key redacted. +func (p *Provider) transportError(err error) error { + var kind error + switch { + case isTimeout(err): + kind = llm.ErrTimeout + case errors.Is(err, context.Canceled): + kind = context.Canceled + } + var ue *url.Error + if errors.As(err, &ue) { + err = ue.Err + } + text := "transport: " + err.Error() + if kind == llm.ErrTimeout { + text = fmt.Sprintf("%s: %s", llm.ErrTimeout, text) + } + return &transportFailure{text: llm.Redact(text, p.apiKey), kind: kind} +} + +// redactedError stands in for one layer of an error chain that holds the +// API key. Its text is the layer's text with the key redacted, and Unwrap +// returns the layer's causes passed through redact, so no error reached by +// unwrapping has text holding the key. Is consults the original layer and +// its chain, so sentinels still match. As does too. The llm typed errors +// remove the key from their own complete text, fixed words included, so +// they always match; As refuses a match whose text, or whose %#v form, +// holds the key, so it never hands back an error that can print it. +type redactedError struct { + text string + err error + key string +} + +func (e *redactedError) Error() string { return e.text } + +// GoString keeps %#v from printing the fields, which hold the key. +func (e *redactedError) GoString() string { return e.text } + +func (e *redactedError) Unwrap() error { return redact(errors.Unwrap(e.err), e.key) } + +func (e *redactedError) Is(target error) bool { return errors.Is(e.err, target) } + +func (e *redactedError) As(target any) bool { + if !errors.As(e.err, target) { + return false + } + found := reflect.ValueOf(target).Elem() + matched, isErr := found.Interface().(error) + if (isErr && holdsKey(matched, e.key)) || strings.Contains(fmt.Sprintf("%#v", found.Interface()), e.key) { + found.SetZero() + return false + } + return true +} + +// redactedJoin is redactedError for a layer with several causes, such as +// one built by errors.Join or by fmt.Errorf with more than one %w. +type redactedJoin struct { + redactedError +} + +func (e *redactedJoin) Unwrap() []error { + causes := e.err.(interface{ Unwrap() []error }).Unwrap() + safe := make([]error, len(causes)) + for i, c := range causes { + safe[i] = redact(c, e.key) + } + return safe +} + +// redact is the backstop every exported method applies to its error: with +// a non-empty key, it returns a stand-in whose text, and the text of every +// error it unwraps to, has the key redacted. Every non-nil err is wrapped, +// even one whose text lacks the key, because %#v of an unwrapped error +// prints its struct field names, and a key may spell one. +func redact(err error, key string) error { + if err == nil || key == "" { + return err + } + re := redactedError{text: llm.Redact(err.Error(), key), err: err, key: key} + if _, ok := err.(interface{ Unwrap() []error }); ok { + return &redactedJoin{re} + } + return &re +} + +// holdsKey reports whether the text of err or of any error in its chain +// contains key. +func holdsKey(err error, key string) bool { + if err == nil { + return false + } + if strings.Contains(err.Error(), key) { + return true + } + switch u := err.(type) { + case interface{ Unwrap() error }: + return holdsKey(u.Unwrap(), key) + case interface{ Unwrap() []error }: + for _, c := range u.Unwrap() { + if holdsKey(c, key) { + return true + } + } + } + return false +} diff --git a/internal/ai/llm/anthropic/anthropic_test.go b/internal/ai/llm/anthropic/anthropic_test.go new file mode 100644 index 0000000..e2bab59 --- /dev/null +++ b/internal/ai/llm/anthropic/anthropic_test.go @@ -0,0 +1,1025 @@ +package anthropic + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm" +) + +const testKey = "sk-ant-test-9f8e7d6c5b4a39281706f5e4d3c2b1a0" + +// recorded is one request the test server received. +type recorded struct { + method string + path string + query url.Values + header http.Header + body []byte +} + +// reply is a response the test server replays. +type reply struct { + status int + header map[string]string + fixture string // testdata file; {{KEY}} is replaced with the key + body string // used when fixture is empty + delay time.Duration +} + +// vendorServer replays its replies in order, repeating the last, and +// records every request. +type vendorServer struct { + *httptest.Server + mu sync.Mutex + got []recorded +} + +func newVendorServer(t *testing.T, replies ...reply) *vendorServer { + return newVendorServerWithKey(t, testKey, replies...) +} + +func newVendorServerWithKey(t *testing.T, key string, replies ...reply) *vendorServer { + t.Helper() + payloads := make([][]byte, len(replies)) + for i, r := range replies { + payloads[i] = []byte(r.body) + if r.fixture != "" { + raw, err := os.ReadFile(filepath.Join("testdata", r.fixture)) + if err != nil { + t.Fatal(err) + } + payloads[i] = bytes.ReplaceAll(raw, []byte("{{KEY}}"), []byte(key)) + } + } + vs := &vendorServer{} + vs.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + body, _ := io.ReadAll(req.Body) + vs.mu.Lock() + n := len(vs.got) + vs.got = append(vs.got, recorded{req.Method, req.URL.Path, req.URL.Query(), req.Header.Clone(), body}) + vs.mu.Unlock() + i := min(n, len(replies)-1) + r := replies[i] + if r.delay > 0 { + select { + case <-time.After(r.delay): + case <-req.Context().Done(): + return + } + } + for k, v := range r.header { + w.Header().Set(k, v) + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(r.status) + w.Write(payloads[i]) + })) + t.Cleanup(vs.Close) + return vs +} + +func (vs *vendorServer) requests() []recorded { + vs.mu.Lock() + defer vs.mu.Unlock() + return append([]recorded(nil), vs.got...) +} + +func (vs *vendorServer) only(t *testing.T) recorded { + t.Helper() + got := vs.requests() + if len(got) != 1 { + t.Fatalf("server got %d requests, want 1", len(got)) + } + return got[0] +} + +func newProvider(t *testing.T, base string) *Provider { + return newProviderWithKey(t, base, testKey) +} + +func newProviderWithKey(t *testing.T, base, key string) *Provider { + t.Helper() + p, err := New(Config{BaseURL: base, APIKey: key, Model: "test-model", MaxTokens: 1024}) + if err != nil { + t.Fatal(err) + } + return p +} + +var dinnersSchema = json.RawMessage(`{"type":"object","properties":{"dinners":{"type":"array"}},"required":["dinners"]}`) + +func schemaRequest() llm.Request { + temp := 0.0 + return llm.Request{ + System: "You plan dinners for Our kitchen.", + Messages: []llm.Message{ + {Role: llm.RoleUser, Content: "Suggest a Side quest."}, + {Role: llm.RoleAssistant, Content: "Something quick?"}, + {Role: llm.RoleUser, Content: "Yes, a Quick dinner."}, + }, + Temperature: &temp, + MaxTokens: 400, + Schema: &llm.JSONSchema{Name: "side_quests", Schema: dinnersSchema}, + } +} + +func textRequest() llm.Request { + return llm.Request{Messages: []llm.Message{{Role: llm.RoleUser, Content: "What should we cook tonight?"}}} +} + +// sentBody decodes a request body into a generic map. +func sentBody(t *testing.T, r recorded) map[string]any { + t.Helper() + var m map[string]any + if err := json.Unmarshal(r.body, &m); err != nil { + t.Fatalf("request body %q: %v", r.body, err) + } + return m +} + +// canonical re-encodes JSON so two encodings of one value compare equal. +func canonical(t *testing.T, raw []byte) string { + t.Helper() + var v any + if err := json.Unmarshal(raw, &v); err != nil { + t.Fatalf("JSON %q: %v", raw, err) + } + out, _ := json.Marshal(v) + return string(out) +} + +func assertHeaders(t *testing.T, r recorded, key string) { + t.Helper() + if got := r.header.Values("X-Api-Key"); len(got) != 1 || got[0] != key { + t.Errorf("x-api-key = %q", got) + } + if got := r.header.Values("Anthropic-Version"); len(got) != 1 || got[0] != "2023-06-01" { + t.Errorf("anthropic-version = %q", got) + } + if got := r.header.Get("Authorization"); got != "" { + t.Errorf("Authorization = %q, want none", got) + } +} + +func TestGenerateToolForcedJSON(t *testing.T) { + vs := newVendorServer(t, reply{status: 200, fixture: "messages_tool_use.json"}) + p := newProvider(t, vs.URL+"/") + + resp, err := p.Generate(context.Background(), schemaRequest()) + if err != nil { + t.Fatal(err) + } + if got, want := canonical(t, resp.JSON), `{"dinners":[{"name":"Sheet-pan gnocchi","why":"Like your Regular tray bake, but quicker."}]}`; got != want { + t.Errorf("Response.JSON = %s, want %s", got, want) + } + if resp.Text != "" { + t.Errorf("Text = %q, want none", resp.Text) + } + if resp.Usage != (llm.Usage{InputTokens: 472, OutputTokens: 61}) || resp.Model != "claude-sonnet-5" || resp.FinishReason != "tool_use" { + t.Errorf("Usage, Model, FinishReason = %+v, %q, %q", resp.Usage, resp.Model, resp.FinishReason) + } + + got := vs.only(t) + if got.method != http.MethodPost || got.path != "/v1/messages" || len(got.query) != 0 { + t.Errorf("request = %s %s?%s", got.method, got.path, got.query.Encode()) + } + assertHeaders(t, got, testKey) + if h := got.header.Get("Content-Type"); h != "application/json" { + t.Errorf("Content-Type = %q", h) + } + body := sentBody(t, got) + if body["model"] != "test-model" || body["temperature"] != 0.0 || body["max_tokens"] != 400.0 { + t.Errorf("model, temperature, max_tokens = %v, %v, %v", body["model"], body["temperature"], body["max_tokens"]) + } + if body["system"] != "You plan dinners for Our kitchen." { + t.Errorf("system = %v", body["system"]) + } + msgs := body["messages"].([]any) + wantMsgs := []string{"user:Suggest a Side quest.", "assistant:Something quick?", "user:Yes, a Quick dinner."} + if len(msgs) != len(wantMsgs) { + t.Fatalf("messages = %v", msgs) + } + for i, m := range msgs { + mm := m.(map[string]any) + if got := fmt.Sprintf("%v:%v", mm["role"], mm["content"]); got != wantMsgs[i] { + t.Errorf("message %d = %s, want %s", i, got, wantMsgs[i]) + } + } + tools := body["tools"].([]any) + if len(tools) != 1 { + t.Fatalf("tools = %v", tools) + } + tool := tools[0].(map[string]any) + schema, _ := json.Marshal(tool["input_schema"]) + if tool["name"] != "side_quests" || string(schema) != canonical(t, dinnersSchema) { + t.Errorf("tool = %v", tool) + } + choice := body["tool_choice"].(map[string]any) + if len(choice) != 2 || choice["type"] != "tool" || choice["name"] != "side_quests" { + t.Errorf("tool_choice = %v", choice) + } +} + +func TestGenerateText(t *testing.T) { + vs := newVendorServer(t, reply{status: 200, fixture: "messages_text.json"}) + p := newProvider(t, vs.URL) + + resp, err := p.Generate(context.Background(), textRequest()) + if err != nil { + t.Fatal(err) + } + if resp.Text != "Try a crock pot chili: it is a Quick dinner." || resp.JSON != nil { + t.Errorf("Text, JSON = %q, %s", resp.Text, resp.JSON) + } + if resp.Usage != (llm.Usage{InputTokens: 2095, OutputTokens: 503}) || resp.Model != "claude-sonnet-5" || resp.FinishReason != "end_turn" { + t.Errorf("Usage, Model, FinishReason = %+v, %q, %q", resp.Usage, resp.Model, resp.FinishReason) + } + + got := vs.only(t) + if got.path != "/v1/messages" { + t.Errorf("path = %s", got.path) + } + assertHeaders(t, got, testKey) + body := sentBody(t, got) + if body["max_tokens"] != 1024.0 { + t.Errorf("max_tokens = %v, want the configured 1024", body["max_tokens"]) + } + for _, field := range []string{"system", "temperature", "tools", "tool_choice"} { + if v, ok := body[field]; ok { + t.Errorf("%s = %v, want it omitted", field, v) + } + } +} + +func TestGenerateTextKeepsStopReason(t *testing.T) { + vs := newVendorServer(t, reply{status: 200, fixture: "messages_max_tokens.json"}) + resp, err := newProvider(t, vs.URL).Generate(context.Background(), textRequest()) + if err != nil { + t.Fatal(err) + } + if resp.FinishReason != "max_tokens" || resp.Usage.OutputTokens != 400 { + t.Errorf("FinishReason, Usage = %q, %+v", resp.FinishReason, resp.Usage) + } +} + +func TestGenerateBadOutput(t *testing.T) { + cases := map[string]string{ + "text instead of tool": `{"type":"message","model":"m","content":[{"type":"text","text":"Sure!"}],"stop_reason":"end_turn","usage":{"input_tokens":1,"output_tokens":2}}`, + "other tool": `{"type":"message","model":"m","content":[{"type":"tool_use","id":"t","name":"other","input":{"a":1}}],"stop_reason":"tool_use","usage":{}}`, + "input not an object": `{"type":"message","model":"m","content":[{"type":"tool_use","id":"t","name":"side_quests","input":[1]}],"stop_reason":"tool_use","usage":{}}`, + "no content": `{"type":"message","model":"m","content":[],"stop_reason":"max_tokens","usage":{}}`, + } + for name, body := range cases { + t.Run(name, func(t *testing.T) { + vs := newVendorServer(t, reply{status: 200, body: body}) + resp, err := newProvider(t, vs.URL).Generate(context.Background(), schemaRequest()) + if !errors.Is(err, llm.ErrBadOutput) { + t.Fatalf("err = %v, want ErrBadOutput", err) + } + if resp.JSON != nil || resp.Model != "m" { + t.Fatalf("Response = %+v, want no JSON and the model kept", resp) + } + }) + } + vs := newVendorServer(t, reply{status: 200, fixture: "messages_max_tokens.json"}) + resp, err := newProvider(t, vs.URL).Generate(context.Background(), schemaRequest()) + if !errors.Is(err, llm.ErrBadOutput) || resp.FinishReason != "max_tokens" { + t.Fatalf("truncated: resp, err = %+v, %v", resp, err) + } +} + +func TestGenerateRejectsNonMessage(t *testing.T) { + vs := newVendorServer(t, reply{status: 200, fixture: "error_overloaded.json"}) + _, err := newProvider(t, vs.URL).Generate(context.Background(), textRequest()) + if err == nil || errors.Is(err, llm.ErrBadOutput) { + t.Fatalf("err = %v, want an untyped malformed-response error", err) + } +} + +func TestGenerateRejectsInvalidRequests(t *testing.T) { + vs := newVendorServer(t, reply{status: 200, fixture: "messages_text.json"}) + p := newProvider(t, vs.URL) + if _, err := p.Generate(context.Background(), llm.Request{}); err == nil { + t.Error("Generate accepted a request with no messages") + } + for _, name := range []string{"side quests", "", strings.Repeat("a", 65), "plan.v2"} { + req := schemaRequest() + req.Schema.Name = name + if _, err := p.Generate(context.Background(), req); err == nil { + t.Errorf("Generate accepted tool name %q", name) + } + } + noModel, err := New(Config{BaseURL: vs.URL, APIKey: testKey, MaxTokens: 1}) + if err != nil { + t.Fatal(err) + } + if _, err := noModel.Generate(context.Background(), textRequest()); err == nil { + t.Error("Generate succeeded with no model configured") + } + if n := len(vs.requests()); n != 0 { + t.Errorf("invalid requests reached the vendor %d times", n) + } +} + +func TestListModelsFollowsPages(t *testing.T) { + vs := newVendorServer(t, + reply{status: 200, fixture: "models_page1.json"}, + reply{status: 200, fixture: "models_page2.json"}, + ) + models, err := newProvider(t, vs.URL).ListModels(context.Background()) + if err != nil { + t.Fatal(err) + } + want := []llm.Model{ + {ID: "claude-sonnet-5", DisplayName: "Claude Sonnet 5"}, + {ID: "claude-haiku-4-5-20251001", DisplayName: "Claude Haiku 4.5"}, + {ID: "claude-opus-5-5", DisplayName: "claude-opus-5-5"}, + } + if len(models) != len(want) { + t.Fatalf("models = %+v", models) + } + for i := range want { + if models[i] != want[i] { + t.Errorf("model %d = %+v, want %+v", i, models[i], want[i]) + } + } + got := vs.requests() + if len(got) != 2 { + t.Fatalf("server got %d requests, want 2", len(got)) + } + for i, wantAfter := range []string{"", "claude-sonnet-5"} { + r := got[i] + if r.method != http.MethodGet || r.path != "/v1/models" || r.query.Get("limit") != "1000" || r.query.Get("after_id") != wantAfter { + t.Errorf("request %d = %s %s?%s", i, r.method, r.path, r.query.Encode()) + } + assertHeaders(t, r, testKey) + if len(r.body) != 0 || r.header.Get("Content-Type") != "" { + t.Errorf("request %d sent a body", i) + } + } +} + +func TestListModelsRejectsBadPages(t *testing.T) { + cases := map[string]string{ + "no data": `{"has_more":false}`, + "no id": `{"data":[{"type":"model","display_name":"x"}],"has_more":false}`, + "no cursor": `{"data":[{"id":"a"}],"has_more":true}`, + "stuck cursor": `{"data":[{"id":"a"}],"has_more":true,"last_id":"a"}`, + "invalid": `{"data":`, + } + for name, body := range cases { + vs := newVendorServer(t, reply{status: 200, body: body}) + if models, err := newProvider(t, vs.URL).ListModels(context.Background()); err == nil { + t.Errorf("%s: ListModels() = %+v, nil error", name, models) + } + } +} + +func TestListModelsEmpty(t *testing.T) { + vs := newVendorServer(t, reply{status: 200, body: `{"data":[],"has_more":false}`}) + models, err := newProvider(t, vs.URL).ListModels(context.Background()) + if err != nil || models == nil || len(models) != 0 { + t.Fatalf("ListModels() = %#v, %v", models, err) + } +} + +func TestVerify(t *testing.T) { + ok := newVendorServer(t, reply{status: 200, fixture: "models_page1.json"}) + if err := newProvider(t, ok.URL).Verify(context.Background()); err != nil { + t.Fatalf("Verify() = %v", err) + } + got := ok.only(t) + if got.method != http.MethodGet || got.path != "/v1/models" || got.query.Get("limit") != "1" || got.query.Has("after_id") { + t.Errorf("Verify called %s %s?%s", got.method, got.path, got.query.Encode()) + } + assertHeaders(t, got, testKey) + + for status, want := range map[int]error{401: llm.ErrAuth, 403: llm.ErrAuth, 529: llm.ErrRateLimited, 500: llm.ErrProvider} { + bad := newVendorServer(t, reply{status: status, fixture: "error_api.json"}) + err := newProvider(t, bad.URL).Verify(context.Background()) + if !errors.Is(err, want) { + t.Errorf("HTTP %d: Verify() = %v, want %v", status, err, want) + } + assertNoSecret(t, err) + } + malformed := newVendorServer(t, reply{status: 200, body: `{}`}) + if err := newProvider(t, malformed.URL).Verify(context.Background()); err == nil { + t.Error("Verify accepted a body with no data list") + } +} + +// assertNoSecret fails if err's text holds the key or a prefix of it. +func assertNoSecret(t *testing.T, err error) { + t.Helper() + for _, secret := range []string{testKey, testKey[:16]} { + if strings.Contains(strings.ToLower(err.Error()), strings.ToLower(secret)) { + t.Fatalf("error leaks %q: %v", secret, err) + } + } +} + +func TestErrorMapping(t *testing.T) { + rateLimited := func(want time.Duration) func(*testing.T, error) { + return func(t *testing.T, err error) { + var rl *llm.RateLimitedError + if !errors.As(err, &rl) || rl.RetryAfter != want { + t.Fatalf("RetryAfter from %v, want %v", err, want) + } + } + } + providerError := func(status int, excerpt string) func(*testing.T, error) { + return func(t *testing.T, err error) { + var pe *llm.ProviderError + if !errors.As(err, &pe) || pe.Status() != status || !strings.Contains(pe.Body(), excerpt) { + t.Fatalf("ProviderError from %v, want HTTP %d with %q", err, status, excerpt) + } + if !strings.Contains(err.Error(), pe.Body()) { + t.Fatalf("Error() = %q, want it to carry the excerpt", err) + } + } + } + cases := []struct { + name string + reply reply + want error + check func(*testing.T, error) + }{ + {name: "401", reply: reply{status: 401, fixture: "error_authentication.json"}, want: llm.ErrAuth}, + {name: "403", reply: reply{status: 403, fixture: "error_permission.json"}, want: llm.ErrAuth}, + {name: "429 seconds", reply: reply{status: 429, header: map[string]string{"Retry-After": "17"}, fixture: "error_rate_limit.json"}, want: llm.ErrRateLimited, check: rateLimited(17 * time.Second)}, + {name: "429 no Retry-After", reply: reply{status: 429, fixture: "error_rate_limit.json"}, want: llm.ErrRateLimited, check: rateLimited(0)}, + {name: "429 HTTP-date", reply: reply{status: 429, header: map[string]string{"Retry-After": time.Now().Add(90 * time.Second).UTC().Format(http.TimeFormat)}, fixture: "error_rate_limit.json"}, want: llm.ErrRateLimited, + check: func(t *testing.T, err error) { + var rl *llm.RateLimitedError + if !errors.As(err, &rl) || rl.RetryAfter < 85*time.Second || rl.RetryAfter > 91*time.Second { + t.Fatalf("RetryAfter from %v", err) + } + }}, + {name: "529 seconds", reply: reply{status: 529, header: map[string]string{"Retry-After": "4"}, fixture: "error_overloaded.json"}, want: llm.ErrRateLimited, check: rateLimited(4 * time.Second)}, + {name: "529 no Retry-After", reply: reply{status: 529, fixture: "error_overloaded.json"}, want: llm.ErrRateLimited, check: rateLimited(0)}, + {name: "408", reply: reply{status: 408}, want: llm.ErrTimeout}, + {name: "400", reply: reply{status: 400, fixture: "error_invalid_request.json"}, want: llm.ErrProvider, check: providerError(400, "max_tokens: Field required")}, + {name: "404", reply: reply{status: 404, fixture: "error_not_found.json"}, want: llm.ErrProvider, check: providerError(404, "not_found_error")}, + {name: "500 echoing the key", reply: reply{status: 500, fixture: "error_api.json"}, want: llm.ErrProvider, check: providerError(500, "x-api-key: [REDACTED]")}, + {name: "503", reply: reply{status: 503, body: "upstream connect error"}, want: llm.ErrProvider, check: providerError(503, "upstream connect error")}, + } + all := []error{llm.ErrAuth, llm.ErrRateLimited, llm.ErrTimeout, llm.ErrBadOutput, llm.ErrProvider} + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + vs := newVendorServer(t, c.reply) + p := newProvider(t, vs.URL) + _, genErr := p.Generate(context.Background(), schemaRequest()) + _, listErr := p.ListModels(context.Background()) + for _, err := range []error{genErr, listErr} { + for _, sentinel := range all { + if errors.Is(err, sentinel) != (sentinel == c.want) { + t.Fatalf("errors.Is(%v, %v) = %v", err, sentinel, sentinel != c.want) + } + } + assertNoSecret(t, err) + assertChainNoExactKey(t, err, testKey) + if c.check != nil { + c.check(t, err) + } + } + }) + } +} + +func TestTimeouts(t *testing.T) { + vs := newVendorServer(t, reply{status: 200, fixture: "messages_tool_use.json", delay: 5 * time.Second}) + p := newProvider(t, vs.URL) + + req := schemaRequest() + req.Timeout = 50 * time.Millisecond + start := time.Now() + _, err := p.Generate(context.Background(), req) + if !errors.Is(err, llm.ErrTimeout) { + t.Fatalf("Request.Timeout: err = %v, want ErrTimeout", err) + } + if elapsed := time.Since(start); elapsed > 2*time.Second { + t.Fatalf("Request.Timeout not honoured: took %v", elapsed) + } + assertNoSecret(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + if _, err := p.ListModels(ctx); !errors.Is(err, llm.ErrTimeout) { + t.Fatalf("context deadline: err = %v, want ErrTimeout", err) + } + ctx, cancel = context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + if err := p.Verify(ctx); !errors.Is(err, llm.ErrTimeout) { + t.Fatalf("Verify context deadline: err = %v, want ErrTimeout", err) + } + + ctx, cancel = context.WithCancel(context.Background()) + cancel() + if _, err := p.Generate(ctx, schemaRequest()); !errors.Is(err, context.Canceled) || errors.Is(err, llm.ErrTimeout) { + t.Fatalf("cancelled context: err = %v, want context.Canceled only", err) + } +} + +// TestTransportErrorWithKeyInBaseURLPath fails the exchange against a base +// URL whose path holds the key, which New accepts; net/http names the full +// request URL in its transport errors. +func TestTransportErrorWithKeyInBaseURLPath(t *testing.T) { + refused := newVendorServer(t, reply{status: 200}) + refusedBase := refused.URL + refused.Close() + + dropped := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + conn, _, err := w.(http.Hijacker).Hijack() + if err != nil { + t.Error(err) + return + } + conn.Close() + })) + t.Cleanup(dropped.Close) + + for name, base := range map[string]string{ + "connection refused": refusedBase + "/" + testKey, + "connection dropped": dropped.URL + "/" + testKey, + } { + p := newProvider(t, base) + _, genErr := p.Generate(context.Background(), schemaRequest()) + _, listErr := p.ListModels(context.Background()) + verifyErr := p.Verify(context.Background()) + for _, err := range []error{genErr, listErr, verifyErr} { + if err == nil { + t.Fatalf("%s: nil error", name) + } + assertNoSecret(t, err) + assertChainNoExactKey(t, err, testKey) + var ue *url.Error + if errors.As(err, &ue) { + t.Fatalf("%s: error chain exposes the request URL: %v", name, ue) + } + if errors.Is(err, llm.ErrTimeout) || errors.Is(err, llm.ErrProvider) { + t.Fatalf("%s: err = %v, want an untyped transport failure", name, err) + } + } + } + + // The request URL escapes a key with spaces, so redacting the key + // itself would miss that copy. + const spacedKey = "sk ant key" + p := newProviderWithKey(t, refusedBase+"/"+spacedKey, spacedKey) + _, genErr := p.Generate(context.Background(), textRequest()) + _, listErr := p.ListModels(context.Background()) + verifyErr := p.Verify(context.Background()) + for _, err := range []error{genErr, listErr, verifyErr} { + if err == nil { + t.Fatal("escaped key: nil error") + } + assertChainNoExactKey(t, err, spacedKey) + assertChainNoExactKey(t, err, url.PathEscape(spacedKey)) + } +} + +func TestNewRejectsBadConfig(t *testing.T) { + cases := map[string]Config{ + "empty base": {}, + "relative base": {BaseURL: "api.anthropic.com"}, + "ftp base": {BaseURL: "ftp://example.com"}, + "credentials in URL": {BaseURL: "https://user:" + testKey + "@example.com"}, + "key in query": {BaseURL: "https://example.com/?key=" + testKey}, + "key in fragment": {BaseURL: "https://example.com/#" + testKey}, + "key, relative base": {BaseURL: "example.com?key=" + testKey}, + "key, ftp base": {BaseURL: "ftp://example.com/?key=" + testKey}, + "key, unparseable": {BaseURL: "https://example.com/%zz?key=" + testKey}, + "empty query": {BaseURL: "https://example.com?"}, + "empty fragment": {BaseURL: "https://example.com#"}, + "empty both": {BaseURL: "https://example.com/?#"}, + "empty key": {BaseURL: "https://example.com", APIKey: "-"}, + "key with newline": {BaseURL: "https://example.com", APIKey: testKey + "\n"}, + "zero max tokens": {BaseURL: "https://example.com", MaxTokens: -1}, + "negative max": {BaseURL: "https://example.com", MaxTokens: -5}, + } + for name, cfg := range cases { + switch cfg.APIKey { + case "": + cfg.APIKey = testKey + case "-": + cfg.APIKey = "" + } + switch cfg.MaxTokens { + case 0: + cfg.MaxTokens = 1024 + case -1: + cfg.MaxTokens = 0 + } + _, err := New(cfg) + if err == nil { + t.Errorf("%s: New() = nil error", name) + continue + } + assertNoSecret(t, err) + assertChainNoExactKey(t, err, testKey) + } + if _, err := New(Config{BaseURL: "http://anthropic:8080/proxy/", APIKey: testKey, MaxTokens: 1}); err != nil { + t.Errorf("New() rejected a valid config: %v", err) + } +} + +func TestRetryAfter(t *testing.T) { + now := time.Date(2026, 10, 3, 12, 0, 0, 0, time.UTC) + cases := map[string]time.Duration{ + "": 0, + "30": 30 * time.Second, + " 5 ": 5 * time.Second, + "-3": 0, + "soon": 0, + "Sat, 03 Oct 2026 12:02:00 GMT": 2 * time.Minute, + "Sat, 03 Oct 2026 11:59:00 GMT": 0, + "9223372036854775807": time.Duration(maxRetryAfterSeconds) * time.Second, + "99999999999999999999": time.Duration(maxRetryAfterSeconds) * time.Second, + "-99999999999999999999": 0, + } + for in, want := range cases { + if got := retryAfter(in, now); got != want { + t.Errorf("retryAfter(%q) = %v, want %v", in, got, want) + } + } +} + +// assertChainNoExactKey fails if key appears, case-sensitively, in the text +// of err, of any error reached from it by unwrapping, or of a typed error +// errors.As finds in it, including a ProviderError's Body. +func assertChainNoExactKey(t *testing.T, err error, key string) { + t.Helper() + var walk func(error) + walk = func(e error) { + if e == nil { + return + } + if strings.Contains(e.Error(), key) { + t.Fatalf("layer leaks %q: %q", key, e.Error()) + } + switch u := e.(type) { + case interface{ Unwrap() error }: + walk(u.Unwrap()) + case interface{ Unwrap() []error }: + for _, c := range u.Unwrap() { + walk(c) + } + } + } + walk(err) + for layer := err; layer != nil; layer = errors.Unwrap(layer) { + if strings.Contains(layer.Error(), key) { + t.Fatalf("errors.Unwrap layer leaks %q: %q", key, layer.Error()) + } + } + var pe *llm.ProviderError + if errors.As(err, &pe) && (strings.Contains(pe.Body(), key) || strings.Contains(pe.Error(), key)) { + t.Fatalf("ProviderError leaks %q: Error %q, Body %q", key, pe.Error(), pe.Body()) + } + var rl *llm.RateLimitedError + if errors.As(err, &rl) && strings.Contains(rl.Error(), key) { + t.Fatalf("RateLimitedError leaks %q: %q", key, rl.Error()) + } + var syntax *json.SyntaxError + if errors.As(err, &syntax) && strings.Contains(syntax.Error(), key) { + t.Fatalf("errors.As reached a *json.SyntaxError leaking %q: %q", key, syntax.Error()) + } + if got := fmt.Sprintf("%v %+v %#v", err, err, err); strings.Contains(got, key) { + t.Fatalf("formatting leaks %q: %q", key, got) + } +} + +// TestHeaderNameKeyNeverLeaks uses the header name "x-api-key" as the API +// key, a word an error naming the header would print, and drives every +// construction and request error path with it. +func TestHeaderNameKeyNeverLeaks(t *testing.T) { + const key = "x-api-key" + construction := map[string]Config{ + "unparseable base": {BaseURL: "https://example.com/%zz/" + key, APIKey: key, MaxTokens: 1}, + "relative base": {BaseURL: key, APIKey: key, MaxTokens: 1}, + "query in base": {BaseURL: "https://example.com/?" + key, APIKey: key, MaxTokens: 1}, + "userinfo in base": {BaseURL: "https://" + key + "@example.com", APIKey: key, MaxTokens: 1}, + "no max tokens": {BaseURL: "https://example.com", APIKey: key}, + "key with newline": {BaseURL: "https://example.com", APIKey: key + "\r\n", MaxTokens: 1}, + } + for name, cfg := range construction { + _, err := New(cfg) + if err == nil { + t.Fatalf("%s: New() = nil error", name) + } + assertChainNoExactKey(t, err, key) + } + + body := `{"type":"error","error":{"type":"x","message":"bad x-api-key header: x-api-key"}}` + statuses := []int{400, 401, 403, 404, 408, 429, 500, 529} + for _, status := range statuses { + vs := newVendorServerWithKey(t, key, reply{status: status, body: body, header: map[string]string{"Retry-After": "3"}}) + p := newProviderWithKey(t, vs.URL, key) + _, genErr := p.Generate(context.Background(), schemaRequest()) + _, listErr := p.ListModels(context.Background()) + verifyErr := p.Verify(context.Background()) + for _, err := range []error{genErr, listErr, verifyErr} { + if err == nil { + t.Fatalf("HTTP %d: nil error", status) + } + assertChainNoExactKey(t, err, key) + } + } + + ok := newVendorServerWithKey(t, key, reply{status: 200, fixture: "messages_text.json"}) + p := newProviderWithKey(t, ok.URL, key) + requests := map[string]llm.Request{ + "unknown role": {Messages: []llm.Message{{Role: llm.Role(key), Content: "hi"}}}, + "bad tool name": {Messages: textRequest().Messages, Schema: &llm.JSONSchema{Name: key + "!", Schema: dinnersSchema}}, + "schema array": {Messages: textRequest().Messages, Schema: &llm.JSONSchema{Name: key, Schema: json.RawMessage(`[]`)}}, + "bad output": {Messages: textRequest().Messages, Schema: &llm.JSONSchema{Name: key, Schema: dinnersSchema}}, + } + for name, req := range requests { + _, err := p.Generate(context.Background(), req) + if err == nil { + t.Fatalf("%s: nil error", name) + } + assertChainNoExactKey(t, err, key) + } + noModel, err := New(Config{BaseURL: ok.URL, APIKey: key, MaxTokens: 1}) + if err != nil { + t.Fatal(err) + } + _, err = noModel.Generate(context.Background(), textRequest()) + if err == nil { + t.Fatal("Generate succeeded with no model") + } + assertChainNoExactKey(t, err, key) + + refused := newVendorServer(t, reply{status: 200}) + refusedBase := refused.URL + refused.Close() + _, err = newProviderWithKey(t, refusedBase+"/"+key, key).Generate(context.Background(), textRequest()) + if err == nil { + t.Fatal("Generate against a closed server succeeded") + } + assertChainNoExactKey(t, err, key) +} + +// TestAdapterNameKeyOn401 uses the adapter's own error prefix "anthropic" as +// the API key. +func TestAdapterNameKeyOn401(t *testing.T) { + const key = "anthropic" + vs := newVendorServerWithKey(t, key, reply{status: 401, fixture: "error_authentication.json"}) + p := newProviderWithKey(t, vs.URL, key) + _, genErr := p.Generate(context.Background(), schemaRequest()) + _, listErr := p.ListModels(context.Background()) + verifyErr := p.Verify(context.Background()) + for _, err := range []error{genErr, listErr, verifyErr} { + if !errors.Is(err, llm.ErrAuth) { + t.Fatalf("err = %v, want ErrAuth", err) + } + assertChainNoExactKey(t, err, key) + } +} + +// TestFixedWordKeyOn500 uses the fixed word "HTTP" of ProviderError's text +// as the API key: errors.As must still reach the ProviderError, whose Body +// and Error must drop it. +func TestFixedWordKeyOn500(t *testing.T) { + const key = "HTTP" + vs := newVendorServerWithKey(t, key, reply{status: 500, body: `{"type":"error","error":{"type":"api_error","message":"HTTP backend down"}}`}) + _, err := newProviderWithKey(t, vs.URL, key).Generate(context.Background(), schemaRequest()) + var pe *llm.ProviderError + if !errors.Is(err, llm.ErrProvider) || !errors.As(err, &pe) || pe.Status() != 500 { + t.Fatalf("err = %v, want a 500 ProviderError", err) + } + if strings.Contains(pe.Error(), key) || strings.Contains(pe.Body(), key) { + t.Fatalf("ProviderError leaks the key: Error %q, Body %q", pe.Error(), pe.Body()) + } + if !strings.Contains(pe.Body(), "backend down") { + t.Fatalf("Body() = %q, want the redacted excerpt", pe.Body()) + } + assertChainNoExactKey(t, err, key) +} + +// TestRateLimitedFixedWordKey uses the fixed word "rate" of +// RateLimitedError's text as the API key, on both rate-limit statuses. +func TestRateLimitedFixedWordKey(t *testing.T) { + const key = "rate" + for _, status := range []int{429, 529} { + vs := newVendorServerWithKey(t, key, reply{status: status, header: map[string]string{"Retry-After": "17"}, fixture: "error_rate_limit.json"}) + _, err := newProviderWithKey(t, vs.URL, key).Generate(context.Background(), schemaRequest()) + var rl *llm.RateLimitedError + if !errors.Is(err, llm.ErrRateLimited) || !errors.As(err, &rl) || rl.RetryAfter != 17*time.Second { + t.Fatalf("HTTP %d: err = %v, want a RateLimitedError with RetryAfter 17s", status, err) + } + assertChainNoExactKey(t, err, key) + } +} + +// TestResponseBodyWithKeyNeverLeaks has the vendor echo the key in error +// and success bodies, including ones that fail to decode, and checks +// Error(), every errors.Unwrap layer and every errors.As target. +func TestResponseBodyWithKeyNeverLeaks(t *testing.T) { + bodies := []reply{ + {status: 500, fixture: "error_api.json"}, + {status: 400, body: `{"type":"error","error":{"type":"invalid_request_error","message":"key ` + testKey + ` rejected"}}`}, + {status: 401, fixture: "error_authentication.json"}, + {status: 429, fixture: "error_rate_limit.json"}, + {status: 200, body: `{"type": ` + testKey + `}`}, + {status: 200, body: testKey}, + {status: 200, body: `{"type":"message","content":[{"type":"text","text":"` + testKey + `"}],"stop_reason":"` + testKey + `"}`}, + {status: 200, body: `{"data": [` + testKey + `]}`}, + } + for i, r := range bodies { + vs := newVendorServer(t, r) + p := newProvider(t, vs.URL) + _, genErr := p.Generate(context.Background(), schemaRequest()) + _, listErr := p.ListModels(context.Background()) + verifyErr := p.Verify(context.Background()) + for _, err := range []error{genErr, listErr, verifyErr} { + if err == nil { + t.Fatalf("body %d: nil error", i) + } + assertNoSecret(t, err) + assertChainNoExactKey(t, err, testKey) + } + } + // A short key that decode errors quote from a malformed body. + const key = "Z" + vs := newVendorServerWithKey(t, key, reply{status: 200, body: `{"type": Z}`}) + _, err := newProviderWithKey(t, vs.URL, key).Generate(context.Background(), textRequest()) + var syntax *json.SyntaxError + if err == nil || errors.As(err, &syntax) { + t.Fatalf("err = %v, want no reachable *json.SyntaxError quoting the key", err) + } + assertChainNoExactKey(t, err, key) +} + +// TestRedactKeepsChainMatches checks that the redaction backstop keeps +// errors.Is and errors.As working through a chain that holds the key. +func TestRedactKeepsChainMatches(t *testing.T) { + pe := llm.NewProviderError(500, []byte(`{"error":"down"}`), testKey) + rl := llm.NewRateLimitedError(time.Second, testKey) + inner := fmt.Errorf("decode %s: %w", testKey, llm.ErrBadOutput) + joined := errors.Join(inner, pe, rl, context.Canceled, &transportFailure{text: "llm: request timed out: transport: slow", kind: llm.ErrTimeout}, fmt.Errorf("HTTP 401: %w", llm.ErrAuth)) + err := redact(fmt.Errorf("anthropic %s: %w", testKey, joined), testKey) + assertChainNoExactKey(t, err, testKey) + for _, target := range []error{llm.ErrBadOutput, llm.ErrProvider, llm.ErrRateLimited, llm.ErrTimeout, llm.ErrAuth, context.Canceled} { + if !errors.Is(err, target) { + t.Fatalf("errors.Is(%v, %v) = false", err, target) + } + } + var gotPE *llm.ProviderError + var gotRL *llm.RateLimitedError + if !errors.As(err, &gotPE) || gotPE != pe || !errors.As(err, &gotRL) || gotRL != rl { + t.Fatalf("errors.As lost a typed error in %v", err) + } + clean := errors.New("anthropic: generate: no model configured") + if got := redact(clean, testKey); got.Error() != clean.Error() || !errors.Is(got, clean) { + t.Fatalf("redact(%v) = %v, want the same text and chain", clean, got) + } +} + +// TestRedactedErrorsFormatWithoutKey formats every error type that holds the +// key, or a cause holding it, with %v, %+v and %#v; %#v prints GoString. +func TestRedactedErrorsFormatWithoutKey(t *testing.T) { + const key = "sk-distinctive-anthropic-format-7c1e" + single := redact(fmt.Errorf("anthropic %s: %w", key, llm.ErrBadOutput), key) + joined := redact(errors.Join(fmt.Errorf("decode %s: %w", key, llm.ErrAuth), llm.NewProviderError(500, []byte("down "+key), key)), key) + if _, ok := single.(*redactedError); !ok { + t.Fatalf("single = %T, want *redactedError", single) + } + if _, ok := joined.(*redactedJoin); !ok { + t.Fatalf("joined = %T, want *redactedJoin", joined) + } + p := &Provider{apiKey: key} + transport := p.transportError(fmt.Errorf("dial %s: %w", key, context.DeadlineExceeded)) + for _, err := range []error{single, joined, transport} { + for _, verb := range []string{"%v", "%+v", "%#v"} { + if got := fmt.Sprintf(verb, err); strings.Contains(got, key) { + t.Fatalf("%s of %T leaks the key: %q", verb, err, got) + } + } + } +} + +func TestRateLimitedHugeRetryAfterIsCapped(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Retry-After", "9223372036854775807") + w.WriteHeader(http.StatusTooManyRequests) + })) + t.Cleanup(srv.Close) + p, err := New(Config{BaseURL: srv.URL, APIKey: testKey, Model: "test-model", MaxTokens: 1024}) + if err != nil { + t.Fatal(err) + } + err = p.Verify(context.Background()) + var rl *llm.RateLimitedError + if !errors.Is(err, llm.ErrRateLimited) || !errors.As(err, &rl) || rl.RetryAfter != time.Duration(maxRetryAfterSeconds)*time.Second { + t.Fatalf("err = %v, want a RateLimitedError with RetryAfter capped to %v", err, time.Duration(maxRetryAfterSeconds)*time.Second) + } +} + +func TestRedirectIsNotFollowed(t *testing.T) { + var mu sync.Mutex + var secondCalls int + var secondHeader http.Header + second := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + mu.Lock() + secondCalls++ + secondHeader = req.Header.Clone() + mu.Unlock() + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(second.Close) + first := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + http.Redirect(w, req, second.URL+req.URL.Path, http.StatusFound) + })) + t.Cleanup(first.Close) + p, err := New(Config{BaseURL: first.URL, APIKey: testKey, Model: "test-model", MaxTokens: 1024}) + if err != nil { + t.Fatal(err) + } + + err = p.Verify(context.Background()) + var pe *llm.ProviderError + if !errors.Is(err, llm.ErrProvider) || !errors.As(err, &pe) || pe.Status() != http.StatusFound { + t.Fatalf("err = %v, want a ProviderError for HTTP 302", err) + } + mu.Lock() + defer mu.Unlock() + if secondCalls != 0 { + for name, values := range secondHeader { + for _, v := range values { + if strings.Contains(v, testKey) { + t.Errorf("redirect target saw the key in %s", name) + } + } + } + t.Fatalf("redirect target got %d requests, want 0", secondCalls) + } +} + +// assertGoStringNoKey checks that %#v of err, of every layer reached by +// unwrapping and of the typed errors reached by errors.As holds no key. +func assertGoStringNoKey(t *testing.T, err error, key string) { + t.Helper() + var walk func(error) + walk = func(e error) { + if e == nil { + return + } + if got := fmt.Sprintf("%#v", e); strings.Contains(got, key) { + t.Fatalf("%%#v of %T leaks %q: %q", e, key, got) + } + switch u := e.(type) { + case interface{ Unwrap() error }: + walk(u.Unwrap()) + case interface{ Unwrap() []error }: + for _, c := range u.Unwrap() { + walk(c) + } + } + } + walk(err) + var pe *llm.ProviderError + if errors.As(err, &pe) { + walk(pe) + } + var rl *llm.RateLimitedError + if errors.As(err, &rl) { + walk(rl) + } +} + +// TestFieldNameKeyNeverLeaksThroughGoString uses struct field names of the +// returned error types as the API key; %#v must not print any of them. +func TestFieldNameKeyNeverLeaksThroughGoString(t *testing.T) { + for _, key := range []string{"body", "status", "text", "RetryAfter", "msg", "err", "key", "kind"} { + vs := newVendorServerWithKey(t, key, reply{status: 500, body: "down"}, reply{status: 429, header: map[string]string{"Retry-After": "17"}, body: "slow"}) + p := newProviderWithKey(t, vs.URL, key) + _, err := p.Generate(context.Background(), schemaRequest()) + var pe *llm.ProviderError + if !errors.As(err, &pe) || pe.Status() != 500 { + t.Fatalf("key %q: err = %v, want a 500 ProviderError", key, err) + } + assertGoStringNoKey(t, err, key) + _, err = p.Generate(context.Background(), schemaRequest()) + var rl *llm.RateLimitedError + if !errors.As(err, &rl) || rl.RetryAfter != 17*time.Second { + t.Fatalf("key %q: err = %v, want a RateLimitedError with RetryAfter 17s", key, err) + } + assertGoStringNoKey(t, err, key) + } +} diff --git a/internal/ai/llm/anthropic/testdata/README.md b/internal/ai/llm/anthropic/testdata/README.md new file mode 100644 index 0000000..c69c707 --- /dev/null +++ b/internal/ai/llm/anthropic/testdata/README.md @@ -0,0 +1,41 @@ +# Anthropic adapter fixtures + +These are the Messages API response bodies the contract tests in +`anthropic_test.go` replay. They are **not live captures**. Product AI is +tested with fakes, and live vendor calls are opt-in and never use +credentials, so no request was sent to Anthropic to make them. Each fixture +is a sanitized adaptation of the response shapes in Anthropic's public API +documentation: + +- Messages: +- Tool use, including `tool_choice` `{type: tool}`: + +- List models: +- Errors: + +Sanitizing: + +- IDs, request IDs, timestamps and token counts are illustrative values in + the documented format, not values from a real account. +- Message content and tool input are replaced with MealQuest dinner text. +- No real credential appears anywhere. `{{KEY}}` is a placeholder the tests + replace with a fake key, to prove the adapter redacts a key echoed back in + a vendor body. + +Below, *not in the public example* marks fields or values that were added +for a test case and do not come from the documented example. + +| Fixture | Shape | Not in the public example | +| --- | --- | --- | +| `messages_tool_use.json` | Message with one `tool_use` block, `stop_reason: tool_use`, `usage` with cache fields. | The tool name and input. | +| `messages_text.json` | Message with `text` blocks, `stop_reason: end_turn`. | A second text block, to test joining. | +| `messages_max_tokens.json` | Message with a `text` block, `stop_reason: max_tokens`. | — | +| `models_page1.json` | Models list page: `data`, `has_more: true`, `first_id`, `last_id`. | — | +| `models_page2.json` | Last models list page, `has_more: false`. | The second model has no `display_name`, to test the fallback to `id`. | +| `error_authentication.json` | Error, `authentication_error` (401). | The `{{KEY}}` text; added to test redaction. | +| `error_permission.json` | Error, `permission_error` (403). | — | +| `error_rate_limit.json` | Error, `rate_limit_error` (429). | The `{{KEY}}` text; added to test redaction. | +| `error_overloaded.json` | Error, `overloaded_error` (529). | — | +| `error_not_found.json` | Error, `not_found_error` (404). | — | +| `error_invalid_request.json` | Error, `invalid_request_error` (400). | — | +| `error_api.json` | Error, `api_error` (500). | The `x-api-key: {{KEY}}` text; added to test redaction. | diff --git a/internal/ai/llm/anthropic/testdata/error_api.json b/internal/ai/llm/anthropic/testdata/error_api.json new file mode 100644 index 0000000..56710aa --- /dev/null +++ b/internal/ai/llm/anthropic/testdata/error_api.json @@ -0,0 +1 @@ +{"type": "error", "error": {"type": "api_error", "message": "An unexpected error has occurred internal to Anthropic's systems. x-api-key: {{KEY}}"}, "request_id": "req_011CSHoEeqs5C35K2UUqR7G4"} diff --git a/internal/ai/llm/anthropic/testdata/error_authentication.json b/internal/ai/llm/anthropic/testdata/error_authentication.json new file mode 100644 index 0000000..a6e1e61 --- /dev/null +++ b/internal/ai/llm/anthropic/testdata/error_authentication.json @@ -0,0 +1 @@ +{"type": "error", "error": {"type": "authentication_error", "message": "invalid x-api-key: {{KEY}}"}, "request_id": "req_011CSHoEeqs5C35K2UUqR7Fy"} diff --git a/internal/ai/llm/anthropic/testdata/error_invalid_request.json b/internal/ai/llm/anthropic/testdata/error_invalid_request.json new file mode 100644 index 0000000..977b7fc --- /dev/null +++ b/internal/ai/llm/anthropic/testdata/error_invalid_request.json @@ -0,0 +1 @@ +{"type": "error", "error": {"type": "invalid_request_error", "message": "max_tokens: Field required"}, "request_id": "req_011CSHoEeqs5C35K2UUqR7G3"} diff --git a/internal/ai/llm/anthropic/testdata/error_not_found.json b/internal/ai/llm/anthropic/testdata/error_not_found.json new file mode 100644 index 0000000..f54171f --- /dev/null +++ b/internal/ai/llm/anthropic/testdata/error_not_found.json @@ -0,0 +1 @@ +{"type": "error", "error": {"type": "not_found_error", "message": "model: claude-nonexistent"}, "request_id": "req_011CSHoEeqs5C35K2UUqR7G2"} diff --git a/internal/ai/llm/anthropic/testdata/error_overloaded.json b/internal/ai/llm/anthropic/testdata/error_overloaded.json new file mode 100644 index 0000000..013e4b3 --- /dev/null +++ b/internal/ai/llm/anthropic/testdata/error_overloaded.json @@ -0,0 +1 @@ +{"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}, "request_id": "req_011CSHoEeqs5C35K2UUqR7G1"} diff --git a/internal/ai/llm/anthropic/testdata/error_permission.json b/internal/ai/llm/anthropic/testdata/error_permission.json new file mode 100644 index 0000000..80941f6 --- /dev/null +++ b/internal/ai/llm/anthropic/testdata/error_permission.json @@ -0,0 +1 @@ +{"type": "error", "error": {"type": "permission_error", "message": "Your API key does not have permission to use the specified resource."}, "request_id": "req_011CSHoEeqs5C35K2UUqR7Fz"} diff --git a/internal/ai/llm/anthropic/testdata/error_rate_limit.json b/internal/ai/llm/anthropic/testdata/error_rate_limit.json new file mode 100644 index 0000000..136220f --- /dev/null +++ b/internal/ai/llm/anthropic/testdata/error_rate_limit.json @@ -0,0 +1 @@ +{"type": "error", "error": {"type": "rate_limit_error", "message": "Number of request tokens has exceeded your per-minute rate limit for key {{KEY}}"}, "request_id": "req_011CSHoEeqs5C35K2UUqR7G0"} diff --git a/internal/ai/llm/anthropic/testdata/messages_max_tokens.json b/internal/ai/llm/anthropic/testdata/messages_max_tokens.json new file mode 100644 index 0000000..64f86c1 --- /dev/null +++ b/internal/ai/llm/anthropic/testdata/messages_max_tokens.json @@ -0,0 +1,12 @@ +{ + "id": "msg_01XFDUDYJgAACzvnptvVoYEL", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5", + "content": [ + {"type": "text", "text": "Here are some Side quests for Our kitchen"} + ], + "stop_reason": "max_tokens", + "stop_sequence": null, + "usage": {"input_tokens": 472, "output_tokens": 400} +} diff --git a/internal/ai/llm/anthropic/testdata/messages_text.json b/internal/ai/llm/anthropic/testdata/messages_text.json new file mode 100644 index 0000000..16a947d --- /dev/null +++ b/internal/ai/llm/anthropic/testdata/messages_text.json @@ -0,0 +1,18 @@ +{ + "id": "msg_013Zva2CMHLNnXjNJJKqJ2EF", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5", + "content": [ + {"type": "text", "text": "Try a crock pot chili: "}, + {"type": "text", "text": "it is a Quick dinner."} + ], + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": { + "input_tokens": 2095, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "output_tokens": 503 + } +} diff --git a/internal/ai/llm/anthropic/testdata/messages_tool_use.json b/internal/ai/llm/anthropic/testdata/messages_tool_use.json new file mode 100644 index 0000000..5a092b3 --- /dev/null +++ b/internal/ai/llm/anthropic/testdata/messages_tool_use.json @@ -0,0 +1,22 @@ +{ + "id": "msg_01Aq9w938a90dw8q", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5", + "content": [ + { + "type": "tool_use", + "id": "toolu_01A09q90qw90lq917835lq9", + "name": "side_quests", + "input": {"dinners": [{"name": "Sheet-pan gnocchi", "why": "Like your Regular tray bake, but quicker."}]} + } + ], + "stop_reason": "tool_use", + "stop_sequence": null, + "usage": { + "input_tokens": 472, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "output_tokens": 61 + } +} diff --git a/internal/ai/llm/anthropic/testdata/models_page1.json b/internal/ai/llm/anthropic/testdata/models_page1.json new file mode 100644 index 0000000..5d7c9ba --- /dev/null +++ b/internal/ai/llm/anthropic/testdata/models_page1.json @@ -0,0 +1,13 @@ +{ + "data": [ + { + "type": "model", + "id": "claude-sonnet-5", + "display_name": "Claude Sonnet 5", + "created_at": "2026-02-24T00:00:00Z" + } + ], + "has_more": true, + "first_id": "claude-sonnet-5", + "last_id": "claude-sonnet-5" +} diff --git a/internal/ai/llm/anthropic/testdata/models_page2.json b/internal/ai/llm/anthropic/testdata/models_page2.json new file mode 100644 index 0000000..126cf98 --- /dev/null +++ b/internal/ai/llm/anthropic/testdata/models_page2.json @@ -0,0 +1,18 @@ +{ + "data": [ + { + "type": "model", + "id": "claude-haiku-4-5-20251001", + "display_name": "Claude Haiku 4.5", + "created_at": "2025-10-01T00:00:00Z" + }, + { + "type": "model", + "id": "claude-opus-5-5", + "created_at": "2026-05-01T00:00:00Z" + } + ], + "has_more": false, + "first_id": "claude-haiku-4-5-20251001", + "last_id": "claude-opus-5-5" +} diff --git a/internal/ai/llm/errors.go b/internal/ai/llm/errors.go index 07fddef..e4f1bd2 100644 --- a/internal/ai/llm/errors.go +++ b/internal/ai/llm/errors.go @@ -58,6 +58,9 @@ func (e *RateLimitedError) Error() string { return fmt.Sprintf("%s: retry after %v", ErrRateLimited, e.RetryAfter) } +// GoString keeps %#v to the redacted Error text instead of the fields. +func (e *RateLimitedError) GoString() string { return e.Error() } + // Is reports whether target is ErrRateLimited. func (e *RateLimitedError) Is(target error) bool { return target == ErrRateLimited } @@ -128,5 +131,9 @@ func (e *ProviderError) Body() string { return e.body } func (e *ProviderError) Error() string { return e.text } +// GoString keeps %#v to the redacted Error text instead of the fields, +// whose names could spell the credential. +func (e *ProviderError) GoString() string { return e.text } + // Is reports whether target is ErrProvider. func (e *ProviderError) Is(target error) bool { return target == ErrProvider } diff --git a/internal/ai/llm/openai/openai.go b/internal/ai/llm/openai/openai.go index e6493de..4f21be7 100644 --- a/internal/ai/llm/openai/openai.go +++ b/internal/ai/llm/openai/openai.go @@ -15,6 +15,7 @@ import ( "errors" "fmt" "io" + "math" "net" "net/http" "net/textproto" @@ -132,7 +133,7 @@ func newFromConfig(cfg Config) (*Provider, error) { model: cfg.Model, headers: headers, mode: cfg.StructuredOutput, - client: &http.Client{}, + client: noRedirectClient(), }, nil } @@ -462,10 +463,15 @@ func retryAfter(value string, now time.Time) time.Duration { if value == "" { return 0 } - if secs, err := strconv.Atoi(value); err == nil { + if secs, err := strconv.ParseInt(value, 10, 64); err == nil || errors.Is(err, strconv.ErrRange) { if secs < 0 { return 0 } + // time.Duration counts nanoseconds in an int64; a larger wait is + // capped to the most whole seconds it holds instead of overflowing. + if secs > maxRetryAfterSeconds { + secs = maxRetryAfterSeconds + } return time.Duration(secs) * time.Second } if at, err := http.ParseTime(value); err == nil && at.After(now) { @@ -474,6 +480,22 @@ func retryAfter(value string, now time.Time) time.Duration { return 0 } +// maxRetryAfterSeconds is the largest delay-seconds value retryAfter +// returns unchanged: the most whole seconds a time.Duration holds. +const maxRetryAfterSeconds = math.MaxInt64 / int64(time.Second) + +// noRedirectClient returns an HTTP client that never follows a redirect. +// A followed redirect would resend the credential headers, which net/http +// strips only for its own list of sensitive headers and only across hosts; +// instead the 3xx response is returned and becomes a provider error. +func noRedirectClient() *http.Client { + return &http.Client{ + CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }, + } +} + func isTimeout(err error) bool { var ne net.Error return errors.Is(err, context.DeadlineExceeded) || (errors.As(err, &ne) && ne.Timeout()) @@ -483,16 +505,14 @@ func isTimeout(err error) bool { // with the request URL removed and the API key redacted, and unwraps only to // the cause's category: llm.ErrTimeout, context.Canceled or nothing. type transportFailure struct { - detail string - kind error + text string + kind error } -func (e *transportFailure) Error() string { - if e.kind == llm.ErrTimeout { - return fmt.Sprintf("%s: transport: %s", llm.ErrTimeout, e.detail) - } - return "transport: " + e.detail -} +func (e *transportFailure) Error() string { return e.text } + +// GoString keeps %#v to the redacted Error text instead of the fields. +func (e *transportFailure) GoString() string { return e.text } func (e *transportFailure) Unwrap() error { return e.kind } @@ -511,7 +531,11 @@ func (p *Provider) transportError(err error) error { if errors.As(err, &ue) { err = ue.Err } - return &transportFailure{detail: llm.Redact(err.Error(), p.apiKey), kind: kind} + text := "transport: " + err.Error() + if kind == llm.ErrTimeout { + text = fmt.Sprintf("%s: %s", llm.ErrTimeout, text) + } + return &transportFailure{text: llm.Redact(text, p.apiKey), kind: kind} } // redactedError stands in for one layer of an error chain that holds the @@ -520,8 +544,8 @@ func (p *Provider) transportError(err error) error { // unwrapping has text holding the key. Is consults the original layer and // its chain, so sentinels still match. As does too. The llm typed errors // remove the key from their own complete text, fixed words included, so -// they always match; As refuses only a match from outside llm whose text -// holds the key, so it never hands back an unredacted error. +// they always match; As refuses a match whose text, or whose %#v form, +// holds the key, so it never hands back an error that can print it. type redactedError struct { text string err error @@ -530,6 +554,9 @@ type redactedError struct { func (e *redactedError) Error() string { return e.text } +// GoString keeps %#v from printing the fields, which hold the key. +func (e *redactedError) GoString() string { return e.text } + func (e *redactedError) Unwrap() error { return redact(errors.Unwrap(e.err), e.key) } func (e *redactedError) Is(target error) bool { return errors.Is(e.err, target) } @@ -539,7 +566,8 @@ func (e *redactedError) As(target any) bool { return false } found := reflect.ValueOf(target).Elem() - if matched, ok := found.Interface().(error); ok && holdsKey(matched, e.key) { + matched, isErr := found.Interface().(error) + if (isErr && holdsKey(matched, e.key)) || strings.Contains(fmt.Sprintf("%#v", found.Interface()), e.key) { found.SetZero() return false } @@ -561,12 +589,13 @@ func (e *redactedJoin) Unwrap() []error { return safe } -// redact is the backstop every exported method applies to its error: when -// the text of err or of any error in its chain holds a non-empty key, it -// returns a stand-in whose text, and the text of every error it unwraps -// to, has the key redacted. Any other err is returned unchanged. +// redact is the backstop every exported method applies to its error: with +// a non-empty key, it returns a stand-in whose text, and the text of every +// error it unwraps to, has the key redacted. Every non-nil err is wrapped, +// even one whose text lacks the key, because %#v of an unwrapped error +// prints its struct field names, and a key may spell one. func redact(err error, key string) error { - if err == nil || key == "" || !holdsKey(err, key) { + if err == nil || key == "" { return err } re := redactedError{text: llm.Redact(err.Error(), key), err: err, key: key} diff --git a/internal/ai/llm/openai/openai_test.go b/internal/ai/llm/openai/openai_test.go index 8f0a248..c953fe1 100644 --- a/internal/ai/llm/openai/openai_test.go +++ b/internal/ai/llm/openai/openai_test.go @@ -779,6 +779,9 @@ func TestRetryAfter(t *testing.T) { "soon": 0, "Sat, 03 Oct 2026 12:02:00 GMT": 2 * time.Minute, "Sat, 03 Oct 2026 11:59:00 GMT": 0, + "9223372036854775807": time.Duration(maxRetryAfterSeconds) * time.Second, + "99999999999999999999": time.Duration(maxRetryAfterSeconds) * time.Second, + "-99999999999999999999": 0, } for in, want := range cases { if got := retryAfter(in, now); got != want { @@ -882,8 +885,8 @@ func TestRedactKeepsChainAndHidesKey(t *testing.T) { t.Fatal("redact changed an error for the empty key") } clean := errors.New("openai: generate: no model configured") - if redact(clean, testKey) != clean { - t.Fatal("redact wrapped an error that holds no key") + if got := redact(clean, testKey); got.Error() != clean.Error() || !errors.Is(got, clean) { + t.Fatalf("redact(%v) = %v, want the same text and chain", clean, got) } // A key inside the redaction marker would survive replacement. if got := redact(errors.New("x E y"), "E").Error(); strings.Contains(got, "E") { @@ -938,7 +941,7 @@ func TestRedactMakesEveryUnwrappedLayerSafe(t *testing.T) { fmt.Errorf("openai: generate: %w", inner), fmt.Errorf("openai: generate: %w", hidden), fmt.Errorf("openai: generate: %w", joined), - fmt.Errorf("openai: %w and %w", inner, &transportFailure{detail: "slow", kind: llm.ErrTimeout}), + fmt.Errorf("openai: %w and %w", inner, &transportFailure{text: "llm: request timed out: transport: slow", kind: llm.ErrTimeout}), } for _, c := range cases { err := redact(c, testKey) @@ -964,7 +967,7 @@ func TestRedactMakesEveryUnwrappedLayerSafe(t *testing.T) { if !errors.Is(err, llm.ErrAuth) { t.Fatalf("errors.Is lost ErrAuth in %v", err) } - err = redact(fmt.Errorf("openai %s: %w", testKey, &transportFailure{detail: "slow", kind: llm.ErrTimeout}), testKey) + err = redact(fmt.Errorf("openai %s: %w", testKey, &transportFailure{text: "llm: request timed out: transport: slow", kind: llm.ErrTimeout}), testKey) if !errors.Is(err, llm.ErrTimeout) { t.Fatalf("errors.Is lost ErrTimeout in %v", err) } @@ -1119,3 +1122,134 @@ func TestGenerateWithAdapterNameKeyNeverLeaksKey(t *testing.T) { assertChainNoExactKey(t, err, key) }) } + +// TestRedactedErrorsFormatWithoutKey formats every error type that holds the +// key, or a cause holding it, with %v, %+v and %#v; %#v prints GoString. +func TestRedactedErrorsFormatWithoutKey(t *testing.T) { + const key = "sk-distinctive-openai-format-7c1e" + single := redact(fmt.Errorf("openai %s: %w", key, llm.ErrBadOutput), key) + joined := redact(errors.Join(fmt.Errorf("decode %s: %w", key, llm.ErrAuth), llm.NewProviderError(500, []byte("down "+key), key)), key) + if _, ok := single.(*redactedError); !ok { + t.Fatalf("single = %T, want *redactedError", single) + } + if _, ok := joined.(*redactedJoin); !ok { + t.Fatalf("joined = %T, want *redactedJoin", joined) + } + p := &Provider{apiKey: key} + transport := p.transportError(fmt.Errorf("dial %s: %w", key, context.DeadlineExceeded)) + for _, err := range []error{single, joined, transport} { + for _, verb := range []string{"%v", "%+v", "%#v"} { + if got := fmt.Sprintf(verb, err); strings.Contains(got, key) { + t.Fatalf("%s of %T leaks the key: %q", verb, err, got) + } + } + } +} + +func TestRateLimitedHugeRetryAfterIsCapped(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Retry-After", "9223372036854775807") + w.WriteHeader(http.StatusTooManyRequests) + })) + t.Cleanup(srv.Close) + p, err := New(Config{BaseURL: srv.URL + "/v1", APIKey: testKey, Model: "test-model", Headers: map[string]string{"X-Title": testKey}, StructuredOutput: PromptOnly}) + if err != nil { + t.Fatal(err) + } + err = p.Verify(context.Background()) + var rl *llm.RateLimitedError + if !errors.Is(err, llm.ErrRateLimited) || !errors.As(err, &rl) || rl.RetryAfter != time.Duration(maxRetryAfterSeconds)*time.Second { + t.Fatalf("err = %v, want a RateLimitedError with RetryAfter capped to %v", err, time.Duration(maxRetryAfterSeconds)*time.Second) + } +} + +func TestRedirectIsNotFollowed(t *testing.T) { + var mu sync.Mutex + var secondCalls int + var secondHeader http.Header + second := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + mu.Lock() + secondCalls++ + secondHeader = req.Header.Clone() + mu.Unlock() + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(second.Close) + first := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + http.Redirect(w, req, second.URL+req.URL.Path, http.StatusFound) + })) + t.Cleanup(first.Close) + p, err := New(Config{BaseURL: first.URL + "/v1", APIKey: testKey, Model: "test-model", Headers: map[string]string{"X-Title": testKey}, StructuredOutput: PromptOnly}) + if err != nil { + t.Fatal(err) + } + + err = p.Verify(context.Background()) + var pe *llm.ProviderError + if !errors.Is(err, llm.ErrProvider) || !errors.As(err, &pe) || pe.Status() != http.StatusFound { + t.Fatalf("err = %v, want a ProviderError for HTTP 302", err) + } + mu.Lock() + defer mu.Unlock() + if secondCalls != 0 { + for name, values := range secondHeader { + for _, v := range values { + if strings.Contains(v, testKey) { + t.Errorf("redirect target saw the key in %s", name) + } + } + } + t.Fatalf("redirect target got %d requests, want 0", secondCalls) + } +} + +// assertGoStringNoKey checks that %#v of err, of every layer reached by +// unwrapping and of the typed errors reached by errors.As holds no key. +func assertGoStringNoKey(t *testing.T, err error, key string) { + t.Helper() + var walk func(error) + walk = func(e error) { + if e == nil { + return + } + if got := fmt.Sprintf("%#v", e); strings.Contains(got, key) { + t.Fatalf("%%#v of %T leaks %q: %q", e, key, got) + } + switch u := e.(type) { + case interface{ Unwrap() error }: + walk(u.Unwrap()) + case interface{ Unwrap() []error }: + for _, c := range u.Unwrap() { + walk(c) + } + } + } + walk(err) + var pe *llm.ProviderError + if errors.As(err, &pe) { + walk(pe) + } + var rl *llm.RateLimitedError + if errors.As(err, &rl) { + walk(rl) + } +} + +// TestFieldNameKeyNeverLeaksThroughGoString uses struct field names of the +// returned error types as the API key; %#v must not print any of them. +func TestFieldNameKeyNeverLeaksThroughGoString(t *testing.T) { + for _, key := range []string{"body", "status", "text", "RetryAfter", "msg", "err", "key", "kind"} { + _, err := keyServerWithKey(t, key, 500, "down").Generate(context.Background(), schemaRequest()) + var pe *llm.ProviderError + if !errors.As(err, &pe) || pe.Status() != 500 { + t.Fatalf("key %q: err = %v, want a 500 ProviderError", key, err) + } + assertGoStringNoKey(t, err, key) + err = keyServerWithKey(t, key, 429, "slow").Verify(context.Background()) + var rl *llm.RateLimitedError + if !errors.As(err, &rl) { + t.Fatalf("key %q: err = %v, want a RateLimitedError", key, err) + } + assertGoStringNoKey(t, err, key) + } +} -- 2.55.0 From 32ab0c0b6d228d085fb9d97e791306652c32cf33 Mon Sep 17 00:00:00 2001 From: dev_agent Date: Sun, 4 Oct 2026 08:47:05 -0400 Subject: [PATCH 4/7] Ollama native adapter (#41) Co-authored-by: dev_agent Co-committed-by: dev_agent --- internal/ai/llm/ollama/ollama.go | 325 +++++++++ internal/ai/llm/ollama/ollama_test.go | 615 ++++++++++++++++++ internal/ai/llm/ollama/testdata/README.md | 36 + .../ai/llm/ollama/testdata/chat_schema.json | 16 + .../ai/llm/ollama/testdata/chat_text.json | 16 + .../testdata/error_model_not_found.json | 1 + internal/ai/llm/ollama/testdata/tags.json | 34 + 7 files changed, 1043 insertions(+) create mode 100644 internal/ai/llm/ollama/ollama.go create mode 100644 internal/ai/llm/ollama/ollama_test.go create mode 100644 internal/ai/llm/ollama/testdata/README.md create mode 100644 internal/ai/llm/ollama/testdata/chat_schema.json create mode 100644 internal/ai/llm/ollama/testdata/chat_text.json create mode 100644 internal/ai/llm/ollama/testdata/error_model_not_found.json create mode 100644 internal/ai/llm/ollama/testdata/tags.json diff --git a/internal/ai/llm/ollama/ollama.go b/internal/ai/llm/ollama/ollama.go new file mode 100644 index 0000000..e284a13 --- /dev/null +++ b/internal/ai/llm/ollama/ollama.go @@ -0,0 +1,325 @@ +// Package ollama is the llm.Provider adapter for Ollama's native API. It +// calls POST {base}/api/chat and GET {base}/api/tags over plain net/http. +// +// Ollama takes no credentials: the adapter sends no Authorization header +// and never logs. The base URL may be a LAN address, and errors for an +// unreachable server name it, so New refuses a base URL that could carry a +// credential. +package ollama + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "strings" + + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm" +) + +// Config configures a Provider. +type Config struct { + // BaseURL is the Ollama server root the /api endpoint paths are + // appended to, e.g. "http://192.168.1.20:11434" or + // "http://ollama:11434". It is required. + BaseURL string + // Model is the model Generate requests, e.g. "llama3.2". ListModels + // and Verify do not need it. + Model string +} + +// Provider is an Ollama native API llm.Provider. +type Provider struct { + base string + model string + client *http.Client +} + +var _ llm.Provider = (*Provider)(nil) + +// New validates cfg and returns a Provider. +func New(cfg Config) (*Provider, error) { + // Base URL errors never echo the URL: a rejected one may hold a + // credential. + u, err := url.Parse(cfg.BaseURL) + if err != nil { + return nil, errors.New("ollama: base URL does not parse") + } + if (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + return nil, errors.New("ollama: base URL must be an absolute http or https URL") + } + // A trailing "?" or "#" parses to an empty query or fragment, yet the + // endpoint path appended to it would land in the query or fragment. + if u.User != nil || strings.ContainsAny(cfg.BaseURL, "?#") { + return nil, errors.New("ollama: base URL must not carry credentials, a query or a fragment") + } + return &Provider{ + base: strings.TrimRight(cfg.BaseURL, "/"), + model: cfg.Model, + client: noRedirectClient(), + }, nil +} + +// maxResponseBody caps a successful response body; a larger one is an error. +const maxResponseBody = 16 << 20 + +// maxErrorBody caps how much of an error response body is read; only an +// llm.MaxBodyExcerpt of it is kept. +const maxErrorBody = 64 << 10 + +type message struct { + Role string `json:"role"` + Content string `json:"content"` +} + +type options struct { + Temperature *float64 `json:"temperature,omitempty"` + NumPredict int `json:"num_predict,omitempty"` +} + +type chatRequest struct { + Model string `json:"model"` + Messages []message `json:"messages"` + Stream bool `json:"stream"` + Format json.RawMessage `json:"format,omitempty"` + Options *options `json:"options,omitempty"` +} + +type chatResponse struct { + Model string `json:"model"` + Message *struct { + Role string `json:"role"` + Content string `json:"content"` + } `json:"message"` + Done bool `json:"done"` + DoneReason string `json:"done_reason"` + PromptEvalCount int `json:"prompt_eval_count"` + EvalCount int `json:"eval_count"` +} + +// Generate sends req to {base}/api/chat without streaming. A Request with a +// Schema sends it as "format", so Ollama constrains the reply to it; +// Response.JSON is then the reply text, which must be a JSON object. +func (p *Provider) Generate(ctx context.Context, req llm.Request) (llm.Response, error) { + if err := req.Validate(); err != nil { + return llm.Response{}, fmt.Errorf("ollama: generate: %w", err) + } + if p.model == "" { + return llm.Response{}, errors.New("ollama: generate: no model configured") + } + body := chatRequest{Model: p.model, Stream: false} + if req.System != "" { + body.Messages = append(body.Messages, message{Role: "system", Content: req.System}) + } + for _, m := range req.Messages { + body.Messages = append(body.Messages, message{Role: string(m.Role), Content: m.Content}) + } + if req.Schema != nil { + body.Format = req.Schema.Schema + } + if req.Temperature != nil || req.MaxTokens > 0 { + body.Options = &options{Temperature: req.Temperature, NumPredict: req.MaxTokens} + } + payload, err := json.Marshal(body) + if err != nil { + return llm.Response{}, fmt.Errorf("ollama: generate: encode request: %w", err) + } + + if req.Timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, req.Timeout) + defer cancel() + } + raw, err := p.do(ctx, http.MethodPost, "/api/chat", payload) + if err != nil { + var pe *llm.ProviderError + if errors.As(err, &pe) && pe.Status() == http.StatusNotFound && p.modelNotFound(pe.Body()) { + // The body is not quoted: the model name is all the caller + // needs. Any other 404 keeps its body excerpt below. + return llm.Response{}, fmt.Errorf("ollama: generate: model %q not found: %w", p.model, llm.NewProviderError(pe.Status(), nil, "")) + } + return llm.Response{}, fmt.Errorf("ollama: generate: %w", err) + } + + var cr chatResponse + if err := json.Unmarshal(raw, &cr); err != nil { + return llm.Response{}, fmt.Errorf("ollama: generate: decode response: %w", err) + } + if !cr.Done || cr.Message == nil { + return llm.Response{}, errors.New("ollama: generate: response is not a finished message") + } + resp := llm.Response{ + Text: cr.Message.Content, + Usage: llm.Usage{InputTokens: cr.PromptEvalCount, OutputTokens: cr.EvalCount}, + Model: cr.Model, + FinishReason: cr.DoneReason, + } + if req.Schema == nil { + return resp, nil + } + if text := strings.TrimSpace(resp.Text); isJSONObject(text) { + resp.JSON = json.RawMessage(text) + return resp, nil + } + return resp, fmt.Errorf("ollama: generate: reply is not a JSON object: %w", llm.ErrBadOutput) +} + +// modelNotFound reports whether body is Ollama's error for the configured +// model not being pulled: {"error": "model \"name\" not found, ..."}, or +// with the name in single quotes as newer servers send it. A 404 from +// anything else, such as a proxy without /api/chat, is not. +func (p *Provider) modelNotFound(body string) bool { + var e struct { + Error string `json:"error"` + } + if err := json.Unmarshal([]byte(body), &e); err != nil { + return false + } + return strings.HasPrefix(e.Error, fmt.Sprintf("model %q not found", p.model)) || + strings.HasPrefix(e.Error, fmt.Sprintf("model '%s' not found", p.model)) +} + +// isJSONObject reports whether s is exactly one JSON object. +func isJSONObject(s string) bool { + return strings.HasPrefix(s, "{") && json.Valid([]byte(s)) +} + +type tagsResponse struct { + Models []struct { + Name string `json:"name"` + } `json:"models"` +} + +// ListModels returns every model {base}/api/tags lists. +func (p *Provider) ListModels(ctx context.Context) ([]llm.Model, error) { + models, err := p.tags(ctx) + if err != nil { + return nil, fmt.Errorf("ollama: list models: %w", err) + } + return models, nil +} + +// Verify checks that the server answers GET {base}/api/tags with a model +// list. +func (p *Provider) Verify(ctx context.Context) error { + if _, err := p.tags(ctx); err != nil { + return fmt.Errorf("ollama: verify: %w", err) + } + return nil +} + +func (p *Provider) tags(ctx context.Context) ([]llm.Model, error) { + raw, err := p.do(ctx, http.MethodGet, "/api/tags", nil) + if err != nil { + return nil, err + } + var tr tagsResponse + if err := json.Unmarshal(raw, &tr); err != nil { + return nil, fmt.Errorf("decode response: %w", err) + } + if tr.Models == nil { + return nil, errors.New("response has no models list") + } + models := make([]llm.Model, 0, len(tr.Models)) + for i, m := range tr.Models { + if m.Name == "" { + return nil, fmt.Errorf("model %d has no name", i) + } + models = append(models, llm.Model{ID: m.Name, DisplayName: m.Name}) + } + return models, nil +} + +// do sends one request and returns the body of a 2xx response. Any other +// status, a 3xx included, is an *llm.ProviderError. +func (p *Provider) do(ctx context.Context, method, path string, payload []byte) ([]byte, error) { + var reqBody io.Reader + if payload != nil { + reqBody = bytes.NewReader(payload) + } + httpReq, err := http.NewRequestWithContext(ctx, method, p.base+path, reqBody) + if err != nil { + return nil, fmt.Errorf("build request: %w", err) + } + httpReq.Header.Set("Accept", "application/json") + if payload != nil { + httpReq.Header.Set("Content-Type", "application/json") + } + + httpResp, err := p.client.Do(httpReq) + if err != nil { + return nil, p.transportError(err) + } + defer httpResp.Body.Close() + + if httpResp.StatusCode < 200 || httpResp.StatusCode > 299 { + body, err := io.ReadAll(io.LimitReader(httpResp.Body, maxErrorBody)) + if err != nil { + return nil, p.transportError(err) + } + return nil, llm.NewProviderError(httpResp.StatusCode, body, "") + } + body, err := io.ReadAll(io.LimitReader(httpResp.Body, maxResponseBody+1)) + if err != nil { + return nil, p.transportError(err) + } + if len(body) > maxResponseBody { + return nil, fmt.Errorf("response body exceeds %d bytes", maxResponseBody) + } + return body, nil +} + +// noRedirectClient returns an HTTP client that never follows a redirect: +// the 3xx response is returned and becomes a provider error, so a request +// is only ever sent to the configured server. +func noRedirectClient() *http.Client { + return &http.Client{ + CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }, + } +} + +func isTimeout(err error) bool { + var ne net.Error + return errors.Is(err, context.DeadlineExceeded) || (errors.As(err, &ne) && ne.Timeout()) +} + +// transportFailure is a failed HTTP exchange. Its text names the base URL +// and the cause; it unwraps only to the cause's category, llm.ErrTimeout or +// context.Canceled. +type transportFailure struct { + text string + kind error +} + +func (e *transportFailure) Error() string { return e.text } + +func (e *transportFailure) Unwrap() error { return e.kind } + +// transportError reports a failure to complete the HTTP exchange. A +// cancelled context is the caller's choice and stays context.Canceled. +// Every other failure, a refused connection or a timeout alike, means the +// server could not be reached in time: it matches llm.ErrTimeout and says +// "Ollama not reachable at {base}". +func (p *Provider) transportError(err error) error { + timedOut := isTimeout(err) + // The *url.Error repeats the request URL; the base is named once below. + var ue *url.Error + if errors.As(err, &ue) { + err = ue.Err + } + if !timedOut && errors.Is(err, context.Canceled) { + return &transportFailure{text: "request canceled: " + err.Error(), kind: context.Canceled} + } + text := fmt.Sprintf("Ollama not reachable at %s: %s", p.base, err) + if timedOut { + text = fmt.Sprintf("Ollama not reachable at %s: %s: %s", p.base, llm.ErrTimeout, err) + } + return &transportFailure{text: text, kind: llm.ErrTimeout} +} diff --git a/internal/ai/llm/ollama/ollama_test.go b/internal/ai/llm/ollama/ollama_test.go new file mode 100644 index 0000000..ec0b434 --- /dev/null +++ b/internal/ai/llm/ollama/ollama_test.go @@ -0,0 +1,615 @@ +package ollama + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm" +) + +// recorded is one request the test server received. +type recorded struct { + method string + path string + query url.Values + header http.Header + body []byte +} + +// reply is a response the test server replays. +type reply struct { + status int + fixture string // testdata file + body string // used when fixture is empty + delay time.Duration +} + +// ollamaServer replays its replies in order, repeating the last, and +// records every request. +type ollamaServer struct { + *httptest.Server + mu sync.Mutex + got []recorded +} + +func newOllamaServer(t *testing.T, replies ...reply) *ollamaServer { + t.Helper() + payloads := make([][]byte, len(replies)) + for i, r := range replies { + payloads[i] = []byte(r.body) + if r.fixture != "" { + raw, err := os.ReadFile(filepath.Join("testdata", r.fixture)) + if err != nil { + t.Fatal(err) + } + payloads[i] = raw + } + } + s := &ollamaServer{} + s.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + body, _ := io.ReadAll(req.Body) + s.mu.Lock() + n := len(s.got) + s.got = append(s.got, recorded{req.Method, req.URL.Path, req.URL.Query(), req.Header.Clone(), body}) + s.mu.Unlock() + i := min(n, len(replies)-1) + r := replies[i] + if r.delay > 0 { + select { + case <-time.After(r.delay): + case <-req.Context().Done(): + return + } + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(r.status) + w.Write(payloads[i]) + })) + t.Cleanup(s.Close) + return s +} + +func (s *ollamaServer) requests() []recorded { + s.mu.Lock() + defer s.mu.Unlock() + return append([]recorded(nil), s.got...) +} + +func (s *ollamaServer) only(t *testing.T) recorded { + t.Helper() + got := s.requests() + if len(got) != 1 { + t.Fatalf("server got %d requests, want 1", len(got)) + } + return got[0] +} + +func newProvider(t *testing.T, base string) *Provider { + t.Helper() + p, err := New(Config{BaseURL: base, Model: "llama3.2"}) + if err != nil { + t.Fatal(err) + } + return p +} + +var dinnersSchema = json.RawMessage(`{"type":"object","properties":{"dinners":{"type":"array"}},"required":["dinners"]}`) + +func schemaRequest() llm.Request { + temp := 0.0 + return llm.Request{ + System: "You plan dinners for Our kitchen.", + Messages: []llm.Message{ + {Role: llm.RoleUser, Content: "Suggest a Side quest."}, + {Role: llm.RoleAssistant, Content: "Something quick?"}, + {Role: llm.RoleUser, Content: "Yes, a Quick dinner."}, + }, + Temperature: &temp, + MaxTokens: 400, + Schema: &llm.JSONSchema{Name: "side_quests", Schema: dinnersSchema}, + } +} + +func textRequest() llm.Request { + return llm.Request{Messages: []llm.Message{{Role: llm.RoleUser, Content: "What should we cook tonight?"}}} +} + +// sentBody decodes a request body into a generic map. +func sentBody(t *testing.T, r recorded) map[string]any { + t.Helper() + var m map[string]any + if err := json.Unmarshal(r.body, &m); err != nil { + t.Fatalf("request body %q: %v", r.body, err) + } + return m +} + +// canonical re-encodes JSON so two encodings of one value compare equal. +func canonical(t *testing.T, raw []byte) string { + t.Helper() + var v any + if err := json.Unmarshal(raw, &v); err != nil { + t.Fatalf("JSON %q: %v", raw, err) + } + out, _ := json.Marshal(v) + return string(out) +} + +func assertNoAuth(t *testing.T, r recorded) { + t.Helper() + if got := r.header.Values("Authorization"); len(got) != 0 { + t.Errorf("Authorization = %q, want none", got) + } +} + +func TestGenerateText(t *testing.T) { + s := newOllamaServer(t, reply{status: 200, fixture: "chat_text.json"}) + p := newProvider(t, s.URL+"/") + + resp, err := p.Generate(context.Background(), textRequest()) + if err != nil { + t.Fatal(err) + } + if resp.Text != "Try a crock pot chili: it is a Quick dinner." || resp.JSON != nil { + t.Errorf("Text, JSON = %q, %s", resp.Text, resp.JSON) + } + if resp.Usage != (llm.Usage{InputTokens: 26, OutputTokens: 298}) || resp.Model != "llama3.2" || resp.FinishReason != "stop" { + t.Errorf("Usage, Model, FinishReason = %+v, %q, %q", resp.Usage, resp.Model, resp.FinishReason) + } + + got := s.only(t) + if got.method != http.MethodPost || got.path != "/api/chat" || len(got.query) != 0 { + t.Errorf("request = %s %s?%s", got.method, got.path, got.query.Encode()) + } + assertNoAuth(t, got) + body := sentBody(t, got) + if stream, ok := body["stream"]; !ok || stream != false { + t.Errorf("stream = %v (present %v), want false", stream, ok) + } + if body["model"] != "llama3.2" { + t.Errorf("model = %v", body["model"]) + } + for _, field := range []string{"format", "options"} { + if v, ok := body[field]; ok { + t.Errorf("%s = %v, want it omitted", field, v) + } + } + if got, want := canonical(t, mustMarshal(t, body["messages"])), `[{"content":"What should we cook tonight?","role":"user"}]`; got != want { + t.Errorf("messages = %s, want %s", got, want) + } +} + +func mustMarshal(t *testing.T, v any) []byte { + t.Helper() + raw, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + return raw +} + +func TestGenerateSchemaOutput(t *testing.T) { + s := newOllamaServer(t, reply{status: 200, fixture: "chat_schema.json"}) + p := newProvider(t, s.URL) + + resp, err := p.Generate(context.Background(), schemaRequest()) + if err != nil { + t.Fatal(err) + } + want := `{"dinners":[{"name":"Sheet-pan gnocchi","why":"Like your Regular tray bake, but quicker."}]}` + if got := canonical(t, resp.JSON); got != want { + t.Errorf("Response.JSON = %s, want %s", got, want) + } + if canonical(t, []byte(resp.Text)) != want { + t.Errorf("Text = %q, want the reply content", resp.Text) + } + if resp.Usage != (llm.Usage{InputTokens: 147, OutputTokens: 41}) || resp.FinishReason != "stop" { + t.Errorf("Usage, FinishReason = %+v, %q", resp.Usage, resp.FinishReason) + } + + got := s.only(t) + assertNoAuth(t, got) + body := sentBody(t, got) + if body["stream"] != false { + t.Errorf("stream = %v, want false", body["stream"]) + } + if got, want := canonical(t, mustMarshal(t, body["format"])), canonical(t, dinnersSchema); got != want { + t.Errorf("format = %s, want the schema %s", got, want) + } + opts, _ := body["options"].(map[string]any) + if len(opts) != 2 || opts["temperature"] != 0.0 || opts["num_predict"] != 400.0 { + t.Errorf("options = %v, want temperature 0 and num_predict 400", body["options"]) + } + wantMessages := `[{"content":"You plan dinners for Our kitchen.","role":"system"},` + + `{"content":"Suggest a Side quest.","role":"user"},` + + `{"content":"Something quick?","role":"assistant"},` + + `{"content":"Yes, a Quick dinner.","role":"user"}]` + if got := canonical(t, mustMarshal(t, body["messages"])); got != wantMessages { + t.Errorf("messages = %s, want %s", got, wantMessages) + } +} + +func TestGenerateTemperatureOnlyWhenSet(t *testing.T) { + s := newOllamaServer(t, reply{status: 200, fixture: "chat_text.json"}) + req := textRequest() + temp := 0.7 + req.Temperature = &temp + if _, err := newProvider(t, s.URL).Generate(context.Background(), req); err != nil { + t.Fatal(err) + } + opts, _ := sentBody(t, s.only(t))["options"].(map[string]any) + if len(opts) != 1 || opts["temperature"] != 0.7 { + t.Errorf("options = %v, want only temperature 0.7", opts) + } +} + +func TestGenerateBadOutput(t *testing.T) { + cases := map[string]string{ + "prose": `{"model":"m","message":{"role":"assistant","content":"Sure! Try chili."},"done":true,"done_reason":"stop"}`, + "array": `{"model":"m","message":{"role":"assistant","content":"[1,2]"},"done":true,"done_reason":"stop"}`, + "cut-off JSON": `{"model":"m","message":{"role":"assistant","content":"{\"dinners\": ["},"done":true,"done_reason":"length"}`, + } + for name, body := range cases { + t.Run(name, func(t *testing.T) { + s := newOllamaServer(t, reply{status: 200, body: body}) + resp, err := newProvider(t, s.URL).Generate(context.Background(), schemaRequest()) + if !errors.Is(err, llm.ErrBadOutput) { + t.Fatalf("err = %v, want ErrBadOutput", err) + } + if resp.JSON != nil || resp.Model != "m" || resp.Text == "" { + t.Fatalf("Response = %+v, want no JSON and the reply kept", resp) + } + }) + } +} + +func TestGenerateRejectsMalformedResponses(t *testing.T) { + for name, body := range map[string]string{ + "not JSON": `not json`, + "not finished": `{"model":"m","message":{"role":"assistant","content":"hi"},"done":false}`, + "no message": `{"model":"m","done":true}`, + } { + s := newOllamaServer(t, reply{status: 200, body: body}) + _, err := newProvider(t, s.URL).Generate(context.Background(), textRequest()) + if err == nil || errors.Is(err, llm.ErrBadOutput) || errors.Is(err, llm.ErrProvider) { + t.Errorf("%s: err = %v, want an untyped malformed-response error", name, err) + } + } +} + +func TestGenerateRejectsInvalidRequests(t *testing.T) { + s := newOllamaServer(t, reply{status: 200, fixture: "chat_text.json"}) + if _, err := newProvider(t, s.URL).Generate(context.Background(), llm.Request{}); err == nil { + t.Error("Generate accepted a request with no messages") + } + noModel, err := New(Config{BaseURL: s.URL}) + if err != nil { + t.Fatal(err) + } + if _, err := noModel.Generate(context.Background(), textRequest()); err == nil { + t.Error("Generate ran with no model configured") + } + if n := len(s.requests()); n != 0 { + t.Errorf("server got %d requests, want 0", n) + } +} + +func TestGenerateModelMissing(t *testing.T) { + s := newOllamaServer(t, reply{status: 404, fixture: "error_model_not_found.json"}) + p, err := New(Config{BaseURL: s.URL, Model: "llama9"}) + if err != nil { + t.Fatal(err) + } + _, err = p.Generate(context.Background(), textRequest()) + var pe *llm.ProviderError + if !errors.Is(err, llm.ErrProvider) || !errors.As(err, &pe) || pe.Status() != http.StatusNotFound { + t.Fatalf("err = %v, want a ProviderError for HTTP 404", err) + } + if errors.Is(err, llm.ErrTimeout) || errors.Is(err, llm.ErrBadOutput) { + t.Fatalf("err = %v matches an unrelated sentinel", err) + } + if want := `ollama: generate: model "llama9" not found: llm: provider error: HTTP 404`; err.Error() != want { + t.Fatalf("Error() = %q, want %q", err, want) + } + if pe.Body() != "" { + t.Fatalf("Body() = %q, want the response body left out", pe.Body()) + } +} + +func TestGenerateNotFoundUnrelatedToModel(t *testing.T) { + for name, body := range map[string]string{ + "proxy page": "404 page not found", + "other error": `{"error":"no route for /api/chat"}`, + "other model": `{"error":"model \"llama3\" not found, try pulling it first"}`, + "empty": "", + } { + t.Run(name, func(t *testing.T) { + s := newOllamaServer(t, reply{status: 404, body: body}) + p, err := New(Config{BaseURL: s.URL, Model: "llama9"}) + if err != nil { + t.Fatal(err) + } + _, err = p.Generate(context.Background(), textRequest()) + var pe *llm.ProviderError + if !errors.Is(err, llm.ErrProvider) || !errors.As(err, &pe) || pe.Status() != http.StatusNotFound { + t.Fatalf("err = %v, want a ProviderError for HTTP 404", err) + } + if strings.Contains(err.Error(), "not found") && !strings.Contains(body, "not found") { + t.Fatalf("Error() = %q claims something was not found", err) + } + if strings.Contains(err.Error(), "llama9") { + t.Fatalf("Error() = %q claims the model was not found", err) + } + if pe.Body() != strings.TrimSpace(body) { + t.Fatalf("Body() = %q, want the response body %q", pe.Body(), body) + } + }) + } +} + +func TestGenerateModelMissingSingleQuoted(t *testing.T) { + s := newOllamaServer(t, reply{status: 404, body: `{"error":"model 'llama9' not found"}`}) + p, err := New(Config{BaseURL: s.URL, Model: "llama9"}) + if err != nil { + t.Fatal(err) + } + _, err = p.Generate(context.Background(), textRequest()) + if want := `ollama: generate: model "llama9" not found: llm: provider error: HTTP 404`; err == nil || err.Error() != want { + t.Fatalf("err = %v, want %q", err, want) + } +} + +func TestListModels(t *testing.T) { + s := newOllamaServer(t, reply{status: 200, fixture: "tags.json"}) + models, err := newProvider(t, s.URL).ListModels(context.Background()) + if err != nil { + t.Fatal(err) + } + want := []llm.Model{{ID: "llama3.2:latest", DisplayName: "llama3.2:latest"}, {ID: "qwen3:8b", DisplayName: "qwen3:8b"}} + if len(models) != len(want) || models[0] != want[0] || models[1] != want[1] { + t.Fatalf("ListModels() = %+v, want %+v", models, want) + } + got := s.only(t) + if got.method != http.MethodGet || got.path != "/api/tags" || len(got.query) != 0 { + t.Errorf("request = %s %s?%s", got.method, got.path, got.query.Encode()) + } + assertNoAuth(t, got) + + empty := newOllamaServer(t, reply{status: 200, body: `{"models":[]}`}) + if models, err := newProvider(t, empty.URL).ListModels(context.Background()); err != nil || models == nil || len(models) != 0 { + t.Fatalf("empty list: %v, %v", models, err) + } + for name, body := range map[string]string{ + "no models list": `{}`, + "unnamed model": `{"models":[{"model":"x"}]}`, + "not JSON": ``, + } { + bad := newOllamaServer(t, reply{status: 200, body: body}) + if _, err := newProvider(t, bad.URL).ListModels(context.Background()); err == nil { + t.Errorf("%s: ListModels accepted it", name) + } + } +} + +func TestVerify(t *testing.T) { + ok := newOllamaServer(t, reply{status: 200, fixture: "tags.json"}) + if err := newProvider(t, ok.URL).Verify(context.Background()); err != nil { + t.Fatalf("Verify() = %v", err) + } + got := ok.only(t) + if got.method != http.MethodGet || got.path != "/api/tags" { + t.Errorf("Verify called %s %s", got.method, got.path) + } + assertNoAuth(t, got) + + bad := newOllamaServer(t, reply{status: 500, body: `{"error":"something went wrong"}`}) + err := newProvider(t, bad.URL).Verify(context.Background()) + var pe *llm.ProviderError + if !errors.As(err, &pe) || pe.Status() != 500 || !strings.Contains(err.Error(), "something went wrong") { + t.Fatalf("Verify() = %v, want a ProviderError for HTTP 500", err) + } + malformed := newOllamaServer(t, reply{status: 200, body: `{}`}) + if err := newProvider(t, malformed.URL).Verify(context.Background()); err == nil { + t.Error("Verify accepted a body with no models list") + } +} + +func TestProviderErrors(t *testing.T) { + cases := map[string]reply{ + "400": {status: 400, body: `{"error":"invalid format"}`}, + "500": {status: 500, body: `{"error":"llama runner process has terminated"}`}, + "503": {status: 503, body: `{"error":"server busy, please try again"}`}, + "huge": {status: 500, body: strings.Repeat("x", 1<<20)}, + } + for name, r := range cases { + t.Run(name, func(t *testing.T) { + s := newOllamaServer(t, r) + p := newProvider(t, s.URL) + _, genErr := p.Generate(context.Background(), schemaRequest()) + _, listErr := p.ListModels(context.Background()) + for _, err := range []error{genErr, listErr} { + var pe *llm.ProviderError + if !errors.As(err, &pe) || pe.Status() != r.status { + t.Fatalf("err = %v, want a ProviderError for HTTP %d", err, r.status) + } + if errors.Is(err, llm.ErrTimeout) || strings.Contains(err.Error(), "not found") { + t.Fatalf("err = %v, want only a provider error", err) + } + if len(err.Error()) > llm.MaxBodyExcerpt+200 { + t.Fatalf("error text has %d bytes, want a bounded excerpt", len(err.Error())) + } + } + }) + } +} + +// assertUnreachable checks the error a request to an unreachable base +// returns: it matches llm.ErrTimeout and names the base. +func assertUnreachable(t *testing.T, err error, base string) { + t.Helper() + if !errors.Is(err, llm.ErrTimeout) { + t.Fatalf("err = %v, want ErrTimeout", err) + } + if errors.Is(err, llm.ErrProvider) || errors.Is(err, context.Canceled) { + t.Fatalf("err = %v matches an unrelated sentinel", err) + } + if want := "Ollama not reachable at " + base; !strings.Contains(err.Error(), want) { + t.Fatalf("Error() = %q, want it to contain %q", err, want) + } + var ue *url.Error + if errors.As(err, &ue) { + t.Fatalf("error chain exposes the url.Error: %v", ue) + } +} + +func TestUnreachableHost(t *testing.T) { + closed := httptest.NewServer(http.NotFoundHandler()) + base := closed.URL + closed.Close() + + p := newProvider(t, base+"/") + _, genErr := p.Generate(context.Background(), schemaRequest()) + _, listErr := p.ListModels(context.Background()) + verifyErr := p.Verify(context.Background()) + for _, err := range []error{genErr, listErr, verifyErr} { + assertUnreachable(t, err, base) + if !strings.Contains(err.Error(), "connection refused") { + t.Errorf("Error() = %q, want the cause", err) + } + } + + dropped := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + conn, _, err := w.(http.Hijacker).Hijack() + if err != nil { + t.Error(err) + return + } + conn.Close() + })) + t.Cleanup(dropped.Close) + _, err := newProvider(t, dropped.URL).Generate(context.Background(), textRequest()) + assertUnreachable(t, err, dropped.URL) +} + +func TestTimeouts(t *testing.T) { + s := newOllamaServer(t, reply{status: 200, fixture: "chat_schema.json", delay: 5 * time.Second}) + p := newProvider(t, s.URL) + + req := schemaRequest() + req.Timeout = 50 * time.Millisecond + start := time.Now() + _, err := p.Generate(context.Background(), req) + assertUnreachable(t, err, s.URL) + if !strings.Contains(err.Error(), llm.ErrTimeout.Error()) { + t.Errorf("Error() = %q, want it to say the request timed out", err) + } + if elapsed := time.Since(start); elapsed > 2*time.Second { + t.Fatalf("Request.Timeout not honoured: took %v", elapsed) + } + + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + _, err = p.ListModels(ctx) + assertUnreachable(t, err, s.URL) + ctx, cancel = context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + assertUnreachable(t, p.Verify(ctx), s.URL) + + ctx, cancel = context.WithCancel(context.Background()) + cancel() + _, err = p.Generate(ctx, schemaRequest()) + if !errors.Is(err, context.Canceled) || errors.Is(err, llm.ErrTimeout) || strings.Contains(err.Error(), "not reachable") { + t.Fatalf("cancelled context: err = %v, want context.Canceled only", err) + } +} + +func TestRedirectIsNotFollowed(t *testing.T) { + var mu sync.Mutex + var secondCalls int + second := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + mu.Lock() + secondCalls++ + mu.Unlock() + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(second.Close) + first := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + http.Redirect(w, req, second.URL+req.URL.Path, http.StatusFound) + })) + t.Cleanup(first.Close) + p := newProvider(t, first.URL) + + _, genErr := p.Generate(context.Background(), textRequest()) + verifyErr := p.Verify(context.Background()) + for _, err := range []error{genErr, verifyErr} { + var pe *llm.ProviderError + if !errors.Is(err, llm.ErrProvider) || !errors.As(err, &pe) || pe.Status() != http.StatusFound { + t.Fatalf("err = %v, want a ProviderError for HTTP 302", err) + } + } + mu.Lock() + defer mu.Unlock() + if secondCalls != 0 { + t.Fatalf("redirect target got %d requests, want 0", secondCalls) + } +} + +func TestNewRejectsBadConfig(t *testing.T) { + const secret = "hunter2-secret" + cases := map[string]string{ + "empty base": "", + "relative base": "ollama:11434", + "host only": "192.168.1.20", + "ftp base": "ftp://192.168.1.20:11434", + "no host": "http://", + "credentials in URL": "http://user:" + secret + "@192.168.1.20:11434", + "user in URL": "http://" + secret + "@ollama:11434", + "secret in query": "http://ollama:11434/?token=" + secret, + "secret in fragment": "http://ollama:11434/#" + secret, + "unparseable": "http://ollama:11434/%zz?token=" + secret, + "empty query": "http://ollama:11434?", + "empty fragment": "http://ollama:11434#", + "empty both": "http://ollama:11434/?#", + } + for name, base := range cases { + _, err := New(Config{BaseURL: base, Model: "llama3.2"}) + if err == nil { + t.Errorf("%s: New() = nil error", name) + continue + } + if strings.Contains(err.Error(), secret) { + t.Errorf("%s: error echoes the URL: %v", name, err) + } + } + for _, base := range []string{ + "http://192.168.1.20:11434", + "http://ollama:11434", + "http://localhost:11434/", + "https://ollama.lan/proxy/", + } { + if _, err := New(Config{BaseURL: base, Model: "llama3.2"}); err != nil { + t.Errorf("New(%q) = %v", base, err) + } + } +} + +func TestBaseWithPath(t *testing.T) { + s := newOllamaServer(t, reply{status: 200, fixture: "tags.json"}) + if err := newProvider(t, s.URL+"/proxy/").Verify(context.Background()); err != nil { + t.Fatal(err) + } + if got := s.only(t).path; got != "/proxy/api/tags" { + t.Fatalf("path = %s, want /proxy/api/tags", got) + } +} diff --git a/internal/ai/llm/ollama/testdata/README.md b/internal/ai/llm/ollama/testdata/README.md new file mode 100644 index 0000000..6b7e194 --- /dev/null +++ b/internal/ai/llm/ollama/testdata/README.md @@ -0,0 +1,36 @@ +# Ollama adapter fixtures + +These are the native Ollama API response bodies the contract tests in +`ollama_test.go` replay. They are **not live captures**. Product AI is tested +with fakes and live vendor calls are opt-in, so no request was sent to an +Ollama server to make them. Each fixture is a sanitized adaptation of the +response shapes in Ollama's public API documentation +(): + +- Chat, non-streaming: `POST /api/chat` with `"stream": false`. +- Structured outputs: `POST /api/chat` with a JSON schema in `"format"`. +- List local models: `GET /api/tags`. +- Errors: a JSON `{"error": "..."}` body with a non-2xx status; a model + that is not pulled is HTTP 404. + +Sanitizing: + +- Timestamps, durations, sizes, digests and token counts are illustrative + values in the documented format, not values from a real server. +- Message content is replaced with MealQuest dinner text. +- Ollama takes no credentials, so none appear. + +Below, *not in the public example* marks values that were added for a test +case and do not come from the documented example. + +| Fixture | Case | Shape | Not in the public example | +| --- | --- | --- | --- | +| `chat_text.json` | Success | Finished chat message, `done_reason: stop`, `prompt_eval_count` and `eval_count`. | — | +| `chat_schema.json` | Schema output | Finished chat message whose content is a JSON object string. | The dinner object. | +| `error_model_not_found.json` | Model missing | `{"error": "model \"…\" not found, try pulling it first"}`, served with HTTP 404. | — | +| `tags.json` | Model list | `{"models": [...]}` with `name`, `model`, `digest` and `details`. | — | + +The unreachable-host case has no fixture: there is no response to record. +The tests close an `httptest` server and dial its address, so the +connection is refused, and they hold a reply past a short deadline for the +timeout. diff --git a/internal/ai/llm/ollama/testdata/chat_schema.json b/internal/ai/llm/ollama/testdata/chat_schema.json new file mode 100644 index 0000000..6fd8173 --- /dev/null +++ b/internal/ai/llm/ollama/testdata/chat_schema.json @@ -0,0 +1,16 @@ +{ + "model": "llama3.2", + "created_at": "2026-10-04T12:20:11.009132Z", + "message": { + "role": "assistant", + "content": "{\"dinners\": [{\"name\": \"Sheet-pan gnocchi\", \"why\": \"Like your Regular tray bake, but quicker.\"}]}" + }, + "done_reason": "stop", + "done": true, + "total_duration": 1416823291, + "load_duration": 12309083, + "prompt_eval_count": 147, + "prompt_eval_duration": 271802000, + "eval_count": 41, + "eval_duration": 1130948000 +} diff --git a/internal/ai/llm/ollama/testdata/chat_text.json b/internal/ai/llm/ollama/testdata/chat_text.json new file mode 100644 index 0000000..cbb988c --- /dev/null +++ b/internal/ai/llm/ollama/testdata/chat_text.json @@ -0,0 +1,16 @@ +{ + "model": "llama3.2", + "created_at": "2026-10-04T12:13:43.416799Z", + "message": { + "role": "assistant", + "content": "Try a crock pot chili: it is a Quick dinner." + }, + "done": true, + "done_reason": "stop", + "total_duration": 5191566416, + "load_duration": 2154458, + "prompt_eval_count": 26, + "prompt_eval_duration": 383809000, + "eval_count": 298, + "eval_duration": 4799921000 +} diff --git a/internal/ai/llm/ollama/testdata/error_model_not_found.json b/internal/ai/llm/ollama/testdata/error_model_not_found.json new file mode 100644 index 0000000..9b7a757 --- /dev/null +++ b/internal/ai/llm/ollama/testdata/error_model_not_found.json @@ -0,0 +1 @@ +{"error":"model \"llama9\" not found, try pulling it first"} diff --git a/internal/ai/llm/ollama/testdata/tags.json b/internal/ai/llm/ollama/testdata/tags.json new file mode 100644 index 0000000..d271139 --- /dev/null +++ b/internal/ai/llm/ollama/testdata/tags.json @@ -0,0 +1,34 @@ +{ + "models": [ + { + "name": "llama3.2:latest", + "model": "llama3.2:latest", + "modified_at": "2026-09-30T08:06:48.639712648-07:00", + "size": 2019393189, + "digest": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72", + "details": { + "parent_model": "", + "format": "gguf", + "family": "llama", + "families": ["llama"], + "parameter_size": "3.2B", + "quantization_level": "Q4_K_M" + } + }, + { + "name": "qwen3:8b", + "model": "qwen3:8b", + "modified_at": "2026-09-28T19:41:02.112408113-07:00", + "size": 5225388164, + "digest": "500a1f067a9f782620b40bee6f7b0c89e17ae61f686b92c24933e4ca4b2b8b41", + "details": { + "parent_model": "", + "format": "gguf", + "family": "qwen3", + "families": ["qwen3"], + "parameter_size": "8.2B", + "quantization_level": "Q4_K_M" + } + } + ] +} -- 2.55.0 From 9ace3aba9f2d2bc866f23c40a90455ddd002e262 Mon Sep 17 00:00:00 2001 From: dev_agent Date: Sun, 4 Oct 2026 09:18:34 -0400 Subject: [PATCH 5/7] Provider presets and factory (#44) Co-authored-by: dev_agent Co-committed-by: dev_agent --- internal/ai/llm/openai/openai.go | 4 + internal/ai/llm/openai/openai_test.go | 14 + internal/ai/llm/presets/presets.go | 319 ++++++++++++++++ internal/ai/llm/presets/presets_test.go | 472 ++++++++++++++++++++++++ internal/ai/llm/settings.go | 68 ++++ internal/store/migrate_test.go | 95 ++++- internal/store/schema_test.go | 21 +- migrations/0004_ai_provider_presets.sql | 68 ++++ 8 files changed, 1055 insertions(+), 6 deletions(-) create mode 100644 internal/ai/llm/presets/presets.go create mode 100644 internal/ai/llm/presets/presets_test.go create mode 100644 internal/ai/llm/settings.go create mode 100644 migrations/0004_ai_provider_presets.sql diff --git a/internal/ai/llm/openai/openai.go b/internal/ai/llm/openai/openai.go index 4f21be7..70c94e9 100644 --- a/internal/ai/llm/openai/openai.go +++ b/internal/ai/llm/openai/openai.go @@ -53,6 +53,7 @@ type Config struct { BaseURL string // APIKey is sent as "Authorization: Bearer ". Empty sends no // Authorization header, for endpoints such as Ollama that need none. + // It must not have a control character other than tab. APIKey string // Model is the model Generate requests. ListModels and Verify do not // need it. @@ -102,6 +103,9 @@ func newFromConfig(cfg Config) (*Provider, error) { if u.User != nil || strings.ContainsAny(cfg.BaseURL, "?#") { return nil, errors.New("openai: base URL must not carry credentials, a query or a fragment") } + if !validHeaderValue(cfg.APIKey) { + return nil, errors.New("openai: API key has a control character") + } // No error from New quotes caller-supplied text or a name derived from // it: any such text may equal or contain the API key, and redaction // cannot catch a fragment or a case-changed copy of it. diff --git a/internal/ai/llm/openai/openai_test.go b/internal/ai/llm/openai/openai_test.go index c953fe1..12f2c09 100644 --- a/internal/ai/llm/openai/openai_test.go +++ b/internal/ai/llm/openai/openai_test.go @@ -628,6 +628,20 @@ func TestNewRejectsBadHeaderWithoutEchoingValue(t *testing.T) { assertNoSecret(t, err) } +func TestNewRejectsKeyWithControlCharacter(t *testing.T) { + for _, key := range []string{testKey + "\n", testKey + "\r\nX-Injected: 1", "\x00" + testKey, testKey + "\x7f"} { + _, err := New(Config{BaseURL: "https://example.com/v1", APIKey: key, StructuredOutput: JSONSchema}) + if err == nil { + t.Errorf("New(%q) = nil error", key) + continue + } + assertNoSecret(t, err) + if strings.Contains(err.Error()+fmt.Sprintf("%#v", err), key) { + t.Errorf("error leaks the key: %v", err) + } + } +} + // TestNewConfigErrorsNeverLeakKey covers every New error that names // caller-supplied text, each with the API key embedded in that text. func TestNewConfigErrorsNeverLeakKey(t *testing.T) { diff --git a/internal/ai/llm/presets/presets.go b/internal/ai/llm/presets/presets.go new file mode 100644 index 0000000..d33d613 --- /dev/null +++ b/internal/ai/llm/presets/presets.go @@ -0,0 +1,319 @@ +// Package presets holds what MealQuest knows about each language-model +// vendor — base URL, adapter, credentials, headers, structured output mode +// and model quirks — and FromSettings, which turns the household's +// llm.Settings into a ready llm.Provider. +// +// FromSettings lives here rather than in package llm because it builds the +// adapters, and every adapter imports llm. +// +// Nothing in this package logs, and no error it returns or unwraps to +// contains the API key, even where the key equals a setting's or request's +// value or text the error spells anyway. +package presets + +import ( + "context" + "errors" + "fmt" + "math" + "regexp" + "slices" + + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm" + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm/anthropic" + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm/ollama" + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm/openai" +) + +// Adapter names the llm adapter package a preset builds. +type Adapter string + +// The adapters. +const ( + AdapterAnthropic Adapter = "anthropic" + AdapterOpenAI Adapter = "openai" + AdapterOllama Adapter = "ollama" +) + +// KeyPolicy is whether a preset sends an API key. +type KeyPolicy int + +// The key policies. +const ( + // KeyRequired presets fail without a key. + KeyRequired KeyPolicy = iota + // KeyOptional presets send a key when one is given. + KeyOptional + // KeyNone presets send no key and refuse one. + KeyNone +) + +// The reasoning models that reject the temperature field are OpenAI's o1 +// and o3 families: the bare family name or the name followed by a "-" +// variant (o1, o1-mini, o3-2025-04-16), never a longer word such as o123. +// OpenRouter names them with an "openai/" prefix and may add a ":" variant +// (openai/o1:online). A preset whose OmitTemperature holds a pattern never +// sends a temperature to a matching model. +var ( + openAIReasoningModels = regexp.MustCompile(`^o[13](-.*)?$`) + openRouterReasoningModels = regexp.MustCompile(`^openai/o[13]([-:].*)?$`) +) + +// Preset is one vendor's fixed configuration. +type Preset struct { + // Name is the Settings.Provider value that selects the preset. + Name string + // Adapter is the adapter the preset builds. + Adapter Adapter + // BaseURL is the API root the adapter appends its endpoint paths to. + // Empty means the household must supply it in Settings.BaseURL. + BaseURL string + // Key is whether the preset sends an API key. + Key KeyPolicy + // RequiresPublicURL means the preset needs Settings.PublicURL for its + // headers. + RequiresPublicURL bool + // Headers returns the extra request headers an openai-adapter preset + // sends; nil means none. + Headers func(llm.Settings) map[string]string + // StructuredOutput is the default mode of an openai-adapter preset. + StructuredOutput openai.StructuredOutput + // FallbackStructuredOutput is the mode the household can select when + // the default fails for its model; empty means none. FromSettings + // builds one mode and never retries a request in another. + FallbackStructuredOutput openai.StructuredOutput + // StructuredOutputs are the modes Settings.StructuredOutput may select. + StructuredOutputs []openai.StructuredOutput + // MaxTokens is the max_tokens the anthropic adapter sends when a + // Request sets none. + MaxTokens int + // OmitTemperature are model-id patterns the preset never sends a + // temperature to. + OmitTemperature []*regexp.Regexp +} + +// openRouterHeaders identify MealQuest to OpenRouter: HTTP-Referer is +// MEALQUEST_PUBLIC_URL and X-Title is the app name. +func openRouterHeaders(s llm.Settings) map[string]string { + return map[string]string{ + "HTTP-Referer": s.PublicURL, + "X-Title": "MealQuest", + } +} + +// omitsTemperature reports whether model matches one of the preset's +// OmitTemperature patterns. +func (p Preset) omitsTemperature(model string) bool { + return slices.ContainsFunc(p.OmitTemperature, func(re *regexp.Regexp) bool { + return re.MatchString(model) + }) +} + +// Lookup returns the preset Settings.Provider name selects. +func Lookup(name string) (Preset, bool) { + p, ok := all[name] + return p, ok +} + +// all is every preset, by Settings.Provider name. +var all = map[string]Preset{ + "anthropic": { + Name: "anthropic", + Adapter: AdapterAnthropic, + BaseURL: "https://api.anthropic.com", + Key: KeyRequired, + MaxTokens: 4096, + }, + "openai": { + Name: "openai", + Adapter: AdapterOpenAI, + BaseURL: "https://api.openai.com/v1", + Key: KeyRequired, + StructuredOutput: openai.JSONSchema, + StructuredOutputs: []openai.StructuredOutput{openai.JSONSchema}, + OmitTemperature: []*regexp.Regexp{openAIReasoningModels}, + }, + "openrouter": { + Name: "openrouter", + Adapter: AdapterOpenAI, + BaseURL: "https://openrouter.ai/api/v1", + Key: KeyRequired, + RequiresPublicURL: true, + Headers: openRouterHeaders, + StructuredOutput: openai.JSONSchema, + FallbackStructuredOutput: openai.JSONObject, + StructuredOutputs: []openai.StructuredOutput{openai.JSONSchema, openai.JSONObject}, + OmitTemperature: []*regexp.Regexp{openRouterReasoningModels}, + }, + "ollama": { + Name: "ollama", + Adapter: AdapterOllama, + Key: KeyNone, + }, + "openai_compatible": { + Name: "openai_compatible", + Adapter: AdapterOpenAI, + Key: KeyOptional, + StructuredOutput: openai.JSONSchema, + StructuredOutputs: []openai.StructuredOutput{openai.JSONSchema, openai.JSONObject, openai.PromptOnly}, + // The server may be OpenAI itself or a gateway that names models + // the OpenRouter way. + OmitTemperature: []*regexp.Regexp{openAIReasoningModels, openRouterReasoningModels}, + }, +} + +// FromSettings returns the Provider s describes, using key as its API key. +// It fails with a configuration error from package llm — ErrUnknownProvider, +// ErrMissingKey, ErrUnexpectedKey, ErrInvalidKey, ErrMissingBaseURL, ErrMissingModel, +// ErrMissingPublicURL or ErrInvalidSetting — before contacting any vendor. +// +// The Provider sends s.Temperature with every request whose Temperature is +// nil; a request that sets one keeps it. Neither is sent to a model the +// preset's OmitTemperature matches. +func FromSettings(s llm.Settings, key string) (llm.Provider, error) { + cp, err := fromSettings(s, key) + if err != nil { + return nil, redact(err, key) + } + return cp, nil +} + +func fromSettings(s llm.Settings, key string) (*configured, error) { + p, ok := Lookup(s.Provider) + if !ok { + return nil, llm.ErrUnknownProvider + } + if err := check(p, s, key); err != nil { + return nil, fmt.Errorf("%s: %w", p.Name, err) + } + base := p.BaseURL + if base == "" { + base = s.BaseURL + } + + var inner llm.Provider + var err error + switch p.Adapter { + case AdapterAnthropic: + inner, err = anthropic.New(anthropic.Config{BaseURL: base, APIKey: key, Model: s.Model, MaxTokens: p.MaxTokens}) + case AdapterOllama: + inner, err = ollama.New(ollama.Config{BaseURL: base, Model: s.Model}) + case AdapterOpenAI: + mode := p.StructuredOutput + if s.StructuredOutput != "" { + mode = openai.StructuredOutput(s.StructuredOutput) + } + cfg := openai.Config{BaseURL: base, APIKey: key, Model: s.Model, StructuredOutput: mode} + if p.Headers != nil { + cfg.Headers = p.Headers(s) + } + inner, err = openai.New(cfg) + default: + panic("presets: preset " + p.Name + " has unknown adapter " + string(p.Adapter)) + } + if err != nil { + return nil, fmt.Errorf("%s: %w: %w", p.Name, llm.ErrInvalidSetting, err) + } + + cp := &configured{Provider: inner, key: key} + if !p.omitsTemperature(s.Model) { + t := s.Temperature + cp.temperature = &t + } + return cp, nil +} + +// check validates s and key against p. Its errors never quote a value. +func check(p Preset, s llm.Settings, key string) error { + switch { + case p.Key == KeyRequired && key == "": + return llm.ErrMissingKey + case p.Key == KeyNone && key != "": + return llm.ErrUnexpectedKey + case !validHeaderValue(key): + return llm.ErrInvalidKey + } + switch { + case p.BaseURL == "" && s.BaseURL == "": + return llm.ErrMissingBaseURL + case p.BaseURL != "" && s.BaseURL != "" && s.BaseURL != p.BaseURL: + return fmt.Errorf("%w: base URL is fixed by the preset", llm.ErrInvalidSetting) + } + if s.Model == "" { + return llm.ErrMissingModel + } + if p.RequiresPublicURL && s.PublicURL == "" { + return llm.ErrMissingPublicURL + } + if s.StructuredOutput != "" && !slices.Contains(p.StructuredOutputs, openai.StructuredOutput(s.StructuredOutput)) { + return fmt.Errorf("%w: structured output mode is not supported by the preset", llm.ErrInvalidSetting) + } + if math.IsNaN(s.Temperature) || math.IsInf(s.Temperature, 0) || s.Temperature < 0 { + return fmt.Errorf("%w: temperature must be a non-negative number", llm.ErrInvalidSetting) + } + return nil +} + +// validHeaderValue reports whether value has no control character other +// than tab, so it can be sent as an HTTP header value. +func validHeaderValue(value string) bool { + for i := 0; i < len(value); i++ { + if c := value[i]; (c < 0x20 && c != '\t') || c == 0x7f { + return false + } + } + return true +} + +// configured applies the household temperature and the preset's +// temperature quirk to every Generate. +type configured struct { + llm.Provider + // key is the API key, redacted from validation errors; the adapter + // redacts its own. + key string + // temperature is sent when a request sets none; nil means the model + // rejects temperature, so none is ever sent. + temperature *float64 +} + +// Generate validates req as sent by the caller before changing its +// temperature, so clearing it for a model that rejects temperature never +// hides an invalid one. +func (c *configured) Generate(ctx context.Context, req llm.Request) (llm.Response, error) { + if err := req.Validate(); err != nil { + return llm.Response{}, redact(fmt.Errorf("presets: generate: %w", err), c.key) + } + if c.temperature == nil { + req.Temperature = nil + } else if req.Temperature == nil { + t := *c.temperature + req.Temperature = &t + } + return c.Provider.Generate(ctx, req) +} + +// redactedError is an error with the API key removed from its text. It +// does not unwrap, because every layer below may hold the key; Is still +// matches the sentinels the original error wraps. +type redactedError struct { + text string + err error +} + +func (e *redactedError) Error() string { return e.text } + +// GoString keeps %#v to the redacted Error text instead of the fields. +func (e *redactedError) GoString() string { return e.text } + +// Is reports whether the original error matches target. +func (e *redactedError) Is(target error) bool { return errors.Is(e.err, target) } + +// redact returns err with key removed from its text; a nil err or an empty +// key returns err unchanged. +func redact(err error, key string) error { + if err == nil || key == "" { + return err + } + return &redactedError{text: llm.Redact(err.Error(), key), err: err} +} diff --git a/internal/ai/llm/presets/presets_test.go b/internal/ai/llm/presets/presets_test.go new file mode 100644 index 0000000..47f98c5 --- /dev/null +++ b/internal/ai/llm/presets/presets_test.go @@ -0,0 +1,472 @@ +package presets + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + "testing" + + "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm" +) + +const testKey = "sk-test-0123456789" + +// captured is the HTTP request an adapter sent. +type captured struct { + url string + header http.Header + body map[string]json.RawMessage +} + +var errStop = errors.New("recorded") + +// recordingTransport records each request and fails it, so no request +// leaves the process. +type recordingTransport struct{ reqs []captured } + +func (rt *recordingTransport) RoundTrip(r *http.Request) (*http.Response, error) { + c := captured{url: r.URL.String(), header: r.Header.Clone()} + if r.Body != nil { + raw, err := io.ReadAll(r.Body) + if err != nil { + return nil, err + } + if err := json.Unmarshal(raw, &c.body); err != nil { + return nil, err + } + } + rt.reqs = append(rt.reqs, c) + return nil, errStop +} + +// generate builds the provider s describes and returns the request its +// Generate sends for req. The adapters use http.DefaultTransport, which the +// test replaces, so the real preset base URLs are never contacted. +func generate(t *testing.T, s llm.Settings, key string, req llm.Request) captured { + t.Helper() + rt := &recordingTransport{} + orig := http.DefaultTransport + http.DefaultTransport = rt + t.Cleanup(func() { http.DefaultTransport = orig }) + + p, err := FromSettings(s, key) + if err != nil { + t.Fatalf("FromSettings: %v", err) + } + if _, err := p.Generate(context.Background(), req); err == nil { + t.Fatal("Generate succeeded through the recording transport") + } + if len(rt.reqs) != 1 { + t.Fatalf("sent %d requests, want 1", len(rt.reqs)) + } + return rt.reqs[0] +} + +func plainRequest() llm.Request { + return llm.Request{Messages: []llm.Message{{Role: llm.RoleUser, Content: "dinner?"}}} +} + +func schemaRequest() llm.Request { + r := plainRequest() + r.Schema = &llm.JSONSchema{Name: "side_quests", Schema: json.RawMessage(`{"type":"object"}`)} + return r +} + +func ptr(f float64) *float64 { return &f } + +func responseFormatType(t *testing.T, c captured) string { + t.Helper() + raw, ok := c.body["response_format"] + if !ok { + return "" + } + var rf struct{ Type string } + if err := json.Unmarshal(raw, &rf); err != nil { + t.Fatalf("response_format: %v", err) + } + return rf.Type +} + +func temperature(t *testing.T, c captured) (float64, bool) { + t.Helper() + raw, ok := c.body["temperature"] + if !ok { + return 0, false + } + var f float64 + if err := json.Unmarshal(raw, &f); err != nil { + t.Fatalf("temperature: %v", err) + } + return f, true +} + +func TestAnthropicPreset(t *testing.T) { + c := generate(t, llm.Settings{Provider: "anthropic", Model: "claude-sonnet-5", Temperature: 0.7}, testKey, plainRequest()) + if c.url != "https://api.anthropic.com/v1/messages" { + t.Fatalf("URL = %q", c.url) + } + if got := c.header.Get("X-Api-Key"); got != testKey { + t.Fatalf("x-api-key = %q", got) + } + if got := c.header.Get("Authorization"); got != "" { + t.Fatalf("Authorization = %q, want none", got) + } + if got, ok := temperature(t, c); !ok || got != 0.7 { + t.Fatalf("temperature = %v, %v; want 0.7", got, ok) + } + if string(c.body["max_tokens"]) != "4096" { + t.Fatalf("max_tokens = %s", c.body["max_tokens"]) + } +} + +func TestAnthropicPresetAcceptsItsOwnBaseURL(t *testing.T) { + c := generate(t, llm.Settings{Provider: "anthropic", BaseURL: "https://api.anthropic.com", Model: "claude-sonnet-5"}, testKey, plainRequest()) + if c.url != "https://api.anthropic.com/v1/messages" { + t.Fatalf("URL = %q", c.url) + } +} + +func TestOpenAIPreset(t *testing.T) { + c := generate(t, llm.Settings{Provider: "openai", Model: "gpt-4o", Temperature: 0.4}, testKey, schemaRequest()) + if c.url != "https://api.openai.com/v1/chat/completions" { + t.Fatalf("URL = %q", c.url) + } + if got := c.header.Get("Authorization"); got != "Bearer "+testKey { + t.Fatalf("Authorization = %q", got) + } + if got := responseFormatType(t, c); got != "json_schema" { + t.Fatalf("response_format type = %q, want json_schema", got) + } + if c.header.Get("HTTP-Referer") != "" || c.header.Get("X-Title") != "" { + t.Fatal("openai sends OpenRouter headers") + } +} + +func TestOpenRouterPreset(t *testing.T) { + s := llm.Settings{Provider: "openrouter", Model: "openai/gpt-4o", PublicURL: "https://meals.example.com/"} + c := generate(t, s, testKey, schemaRequest()) + if c.url != "https://openrouter.ai/api/v1/chat/completions" { + t.Fatalf("URL = %q", c.url) + } + if got := c.header.Get("Authorization"); got != "Bearer "+testKey { + t.Fatalf("Authorization = %q", got) + } + if got := c.header.Get("HTTP-Referer"); got != "https://meals.example.com/" { + t.Fatalf("HTTP-Referer = %q", got) + } + if got := c.header.Get("X-Title"); got != "MealQuest" { + t.Fatalf("X-Title = %q", got) + } + if got := responseFormatType(t, c); got != "json_schema" { + t.Fatalf("default response_format type = %q, want json_schema", got) + } +} + +func TestOpenRouterFallbackModeIsJSONObject(t *testing.T) { + p, _ := Lookup("openrouter") + s := llm.Settings{ + Provider: "openrouter", + Model: "meta-llama/llama-3.3-70b-instruct", + PublicURL: "https://meals.example.com", + StructuredOutput: string(p.FallbackStructuredOutput), + } + c := generate(t, s, testKey, schemaRequest()) + if got := responseFormatType(t, c); got != "json_object" { + t.Fatalf("response_format type = %q, want json_object", got) + } +} + +func TestOllamaPreset(t *testing.T) { + s := llm.Settings{Provider: "ollama", BaseURL: "http://ollama.lan:11434", Model: "llama3.2", Temperature: 0.5} + c := generate(t, s, "", plainRequest()) + if c.url != "http://ollama.lan:11434/api/chat" { + t.Fatalf("URL = %q", c.url) + } + if got := c.header.Get("Authorization"); got != "" { + t.Fatalf("Authorization = %q, want none", got) + } + var opts struct{ Temperature *float64 } + if err := json.Unmarshal(c.body["options"], &opts); err != nil || opts.Temperature == nil || *opts.Temperature != 0.5 { + t.Fatalf("options = %s", c.body["options"]) + } +} + +func TestOpenAICompatiblePreset(t *testing.T) { + base := llm.Settings{Provider: "openai_compatible", BaseURL: "http://llm.lan:8000/v1", Model: "qwen2.5"} + + c := generate(t, base, "", schemaRequest()) + if c.url != "http://llm.lan:8000/v1/chat/completions" { + t.Fatalf("URL = %q", c.url) + } + if got := c.header.Get("Authorization"); got != "" { + t.Fatalf("Authorization without key = %q, want none", got) + } + if got := responseFormatType(t, c); got != "json_schema" { + t.Fatalf("default response_format type = %q, want json_schema", got) + } + + c = generate(t, base, testKey, plainRequest()) + if got := c.header.Get("Authorization"); got != "Bearer "+testKey { + t.Fatalf("Authorization with key = %q", got) + } + + for mode, want := range map[string]string{"json_schema": "json_schema", "json_object": "json_object", "prompt_only": ""} { + s := base + s.StructuredOutput = mode + if got := responseFormatType(t, generate(t, s, "", schemaRequest())); got != want { + t.Errorf("mode %s: response_format type = %q, want %q", mode, got, want) + } + } +} + +func TestTemperatureOmittedForRejectingModels(t *testing.T) { + cases := map[string][]string{ + "openai": {"o1", "o1-mini", "o1-2024-12-17", "o3", "o3-mini"}, + "openai_compatible": {"o1", "o1-mini", "o3-mini", "openai/o1"}, + "openrouter": {"openai/o1", "openai/o1-mini", "openai/o3", "openai/o3-mini", "openai/o1:online"}, + } + for provider, models := range cases { + for _, model := range models { + s := llm.Settings{Provider: provider, Model: model, Temperature: 0.8} + switch provider { + case "openai_compatible": + s.BaseURL = "http://llm.lan/v1" + case "openrouter": + s.PublicURL = "https://mealquest.example" + } + for _, req := range []llm.Request{plainRequest(), func() llm.Request { r := plainRequest(); r.Temperature = ptr(0.2); return r }()} { + if got, ok := temperature(t, generate(t, s, testKey, req)); ok { + t.Errorf("%s %s: temperature %v sent, want omitted", provider, model, got) + } + } + } + } +} + +func TestTemperatureSentForOtherModels(t *testing.T) { + cases := map[string][]string{ + "openai": {"gpt-4o", "gpt-4o-mini", "chatgpt-o1-like", "o123", "o3custom", "o1mini", "openai/o1"}, + "openrouter": {"openai/gpt-4o", "openai/o123", "openai/o3custom", "acme/o1", "o1-mini"}, + } + for provider, models := range cases { + for _, model := range models { + s := llm.Settings{Provider: provider, Model: model, Temperature: 0.8} + if provider == "openrouter" { + s.PublicURL = "https://mealquest.example" + } + if got, ok := temperature(t, generate(t, s, testKey, plainRequest())); !ok || got != 0.8 { + t.Errorf("%s %s: temperature = %v, %v; want 0.8", provider, model, got, ok) + } + } + } + // A request's own temperature wins over the household's. + req := plainRequest() + req.Temperature = ptr(0.1) + c := generate(t, llm.Settings{Provider: "openai", Model: "gpt-4o", Temperature: 0.8}, testKey, req) + if got, ok := temperature(t, c); !ok || got != 0.1 { + t.Fatalf("temperature = %v, %v; want 0.1", got, ok) + } +} + +// A request is validated before the temperature quirk clears its +// temperature, so a negative one is an error and nothing is sent. +func TestNegativeTemperatureRejectedForRejectingModels(t *testing.T) { + rt := &recordingTransport{} + orig := http.DefaultTransport + http.DefaultTransport = rt + t.Cleanup(func() { http.DefaultTransport = orig }) + + for _, s := range []llm.Settings{ + {Provider: "openai", Model: "o1"}, + {Provider: "openrouter", Model: "openai/o1", PublicURL: "https://mealquest.example"}, + } { + p, err := FromSettings(s, testKey) + if err != nil { + t.Fatalf("%s: FromSettings: %v", s.Model, err) + } + req := plainRequest() + req.Temperature = ptr(-0.5) + if _, err := p.Generate(context.Background(), req); err == nil || errors.Is(err, errStop) || !strings.Contains(err.Error(), "temperature") { + t.Errorf("%s: Generate = %v, want a temperature validation error", s.Model, err) + } + } + if len(rt.reqs) != 0 { + t.Fatalf("sent %d requests, want 0", len(rt.reqs)) + } +} + +func TestTemperatureQuirkPresets(t *testing.T) { + want := map[string]bool{"openai": true, "openrouter": true, "openai_compatible": true} + for name, p := range all { + if got := len(p.OmitTemperature) > 0; got != want[name] { + t.Errorf("%s: lists temperature quirk = %v, want %v", name, got, want[name]) + } + } +} + +func TestPresetTable(t *testing.T) { + cases := []struct { + name string + adapter Adapter + baseURL string + key KeyPolicy + }{ + {"anthropic", AdapterAnthropic, "https://api.anthropic.com", KeyRequired}, + {"openai", AdapterOpenAI, "https://api.openai.com/v1", KeyRequired}, + {"openrouter", AdapterOpenAI, "https://openrouter.ai/api/v1", KeyRequired}, + {"ollama", AdapterOllama, "", KeyNone}, + {"openai_compatible", AdapterOpenAI, "", KeyOptional}, + } + if len(all) != len(cases) { + t.Fatalf("%d presets, want %d", len(all), len(cases)) + } + for _, c := range cases { + p, ok := Lookup(c.name) + if !ok { + t.Fatalf("no preset %q", c.name) + } + if p.Name != c.name || p.Adapter != c.adapter || p.BaseURL != c.baseURL || p.Key != c.key { + t.Errorf("%s = {%s %s %q %d}", c.name, p.Name, p.Adapter, p.BaseURL, p.Key) + } + } +} + +func TestConfigErrors(t *testing.T) { + sentinels := []error{ + llm.ErrUnknownProvider, llm.ErrMissingKey, llm.ErrUnexpectedKey, llm.ErrInvalidKey, llm.ErrMissingBaseURL, + llm.ErrMissingModel, llm.ErrMissingPublicURL, llm.ErrInvalidSetting, + } + cases := []struct { + name string + s llm.Settings + key string + want error + }{ + {"unknown provider", llm.Settings{Provider: "gemini", Model: "m"}, testKey, llm.ErrUnknownProvider}, + {"anthropic no key", llm.Settings{Provider: "anthropic", Model: "m"}, "", llm.ErrMissingKey}, + {"openai no key", llm.Settings{Provider: "openai", Model: "m"}, "", llm.ErrMissingKey}, + {"openrouter no key", llm.Settings{Provider: "openrouter", Model: "m", PublicURL: "https://x.example"}, "", llm.ErrMissingKey}, + {"ollama with key", llm.Settings{Provider: "ollama", BaseURL: "http://o:11434", Model: "m"}, testKey, llm.ErrUnexpectedKey}, + {"anthropic key with newline", llm.Settings{Provider: "anthropic", Model: "m"}, testKey + "\n", llm.ErrInvalidKey}, + {"openai key with newline", llm.Settings{Provider: "openai", Model: "m"}, testKey + "\n", llm.ErrInvalidKey}, + {"openrouter key with CR LF", llm.Settings{Provider: "openrouter", Model: "m", PublicURL: "https://x.example"}, testKey + "\r\nX-Injected: 1", llm.ErrInvalidKey}, + {"openai_compatible key with NUL", llm.Settings{Provider: "openai_compatible", BaseURL: "http://l/v1", Model: "m"}, "\x00" + testKey, llm.ErrInvalidKey}, + {"ollama no base URL", llm.Settings{Provider: "ollama", Model: "m"}, "", llm.ErrMissingBaseURL}, + {"openai_compatible no base URL", llm.Settings{Provider: "openai_compatible", Model: "m"}, "", llm.ErrMissingBaseURL}, + {"anthropic no model", llm.Settings{Provider: "anthropic"}, testKey, llm.ErrMissingModel}, + {"openai no model", llm.Settings{Provider: "openai"}, testKey, llm.ErrMissingModel}, + {"openrouter no model", llm.Settings{Provider: "openrouter", PublicURL: "https://x.example"}, testKey, llm.ErrMissingModel}, + {"ollama no model", llm.Settings{Provider: "ollama", BaseURL: "http://o:11434"}, "", llm.ErrMissingModel}, + {"openai_compatible no model", llm.Settings{Provider: "openai_compatible", BaseURL: "http://l/v1"}, "", llm.ErrMissingModel}, + {"openrouter no public URL", llm.Settings{Provider: "openrouter", Model: "m"}, testKey, llm.ErrMissingPublicURL}, + {"fixed base URL overridden", llm.Settings{Provider: "openai", BaseURL: "https://evil.example/v1", Model: "m"}, testKey, llm.ErrInvalidSetting}, + {"openai json_object", llm.Settings{Provider: "openai", Model: "m", StructuredOutput: "json_object"}, testKey, llm.ErrInvalidSetting}, + {"anthropic mode", llm.Settings{Provider: "anthropic", Model: "m", StructuredOutput: "json_schema"}, testKey, llm.ErrInvalidSetting}, + {"unknown mode", llm.Settings{Provider: "openai_compatible", BaseURL: "http://l/v1", Model: "m", StructuredOutput: "xml"}, "", llm.ErrInvalidSetting}, + {"negative temperature", llm.Settings{Provider: "openai", Model: "m", Temperature: -0.1}, testKey, llm.ErrInvalidSetting}, + {"bad base URL", llm.Settings{Provider: "ollama", BaseURL: "ftp://o", Model: "m"}, "", llm.ErrInvalidSetting}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + p, err := FromSettings(c.s, c.key) + if p != nil || !errors.Is(err, c.want) { + t.Fatalf("FromSettings = %v, %v; want %v", p, err, c.want) + } + for _, other := range sentinels { + if other != c.want && errors.Is(err, other) { + t.Fatalf("error %v also matches %v", err, other) + } + } + if c.key != "" && strings.Contains(err.Error(), c.key) { + t.Fatalf("error leaks the key: %v", err) + } + }) + } +} + +func TestReadyProviderForEveryPreset(t *testing.T) { + for _, s := range []struct { + s llm.Settings + key string + }{ + {llm.Settings{Provider: "anthropic", Model: "m"}, testKey}, + {llm.Settings{Provider: "openai", Model: "m"}, testKey}, + {llm.Settings{Provider: "openrouter", Model: "m", PublicURL: "https://x.example"}, testKey}, + {llm.Settings{Provider: "ollama", BaseURL: "http://o:11434", Model: "m"}, ""}, + {llm.Settings{Provider: "openai_compatible", BaseURL: "http://l/v1", Model: "m"}, ""}, + } { + if p, err := FromSettings(s.s, s.key); err != nil || p == nil { + t.Errorf("%s: FromSettings = %v, %v", s.s.Provider, p, err) + } + } +} + +// A key that matches a value the request carries, or text the error spells +// anyway, never appears in a validation error, and nothing is sent. +func TestGenerateValidationErrorsNeverLeakKey(t *testing.T) { + rt := &recordingTransport{} + orig := http.DefaultTransport + http.DefaultTransport = rt + t.Cleanup(func() { http.DefaultTransport = orig }) + + for _, key := range []string{"-0.5", "temperature", "llm", "negative"} { + p, err := FromSettings(llm.Settings{Provider: "openai_compatible", BaseURL: "http://l/v1", Model: "m"}, key) + if err != nil { + t.Fatalf("%s: FromSettings: %v", key, err) + } + req := plainRequest() + req.Temperature = ptr(-0.5) + _, err = p.Generate(context.Background(), req) + if err == nil || errors.Is(err, errStop) { + t.Fatalf("%s: Generate = %v, want a validation error", key, err) + } + for e := error(err); e != nil; e = errors.Unwrap(e) { + if s := e.Error() + fmt.Sprintf("%#v", e); strings.Contains(s, key) { + t.Fatalf("%s: error leaks the key: %s", key, s) + } + } + } + if len(rt.reqs) != 0 { + t.Fatalf("sent %d requests, want 0", len(rt.reqs)) + } +} + +// A configuration error keeps its sentinel but never holds the key, even +// when the key is also a setting's value or text the error spells anyway. +func TestConfigErrorsNeverLeakKey(t *testing.T) { + cases := []struct { + name string + s llm.Settings + key string + want error + }{ + {"unknown provider named by key", llm.Settings{Provider: "llm", Model: "m"}, "llm", llm.ErrUnknownProvider}, + {"no base URL", llm.Settings{Provider: "openai_compatible", Model: "m"}, "base URL", llm.ErrMissingBaseURL}, + {"no model", llm.Settings{Provider: "openai_compatible", BaseURL: "http://l/v1"}, "model", llm.ErrMissingModel}, + {"key as model, no public URL", llm.Settings{Provider: "openrouter", Model: "missing"}, "missing", llm.ErrMissingPublicURL}, + {"key as base URL", llm.Settings{Provider: "openai", BaseURL: "llm", Model: "m"}, "llm", llm.ErrInvalidSetting}, + {"key as mode", llm.Settings{Provider: "openai_compatible", BaseURL: "http://l/v1", Model: "m", StructuredOutput: "xml"}, "xml", llm.ErrInvalidSetting}, + {"key as temperature", llm.Settings{Provider: "openai", Model: "m", Temperature: -0.1}, "-0.1", llm.ErrInvalidSetting}, + {"key in bad base URL", llm.Settings{Provider: "openai_compatible", BaseURL: "ftp://k3y", Model: "m"}, "k3y", llm.ErrInvalidSetting}, + {"ollama with key", llm.Settings{Provider: "ollama", BaseURL: "http://o:11434", Model: "m"}, "key", llm.ErrUnexpectedKey}, + {"key with newline", llm.Settings{Provider: "openai", Model: "m"}, "sk-test\n", llm.ErrInvalidKey}, + {"key with newline as text", llm.Settings{Provider: "openai", Model: "m"}, "header\n", llm.ErrInvalidKey}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + p, err := FromSettings(c.s, c.key) + if p != nil || !errors.Is(err, c.want) { + t.Fatalf("FromSettings = %v, %v; want %v", p, err, c.want) + } + for e := error(err); e != nil; e = errors.Unwrap(e) { + if s := e.Error() + fmt.Sprintf("%#v", e); strings.Contains(s, c.key) { + t.Fatalf("error leaks the key: %s", s) + } + } + }) + } +} diff --git a/internal/ai/llm/settings.go b/internal/ai/llm/settings.go new file mode 100644 index 0000000..adde36c --- /dev/null +++ b/internal/ai/llm/settings.go @@ -0,0 +1,68 @@ +package llm + +import "errors" + +// Settings is the household's AI configuration, the input to +// presets.FromSettings. The caller fills it from the db.AiSetting row and +// the service config; this package does not import internal/db. +// +// llm.Settings{ +// Provider: row.Provider, +// BaseURL: row.BaseUrl, +// Model: row.Model, +// Temperature: row.Temperature, +// PublicURL: cfg.PublicURL.String(), // MEALQUEST_PUBLIC_URL +// } +// +// Settings never holds the API key: the caller decrypts it and passes it +// to presets.FromSettings separately. +type Settings struct { + // Provider names the preset: "anthropic", "openai", "openrouter", + // "ollama" or "openai_compatible". + Provider string + // BaseURL is the API root. It is required for ollama and + // openai_compatible; the other presets fix their own, and a non-empty + // value must equal it. + BaseURL string + // Model is the model id Generate requests. It is required. + Model string + // Temperature is the household's sampling temperature. It is sent with + // every request that sets none, except to models that reject it. + Temperature float64 + // PublicURL is MEALQUEST_PUBLIC_URL. OpenRouter requires it and + // receives it as the HTTP-Referer header; other presets ignore it. + PublicURL string + // StructuredOutput selects how an OpenAI-compatible preset asks for + // JSON: "json_schema", "json_object" or "prompt_only". Empty selects + // the preset's default, json_schema. Each preset allows only the modes + // it lists; anthropic and ollama allow none, so it must be empty. + StructuredOutput string +} + +// Configuration errors from presets.FromSettings. Each is distinct and +// matches with errors.Is. +var ( + // ErrUnknownProvider means Settings.Provider names no preset. + ErrUnknownProvider = errors.New("llm: unknown provider") + // ErrMissingKey means the preset requires an API key and none was + // given. + ErrMissingKey = errors.New("llm: API key is required") + // ErrUnexpectedKey means an API key was given to a preset that sends + // none. + ErrUnexpectedKey = errors.New("llm: provider takes no API key") + // ErrInvalidKey means the API key is not a valid HTTP header value: + // it has a control character such as CR or LF. + ErrInvalidKey = errors.New("llm: API key is not a valid header value") + // ErrMissingBaseURL means the preset requires a base URL and none was + // given. + ErrMissingBaseURL = errors.New("llm: base URL is required") + // ErrMissingModel means Settings.Model is empty. + ErrMissingModel = errors.New("llm: model is required") + // ErrMissingPublicURL means the preset requires Settings.PublicURL and + // it is empty. + ErrMissingPublicURL = errors.New("llm: public URL is required") + // ErrInvalidSetting means a setting has a value the preset does not + // accept, such as a base URL other than the preset's, an unsupported + // structured output mode or a negative temperature. + ErrInvalidSetting = errors.New("llm: invalid setting") +) diff --git a/internal/store/migrate_test.go b/internal/store/migrate_test.go index e104996..444399a 100644 --- a/internal/store/migrate_test.go +++ b/internal/store/migrate_test.go @@ -175,8 +175,8 @@ func TestMigrateEmbeddedMigrations(t *testing.T) { if err := Migrate(context.Background(), s.Writer, migrations.FS); err != nil { t.Fatal(err) } - if got := versions(migrationRows(t, s)); got != "0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status" { - t.Errorf("versions = %q, want 0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status", got) + if got := versions(migrationRows(t, s)); got != "0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets" { + t.Errorf("versions = %q, want 0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets", got) } } @@ -196,8 +196,8 @@ func TestMigrateBootstrapThenEmbeddedInitApplies(t *testing.T) { if err := Migrate(ctx, s.Writer, migrations.FS); err != nil { t.Fatal(err) } - if got := versions(migrationRows(t, s)); got != "0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status" { - t.Errorf("versions = %q, want 0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status", got) + if got := versions(migrationRows(t, s)); got != "0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets" { + t.Errorf("versions = %q, want 0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets", got) } if !tableExists(t, s, "dinners") { t.Error("table from 0001_init was not created") @@ -309,3 +309,90 @@ func TestMigrateJobsStatusMapsLegacyRows(t *testing.T) { t.Error("jobs_new left behind") } } + +// TestMigrateAIProviderPresetsKeepsRowsAndReferences upgrades a database +// holding ai_settings and ai_runs rows, with suggestion_batches and +// week_plans referencing the run, and checks 0004 carries every row over, +// keeps the references enforced and accepts the new providers. +func TestMigrateAIProviderPresetsKeepsRowsAndReferences(t *testing.T) { + ctx := context.Background() + s := openTestStore(t) + before := fstest.MapFS{} + for _, name := range []string{"0000_bootstrap.sql", "0001_init.sql", "0002_household_settings_updated_by.sql", "0003_jobs_status.sql"} { + body, err := fs.ReadFile(migrations.FS, name) + if err != nil { + t.Fatal(err) + } + before[name] = &fstest.MapFile{Data: body} + } + if err := Migrate(ctx, s.Writer, before); err != nil { + t.Fatal(err) + } + for _, stmt := range []string{ + `INSERT INTO ai_settings (id, provider, base_url, model, temperature, system_prompt, api_key_ct, api_key_last4, key_verified_at, created_at, updated_at) + VALUES (1, 'anthropic', 'https://api.anthropic.com', 'claude', 0.7, 'sp', x'0102', 'abcd', 'v', 'c', 'u')`, + `INSERT INTO ai_runs (id, skill, skill_version, provider, model, temperature, status, latency_ms, tokens_in, tokens_out, error, request_redacted, response_redacted, created_at, updated_at) + VALUES ('run-1', 'suggest', '1', 'ollama', 'llama', 0.5, 'ok', 10, 20, 30, 'e', 'req', 'resp', 'c', 'u')`, + `INSERT INTO suggestion_batches (id, trigger, status, ai_run_id, created_at, updated_at) + VALUES ('batch-1', 'manual', 'done', 'run-1', 'c', 'u')`, + `INSERT INTO week_plans (id, week_start, status, ai_run_id, created_at, updated_at) + VALUES ('plan-1', '2026-10-04', 'draft', 'run-1', 'c', 'u')`, + } { + if _, err := s.Writer.Exec(stmt); err != nil { + t.Fatalf("seed: %v", err) + } + } + + if err := Migrate(ctx, s.Writer, migrations.FS); err != nil { + t.Fatal(err) + } + + var provider, baseURL, model, systemPrompt, last4, verifiedAt, createdAt, updatedAt string + var temperature float64 + var keyCT []byte + if err := s.Reader.QueryRow(`SELECT provider, base_url, model, temperature, system_prompt, api_key_ct, api_key_last4, key_verified_at, created_at, updated_at FROM ai_settings WHERE id = 1`). + Scan(&provider, &baseURL, &model, &temperature, &systemPrompt, &keyCT, &last4, &verifiedAt, &createdAt, &updatedAt); err != nil { + t.Fatalf("read ai_settings: %v", err) + } + settings := []any{provider, baseURL, model, temperature, systemPrompt, string(keyCT), last4, verifiedAt, createdAt, updatedAt} + if want := []any{"anthropic", "https://api.anthropic.com", "claude", 0.7, "sp", "\x01\x02", "abcd", "v", "c", "u"}; !slices.Equal(settings, want) { + t.Errorf("ai_settings = %v, want %v", settings, want) + } + + var skill, skillVersion, status, errText, request, response string + var latency, tokensIn, tokensOut int + if err := s.Reader.QueryRow(`SELECT skill, skill_version, provider, model, temperature, status, latency_ms, tokens_in, tokens_out, error, request_redacted, response_redacted, created_at, updated_at FROM ai_runs WHERE id = 'run-1'`). + Scan(&skill, &skillVersion, &provider, &model, &temperature, &status, &latency, &tokensIn, &tokensOut, &errText, &request, &response, &createdAt, &updatedAt); err != nil { + t.Fatalf("read ai_runs: %v", err) + } + run := []any{skill, skillVersion, provider, model, temperature, status, latency, tokensIn, tokensOut, errText, request, response, createdAt, updatedAt} + if want := []any{"suggest", "1", "ollama", "llama", 0.5, "ok", 10, 20, 30, "e", "req", "resp", "c", "u"}; !slices.Equal(run, want) { + t.Errorf("ai_runs = %v, want %v", run, want) + } + + rows, err := s.Reader.Query("PRAGMA foreign_key_check") + if err != nil { + t.Fatal(err) + } + if rows.Next() { + t.Error("foreign_key_check reported violations after 0004") + } + if err := rows.Close(); err != nil { + t.Fatal(err) + } + if _, err := s.Writer.Exec(`DELETE FROM ai_runs WHERE id = 'run-1'`); err == nil || !strings.Contains(err.Error(), "FOREIGN KEY constraint failed") { + t.Errorf("deleting a referenced ai_run: error %v, want FOREIGN KEY constraint failure", err) + } + + if _, err := s.Writer.Exec(`UPDATE ai_settings SET provider = 'openrouter' WHERE id = 1`); err != nil { + t.Errorf("ai_settings.provider openrouter rejected: %v", err) + } + if _, err := s.Writer.Exec(`UPDATE ai_runs SET provider = 'openai' WHERE id = 'run-1'`); err != nil { + t.Errorf("ai_runs.provider openai rejected: %v", err) + } + for _, table := range []string{"ai_settings_copy", "ai_runs_copy"} { + if tableExists(t, s, table) { + t.Errorf("%s left behind", table) + } + } +} diff --git a/internal/store/schema_test.go b/internal/store/schema_test.go index 2b36ad1..1ce39a4 100644 --- a/internal/store/schema_test.go +++ b/internal/store/schema_test.go @@ -305,7 +305,7 @@ func TestSchemaEnumChecksRejectInvalidValues(t *testing.T) { {"household_settings", "plan_draft_dow", 5.5}, {"household_settings", "suggestions_dow", 0.5}, {"household_settings", "chat_autosend_dow", 3.5}, - {"ai_runs", "provider", "openai"}, + {"ai_runs", "provider", "gemini"}, {"ai_runs", "status", "done"}, {"dinners", "status", "deck"}, {"dinners", "origin", "ai"}, @@ -322,7 +322,7 @@ func TestSchemaEnumChecksRejectInvalidValues(t *testing.T) { {"rotation_reviews", "decision", "promote"}, {"shopping_lists", "status", "frozen"}, {"shopping_items", "needed", 2}, - {"ai_settings", "provider", "openai"}, + {"ai_settings", "provider", "gemini"}, {"recipe_search_settings", "provider", "google"}, {"chat_settings", "last_test_status", "pending"}, {"notification_prefs", "event", "weekly_digest"}, @@ -360,6 +360,23 @@ func TestSchemaRotationReviewEventAllowed(t *testing.T) { } } +// TestSchemaAIProvidersAllowed checks every provider preset can be stored in +// both ai_settings and ai_runs. +func TestSchemaAIProvidersAllowed(t *testing.T) { + s := seededStore(t) + for _, provider := range []string{"openai_compatible", "anthropic", "ollama", "openai", "openrouter"} { + for _, table := range []string{"ai_settings", "ai_runs"} { + inRolledBackTx(t, s, func(tx *sql.Tx) { + row := testRow(table, 1) + row["provider"] = provider + if err := insertSchemaRow(tx, table, row); err != nil { + t.Errorf("%s.provider %s rejected: %v", table, provider, err) + } + }) + } + } +} + func TestSchemaDinnerStatuses(t *testing.T) { s := seededStore(t) for _, status := range []string{"suggested", "quest_log", "tried", "regular", "archived"} { diff --git a/migrations/0004_ai_provider_presets.sql b/migrations/0004_ai_provider_presets.sql new file mode 100644 index 0000000..965ca79 --- /dev/null +++ b/migrations/0004_ai_provider_presets.sql @@ -0,0 +1,68 @@ +-- 0004_ai_provider_presets: ai_settings.provider and ai_runs.provider become +-- openai_compatible, anthropic, ollama, openai, openrouter, the presets in +-- internal/ai/llm/presets. Every 0001 value is kept, so existing rows copy +-- unchanged. SQLite cannot change a CHECK in place, so both tables are +-- rebuilt: copy the rows aside, drop the table, create it again with the +-- wider CHECK, copy the rows back and drop the copy. Neither table has +-- indexes or triggers. +-- +-- suggestion_batches.ai_run_id and week_plans.ai_run_id reference ai_runs, +-- so the rebuild cannot use 0003's create-new-then-rename: dropping ai_runs +-- counts every referencing row as a foreign key violation, and only inserting +-- the matching ai_runs rows under the name ai_runs clears them. Deferring +-- foreign keys holds the check until commit, after the rows are back. The +-- pragma lasts only for this migration's transaction. + +PRAGMA defer_foreign_keys = ON; + +CREATE TABLE ai_settings_copy AS SELECT * FROM ai_settings; + +DROP TABLE ai_settings; + +CREATE TABLE ai_settings ( + id INTEGER PRIMARY KEY CHECK (id = 1), + provider TEXT NOT NULL CHECK (provider IN ('openai_compatible', 'anthropic', 'ollama', 'openai', 'openrouter')), + base_url TEXT NOT NULL, + model TEXT NOT NULL, + temperature REAL NOT NULL, + system_prompt TEXT, + api_key_ct BLOB, + api_key_last4 TEXT, + key_verified_at TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); + +INSERT INTO ai_settings (id, provider, base_url, model, temperature, system_prompt, api_key_ct, api_key_last4, key_verified_at, created_at, updated_at) +SELECT id, provider, base_url, model, temperature, system_prompt, api_key_ct, api_key_last4, key_verified_at, created_at, updated_at +FROM ai_settings_copy; + +DROP TABLE ai_settings_copy; + +CREATE TABLE ai_runs_copy AS SELECT * FROM ai_runs; + +DROP TABLE ai_runs; + +CREATE TABLE ai_runs ( + id TEXT NOT NULL PRIMARY KEY, + skill TEXT NOT NULL, + skill_version TEXT NOT NULL, + provider TEXT NOT NULL CHECK (provider IN ('openai_compatible', 'anthropic', 'ollama', 'openai', 'openrouter')), + model TEXT NOT NULL, + temperature REAL NOT NULL, + status TEXT NOT NULL CHECK (status IN ('running', 'ok', 'error')), + latency_ms INTEGER, + tokens_in INTEGER, + tokens_out INTEGER, + error TEXT, + request_redacted TEXT NOT NULL, + response_redacted TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); + +INSERT INTO ai_runs (id, skill, skill_version, provider, model, temperature, status, latency_ms, tokens_in, tokens_out, error, request_redacted, response_redacted, created_at, updated_at) +SELECT id, skill, skill_version, provider, model, temperature, status, latency_ms, tokens_in, tokens_out, error, request_redacted, response_redacted, created_at, updated_at +FROM ai_runs_copy; + +DROP TABLE ai_runs_copy; -- 2.55.0 From e105cdb9c512cf0c19fffb137c2753fb7d34215b Mon Sep 17 00:00:00 2001 From: dev_agent Date: Sun, 4 Oct 2026 09:36:10 -0400 Subject: [PATCH 6/7] Add xAI Grok as a provider (#46) Co-authored-by: dev_agent Co-committed-by: dev_agent --- internal/ai/llm/presets/presets.go | 9 +++ internal/ai/llm/presets/presets_test.go | 48 ++++++++++++ internal/ai/llm/settings.go | 2 +- internal/store/migrate_test.go | 98 ++++++++++++++++++++++++- internal/store/schema_test.go | 2 +- migrations/0005_ai_provider_xai.sql | 64 ++++++++++++++++ 6 files changed, 217 insertions(+), 6 deletions(-) create mode 100644 migrations/0005_ai_provider_xai.sql diff --git a/internal/ai/llm/presets/presets.go b/internal/ai/llm/presets/presets.go index d33d613..43b0d85 100644 --- a/internal/ai/llm/presets/presets.go +++ b/internal/ai/llm/presets/presets.go @@ -145,6 +145,15 @@ var all = map[string]Preset{ StructuredOutputs: []openai.StructuredOutput{openai.JSONSchema, openai.JSONObject}, OmitTemperature: []*regexp.Regexp{openRouterReasoningModels}, }, + "xai": { + Name: "xai", + Adapter: AdapterOpenAI, + BaseURL: "https://api.x.ai/v1", + Key: KeyRequired, + StructuredOutput: openai.JSONSchema, + FallbackStructuredOutput: openai.JSONObject, + StructuredOutputs: []openai.StructuredOutput{openai.JSONSchema, openai.JSONObject}, + }, "ollama": { Name: "ollama", Adapter: AdapterOllama, diff --git a/internal/ai/llm/presets/presets_test.go b/internal/ai/llm/presets/presets_test.go index 47f98c5..142c564 100644 --- a/internal/ai/llm/presets/presets_test.go +++ b/internal/ai/llm/presets/presets_test.go @@ -180,6 +180,43 @@ func TestOpenRouterFallbackModeIsJSONObject(t *testing.T) { } } +func TestXAIPreset(t *testing.T) { + c := generate(t, llm.Settings{Provider: "xai", Model: "grok-4", Temperature: 0.6}, testKey, schemaRequest()) + if c.url != "https://api.x.ai/v1/chat/completions" { + t.Fatalf("URL = %q", c.url) + } + if got := c.header.Get("Authorization"); got != "Bearer "+testKey { + t.Fatalf("Authorization = %q", got) + } + for _, h := range []string{"X-Api-Key", "HTTP-Referer", "X-Title"} { + if got := c.header.Get(h); got != "" { + t.Fatalf("%s = %q, want none", h, got) + } + } + if got := responseFormatType(t, c); got != "json_schema" { + t.Fatalf("default response_format type = %q, want json_schema", got) + } + if got, ok := temperature(t, c); !ok || got != 0.6 { + t.Fatalf("temperature = %v, %v; want 0.6", got, ok) + } +} + +func TestXAIFallbackModeIsJSONObject(t *testing.T) { + p, _ := Lookup("xai") + s := llm.Settings{Provider: "xai", Model: "grok-4", StructuredOutput: string(p.FallbackStructuredOutput)} + c := generate(t, s, testKey, schemaRequest()) + if got := responseFormatType(t, c); got != "json_object" { + t.Fatalf("response_format type = %q, want json_object", got) + } +} + +func TestXAIPresetAcceptsItsOwnBaseURL(t *testing.T) { + c := generate(t, llm.Settings{Provider: "xai", BaseURL: "https://api.x.ai/v1", Model: "grok-4"}, testKey, plainRequest()) + if c.url != "https://api.x.ai/v1/chat/completions" { + t.Fatalf("URL = %q", c.url) + } +} + func TestOllamaPreset(t *testing.T) { s := llm.Settings{Provider: "ollama", BaseURL: "http://ollama.lan:11434", Model: "llama3.2", Temperature: 0.5} c := generate(t, s, "", plainRequest()) @@ -318,6 +355,7 @@ func TestPresetTable(t *testing.T) { {"anthropic", AdapterAnthropic, "https://api.anthropic.com", KeyRequired}, {"openai", AdapterOpenAI, "https://api.openai.com/v1", KeyRequired}, {"openrouter", AdapterOpenAI, "https://openrouter.ai/api/v1", KeyRequired}, + {"xai", AdapterOpenAI, "https://api.x.ai/v1", KeyRequired}, {"ollama", AdapterOllama, "", KeyNone}, {"openai_compatible", AdapterOpenAI, "", KeyOptional}, } @@ -350,10 +388,12 @@ func TestConfigErrors(t *testing.T) { {"anthropic no key", llm.Settings{Provider: "anthropic", Model: "m"}, "", llm.ErrMissingKey}, {"openai no key", llm.Settings{Provider: "openai", Model: "m"}, "", llm.ErrMissingKey}, {"openrouter no key", llm.Settings{Provider: "openrouter", Model: "m", PublicURL: "https://x.example"}, "", llm.ErrMissingKey}, + {"xai no key", llm.Settings{Provider: "xai", Model: "grok-4"}, "", llm.ErrMissingKey}, {"ollama with key", llm.Settings{Provider: "ollama", BaseURL: "http://o:11434", Model: "m"}, testKey, llm.ErrUnexpectedKey}, {"anthropic key with newline", llm.Settings{Provider: "anthropic", Model: "m"}, testKey + "\n", llm.ErrInvalidKey}, {"openai key with newline", llm.Settings{Provider: "openai", Model: "m"}, testKey + "\n", llm.ErrInvalidKey}, {"openrouter key with CR LF", llm.Settings{Provider: "openrouter", Model: "m", PublicURL: "https://x.example"}, testKey + "\r\nX-Injected: 1", llm.ErrInvalidKey}, + {"xai key with newline", llm.Settings{Provider: "xai", Model: "grok-4"}, testKey + "\n", llm.ErrInvalidKey}, {"openai_compatible key with NUL", llm.Settings{Provider: "openai_compatible", BaseURL: "http://l/v1", Model: "m"}, "\x00" + testKey, llm.ErrInvalidKey}, {"ollama no base URL", llm.Settings{Provider: "ollama", Model: "m"}, "", llm.ErrMissingBaseURL}, {"openai_compatible no base URL", llm.Settings{Provider: "openai_compatible", Model: "m"}, "", llm.ErrMissingBaseURL}, @@ -362,8 +402,11 @@ func TestConfigErrors(t *testing.T) { {"openrouter no model", llm.Settings{Provider: "openrouter", PublicURL: "https://x.example"}, testKey, llm.ErrMissingModel}, {"ollama no model", llm.Settings{Provider: "ollama", BaseURL: "http://o:11434"}, "", llm.ErrMissingModel}, {"openai_compatible no model", llm.Settings{Provider: "openai_compatible", BaseURL: "http://l/v1"}, "", llm.ErrMissingModel}, + {"xai no model", llm.Settings{Provider: "xai"}, testKey, llm.ErrMissingModel}, {"openrouter no public URL", llm.Settings{Provider: "openrouter", Model: "m"}, testKey, llm.ErrMissingPublicURL}, {"fixed base URL overridden", llm.Settings{Provider: "openai", BaseURL: "https://evil.example/v1", Model: "m"}, testKey, llm.ErrInvalidSetting}, + {"xai base URL overridden", llm.Settings{Provider: "xai", BaseURL: "https://api.openai.com/v1", Model: "grok-4"}, testKey, llm.ErrInvalidSetting}, + {"xai prompt_only", llm.Settings{Provider: "xai", Model: "grok-4", StructuredOutput: "prompt_only"}, testKey, llm.ErrInvalidSetting}, {"openai json_object", llm.Settings{Provider: "openai", Model: "m", StructuredOutput: "json_object"}, testKey, llm.ErrInvalidSetting}, {"anthropic mode", llm.Settings{Provider: "anthropic", Model: "m", StructuredOutput: "json_schema"}, testKey, llm.ErrInvalidSetting}, {"unknown mode", llm.Settings{Provider: "openai_compatible", BaseURL: "http://l/v1", Model: "m", StructuredOutput: "xml"}, "", llm.ErrInvalidSetting}, @@ -396,6 +439,7 @@ func TestReadyProviderForEveryPreset(t *testing.T) { {llm.Settings{Provider: "anthropic", Model: "m"}, testKey}, {llm.Settings{Provider: "openai", Model: "m"}, testKey}, {llm.Settings{Provider: "openrouter", Model: "m", PublicURL: "https://x.example"}, testKey}, + {llm.Settings{Provider: "xai", Model: "m"}, testKey}, {llm.Settings{Provider: "ollama", BaseURL: "http://o:11434", Model: "m"}, ""}, {llm.Settings{Provider: "openai_compatible", BaseURL: "http://l/v1", Model: "m"}, ""}, } { @@ -454,6 +498,10 @@ func TestConfigErrorsNeverLeakKey(t *testing.T) { {"key in bad base URL", llm.Settings{Provider: "openai_compatible", BaseURL: "ftp://k3y", Model: "m"}, "k3y", llm.ErrInvalidSetting}, {"ollama with key", llm.Settings{Provider: "ollama", BaseURL: "http://o:11434", Model: "m"}, "key", llm.ErrUnexpectedKey}, {"key with newline", llm.Settings{Provider: "openai", Model: "m"}, "sk-test\n", llm.ErrInvalidKey}, + {"xai key as preset name", llm.Settings{Provider: "xai"}, "xai", llm.ErrMissingModel}, + {"xai key as missing-model text", llm.Settings{Provider: "xai"}, "model is required", llm.ErrMissingModel}, + {"xai key as fixed-URL text", llm.Settings{Provider: "xai", BaseURL: "https://x.example/v1", Model: "m"}, "fixed", llm.ErrInvalidSetting}, + {"xai key as structured-output text", llm.Settings{Provider: "xai", Model: "m", StructuredOutput: "prompt_only"}, "structured output", llm.ErrInvalidSetting}, {"key with newline as text", llm.Settings{Provider: "openai", Model: "m"}, "header\n", llm.ErrInvalidKey}, } for _, c := range cases { diff --git a/internal/ai/llm/settings.go b/internal/ai/llm/settings.go index adde36c..11c2c74 100644 --- a/internal/ai/llm/settings.go +++ b/internal/ai/llm/settings.go @@ -18,7 +18,7 @@ import "errors" // to presets.FromSettings separately. type Settings struct { // Provider names the preset: "anthropic", "openai", "openrouter", - // "ollama" or "openai_compatible". + // "xai", "ollama" or "openai_compatible". Provider string // BaseURL is the API root. It is required for ollama and // openai_compatible; the other presets fix their own, and a non-empty diff --git a/internal/store/migrate_test.go b/internal/store/migrate_test.go index 444399a..624f17e 100644 --- a/internal/store/migrate_test.go +++ b/internal/store/migrate_test.go @@ -175,8 +175,8 @@ func TestMigrateEmbeddedMigrations(t *testing.T) { if err := Migrate(context.Background(), s.Writer, migrations.FS); err != nil { t.Fatal(err) } - if got := versions(migrationRows(t, s)); got != "0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets" { - t.Errorf("versions = %q, want 0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets", got) + if got := versions(migrationRows(t, s)); got != "0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai" { + t.Errorf("versions = %q, want 0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai", got) } } @@ -196,8 +196,8 @@ func TestMigrateBootstrapThenEmbeddedInitApplies(t *testing.T) { if err := Migrate(ctx, s.Writer, migrations.FS); err != nil { t.Fatal(err) } - if got := versions(migrationRows(t, s)); got != "0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets" { - t.Errorf("versions = %q, want 0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets", got) + if got := versions(migrationRows(t, s)); got != "0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai" { + t.Errorf("versions = %q, want 0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai", got) } if !tableExists(t, s, "dinners") { t.Error("table from 0001_init was not created") @@ -396,3 +396,93 @@ func TestMigrateAIProviderPresetsKeepsRowsAndReferences(t *testing.T) { } } } + +// TestMigrateAIProviderXAIKeepsRowsAndReferences upgrades a database at 0004 +// holding ai_settings and ai_runs rows with 0004's providers, with +// suggestion_batches and week_plans referencing the run, and checks 0005 +// carries every row over, keeps the references enforced, accepts xai and +// still rejects an unknown provider. +func TestMigrateAIProviderXAIKeepsRowsAndReferences(t *testing.T) { + ctx := context.Background() + s := openTestStore(t) + before := fstest.MapFS{} + for _, name := range []string{"0000_bootstrap.sql", "0001_init.sql", "0002_household_settings_updated_by.sql", "0003_jobs_status.sql", "0004_ai_provider_presets.sql"} { + body, err := fs.ReadFile(migrations.FS, name) + if err != nil { + t.Fatal(err) + } + before[name] = &fstest.MapFile{Data: body} + } + if err := Migrate(ctx, s.Writer, before); err != nil { + t.Fatal(err) + } + for _, stmt := range []string{ + `INSERT INTO ai_settings (id, provider, base_url, model, temperature, system_prompt, api_key_ct, api_key_last4, key_verified_at, created_at, updated_at) + VALUES (1, 'openrouter', 'https://openrouter.ai/api/v1', 'openai/gpt-4o', 0.7, 'sp', x'0102', 'abcd', 'v', 'c', 'u')`, + `INSERT INTO ai_runs (id, skill, skill_version, provider, model, temperature, status, latency_ms, tokens_in, tokens_out, error, request_redacted, response_redacted, created_at, updated_at) + VALUES ('run-1', 'suggest', '1', 'openai', 'gpt-4o', 0.5, 'ok', 10, 20, 30, 'e', 'req', 'resp', 'c', 'u')`, + `INSERT INTO suggestion_batches (id, trigger, status, ai_run_id, created_at, updated_at) + VALUES ('batch-1', 'manual', 'done', 'run-1', 'c', 'u')`, + `INSERT INTO week_plans (id, week_start, status, ai_run_id, created_at, updated_at) + VALUES ('plan-1', '2026-10-04', 'draft', 'run-1', 'c', 'u')`, + } { + if _, err := s.Writer.Exec(stmt); err != nil { + t.Fatalf("seed: %v", err) + } + } + + if err := Migrate(ctx, s.Writer, migrations.FS); err != nil { + t.Fatal(err) + } + + var provider, baseURL, model, systemPrompt, last4, verifiedAt, createdAt, updatedAt string + var temperature float64 + var keyCT []byte + if err := s.Reader.QueryRow(`SELECT provider, base_url, model, temperature, system_prompt, api_key_ct, api_key_last4, key_verified_at, created_at, updated_at FROM ai_settings WHERE id = 1`). + Scan(&provider, &baseURL, &model, &temperature, &systemPrompt, &keyCT, &last4, &verifiedAt, &createdAt, &updatedAt); err != nil { + t.Fatalf("read ai_settings: %v", err) + } + settings := []any{provider, baseURL, model, temperature, systemPrompt, string(keyCT), last4, verifiedAt, createdAt, updatedAt} + if want := []any{"openrouter", "https://openrouter.ai/api/v1", "openai/gpt-4o", 0.7, "sp", "\x01\x02", "abcd", "v", "c", "u"}; !slices.Equal(settings, want) { + t.Errorf("ai_settings = %v, want %v", settings, want) + } + + var skill, skillVersion, status, errText, request, response string + var latency, tokensIn, tokensOut int + if err := s.Reader.QueryRow(`SELECT skill, skill_version, provider, model, temperature, status, latency_ms, tokens_in, tokens_out, error, request_redacted, response_redacted, created_at, updated_at FROM ai_runs WHERE id = 'run-1'`). + Scan(&skill, &skillVersion, &provider, &model, &temperature, &status, &latency, &tokensIn, &tokensOut, &errText, &request, &response, &createdAt, &updatedAt); err != nil { + t.Fatalf("read ai_runs: %v", err) + } + run := []any{skill, skillVersion, provider, model, temperature, status, latency, tokensIn, tokensOut, errText, request, response, createdAt, updatedAt} + if want := []any{"suggest", "1", "openai", "gpt-4o", 0.5, "ok", 10, 20, 30, "e", "req", "resp", "c", "u"}; !slices.Equal(run, want) { + t.Errorf("ai_runs = %v, want %v", run, want) + } + + rows, err := s.Reader.Query("PRAGMA foreign_key_check") + if err != nil { + t.Fatal(err) + } + if rows.Next() { + t.Error("foreign_key_check reported violations after 0005") + } + if err := rows.Close(); err != nil { + t.Fatal(err) + } + if _, err := s.Writer.Exec(`DELETE FROM ai_runs WHERE id = 'run-1'`); err == nil || !strings.Contains(err.Error(), "FOREIGN KEY constraint failed") { + t.Errorf("deleting a referenced ai_run: error %v, want FOREIGN KEY constraint failure", err) + } + + for _, table := range []string{"ai_settings", "ai_runs"} { + if _, err := s.Writer.Exec(`UPDATE ` + table + ` SET provider = 'xai'`); err != nil { + t.Errorf("%s.provider xai rejected: %v", table, err) + } + if _, err := s.Writer.Exec(`UPDATE ` + table + ` SET provider = 'grok'`); err == nil || !strings.Contains(err.Error(), "CHECK constraint failed") { + t.Errorf("%s.provider grok: error %v, want CHECK constraint failure", table, err) + } + } + for _, table := range []string{"ai_settings_copy", "ai_runs_copy"} { + if tableExists(t, s, table) { + t.Errorf("%s left behind", table) + } + } +} diff --git a/internal/store/schema_test.go b/internal/store/schema_test.go index 1ce39a4..a5651bf 100644 --- a/internal/store/schema_test.go +++ b/internal/store/schema_test.go @@ -364,7 +364,7 @@ func TestSchemaRotationReviewEventAllowed(t *testing.T) { // both ai_settings and ai_runs. func TestSchemaAIProvidersAllowed(t *testing.T) { s := seededStore(t) - for _, provider := range []string{"openai_compatible", "anthropic", "ollama", "openai", "openrouter"} { + for _, provider := range []string{"openai_compatible", "anthropic", "ollama", "openai", "openrouter", "xai"} { for _, table := range []string{"ai_settings", "ai_runs"} { inRolledBackTx(t, s, func(tx *sql.Tx) { row := testRow(table, 1) diff --git a/migrations/0005_ai_provider_xai.sql b/migrations/0005_ai_provider_xai.sql new file mode 100644 index 0000000..6976934 --- /dev/null +++ b/migrations/0005_ai_provider_xai.sql @@ -0,0 +1,64 @@ +-- 0005_ai_provider_xai: ai_settings.provider and ai_runs.provider also +-- accept xai, the xAI Grok preset in internal/ai/llm/presets. Every 0004 +-- value is kept, so existing rows copy unchanged. The rebuild is 0004's: +-- copy the rows aside, drop the table, create it again with the wider CHECK, +-- copy the rows back and drop the copy. Neither table has indexes or +-- triggers. +-- +-- suggestion_batches.ai_run_id and week_plans.ai_run_id reference ai_runs, +-- so foreign keys are deferred until commit, after the ai_runs rows are +-- back. The pragma lasts only for this migration's transaction. + +PRAGMA defer_foreign_keys = ON; + +CREATE TABLE ai_settings_copy AS SELECT * FROM ai_settings; + +DROP TABLE ai_settings; + +CREATE TABLE ai_settings ( + id INTEGER PRIMARY KEY CHECK (id = 1), + provider TEXT NOT NULL CHECK (provider IN ('openai_compatible', 'anthropic', 'ollama', 'openai', 'openrouter', 'xai')), + base_url TEXT NOT NULL, + model TEXT NOT NULL, + temperature REAL NOT NULL, + system_prompt TEXT, + api_key_ct BLOB, + api_key_last4 TEXT, + key_verified_at TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); + +INSERT INTO ai_settings (id, provider, base_url, model, temperature, system_prompt, api_key_ct, api_key_last4, key_verified_at, created_at, updated_at) +SELECT id, provider, base_url, model, temperature, system_prompt, api_key_ct, api_key_last4, key_verified_at, created_at, updated_at +FROM ai_settings_copy; + +DROP TABLE ai_settings_copy; + +CREATE TABLE ai_runs_copy AS SELECT * FROM ai_runs; + +DROP TABLE ai_runs; + +CREATE TABLE ai_runs ( + id TEXT NOT NULL PRIMARY KEY, + skill TEXT NOT NULL, + skill_version TEXT NOT NULL, + provider TEXT NOT NULL CHECK (provider IN ('openai_compatible', 'anthropic', 'ollama', 'openai', 'openrouter', 'xai')), + model TEXT NOT NULL, + temperature REAL NOT NULL, + status TEXT NOT NULL CHECK (status IN ('running', 'ok', 'error')), + latency_ms INTEGER, + tokens_in INTEGER, + tokens_out INTEGER, + error TEXT, + request_redacted TEXT NOT NULL, + response_redacted TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); + +INSERT INTO ai_runs (id, skill, skill_version, provider, model, temperature, status, latency_ms, tokens_in, tokens_out, error, request_redacted, response_redacted, created_at, updated_at) +SELECT id, skill, skill_version, provider, model, temperature, status, latency_ms, tokens_in, tokens_out, error, request_redacted, response_redacted, created_at, updated_at +FROM ai_runs_copy; + +DROP TABLE ai_runs_copy; -- 2.55.0 From b347e1c9110e3a94c7bf54d8018829585642bf36 Mon Sep 17 00:00:00 2001 From: dev_agent Date: Sun, 4 Oct 2026 11:02:55 -0400 Subject: [PATCH 7/7] Retry, backoff and default Generate timeout (#48) Co-authored-by: dev_agent Co-committed-by: dev_agent --- internal/ai/llm/anthropic/anthropic.go | 2 +- internal/ai/llm/errors.go | 29 ++ internal/ai/llm/errors_test.go | 14 + internal/ai/llm/openai/openai.go | 2 +- internal/ai/llm/presets/presets.go | 5 +- internal/ai/llm/presets/presets_test.go | 156 +++++++++- internal/ai/llm/retry.go | 124 ++++++++ internal/ai/llm/retry_test.go | 359 ++++++++++++++++++++++++ 8 files changed, 680 insertions(+), 11 deletions(-) create mode 100644 internal/ai/llm/retry.go create mode 100644 internal/ai/llm/retry_test.go diff --git a/internal/ai/llm/anthropic/anthropic.go b/internal/ai/llm/anthropic/anthropic.go index 37b1239..c41c0f1 100644 --- a/internal/ai/llm/anthropic/anthropic.go +++ b/internal/ai/llm/anthropic/anthropic.go @@ -381,7 +381,7 @@ func (p *Provider) statusError(resp *http.Response, body []byte) error { case http.StatusTooManyRequests, statusOverloaded: return llm.NewRateLimitedError(retryAfter(resp.Header.Get("Retry-After"), time.Now()), p.apiKey) case http.StatusRequestTimeout: - return fmt.Errorf("HTTP %d: %w", resp.StatusCode, llm.ErrTimeout) + return llm.NewTimeoutStatusError(resp.StatusCode, p.apiKey) default: return llm.NewProviderError(resp.StatusCode, body, p.apiKey) } diff --git a/internal/ai/llm/errors.go b/internal/ai/llm/errors.go index e4f1bd2..b4c6480 100644 --- a/internal/ai/llm/errors.go +++ b/internal/ai/llm/errors.go @@ -137,3 +137,32 @@ func (e *ProviderError) GoString() string { return e.text } // Is reports whether target is ErrProvider. func (e *ProviderError) Is(target error) bool { return target == ErrProvider } + +// TimeoutStatusError is an HTTP status, such as 408, by which the vendor +// said the request timed out. It matches ErrTimeout, and Status keeps it +// apart from a transport timeout, so a caller can tell the vendor answered. +// Build it with NewTimeoutStatusError, which redacts its text. +type TimeoutStatusError struct { + status int + text string +} + +// NewTimeoutStatusError returns the error for a timeout HTTP status. +// secret is the credential sent with the request, or "" when the request +// sent none; every occurrence of a non-empty secret is removed from the +// error text, and the secret itself is not kept. +func NewTimeoutStatusError(status int, secret string) *TimeoutStatusError { + text := fmt.Sprintf("HTTP %d: %s", status, ErrTimeout) + return &TimeoutStatusError{status: status, text: Redact(text, secret)} +} + +// Status returns the HTTP status code. +func (e *TimeoutStatusError) Status() int { return e.status } + +func (e *TimeoutStatusError) Error() string { return e.text } + +// GoString keeps %#v to the redacted Error text instead of the fields. +func (e *TimeoutStatusError) GoString() string { return e.text } + +// Unwrap returns ErrTimeout. +func (e *TimeoutStatusError) Unwrap() error { return ErrTimeout } diff --git a/internal/ai/llm/errors_test.go b/internal/ai/llm/errors_test.go index 5ac5fa7..37406df 100644 --- a/internal/ai/llm/errors_test.go +++ b/internal/ai/llm/errors_test.go @@ -196,3 +196,17 @@ func TestTypedErrorsDoNotKeepSecret(t *testing.T) { } } } + +func TestTimeoutStatusError(t *testing.T) { + err := fmt.Errorf("openai: %w", NewTimeoutStatusError(408, "HTTP")) + if !errors.Is(err, ErrTimeout) || errors.Is(err, ErrProvider) { + t.Fatalf("%v: want ErrTimeout only", err) + } + var ts *TimeoutStatusError + if !errors.As(err, &ts) || ts.Status() != 408 { + t.Fatalf("TimeoutStatusError from %v", err) + } + if s := ts.Error() + fmt.Sprintf("%#v", ts); strings.Contains(s, "HTTP") { + t.Fatalf("text holds the secret: %s", s) + } +} diff --git a/internal/ai/llm/openai/openai.go b/internal/ai/llm/openai/openai.go index 70c94e9..d8bf66a 100644 --- a/internal/ai/llm/openai/openai.go +++ b/internal/ai/llm/openai/openai.go @@ -453,7 +453,7 @@ func (p *Provider) statusError(resp *http.Response, body []byte) error { case http.StatusTooManyRequests: return llm.NewRateLimitedError(retryAfter(resp.Header.Get("Retry-After"), time.Now()), p.apiKey) case http.StatusRequestTimeout: - return fmt.Errorf("HTTP %d: %w", resp.StatusCode, llm.ErrTimeout) + return llm.NewTimeoutStatusError(resp.StatusCode, p.apiKey) default: return llm.NewProviderError(resp.StatusCode, body, p.apiKey) } diff --git a/internal/ai/llm/presets/presets.go b/internal/ai/llm/presets/presets.go index 43b0d85..da86eda 100644 --- a/internal/ai/llm/presets/presets.go +++ b/internal/ai/llm/presets/presets.go @@ -178,13 +178,14 @@ var all = map[string]Preset{ // // The Provider sends s.Temperature with every request whose Temperature is // nil; a request that sets one keeps it. Neither is sent to a model the -// preset's OmitTemperature matches. +// preset's OmitTemperature matches. It is wrapped in llm.NewRetrying, so +// Generate retries transient vendor failures and has the default timeout. func FromSettings(s llm.Settings, key string) (llm.Provider, error) { cp, err := fromSettings(s, key) if err != nil { return nil, redact(err, key) } - return cp, nil + return llm.NewRetrying(cp), nil } func fromSettings(s llm.Settings, key string) (*configured, error) { diff --git a/internal/ai/llm/presets/presets_test.go b/internal/ai/llm/presets/presets_test.go index 142c564..41e4b3f 100644 --- a/internal/ai/llm/presets/presets_test.go +++ b/internal/ai/llm/presets/presets_test.go @@ -22,10 +22,8 @@ type captured struct { body map[string]json.RawMessage } -var errStop = errors.New("recorded") - -// recordingTransport records each request and fails it, so no request -// leaves the process. +// recordingTransport records each request and answers it 400, so no +// request leaves the process and the Provider never retries it. type recordingTransport struct{ reqs []captured } func (rt *recordingTransport) RoundTrip(r *http.Request) (*http.Response, error) { @@ -40,7 +38,12 @@ func (rt *recordingTransport) RoundTrip(r *http.Request) (*http.Response, error) } } rt.reqs = append(rt.reqs, c) - return nil, errStop + return &http.Response{ + StatusCode: http.StatusBadRequest, + Header: http.Header{"Content-Type": {"application/json"}}, + Body: io.NopCloser(strings.NewReader(`{}`)), + Request: r, + }, nil } // generate builds the provider s describes and returns the request its @@ -327,7 +330,7 @@ func TestNegativeTemperatureRejectedForRejectingModels(t *testing.T) { } req := plainRequest() req.Temperature = ptr(-0.5) - if _, err := p.Generate(context.Background(), req); err == nil || errors.Is(err, errStop) || !strings.Contains(err.Error(), "temperature") { + if _, err := p.Generate(context.Background(), req); err == nil || errors.Is(err, llm.ErrProvider) || !strings.Contains(err.Error(), "temperature") { t.Errorf("%s: Generate = %v, want a temperature validation error", s.Model, err) } } @@ -465,7 +468,7 @@ func TestGenerateValidationErrorsNeverLeakKey(t *testing.T) { req := plainRequest() req.Temperature = ptr(-0.5) _, err = p.Generate(context.Background(), req) - if err == nil || errors.Is(err, errStop) { + if err == nil || errors.Is(err, llm.ErrProvider) { t.Fatalf("%s: Generate = %v, want a validation error", key, err) } for e := error(err); e != nil; e = errors.Unwrap(e) { @@ -518,3 +521,142 @@ func TestConfigErrorsNeverLeakKey(t *testing.T) { }) } } + +// scriptedTransport answers each request with the next scripted status and +// body, echoing the key in every failure body, and counts the requests. A +// scripted status of 0 is a transport timeout instead of a response. +type scriptedTransport struct { + statuses []int + key string + sent int +} + +func (st *scriptedTransport) RoundTrip(r *http.Request) (*http.Response, error) { + if st.sent == len(st.statuses) { + return nil, fmt.Errorf("unscripted request %d", st.sent+1) + } + status := st.statuses[st.sent] + st.sent++ + if status == 0 { + return nil, transportTimeout{} + } + header := http.Header{"Content-Type": {"application/json"}} + body := `{"error":{"message":"bad key ` + st.key + `"}}` + switch status { + case http.StatusOK: + body = `{"model":"m","choices":[{"message":{"content":"tacos"},"finish_reason":"stop"}]}` + case http.StatusTooManyRequests: + header.Set("Retry-After", "1") + } + return &http.Response{ + StatusCode: status, + Header: header, + Body: io.NopCloser(strings.NewReader(body)), + Request: r, + }, nil +} + +// transportTimeout is a net.Error that timed out. +type transportTimeout struct{} + +func (transportTimeout) Error() string { return "i/o timeout" } +func (transportTimeout) Timeout() bool { return true } +func (transportTimeout) Temporary() bool { return true } + +// useTransport makes the adapters send through rt for the rest of the test. +func useTransport(t *testing.T, rt http.RoundTripper) { + t.Helper() + orig := http.DefaultTransport + http.DefaultTransport = rt + t.Cleanup(func() { http.DefaultTransport = orig }) +} + +// The Provider FromSettings returns retries a rate-limited Generate. +func TestFromSettingsRetriesRateLimited(t *testing.T) { + st := &scriptedTransport{statuses: []int{http.StatusTooManyRequests, http.StatusOK}, key: testKey} + useTransport(t, st) + + p, err := FromSettings(llm.Settings{Provider: "openai", Model: "m"}, testKey) + if err != nil { + t.Fatalf("FromSettings: %v", err) + } + resp, err := p.Generate(context.Background(), plainRequest()) + if err != nil { + t.Fatalf("Generate: %v", err) + } + if resp.Text != "tacos" { + t.Errorf("Text = %q, want %q", resp.Text, "tacos") + } + if st.sent != 2 { + t.Errorf("sent %d requests, want 2", st.sent) + } +} + +// When every retry fails, the error FromSettings's Provider returns keeps +// its sentinel and never holds the key the vendor echoed. +func TestFromSettingsRetryErrorNeverLeaksKey(t *testing.T) { + st := &scriptedTransport{statuses: []int{http.StatusTooManyRequests, http.StatusTooManyRequests, http.StatusServiceUnavailable}, key: testKey} + useTransport(t, st) + + p, err := FromSettings(llm.Settings{Provider: "openai", Model: "m"}, testKey) + if err != nil { + t.Fatalf("FromSettings: %v", err) + } + _, err = p.Generate(context.Background(), plainRequest()) + if !errors.Is(err, llm.ErrProvider) { + t.Fatalf("Generate = %v, want %v", err, llm.ErrProvider) + } + if st.sent != llm.MaxRetries+1 { + t.Errorf("sent %d requests, want %d", st.sent, llm.MaxRetries+1) + } + for e := error(err); e != nil; e = errors.Unwrap(e) { + if s := e.Error() + fmt.Sprintf("%#v", e); strings.Contains(s, testKey) { + t.Fatalf("error leaks the key: %s", s) + } + } +} + +// An HTTP 408 from the vendor is a 4xx answer and is not retried, though it +// matches llm.ErrTimeout. +func TestFromSettingsDoesNotRetryHTTP408(t *testing.T) { + for _, provider := range []string{"openai", "anthropic"} { + t.Run(provider, func(t *testing.T) { + st := &scriptedTransport{statuses: []int{http.StatusRequestTimeout}, key: testKey} + useTransport(t, st) + + p, err := FromSettings(llm.Settings{Provider: provider, Model: "m"}, testKey) + if err != nil { + t.Fatalf("FromSettings: %v", err) + } + _, err = p.Generate(context.Background(), plainRequest()) + var ts *llm.TimeoutStatusError + if !errors.Is(err, llm.ErrTimeout) || !errors.As(err, &ts) || ts.Status() != http.StatusRequestTimeout { + t.Fatalf("Generate = %v, want an HTTP 408 ErrTimeout", err) + } + if st.sent != 1 { + t.Errorf("sent %d requests, want 1", st.sent) + } + }) + } +} + +// A transport timeout carries no status and is retried. +func TestFromSettingsRetriesTransportTimeout(t *testing.T) { + st := &scriptedTransport{statuses: []int{0, http.StatusOK}, key: testKey} + useTransport(t, st) + + p, err := FromSettings(llm.Settings{Provider: "openai", Model: "m"}, testKey) + if err != nil { + t.Fatalf("FromSettings: %v", err) + } + resp, err := p.Generate(context.Background(), plainRequest()) + if err != nil { + t.Fatalf("Generate: %v", err) + } + if resp.Text != "tacos" { + t.Errorf("Text = %q, want %q", resp.Text, "tacos") + } + if st.sent != 2 { + t.Errorf("sent %d requests, want 2", st.sent) + } +} diff --git a/internal/ai/llm/retry.go b/internal/ai/llm/retry.go new file mode 100644 index 0000000..ec50fa5 --- /dev/null +++ b/internal/ai/llm/retry.go @@ -0,0 +1,124 @@ +package llm + +import ( + "context" + "errors" + "math/rand/v2" + "time" +) + +const ( + // DefaultTimeout bounds a whole Generate call, retries and waits + // included, whose Request sets no Timeout. + DefaultTimeout = 60 * time.Second + // MaxRetries is how many times a failed Generate is retried, so a call + // makes at most MaxRetries+1 attempts. + MaxRetries = 2 + // MaxRetryAfter caps the wait a rate-limited vendor may ask for. + MaxRetryAfter = 20 * time.Second + // backoffBase is the first retry's backoff before jitter; each later + // retry doubles it. + backoffBase = time.Second +) + +// Retrying is a Provider that retries transient Generate failures of the +// Provider it wraps. It retries ErrRateLimited, a transport ErrTimeout and +// a 5xx ErrProvider at most MaxRetries times, and nothing else: an error +// carrying a 4xx HTTP status other than a rate limit, such as an HTTP 408 +// TimeoutStatusError, is not retried. A rate-limited retry waits for the +// vendor's RetryAfter, capped at MaxRetryAfter; every other retry, and a +// rate limit that names no wait, waits a jittered exponential backoff. The Request's Timeout, or DefaultTimeout, is one +// deadline for the whole call, retries and waits included; a caller's +// earlier context deadline wins. A retry whose wait would not end before +// that deadline is not made: the last error is returned at once. +// ListModels and Verify pass straight through. +type Retrying struct { + Provider + // now is the clock; tests replace it. + now func() time.Time + // sleep waits d or until ctx is done; tests replace it. + sleep func(ctx context.Context, d time.Duration) error + // jitter returns a random duration in [0, n) for n > 0; tests replace + // it. + jitter func(n time.Duration) time.Duration +} + +// NewRetrying returns inner wrapped with retries and the default Generate +// timeout. +func NewRetrying(inner Provider) *Retrying { + return &Retrying{ + Provider: inner, + now: time.Now, + sleep: sleep, + jitter: rand.N[time.Duration], + } +} + +// Generate sends req to the wrapped Provider, retrying transient failures. +// req.Timeout, or DefaultTimeout when it is 0, sets one deadline for the +// whole call, carried by the context each attempt receives; attempts get +// req with Timeout 0 so none restarts the clock. +func (r *Retrying) Generate(ctx context.Context, req Request) (Response, error) { + timeout := req.Timeout + if timeout == 0 { + timeout = DefaultTimeout + } + if timeout > 0 { + // A negative Timeout stays on req for the wrapped Provider's + // Validate to reject, with its key redaction. + req.Timeout = 0 + } + ctx, cancel := context.WithDeadline(ctx, r.now().Add(timeout)) + defer cancel() + deadline, _ := ctx.Deadline() + for retry := 0; ; retry++ { + resp, err := r.Provider.Generate(ctx, req) + if err == nil || retry == MaxRetries || ctx.Err() != nil { + return resp, err + } + wait, ok := r.wait(err, retry) + if !ok { + return resp, err + } + if !r.now().Add(wait).Before(deadline) { + return resp, err + } + if serr := r.sleep(ctx, wait); serr != nil { + return resp, err + } + } +} + +// wait returns how long to wait before retrying after err, the retry-th +// retry counting from 0, and false when err is not retried. +func (r *Retrying) wait(err error, retry int) (time.Duration, bool) { + var st interface{ Status() int } + if errors.As(err, &st) && st.Status() >= 400 && st.Status() <= 499 { + return 0, false + } + var rl *RateLimitedError + if errors.As(err, &rl) && rl.RetryAfter > 0 { + return min(rl.RetryAfter, MaxRetryAfter), true + } + var pe *ProviderError + switch { + case errors.Is(err, ErrRateLimited), errors.Is(err, ErrTimeout): + case errors.As(err, &pe) && pe.Status() >= 500 && pe.Status() <= 599: + default: + return 0, false + } + half := (backoffBase << retry) / 2 + return half + r.jitter(half), true +} + +// sleep waits d or until ctx is done, returning ctx's error in that case. +func sleep(ctx context.Context, d time.Duration) error { + t := time.NewTimer(d) + defer t.Stop() + select { + case <-t.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} diff --git a/internal/ai/llm/retry_test.go b/internal/ai/llm/retry_test.go new file mode 100644 index 0000000..2095616 --- /dev/null +++ b/internal/ai/llm/retry_test.go @@ -0,0 +1,359 @@ +package llm + +import ( + "context" + "errors" + "fmt" + "slices" + "testing" + "time" +) + +// fakeClock is a clock whose sleeps return at once, advance it and are +// recorded. +type fakeClock struct { + t time.Time + sleeps []time.Duration +} + +func (c *fakeClock) now() time.Time { return c.t } + +func (c *fakeClock) sleep(ctx context.Context, d time.Duration) error { + if err := ctx.Err(); err != nil { + return err + } + c.sleeps = append(c.sleeps, d) + c.t = c.t.Add(d) + return nil +} + +// newTestRetrying returns a Retrying over a fresh Fake with a fake clock +// at the real current time, so real context deadlines compare sensibly, +// and jitter that always returns its maximum. +func newTestRetrying() (*Retrying, *Fake, *fakeClock) { + f := &Fake{} + c := &fakeClock{t: time.Now()} + r := NewRetrying(f) + r.now = c.now + r.sleep = c.sleep + r.jitter = func(n time.Duration) time.Duration { return n - 1 } + return r, f, c +} + +var retryReq = Request{Messages: []Message{{Role: RoleUser, Content: "plan dinners"}}} + +func ok(text string) GenerateResult { return GenerateResult{Response: Response{Text: text}} } + +func fail(err error) GenerateResult { return GenerateResult{Err: err} } + +func assertConsumed(t *testing.T, f *Fake, attempts int) { + t.Helper() + if g, _, _ := f.Remaining(); g != 0 { + t.Errorf("%d scripted Generate results unused", g) + } + if n := len(f.Requests()); n != attempts { + t.Errorf("made %d attempts, want %d", n, attempts) + } +} + +func TestRetryingRateLimitedThenSuccess(t *testing.T) { + r, f, c := newTestRetrying() + f.QueueGenerate(fail(NewRateLimitedError(3*time.Second, "")), ok("dinner")) + + resp, err := r.Generate(context.Background(), retryReq) + if err != nil || resp.Text != "dinner" { + t.Fatalf("Generate = %+v, %v", resp, err) + } + if want := []time.Duration{3 * time.Second}; !slices.Equal(c.sleeps, want) { + t.Errorf("sleeps = %v, want %v", c.sleeps, want) + } + assertConsumed(t, f, 2) +} + +func TestRetryingRateLimitedWaitIsCapped(t *testing.T) { + r, f, c := newTestRetrying() + f.QueueGenerate(fail(NewRateLimitedError(time.Hour, "")), ok("dinner")) + + if _, err := r.Generate(context.Background(), retryReq); err != nil { + t.Fatal(err) + } + if want := []time.Duration{20 * time.Second}; !slices.Equal(c.sleeps, want) { + t.Errorf("sleeps = %v, want %v", c.sleeps, want) + } + assertConsumed(t, f, 2) +} + +func TestRetryingRateLimitedWithoutRetryAfterBacksOff(t *testing.T) { + r, f, c := newTestRetrying() + f.QueueGenerate(fail(NewRateLimitedError(0, "")), ok("dinner")) + + if _, err := r.Generate(context.Background(), retryReq); err != nil { + t.Fatal(err) + } + if want := []time.Duration{time.Second - 1}; !slices.Equal(c.sleeps, want) { + t.Errorf("sleeps = %v, want %v", c.sleeps, want) + } +} + +func TestRetrying5xxTwiceThenSuccess(t *testing.T) { + r, f, c := newTestRetrying() + f.QueueGenerate( + fail(NewProviderError(503, []byte("busy"), "")), + // Adapters wrap the error they return. + fail(fmt.Errorf("openai: generate: %w", NewProviderError(503, []byte("busy"), ""))), + ok("dinner"), + ) + + resp, err := r.Generate(context.Background(), retryReq) + if err != nil || resp.Text != "dinner" { + t.Fatalf("Generate = %+v, %v", resp, err) + } + // Max jitter: [0.5 s, 1 s) then [1 s, 2 s). + if want := []time.Duration{time.Second - 1, 2*time.Second - 1}; !slices.Equal(c.sleeps, want) { + t.Errorf("sleeps = %v, want %v", c.sleeps, want) + } + assertConsumed(t, f, 3) +} + +func TestRetryingBackoffJitterLowerBound(t *testing.T) { + r, f, c := newTestRetrying() + r.jitter = func(time.Duration) time.Duration { return 0 } + f.QueueGenerate(fail(ErrTimeout), fail(ErrTimeout), ok("dinner")) + + if _, err := r.Generate(context.Background(), retryReq); err != nil { + t.Fatal(err) + } + if want := []time.Duration{500 * time.Millisecond, time.Second}; !slices.Equal(c.sleeps, want) { + t.Errorf("sleeps = %v, want %v", c.sleeps, want) + } +} + +func TestRetrying5xxThreeTimesFails(t *testing.T) { + r, f, c := newTestRetrying() + last := NewProviderError(503, []byte("third"), "") + f.QueueGenerate( + fail(NewProviderError(503, []byte("first"), "")), + fail(NewProviderError(503, []byte("second"), "")), + fail(last), + ) + + _, err := r.Generate(context.Background(), retryReq) + if err != last { + t.Fatalf("err = %v, want the last attempt's error", err) + } + if len(c.sleeps) != MaxRetries { + t.Errorf("slept %d times, want %d", len(c.sleeps), MaxRetries) + } + assertConsumed(t, f, 3) +} + +func TestRetryingDoesNotRetryPermanentErrors(t *testing.T) { + for name, err := range map[string]error{ + "401 auth": ErrAuth, + "400": NewProviderError(400, []byte("bad request"), ""), + "404": NewProviderError(404, []byte("no such model"), ""), + "408": NewTimeoutStatusError(408, ""), + "wrapped 408": fmt.Errorf("openai: generate: %w", NewTimeoutStatusError(408, "")), + "bad output": ErrBadOutput, + "other": errors.New("llm: request has no messages"), + } { + t.Run(name, func(t *testing.T) { + r, f, c := newTestRetrying() + f.QueueGenerate(fail(err)) + + if _, got := r.Generate(context.Background(), retryReq); got != err { + t.Fatalf("err = %v, want %v", got, err) + } + if len(c.sleeps) != 0 { + t.Errorf("slept %v", c.sleeps) + } + assertConsumed(t, f, 1) + }) + } +} + +// A transport timeout matches ErrTimeout like an HTTP 408, but carries no +// status, so it is retried. +func TestRetryingTransportTimeoutRetried(t *testing.T) { + r, f, _ := newTestRetrying() + f.QueueGenerate(fail(fmt.Errorf("openai: generate: %w", ErrTimeout)), ok("dinner")) + + if _, err := r.Generate(context.Background(), retryReq); err != nil { + t.Fatal(err) + } + assertConsumed(t, f, 2) +} + +func TestRetryingDeadlineShorterThanRetryAfter(t *testing.T) { + r, f, c := newTestRetrying() + rl := NewRateLimitedError(10*time.Second, "") + f.QueueGenerate(fail(rl)) + ctx, cancel := context.WithDeadline(context.Background(), c.t.Add(5*time.Second)) + defer cancel() + + if _, err := r.Generate(ctx, retryReq); err != rl { + t.Fatalf("err = %v, want the rate-limited error", err) + } + if len(c.sleeps) != 0 { + t.Errorf("slept %v", c.sleeps) + } + assertConsumed(t, f, 1) +} + +func TestRetryingDeadlineShorterThanBackoff(t *testing.T) { + r, f, c := newTestRetrying() + f.QueueGenerate(fail(ErrTimeout)) + // The first backoff with maximum jitter is just under 1 s. + ctx, cancel := context.WithDeadline(context.Background(), c.t.Add(time.Second-1)) + defer cancel() + + if _, err := r.Generate(ctx, retryReq); !errors.Is(err, ErrTimeout) { + t.Fatalf("err = %v, want ErrTimeout", err) + } + if len(c.sleeps) != 0 { + t.Errorf("slept %v", c.sleeps) + } + assertConsumed(t, f, 1) +} + +func TestRetryingRetriesWithinDeadline(t *testing.T) { + r, f, c := newTestRetrying() + f.QueueGenerate(fail(NewRateLimitedError(3*time.Second, "")), ok("dinner")) + ctx, cancel := context.WithDeadline(context.Background(), c.t.Add(time.Minute)) + defer cancel() + + if _, err := r.Generate(ctx, retryReq); err != nil { + t.Fatal(err) + } + assertConsumed(t, f, 2) +} + +func TestRetryingStopsWhenContextDone(t *testing.T) { + r, f, _ := newTestRetrying() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + f.QueueGenerate(fail(ErrTimeout)) + + if _, err := r.Generate(ctx, retryReq); !errors.Is(err, ErrTimeout) { + t.Fatalf("err = %v, want ErrTimeout", err) + } + assertConsumed(t, f, 1) +} + +// slowProvider is a Provider whose every attempt takes d on clock c, +// records the deadline of the context it received, and times out. +type slowProvider struct { + Provider + c *fakeClock + d time.Duration + reqs []Request + deadlines []time.Time +} + +func (p *slowProvider) Generate(ctx context.Context, req Request) (Response, error) { + p.reqs = append(p.reqs, req) + deadline, _ := ctx.Deadline() + p.deadlines = append(p.deadlines, deadline) + p.c.t = p.c.t.Add(p.d) + return Response{}, ErrTimeout +} + +func TestRetryingTimeoutBoundsWholeCall(t *testing.T) { + for _, tc := range []struct { + name string + timeout time.Duration + want time.Duration + }{ + {"default", 0, 60 * time.Second}, + {"custom", 10 * time.Second, 10 * time.Second}, + } { + t.Run(tc.name, func(t *testing.T) { + r, _, c := newTestRetrying() + // Each attempt uses most of the one deadline, so a fresh full + // timeout per attempt would allow every retry; one deadline + // leaves room for exactly one. + p := &slowProvider{c: c, d: tc.want * 5 / 6} + r.Provider = p + start := c.t + req := retryReq + req.Timeout = tc.timeout + + if _, err := r.Generate(context.Background(), req); !errors.Is(err, ErrTimeout) { + t.Fatalf("err = %v, want ErrTimeout", err) + } + if len(p.deadlines) != 2 { + t.Fatalf("made %d attempts, want 2", len(p.deadlines)) + } + for i, d := range p.deadlines { + if !d.Equal(start.Add(tc.want)) { + t.Errorf("attempt %d deadline = start+%v, want start+%v", i, d.Sub(start), tc.want) + } + if p.reqs[i].Timeout != 0 { + t.Errorf("attempt %d Timeout = %v, want 0", i, p.reqs[i].Timeout) + } + } + if want := []time.Duration{time.Second - 1}; !slices.Equal(c.sleeps, want) { + t.Errorf("sleeps = %v, want %v", c.sleeps, want) + } + }) + } +} + +func TestRetryingCallerDeadlineShorterThanTimeout(t *testing.T) { + r, _, c := newTestRetrying() + p := &slowProvider{c: c, d: time.Second} + r.Provider = p + ctx, cancel := context.WithDeadline(context.Background(), c.t.Add(5*time.Second)) + defer cancel() + + if _, err := r.Generate(ctx, retryReq); !errors.Is(err, ErrTimeout) { + t.Fatalf("err = %v, want ErrTimeout", err) + } + for i, d := range p.deadlines { + if want, _ := ctx.Deadline(); !d.Equal(want) { + t.Errorf("attempt %d deadline = %v, want caller's %v", i, d, want) + } + } +} + +func TestRetryingRejectsNegativeTimeout(t *testing.T) { + r, f, _ := newTestRetrying() + f.QueueGenerate(fail(errors.New("llm: timeout is negative"))) + req := retryReq + req.Timeout = -time.Second + + if _, err := r.Generate(context.Background(), req); err == nil { + t.Fatal("Generate accepted a negative timeout") + } + assertConsumed(t, f, 1) + if got := f.Requests()[0].Timeout; got != -time.Second { + t.Errorf("wrapped Provider got Timeout %v, want -1s to reject", got) + } +} + +func TestRetryingPassesThroughListModelsAndVerify(t *testing.T) { + r, f, _ := newTestRetrying() + f.QueueListModels(ListModelsResult{Err: ErrTimeout}) + f.QueueVerify(ErrTimeout) + + if _, err := r.ListModels(context.Background()); !errors.Is(err, ErrTimeout) { + t.Fatalf("ListModels err = %v", err) + } + if err := r.Verify(context.Background()); !errors.Is(err, ErrTimeout) { + t.Fatalf("Verify err = %v", err) + } + if _, l, v := f.Remaining(); l != 0 || v != 0 { + t.Fatalf("Remaining = %d, %d", l, v) + } +} + +func TestSleepStopsWhenContextDone(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := sleep(ctx, time.Hour); !errors.Is(err, context.Canceled) { + t.Fatalf("sleep err = %v", err) + } + if err := sleep(context.Background(), time.Millisecond); err != nil { + t.Fatalf("sleep err = %v", err) + } +} -- 2.55.0