Compare commits

..
Author SHA1 Message Date
dev_agentandClaude Opus 5.5 bcdea85a2d Wire retrying Provider into presets.FromSettings
CI / ci (pull_request) Successful in 1m31s
CI / ci (push) Successful in 1m32s
FromSettings now returns llm.NewRetrying around the configured adapter,
so every Provider it builds retries transient Generate failures and gets
the default timeout; ListModels and Verify pass through.

The test recording transport now answers 400 instead of failing the
round trip: the ollama adapter reports a transport failure as
ErrTimeout, which is retried, so the helper would see three requests.

Tests: a 429 then 200 succeeds on the second request; exhausted retries
return ErrProvider after MaxRetries+1 requests with no key in any error.

Maestro task: 1a882923-fdb4-4576-ab82-9529e9974b26

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-04 10:49:46 -04:00
dev_agentandClaude Opus 5.5 cbf24e63e1 Add retrying Provider wrapper with backoff and default timeout
llm.NewRetrying wraps a Provider and retries ErrRateLimited, ErrTimeout
and 5xx ErrProvider at most twice. A rate limit waits RetryAfter, capped
at 20 s; other retries use jittered exponential backoff. ErrAuth, 4xx and
other errors are never retried, and no retry is made whose wait would not
end before the caller's context deadline. Generate gets a 60 s per-attempt
timeout when Request.Timeout is 0.

Maestro task: 1a882923-fdb4-4576-ab82-9529e9974b26

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-04 10:48:07 -04:00
dev_agent e105cdb9c5 Add xAI Grok as a provider (#46)
CI / ci (push) Successful in 1m28s
Co-authored-by: dev_agent <dev_agent@cubecraftlabs.com>
Co-committed-by: dev_agent <dev_agent@cubecraftlabs.com>
2026-10-04 09:36:10 -04:00
dev_agent 9ace3aba9f Provider presets and factory (#44)
CI / ci (push) Successful in 1m28s
Co-authored-by: dev_agent <dev_agent@cubecraftlabs.com>
Co-committed-by: dev_agent <dev_agent@cubecraftlabs.com>
2026-10-04 09:18:34 -04:00
dev_agent 32ab0c0b6d Ollama native adapter (#41)
CI / ci (push) Successful in 1m29s
Co-authored-by: dev_agent <dev_agent@cubecraftlabs.com>
Co-committed-by: dev_agent <dev_agent@cubecraftlabs.com>
2026-10-04 08:47:05 -04:00
dev_agent 15886f2c55 Anthropic Messages adapter (#40)
CI / ci (push) Successful in 1m27s
Co-authored-by: dev_agent <dev_agent@cubecraftlabs.com>
Co-committed-by: dev_agent <dev_agent@cubecraftlabs.com>
2026-10-04 08:09:20 -04:00
18 changed files with 2774 additions and 6 deletions
+325
View File
@@ -0,0 +1,325 @@
// Package ollama is the llm.Provider adapter for Ollama's native API. It
// calls POST {base}/api/chat and GET {base}/api/tags over plain net/http.
//
// Ollama takes no credentials: the adapter sends no Authorization header
// and never logs. The base URL may be a LAN address, and errors for an
// unreachable server name it, so New refuses a base URL that could carry a
// credential.
package ollama
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"strings"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm"
)
// Config configures a Provider.
type Config struct {
// BaseURL is the Ollama server root the /api endpoint paths are
// appended to, e.g. "http://192.168.1.20:11434" or
// "http://ollama:11434". It is required.
BaseURL string
// Model is the model Generate requests, e.g. "llama3.2". ListModels
// and Verify do not need it.
Model string
}
// Provider is an Ollama native API llm.Provider.
type Provider struct {
base string
model string
client *http.Client
}
var _ llm.Provider = (*Provider)(nil)
// New validates cfg and returns a Provider.
func New(cfg Config) (*Provider, error) {
// Base URL errors never echo the URL: a rejected one may hold a
// credential.
u, err := url.Parse(cfg.BaseURL)
if err != nil {
return nil, errors.New("ollama: base URL does not parse")
}
if (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
return nil, errors.New("ollama: base URL must be an absolute http or https URL")
}
// A trailing "?" or "#" parses to an empty query or fragment, yet the
// endpoint path appended to it would land in the query or fragment.
if u.User != nil || strings.ContainsAny(cfg.BaseURL, "?#") {
return nil, errors.New("ollama: base URL must not carry credentials, a query or a fragment")
}
return &Provider{
base: strings.TrimRight(cfg.BaseURL, "/"),
model: cfg.Model,
client: noRedirectClient(),
}, nil
}
// maxResponseBody caps a successful response body; a larger one is an error.
const maxResponseBody = 16 << 20
// maxErrorBody caps how much of an error response body is read; only an
// llm.MaxBodyExcerpt of it is kept.
const maxErrorBody = 64 << 10
type message struct {
Role string `json:"role"`
Content string `json:"content"`
}
type options struct {
Temperature *float64 `json:"temperature,omitempty"`
NumPredict int `json:"num_predict,omitempty"`
}
type chatRequest struct {
Model string `json:"model"`
Messages []message `json:"messages"`
Stream bool `json:"stream"`
Format json.RawMessage `json:"format,omitempty"`
Options *options `json:"options,omitempty"`
}
type chatResponse struct {
Model string `json:"model"`
Message *struct {
Role string `json:"role"`
Content string `json:"content"`
} `json:"message"`
Done bool `json:"done"`
DoneReason string `json:"done_reason"`
PromptEvalCount int `json:"prompt_eval_count"`
EvalCount int `json:"eval_count"`
}
// Generate sends req to {base}/api/chat without streaming. A Request with a
// Schema sends it as "format", so Ollama constrains the reply to it;
// Response.JSON is then the reply text, which must be a JSON object.
func (p *Provider) Generate(ctx context.Context, req llm.Request) (llm.Response, error) {
if err := req.Validate(); err != nil {
return llm.Response{}, fmt.Errorf("ollama: generate: %w", err)
}
if p.model == "" {
return llm.Response{}, errors.New("ollama: generate: no model configured")
}
body := chatRequest{Model: p.model, Stream: false}
if req.System != "" {
body.Messages = append(body.Messages, message{Role: "system", Content: req.System})
}
for _, m := range req.Messages {
body.Messages = append(body.Messages, message{Role: string(m.Role), Content: m.Content})
}
if req.Schema != nil {
body.Format = req.Schema.Schema
}
if req.Temperature != nil || req.MaxTokens > 0 {
body.Options = &options{Temperature: req.Temperature, NumPredict: req.MaxTokens}
}
payload, err := json.Marshal(body)
if err != nil {
return llm.Response{}, fmt.Errorf("ollama: generate: encode request: %w", err)
}
if req.Timeout > 0 {
var cancel context.CancelFunc
ctx, cancel = context.WithTimeout(ctx, req.Timeout)
defer cancel()
}
raw, err := p.do(ctx, http.MethodPost, "/api/chat", payload)
if err != nil {
var pe *llm.ProviderError
if errors.As(err, &pe) && pe.Status() == http.StatusNotFound && p.modelNotFound(pe.Body()) {
// The body is not quoted: the model name is all the caller
// needs. Any other 404 keeps its body excerpt below.
return llm.Response{}, fmt.Errorf("ollama: generate: model %q not found: %w", p.model, llm.NewProviderError(pe.Status(), nil, ""))
}
return llm.Response{}, fmt.Errorf("ollama: generate: %w", err)
}
var cr chatResponse
if err := json.Unmarshal(raw, &cr); err != nil {
return llm.Response{}, fmt.Errorf("ollama: generate: decode response: %w", err)
}
if !cr.Done || cr.Message == nil {
return llm.Response{}, errors.New("ollama: generate: response is not a finished message")
}
resp := llm.Response{
Text: cr.Message.Content,
Usage: llm.Usage{InputTokens: cr.PromptEvalCount, OutputTokens: cr.EvalCount},
Model: cr.Model,
FinishReason: cr.DoneReason,
}
if req.Schema == nil {
return resp, nil
}
if text := strings.TrimSpace(resp.Text); isJSONObject(text) {
resp.JSON = json.RawMessage(text)
return resp, nil
}
return resp, fmt.Errorf("ollama: generate: reply is not a JSON object: %w", llm.ErrBadOutput)
}
// modelNotFound reports whether body is Ollama's error for the configured
// model not being pulled: {"error": "model \"name\" not found, ..."}, or
// with the name in single quotes as newer servers send it. A 404 from
// anything else, such as a proxy without /api/chat, is not.
func (p *Provider) modelNotFound(body string) bool {
var e struct {
Error string `json:"error"`
}
if err := json.Unmarshal([]byte(body), &e); err != nil {
return false
}
return strings.HasPrefix(e.Error, fmt.Sprintf("model %q not found", p.model)) ||
strings.HasPrefix(e.Error, fmt.Sprintf("model '%s' not found", p.model))
}
// isJSONObject reports whether s is exactly one JSON object.
func isJSONObject(s string) bool {
return strings.HasPrefix(s, "{") && json.Valid([]byte(s))
}
type tagsResponse struct {
Models []struct {
Name string `json:"name"`
} `json:"models"`
}
// ListModels returns every model {base}/api/tags lists.
func (p *Provider) ListModels(ctx context.Context) ([]llm.Model, error) {
models, err := p.tags(ctx)
if err != nil {
return nil, fmt.Errorf("ollama: list models: %w", err)
}
return models, nil
}
// Verify checks that the server answers GET {base}/api/tags with a model
// list.
func (p *Provider) Verify(ctx context.Context) error {
if _, err := p.tags(ctx); err != nil {
return fmt.Errorf("ollama: verify: %w", err)
}
return nil
}
func (p *Provider) tags(ctx context.Context) ([]llm.Model, error) {
raw, err := p.do(ctx, http.MethodGet, "/api/tags", nil)
if err != nil {
return nil, err
}
var tr tagsResponse
if err := json.Unmarshal(raw, &tr); err != nil {
return nil, fmt.Errorf("decode response: %w", err)
}
if tr.Models == nil {
return nil, errors.New("response has no models list")
}
models := make([]llm.Model, 0, len(tr.Models))
for i, m := range tr.Models {
if m.Name == "" {
return nil, fmt.Errorf("model %d has no name", i)
}
models = append(models, llm.Model{ID: m.Name, DisplayName: m.Name})
}
return models, nil
}
// do sends one request and returns the body of a 2xx response. Any other
// status, a 3xx included, is an *llm.ProviderError.
func (p *Provider) do(ctx context.Context, method, path string, payload []byte) ([]byte, error) {
var reqBody io.Reader
if payload != nil {
reqBody = bytes.NewReader(payload)
}
httpReq, err := http.NewRequestWithContext(ctx, method, p.base+path, reqBody)
if err != nil {
return nil, fmt.Errorf("build request: %w", err)
}
httpReq.Header.Set("Accept", "application/json")
if payload != nil {
httpReq.Header.Set("Content-Type", "application/json")
}
httpResp, err := p.client.Do(httpReq)
if err != nil {
return nil, p.transportError(err)
}
defer httpResp.Body.Close()
if httpResp.StatusCode < 200 || httpResp.StatusCode > 299 {
body, err := io.ReadAll(io.LimitReader(httpResp.Body, maxErrorBody))
if err != nil {
return nil, p.transportError(err)
}
return nil, llm.NewProviderError(httpResp.StatusCode, body, "")
}
body, err := io.ReadAll(io.LimitReader(httpResp.Body, maxResponseBody+1))
if err != nil {
return nil, p.transportError(err)
}
if len(body) > maxResponseBody {
return nil, fmt.Errorf("response body exceeds %d bytes", maxResponseBody)
}
return body, nil
}
// noRedirectClient returns an HTTP client that never follows a redirect:
// the 3xx response is returned and becomes a provider error, so a request
// is only ever sent to the configured server.
func noRedirectClient() *http.Client {
return &http.Client{
CheckRedirect: func(*http.Request, []*http.Request) error {
return http.ErrUseLastResponse
},
}
}
func isTimeout(err error) bool {
var ne net.Error
return errors.Is(err, context.DeadlineExceeded) || (errors.As(err, &ne) && ne.Timeout())
}
// transportFailure is a failed HTTP exchange. Its text names the base URL
// and the cause; it unwraps only to the cause's category, llm.ErrTimeout or
// context.Canceled.
type transportFailure struct {
text string
kind error
}
func (e *transportFailure) Error() string { return e.text }
func (e *transportFailure) Unwrap() error { return e.kind }
// transportError reports a failure to complete the HTTP exchange. A
// cancelled context is the caller's choice and stays context.Canceled.
// Every other failure, a refused connection or a timeout alike, means the
// server could not be reached in time: it matches llm.ErrTimeout and says
// "Ollama not reachable at {base}".
func (p *Provider) transportError(err error) error {
timedOut := isTimeout(err)
// The *url.Error repeats the request URL; the base is named once below.
var ue *url.Error
if errors.As(err, &ue) {
err = ue.Err
}
if !timedOut && errors.Is(err, context.Canceled) {
return &transportFailure{text: "request canceled: " + err.Error(), kind: context.Canceled}
}
text := fmt.Sprintf("Ollama not reachable at %s: %s", p.base, err)
if timedOut {
text = fmt.Sprintf("Ollama not reachable at %s: %s: %s", p.base, llm.ErrTimeout, err)
}
return &transportFailure{text: text, kind: llm.ErrTimeout}
}
+615
View File
@@ -0,0 +1,615 @@
package ollama
import (
"context"
"encoding/json"
"errors"
"io"
"net/http"
"net/http/httptest"
"net/url"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm"
)
// recorded is one request the test server received.
type recorded struct {
method string
path string
query url.Values
header http.Header
body []byte
}
// reply is a response the test server replays.
type reply struct {
status int
fixture string // testdata file
body string // used when fixture is empty
delay time.Duration
}
// ollamaServer replays its replies in order, repeating the last, and
// records every request.
type ollamaServer struct {
*httptest.Server
mu sync.Mutex
got []recorded
}
func newOllamaServer(t *testing.T, replies ...reply) *ollamaServer {
t.Helper()
payloads := make([][]byte, len(replies))
for i, r := range replies {
payloads[i] = []byte(r.body)
if r.fixture != "" {
raw, err := os.ReadFile(filepath.Join("testdata", r.fixture))
if err != nil {
t.Fatal(err)
}
payloads[i] = raw
}
}
s := &ollamaServer{}
s.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
body, _ := io.ReadAll(req.Body)
s.mu.Lock()
n := len(s.got)
s.got = append(s.got, recorded{req.Method, req.URL.Path, req.URL.Query(), req.Header.Clone(), body})
s.mu.Unlock()
i := min(n, len(replies)-1)
r := replies[i]
if r.delay > 0 {
select {
case <-time.After(r.delay):
case <-req.Context().Done():
return
}
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(r.status)
w.Write(payloads[i])
}))
t.Cleanup(s.Close)
return s
}
func (s *ollamaServer) requests() []recorded {
s.mu.Lock()
defer s.mu.Unlock()
return append([]recorded(nil), s.got...)
}
func (s *ollamaServer) only(t *testing.T) recorded {
t.Helper()
got := s.requests()
if len(got) != 1 {
t.Fatalf("server got %d requests, want 1", len(got))
}
return got[0]
}
func newProvider(t *testing.T, base string) *Provider {
t.Helper()
p, err := New(Config{BaseURL: base, Model: "llama3.2"})
if err != nil {
t.Fatal(err)
}
return p
}
var dinnersSchema = json.RawMessage(`{"type":"object","properties":{"dinners":{"type":"array"}},"required":["dinners"]}`)
func schemaRequest() llm.Request {
temp := 0.0
return llm.Request{
System: "You plan dinners for Our kitchen.",
Messages: []llm.Message{
{Role: llm.RoleUser, Content: "Suggest a Side quest."},
{Role: llm.RoleAssistant, Content: "Something quick?"},
{Role: llm.RoleUser, Content: "Yes, a Quick dinner."},
},
Temperature: &temp,
MaxTokens: 400,
Schema: &llm.JSONSchema{Name: "side_quests", Schema: dinnersSchema},
}
}
func textRequest() llm.Request {
return llm.Request{Messages: []llm.Message{{Role: llm.RoleUser, Content: "What should we cook tonight?"}}}
}
// sentBody decodes a request body into a generic map.
func sentBody(t *testing.T, r recorded) map[string]any {
t.Helper()
var m map[string]any
if err := json.Unmarshal(r.body, &m); err != nil {
t.Fatalf("request body %q: %v", r.body, err)
}
return m
}
// canonical re-encodes JSON so two encodings of one value compare equal.
func canonical(t *testing.T, raw []byte) string {
t.Helper()
var v any
if err := json.Unmarshal(raw, &v); err != nil {
t.Fatalf("JSON %q: %v", raw, err)
}
out, _ := json.Marshal(v)
return string(out)
}
func assertNoAuth(t *testing.T, r recorded) {
t.Helper()
if got := r.header.Values("Authorization"); len(got) != 0 {
t.Errorf("Authorization = %q, want none", got)
}
}
func TestGenerateText(t *testing.T) {
s := newOllamaServer(t, reply{status: 200, fixture: "chat_text.json"})
p := newProvider(t, s.URL+"/")
resp, err := p.Generate(context.Background(), textRequest())
if err != nil {
t.Fatal(err)
}
if resp.Text != "Try a crock pot chili: it is a Quick dinner." || resp.JSON != nil {
t.Errorf("Text, JSON = %q, %s", resp.Text, resp.JSON)
}
if resp.Usage != (llm.Usage{InputTokens: 26, OutputTokens: 298}) || resp.Model != "llama3.2" || resp.FinishReason != "stop" {
t.Errorf("Usage, Model, FinishReason = %+v, %q, %q", resp.Usage, resp.Model, resp.FinishReason)
}
got := s.only(t)
if got.method != http.MethodPost || got.path != "/api/chat" || len(got.query) != 0 {
t.Errorf("request = %s %s?%s", got.method, got.path, got.query.Encode())
}
assertNoAuth(t, got)
body := sentBody(t, got)
if stream, ok := body["stream"]; !ok || stream != false {
t.Errorf("stream = %v (present %v), want false", stream, ok)
}
if body["model"] != "llama3.2" {
t.Errorf("model = %v", body["model"])
}
for _, field := range []string{"format", "options"} {
if v, ok := body[field]; ok {
t.Errorf("%s = %v, want it omitted", field, v)
}
}
if got, want := canonical(t, mustMarshal(t, body["messages"])), `[{"content":"What should we cook tonight?","role":"user"}]`; got != want {
t.Errorf("messages = %s, want %s", got, want)
}
}
func mustMarshal(t *testing.T, v any) []byte {
t.Helper()
raw, err := json.Marshal(v)
if err != nil {
t.Fatal(err)
}
return raw
}
func TestGenerateSchemaOutput(t *testing.T) {
s := newOllamaServer(t, reply{status: 200, fixture: "chat_schema.json"})
p := newProvider(t, s.URL)
resp, err := p.Generate(context.Background(), schemaRequest())
if err != nil {
t.Fatal(err)
}
want := `{"dinners":[{"name":"Sheet-pan gnocchi","why":"Like your Regular tray bake, but quicker."}]}`
if got := canonical(t, resp.JSON); got != want {
t.Errorf("Response.JSON = %s, want %s", got, want)
}
if canonical(t, []byte(resp.Text)) != want {
t.Errorf("Text = %q, want the reply content", resp.Text)
}
if resp.Usage != (llm.Usage{InputTokens: 147, OutputTokens: 41}) || resp.FinishReason != "stop" {
t.Errorf("Usage, FinishReason = %+v, %q", resp.Usage, resp.FinishReason)
}
got := s.only(t)
assertNoAuth(t, got)
body := sentBody(t, got)
if body["stream"] != false {
t.Errorf("stream = %v, want false", body["stream"])
}
if got, want := canonical(t, mustMarshal(t, body["format"])), canonical(t, dinnersSchema); got != want {
t.Errorf("format = %s, want the schema %s", got, want)
}
opts, _ := body["options"].(map[string]any)
if len(opts) != 2 || opts["temperature"] != 0.0 || opts["num_predict"] != 400.0 {
t.Errorf("options = %v, want temperature 0 and num_predict 400", body["options"])
}
wantMessages := `[{"content":"You plan dinners for Our kitchen.","role":"system"},` +
`{"content":"Suggest a Side quest.","role":"user"},` +
`{"content":"Something quick?","role":"assistant"},` +
`{"content":"Yes, a Quick dinner.","role":"user"}]`
if got := canonical(t, mustMarshal(t, body["messages"])); got != wantMessages {
t.Errorf("messages = %s, want %s", got, wantMessages)
}
}
func TestGenerateTemperatureOnlyWhenSet(t *testing.T) {
s := newOllamaServer(t, reply{status: 200, fixture: "chat_text.json"})
req := textRequest()
temp := 0.7
req.Temperature = &temp
if _, err := newProvider(t, s.URL).Generate(context.Background(), req); err != nil {
t.Fatal(err)
}
opts, _ := sentBody(t, s.only(t))["options"].(map[string]any)
if len(opts) != 1 || opts["temperature"] != 0.7 {
t.Errorf("options = %v, want only temperature 0.7", opts)
}
}
func TestGenerateBadOutput(t *testing.T) {
cases := map[string]string{
"prose": `{"model":"m","message":{"role":"assistant","content":"Sure! Try chili."},"done":true,"done_reason":"stop"}`,
"array": `{"model":"m","message":{"role":"assistant","content":"[1,2]"},"done":true,"done_reason":"stop"}`,
"cut-off JSON": `{"model":"m","message":{"role":"assistant","content":"{\"dinners\": ["},"done":true,"done_reason":"length"}`,
}
for name, body := range cases {
t.Run(name, func(t *testing.T) {
s := newOllamaServer(t, reply{status: 200, body: body})
resp, err := newProvider(t, s.URL).Generate(context.Background(), schemaRequest())
if !errors.Is(err, llm.ErrBadOutput) {
t.Fatalf("err = %v, want ErrBadOutput", err)
}
if resp.JSON != nil || resp.Model != "m" || resp.Text == "" {
t.Fatalf("Response = %+v, want no JSON and the reply kept", resp)
}
})
}
}
func TestGenerateRejectsMalformedResponses(t *testing.T) {
for name, body := range map[string]string{
"not JSON": `not json`,
"not finished": `{"model":"m","message":{"role":"assistant","content":"hi"},"done":false}`,
"no message": `{"model":"m","done":true}`,
} {
s := newOllamaServer(t, reply{status: 200, body: body})
_, err := newProvider(t, s.URL).Generate(context.Background(), textRequest())
if err == nil || errors.Is(err, llm.ErrBadOutput) || errors.Is(err, llm.ErrProvider) {
t.Errorf("%s: err = %v, want an untyped malformed-response error", name, err)
}
}
}
func TestGenerateRejectsInvalidRequests(t *testing.T) {
s := newOllamaServer(t, reply{status: 200, fixture: "chat_text.json"})
if _, err := newProvider(t, s.URL).Generate(context.Background(), llm.Request{}); err == nil {
t.Error("Generate accepted a request with no messages")
}
noModel, err := New(Config{BaseURL: s.URL})
if err != nil {
t.Fatal(err)
}
if _, err := noModel.Generate(context.Background(), textRequest()); err == nil {
t.Error("Generate ran with no model configured")
}
if n := len(s.requests()); n != 0 {
t.Errorf("server got %d requests, want 0", n)
}
}
func TestGenerateModelMissing(t *testing.T) {
s := newOllamaServer(t, reply{status: 404, fixture: "error_model_not_found.json"})
p, err := New(Config{BaseURL: s.URL, Model: "llama9"})
if err != nil {
t.Fatal(err)
}
_, err = p.Generate(context.Background(), textRequest())
var pe *llm.ProviderError
if !errors.Is(err, llm.ErrProvider) || !errors.As(err, &pe) || pe.Status() != http.StatusNotFound {
t.Fatalf("err = %v, want a ProviderError for HTTP 404", err)
}
if errors.Is(err, llm.ErrTimeout) || errors.Is(err, llm.ErrBadOutput) {
t.Fatalf("err = %v matches an unrelated sentinel", err)
}
if want := `ollama: generate: model "llama9" not found: llm: provider error: HTTP 404`; err.Error() != want {
t.Fatalf("Error() = %q, want %q", err, want)
}
if pe.Body() != "" {
t.Fatalf("Body() = %q, want the response body left out", pe.Body())
}
}
func TestGenerateNotFoundUnrelatedToModel(t *testing.T) {
for name, body := range map[string]string{
"proxy page": "404 page not found",
"other error": `{"error":"no route for /api/chat"}`,
"other model": `{"error":"model \"llama3\" not found, try pulling it first"}`,
"empty": "",
} {
t.Run(name, func(t *testing.T) {
s := newOllamaServer(t, reply{status: 404, body: body})
p, err := New(Config{BaseURL: s.URL, Model: "llama9"})
if err != nil {
t.Fatal(err)
}
_, err = p.Generate(context.Background(), textRequest())
var pe *llm.ProviderError
if !errors.Is(err, llm.ErrProvider) || !errors.As(err, &pe) || pe.Status() != http.StatusNotFound {
t.Fatalf("err = %v, want a ProviderError for HTTP 404", err)
}
if strings.Contains(err.Error(), "not found") && !strings.Contains(body, "not found") {
t.Fatalf("Error() = %q claims something was not found", err)
}
if strings.Contains(err.Error(), "llama9") {
t.Fatalf("Error() = %q claims the model was not found", err)
}
if pe.Body() != strings.TrimSpace(body) {
t.Fatalf("Body() = %q, want the response body %q", pe.Body(), body)
}
})
}
}
func TestGenerateModelMissingSingleQuoted(t *testing.T) {
s := newOllamaServer(t, reply{status: 404, body: `{"error":"model 'llama9' not found"}`})
p, err := New(Config{BaseURL: s.URL, Model: "llama9"})
if err != nil {
t.Fatal(err)
}
_, err = p.Generate(context.Background(), textRequest())
if want := `ollama: generate: model "llama9" not found: llm: provider error: HTTP 404`; err == nil || err.Error() != want {
t.Fatalf("err = %v, want %q", err, want)
}
}
func TestListModels(t *testing.T) {
s := newOllamaServer(t, reply{status: 200, fixture: "tags.json"})
models, err := newProvider(t, s.URL).ListModels(context.Background())
if err != nil {
t.Fatal(err)
}
want := []llm.Model{{ID: "llama3.2:latest", DisplayName: "llama3.2:latest"}, {ID: "qwen3:8b", DisplayName: "qwen3:8b"}}
if len(models) != len(want) || models[0] != want[0] || models[1] != want[1] {
t.Fatalf("ListModels() = %+v, want %+v", models, want)
}
got := s.only(t)
if got.method != http.MethodGet || got.path != "/api/tags" || len(got.query) != 0 {
t.Errorf("request = %s %s?%s", got.method, got.path, got.query.Encode())
}
assertNoAuth(t, got)
empty := newOllamaServer(t, reply{status: 200, body: `{"models":[]}`})
if models, err := newProvider(t, empty.URL).ListModels(context.Background()); err != nil || models == nil || len(models) != 0 {
t.Fatalf("empty list: %v, %v", models, err)
}
for name, body := range map[string]string{
"no models list": `{}`,
"unnamed model": `{"models":[{"model":"x"}]}`,
"not JSON": `<html>`,
} {
bad := newOllamaServer(t, reply{status: 200, body: body})
if _, err := newProvider(t, bad.URL).ListModels(context.Background()); err == nil {
t.Errorf("%s: ListModels accepted it", name)
}
}
}
func TestVerify(t *testing.T) {
ok := newOllamaServer(t, reply{status: 200, fixture: "tags.json"})
if err := newProvider(t, ok.URL).Verify(context.Background()); err != nil {
t.Fatalf("Verify() = %v", err)
}
got := ok.only(t)
if got.method != http.MethodGet || got.path != "/api/tags" {
t.Errorf("Verify called %s %s", got.method, got.path)
}
assertNoAuth(t, got)
bad := newOllamaServer(t, reply{status: 500, body: `{"error":"something went wrong"}`})
err := newProvider(t, bad.URL).Verify(context.Background())
var pe *llm.ProviderError
if !errors.As(err, &pe) || pe.Status() != 500 || !strings.Contains(err.Error(), "something went wrong") {
t.Fatalf("Verify() = %v, want a ProviderError for HTTP 500", err)
}
malformed := newOllamaServer(t, reply{status: 200, body: `{}`})
if err := newProvider(t, malformed.URL).Verify(context.Background()); err == nil {
t.Error("Verify accepted a body with no models list")
}
}
func TestProviderErrors(t *testing.T) {
cases := map[string]reply{
"400": {status: 400, body: `{"error":"invalid format"}`},
"500": {status: 500, body: `{"error":"llama runner process has terminated"}`},
"503": {status: 503, body: `{"error":"server busy, please try again"}`},
"huge": {status: 500, body: strings.Repeat("x", 1<<20)},
}
for name, r := range cases {
t.Run(name, func(t *testing.T) {
s := newOllamaServer(t, r)
p := newProvider(t, s.URL)
_, genErr := p.Generate(context.Background(), schemaRequest())
_, listErr := p.ListModels(context.Background())
for _, err := range []error{genErr, listErr} {
var pe *llm.ProviderError
if !errors.As(err, &pe) || pe.Status() != r.status {
t.Fatalf("err = %v, want a ProviderError for HTTP %d", err, r.status)
}
if errors.Is(err, llm.ErrTimeout) || strings.Contains(err.Error(), "not found") {
t.Fatalf("err = %v, want only a provider error", err)
}
if len(err.Error()) > llm.MaxBodyExcerpt+200 {
t.Fatalf("error text has %d bytes, want a bounded excerpt", len(err.Error()))
}
}
})
}
}
// assertUnreachable checks the error a request to an unreachable base
// returns: it matches llm.ErrTimeout and names the base.
func assertUnreachable(t *testing.T, err error, base string) {
t.Helper()
if !errors.Is(err, llm.ErrTimeout) {
t.Fatalf("err = %v, want ErrTimeout", err)
}
if errors.Is(err, llm.ErrProvider) || errors.Is(err, context.Canceled) {
t.Fatalf("err = %v matches an unrelated sentinel", err)
}
if want := "Ollama not reachable at " + base; !strings.Contains(err.Error(), want) {
t.Fatalf("Error() = %q, want it to contain %q", err, want)
}
var ue *url.Error
if errors.As(err, &ue) {
t.Fatalf("error chain exposes the url.Error: %v", ue)
}
}
func TestUnreachableHost(t *testing.T) {
closed := httptest.NewServer(http.NotFoundHandler())
base := closed.URL
closed.Close()
p := newProvider(t, base+"/")
_, genErr := p.Generate(context.Background(), schemaRequest())
_, listErr := p.ListModels(context.Background())
verifyErr := p.Verify(context.Background())
for _, err := range []error{genErr, listErr, verifyErr} {
assertUnreachable(t, err, base)
if !strings.Contains(err.Error(), "connection refused") {
t.Errorf("Error() = %q, want the cause", err)
}
}
dropped := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
conn, _, err := w.(http.Hijacker).Hijack()
if err != nil {
t.Error(err)
return
}
conn.Close()
}))
t.Cleanup(dropped.Close)
_, err := newProvider(t, dropped.URL).Generate(context.Background(), textRequest())
assertUnreachable(t, err, dropped.URL)
}
func TestTimeouts(t *testing.T) {
s := newOllamaServer(t, reply{status: 200, fixture: "chat_schema.json", delay: 5 * time.Second})
p := newProvider(t, s.URL)
req := schemaRequest()
req.Timeout = 50 * time.Millisecond
start := time.Now()
_, err := p.Generate(context.Background(), req)
assertUnreachable(t, err, s.URL)
if !strings.Contains(err.Error(), llm.ErrTimeout.Error()) {
t.Errorf("Error() = %q, want it to say the request timed out", err)
}
if elapsed := time.Since(start); elapsed > 2*time.Second {
t.Fatalf("Request.Timeout not honoured: took %v", elapsed)
}
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
defer cancel()
_, err = p.ListModels(ctx)
assertUnreachable(t, err, s.URL)
ctx, cancel = context.WithTimeout(context.Background(), 50*time.Millisecond)
defer cancel()
assertUnreachable(t, p.Verify(ctx), s.URL)
ctx, cancel = context.WithCancel(context.Background())
cancel()
_, err = p.Generate(ctx, schemaRequest())
if !errors.Is(err, context.Canceled) || errors.Is(err, llm.ErrTimeout) || strings.Contains(err.Error(), "not reachable") {
t.Fatalf("cancelled context: err = %v, want context.Canceled only", err)
}
}
func TestRedirectIsNotFollowed(t *testing.T) {
var mu sync.Mutex
var secondCalls int
second := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
mu.Lock()
secondCalls++
mu.Unlock()
w.WriteHeader(http.StatusOK)
}))
t.Cleanup(second.Close)
first := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
http.Redirect(w, req, second.URL+req.URL.Path, http.StatusFound)
}))
t.Cleanup(first.Close)
p := newProvider(t, first.URL)
_, genErr := p.Generate(context.Background(), textRequest())
verifyErr := p.Verify(context.Background())
for _, err := range []error{genErr, verifyErr} {
var pe *llm.ProviderError
if !errors.Is(err, llm.ErrProvider) || !errors.As(err, &pe) || pe.Status() != http.StatusFound {
t.Fatalf("err = %v, want a ProviderError for HTTP 302", err)
}
}
mu.Lock()
defer mu.Unlock()
if secondCalls != 0 {
t.Fatalf("redirect target got %d requests, want 0", secondCalls)
}
}
func TestNewRejectsBadConfig(t *testing.T) {
const secret = "hunter2-secret"
cases := map[string]string{
"empty base": "",
"relative base": "ollama:11434",
"host only": "192.168.1.20",
"ftp base": "ftp://192.168.1.20:11434",
"no host": "http://",
"credentials in URL": "http://user:" + secret + "@192.168.1.20:11434",
"user in URL": "http://" + secret + "@ollama:11434",
"secret in query": "http://ollama:11434/?token=" + secret,
"secret in fragment": "http://ollama:11434/#" + secret,
"unparseable": "http://ollama:11434/%zz?token=" + secret,
"empty query": "http://ollama:11434?",
"empty fragment": "http://ollama:11434#",
"empty both": "http://ollama:11434/?#",
}
for name, base := range cases {
_, err := New(Config{BaseURL: base, Model: "llama3.2"})
if err == nil {
t.Errorf("%s: New() = nil error", name)
continue
}
if strings.Contains(err.Error(), secret) {
t.Errorf("%s: error echoes the URL: %v", name, err)
}
}
for _, base := range []string{
"http://192.168.1.20:11434",
"http://ollama:11434",
"http://localhost:11434/",
"https://ollama.lan/proxy/",
} {
if _, err := New(Config{BaseURL: base, Model: "llama3.2"}); err != nil {
t.Errorf("New(%q) = %v", base, err)
}
}
}
func TestBaseWithPath(t *testing.T) {
s := newOllamaServer(t, reply{status: 200, fixture: "tags.json"})
if err := newProvider(t, s.URL+"/proxy/").Verify(context.Background()); err != nil {
t.Fatal(err)
}
if got := s.only(t).path; got != "/proxy/api/tags" {
t.Fatalf("path = %s, want /proxy/api/tags", got)
}
}
+36
View File
@@ -0,0 +1,36 @@
# Ollama adapter fixtures
These are the native Ollama API response bodies the contract tests in
`ollama_test.go` replay. They are **not live captures**. Product AI is tested
with fakes and live vendor calls are opt-in, so no request was sent to an
Ollama server to make them. Each fixture is a sanitized adaptation of the
response shapes in Ollama's public API documentation
(<https://github.com/ollama/ollama/blob/main/docs/api.md>):
- Chat, non-streaming: `POST /api/chat` with `"stream": false`.
- Structured outputs: `POST /api/chat` with a JSON schema in `"format"`.
- List local models: `GET /api/tags`.
- Errors: a JSON `{"error": "..."}` body with a non-2xx status; a model
that is not pulled is HTTP 404.
Sanitizing:
- Timestamps, durations, sizes, digests and token counts are illustrative
values in the documented format, not values from a real server.
- Message content is replaced with MealQuest dinner text.
- Ollama takes no credentials, so none appear.
Below, *not in the public example* marks values that were added for a test
case and do not come from the documented example.
| Fixture | Case | Shape | Not in the public example |
| --- | --- | --- | --- |
| `chat_text.json` | Success | Finished chat message, `done_reason: stop`, `prompt_eval_count` and `eval_count`. | — |
| `chat_schema.json` | Schema output | Finished chat message whose content is a JSON object string. | The dinner object. |
| `error_model_not_found.json` | Model missing | `{"error": "model \"…\" not found, try pulling it first"}`, served with HTTP 404. | — |
| `tags.json` | Model list | `{"models": [...]}` with `name`, `model`, `digest` and `details`. | — |
The unreachable-host case has no fixture: there is no response to record.
The tests close an `httptest` server and dial its address, so the
connection is refused, and they hold a reply past a short deadline for the
timeout.
+16
View File
@@ -0,0 +1,16 @@
{
"model": "llama3.2",
"created_at": "2026-10-04T12:20:11.009132Z",
"message": {
"role": "assistant",
"content": "{\"dinners\": [{\"name\": \"Sheet-pan gnocchi\", \"why\": \"Like your Regular tray bake, but quicker.\"}]}"
},
"done_reason": "stop",
"done": true,
"total_duration": 1416823291,
"load_duration": 12309083,
"prompt_eval_count": 147,
"prompt_eval_duration": 271802000,
"eval_count": 41,
"eval_duration": 1130948000
}
+16
View File
@@ -0,0 +1,16 @@
{
"model": "llama3.2",
"created_at": "2026-10-04T12:13:43.416799Z",
"message": {
"role": "assistant",
"content": "Try a crock pot chili: it is a Quick dinner."
},
"done": true,
"done_reason": "stop",
"total_duration": 5191566416,
"load_duration": 2154458,
"prompt_eval_count": 26,
"prompt_eval_duration": 383809000,
"eval_count": 298,
"eval_duration": 4799921000
}
@@ -0,0 +1 @@
{"error":"model \"llama9\" not found, try pulling it first"}
+34
View File
@@ -0,0 +1,34 @@
{
"models": [
{
"name": "llama3.2:latest",
"model": "llama3.2:latest",
"modified_at": "2026-09-30T08:06:48.639712648-07:00",
"size": 2019393189,
"digest": "a80c4f17acd55265feec403c7aef86be0c25983ab279d83f3bcd3abbcb5b8b72",
"details": {
"parent_model": "",
"format": "gguf",
"family": "llama",
"families": ["llama"],
"parameter_size": "3.2B",
"quantization_level": "Q4_K_M"
}
},
{
"name": "qwen3:8b",
"model": "qwen3:8b",
"modified_at": "2026-09-28T19:41:02.112408113-07:00",
"size": 5225388164,
"digest": "500a1f067a9f782620b40bee6f7b0c89e17ae61f686b92c24933e4ca4b2b8b41",
"details": {
"parent_model": "",
"format": "gguf",
"family": "qwen3",
"families": ["qwen3"],
"parameter_size": "8.2B",
"quantization_level": "Q4_K_M"
}
}
]
}
+4
View File
@@ -53,6 +53,7 @@ type Config struct {
BaseURL string
// APIKey is sent as "Authorization: Bearer <APIKey>". Empty sends no
// Authorization header, for endpoints such as Ollama that need none.
// It must not have a control character other than tab.
APIKey string
// Model is the model Generate requests. ListModels and Verify do not
// need it.
@@ -102,6 +103,9 @@ func newFromConfig(cfg Config) (*Provider, error) {
if u.User != nil || strings.ContainsAny(cfg.BaseURL, "?#") {
return nil, errors.New("openai: base URL must not carry credentials, a query or a fragment")
}
if !validHeaderValue(cfg.APIKey) {
return nil, errors.New("openai: API key has a control character")
}
// No error from New quotes caller-supplied text or a name derived from
// it: any such text may equal or contain the API key, and redaction
// cannot catch a fragment or a case-changed copy of it.
+14
View File
@@ -628,6 +628,20 @@ func TestNewRejectsBadHeaderWithoutEchoingValue(t *testing.T) {
assertNoSecret(t, err)
}
func TestNewRejectsKeyWithControlCharacter(t *testing.T) {
for _, key := range []string{testKey + "\n", testKey + "\r\nX-Injected: 1", "\x00" + testKey, testKey + "\x7f"} {
_, err := New(Config{BaseURL: "https://example.com/v1", APIKey: key, StructuredOutput: JSONSchema})
if err == nil {
t.Errorf("New(%q) = nil error", key)
continue
}
assertNoSecret(t, err)
if strings.Contains(err.Error()+fmt.Sprintf("%#v", err), key) {
t.Errorf("error leaks the key: %v", err)
}
}
}
// TestNewConfigErrorsNeverLeakKey covers every New error that names
// caller-supplied text, each with the API key embedded in that text.
func TestNewConfigErrorsNeverLeakKey(t *testing.T) {
+329
View File
@@ -0,0 +1,329 @@
// Package presets holds what MealQuest knows about each language-model
// vendor — base URL, adapter, credentials, headers, structured output mode
// and model quirks — and FromSettings, which turns the household's
// llm.Settings into a ready llm.Provider.
//
// FromSettings lives here rather than in package llm because it builds the
// adapters, and every adapter imports llm.
//
// Nothing in this package logs, and no error it returns or unwraps to
// contains the API key, even where the key equals a setting's or request's
// value or text the error spells anyway.
package presets
import (
"context"
"errors"
"fmt"
"math"
"regexp"
"slices"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm/anthropic"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm/ollama"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm/openai"
)
// Adapter names the llm adapter package a preset builds.
type Adapter string
// The adapters.
const (
AdapterAnthropic Adapter = "anthropic"
AdapterOpenAI Adapter = "openai"
AdapterOllama Adapter = "ollama"
)
// KeyPolicy is whether a preset sends an API key.
type KeyPolicy int
// The key policies.
const (
// KeyRequired presets fail without a key.
KeyRequired KeyPolicy = iota
// KeyOptional presets send a key when one is given.
KeyOptional
// KeyNone presets send no key and refuse one.
KeyNone
)
// The reasoning models that reject the temperature field are OpenAI's o1
// and o3 families: the bare family name or the name followed by a "-"
// variant (o1, o1-mini, o3-2025-04-16), never a longer word such as o123.
// OpenRouter names them with an "openai/" prefix and may add a ":" variant
// (openai/o1:online). A preset whose OmitTemperature holds a pattern never
// sends a temperature to a matching model.
var (
openAIReasoningModels = regexp.MustCompile(`^o[13](-.*)?$`)
openRouterReasoningModels = regexp.MustCompile(`^openai/o[13]([-:].*)?$`)
)
// Preset is one vendor's fixed configuration.
type Preset struct {
// Name is the Settings.Provider value that selects the preset.
Name string
// Adapter is the adapter the preset builds.
Adapter Adapter
// BaseURL is the API root the adapter appends its endpoint paths to.
// Empty means the household must supply it in Settings.BaseURL.
BaseURL string
// Key is whether the preset sends an API key.
Key KeyPolicy
// RequiresPublicURL means the preset needs Settings.PublicURL for its
// headers.
RequiresPublicURL bool
// Headers returns the extra request headers an openai-adapter preset
// sends; nil means none.
Headers func(llm.Settings) map[string]string
// StructuredOutput is the default mode of an openai-adapter preset.
StructuredOutput openai.StructuredOutput
// FallbackStructuredOutput is the mode the household can select when
// the default fails for its model; empty means none. FromSettings
// builds one mode and never retries a request in another.
FallbackStructuredOutput openai.StructuredOutput
// StructuredOutputs are the modes Settings.StructuredOutput may select.
StructuredOutputs []openai.StructuredOutput
// MaxTokens is the max_tokens the anthropic adapter sends when a
// Request sets none.
MaxTokens int
// OmitTemperature are model-id patterns the preset never sends a
// temperature to.
OmitTemperature []*regexp.Regexp
}
// openRouterHeaders identify MealQuest to OpenRouter: HTTP-Referer is
// MEALQUEST_PUBLIC_URL and X-Title is the app name.
func openRouterHeaders(s llm.Settings) map[string]string {
return map[string]string{
"HTTP-Referer": s.PublicURL,
"X-Title": "MealQuest",
}
}
// omitsTemperature reports whether model matches one of the preset's
// OmitTemperature patterns.
func (p Preset) omitsTemperature(model string) bool {
return slices.ContainsFunc(p.OmitTemperature, func(re *regexp.Regexp) bool {
return re.MatchString(model)
})
}
// Lookup returns the preset Settings.Provider name selects.
func Lookup(name string) (Preset, bool) {
p, ok := all[name]
return p, ok
}
// all is every preset, by Settings.Provider name.
var all = map[string]Preset{
"anthropic": {
Name: "anthropic",
Adapter: AdapterAnthropic,
BaseURL: "https://api.anthropic.com",
Key: KeyRequired,
MaxTokens: 4096,
},
"openai": {
Name: "openai",
Adapter: AdapterOpenAI,
BaseURL: "https://api.openai.com/v1",
Key: KeyRequired,
StructuredOutput: openai.JSONSchema,
StructuredOutputs: []openai.StructuredOutput{openai.JSONSchema},
OmitTemperature: []*regexp.Regexp{openAIReasoningModels},
},
"openrouter": {
Name: "openrouter",
Adapter: AdapterOpenAI,
BaseURL: "https://openrouter.ai/api/v1",
Key: KeyRequired,
RequiresPublicURL: true,
Headers: openRouterHeaders,
StructuredOutput: openai.JSONSchema,
FallbackStructuredOutput: openai.JSONObject,
StructuredOutputs: []openai.StructuredOutput{openai.JSONSchema, openai.JSONObject},
OmitTemperature: []*regexp.Regexp{openRouterReasoningModels},
},
"xai": {
Name: "xai",
Adapter: AdapterOpenAI,
BaseURL: "https://api.x.ai/v1",
Key: KeyRequired,
StructuredOutput: openai.JSONSchema,
FallbackStructuredOutput: openai.JSONObject,
StructuredOutputs: []openai.StructuredOutput{openai.JSONSchema, openai.JSONObject},
},
"ollama": {
Name: "ollama",
Adapter: AdapterOllama,
Key: KeyNone,
},
"openai_compatible": {
Name: "openai_compatible",
Adapter: AdapterOpenAI,
Key: KeyOptional,
StructuredOutput: openai.JSONSchema,
StructuredOutputs: []openai.StructuredOutput{openai.JSONSchema, openai.JSONObject, openai.PromptOnly},
// The server may be OpenAI itself or a gateway that names models
// the OpenRouter way.
OmitTemperature: []*regexp.Regexp{openAIReasoningModels, openRouterReasoningModels},
},
}
// FromSettings returns the Provider s describes, using key as its API key.
// It fails with a configuration error from package llm — ErrUnknownProvider,
// ErrMissingKey, ErrUnexpectedKey, ErrInvalidKey, ErrMissingBaseURL, ErrMissingModel,
// ErrMissingPublicURL or ErrInvalidSetting — before contacting any vendor.
//
// The Provider sends s.Temperature with every request whose Temperature is
// nil; a request that sets one keeps it. Neither is sent to a model the
// preset's OmitTemperature matches. It is wrapped in llm.NewRetrying, so
// Generate retries transient vendor failures and has the default timeout.
func FromSettings(s llm.Settings, key string) (llm.Provider, error) {
cp, err := fromSettings(s, key)
if err != nil {
return nil, redact(err, key)
}
return llm.NewRetrying(cp), nil
}
func fromSettings(s llm.Settings, key string) (*configured, error) {
p, ok := Lookup(s.Provider)
if !ok {
return nil, llm.ErrUnknownProvider
}
if err := check(p, s, key); err != nil {
return nil, fmt.Errorf("%s: %w", p.Name, err)
}
base := p.BaseURL
if base == "" {
base = s.BaseURL
}
var inner llm.Provider
var err error
switch p.Adapter {
case AdapterAnthropic:
inner, err = anthropic.New(anthropic.Config{BaseURL: base, APIKey: key, Model: s.Model, MaxTokens: p.MaxTokens})
case AdapterOllama:
inner, err = ollama.New(ollama.Config{BaseURL: base, Model: s.Model})
case AdapterOpenAI:
mode := p.StructuredOutput
if s.StructuredOutput != "" {
mode = openai.StructuredOutput(s.StructuredOutput)
}
cfg := openai.Config{BaseURL: base, APIKey: key, Model: s.Model, StructuredOutput: mode}
if p.Headers != nil {
cfg.Headers = p.Headers(s)
}
inner, err = openai.New(cfg)
default:
panic("presets: preset " + p.Name + " has unknown adapter " + string(p.Adapter))
}
if err != nil {
return nil, fmt.Errorf("%s: %w: %w", p.Name, llm.ErrInvalidSetting, err)
}
cp := &configured{Provider: inner, key: key}
if !p.omitsTemperature(s.Model) {
t := s.Temperature
cp.temperature = &t
}
return cp, nil
}
// check validates s and key against p. Its errors never quote a value.
func check(p Preset, s llm.Settings, key string) error {
switch {
case p.Key == KeyRequired && key == "":
return llm.ErrMissingKey
case p.Key == KeyNone && key != "":
return llm.ErrUnexpectedKey
case !validHeaderValue(key):
return llm.ErrInvalidKey
}
switch {
case p.BaseURL == "" && s.BaseURL == "":
return llm.ErrMissingBaseURL
case p.BaseURL != "" && s.BaseURL != "" && s.BaseURL != p.BaseURL:
return fmt.Errorf("%w: base URL is fixed by the preset", llm.ErrInvalidSetting)
}
if s.Model == "" {
return llm.ErrMissingModel
}
if p.RequiresPublicURL && s.PublicURL == "" {
return llm.ErrMissingPublicURL
}
if s.StructuredOutput != "" && !slices.Contains(p.StructuredOutputs, openai.StructuredOutput(s.StructuredOutput)) {
return fmt.Errorf("%w: structured output mode is not supported by the preset", llm.ErrInvalidSetting)
}
if math.IsNaN(s.Temperature) || math.IsInf(s.Temperature, 0) || s.Temperature < 0 {
return fmt.Errorf("%w: temperature must be a non-negative number", llm.ErrInvalidSetting)
}
return nil
}
// validHeaderValue reports whether value has no control character other
// than tab, so it can be sent as an HTTP header value.
func validHeaderValue(value string) bool {
for i := 0; i < len(value); i++ {
if c := value[i]; (c < 0x20 && c != '\t') || c == 0x7f {
return false
}
}
return true
}
// configured applies the household temperature and the preset's
// temperature quirk to every Generate.
type configured struct {
llm.Provider
// key is the API key, redacted from validation errors; the adapter
// redacts its own.
key string
// temperature is sent when a request sets none; nil means the model
// rejects temperature, so none is ever sent.
temperature *float64
}
// Generate validates req as sent by the caller before changing its
// temperature, so clearing it for a model that rejects temperature never
// hides an invalid one.
func (c *configured) Generate(ctx context.Context, req llm.Request) (llm.Response, error) {
if err := req.Validate(); err != nil {
return llm.Response{}, redact(fmt.Errorf("presets: generate: %w", err), c.key)
}
if c.temperature == nil {
req.Temperature = nil
} else if req.Temperature == nil {
t := *c.temperature
req.Temperature = &t
}
return c.Provider.Generate(ctx, req)
}
// redactedError is an error with the API key removed from its text. It
// does not unwrap, because every layer below may hold the key; Is still
// matches the sentinels the original error wraps.
type redactedError struct {
text string
err error
}
func (e *redactedError) Error() string { return e.text }
// GoString keeps %#v to the redacted Error text instead of the fields.
func (e *redactedError) GoString() string { return e.text }
// Is reports whether the original error matches target.
func (e *redactedError) Is(target error) bool { return errors.Is(e.err, target) }
// redact returns err with key removed from its text; a nil err or an empty
// key returns err unchanged.
func redact(err error, key string) error {
if err == nil || key == "" {
return err
}
return &redactedError{text: llm.Redact(err.Error(), key), err: err}
}
+606
View File
@@ -0,0 +1,606 @@
package presets
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strings"
"testing"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm"
)
const testKey = "sk-test-0123456789"
// captured is the HTTP request an adapter sent.
type captured struct {
url string
header http.Header
body map[string]json.RawMessage
}
// recordingTransport records each request and answers it 400, so no
// request leaves the process and the Provider never retries it.
type recordingTransport struct{ reqs []captured }
func (rt *recordingTransport) RoundTrip(r *http.Request) (*http.Response, error) {
c := captured{url: r.URL.String(), header: r.Header.Clone()}
if r.Body != nil {
raw, err := io.ReadAll(r.Body)
if err != nil {
return nil, err
}
if err := json.Unmarshal(raw, &c.body); err != nil {
return nil, err
}
}
rt.reqs = append(rt.reqs, c)
return &http.Response{
StatusCode: http.StatusBadRequest,
Header: http.Header{"Content-Type": {"application/json"}},
Body: io.NopCloser(strings.NewReader(`{}`)),
Request: r,
}, nil
}
// generate builds the provider s describes and returns the request its
// Generate sends for req. The adapters use http.DefaultTransport, which the
// test replaces, so the real preset base URLs are never contacted.
func generate(t *testing.T, s llm.Settings, key string, req llm.Request) captured {
t.Helper()
rt := &recordingTransport{}
orig := http.DefaultTransport
http.DefaultTransport = rt
t.Cleanup(func() { http.DefaultTransport = orig })
p, err := FromSettings(s, key)
if err != nil {
t.Fatalf("FromSettings: %v", err)
}
if _, err := p.Generate(context.Background(), req); err == nil {
t.Fatal("Generate succeeded through the recording transport")
}
if len(rt.reqs) != 1 {
t.Fatalf("sent %d requests, want 1", len(rt.reqs))
}
return rt.reqs[0]
}
func plainRequest() llm.Request {
return llm.Request{Messages: []llm.Message{{Role: llm.RoleUser, Content: "dinner?"}}}
}
func schemaRequest() llm.Request {
r := plainRequest()
r.Schema = &llm.JSONSchema{Name: "side_quests", Schema: json.RawMessage(`{"type":"object"}`)}
return r
}
func ptr(f float64) *float64 { return &f }
func responseFormatType(t *testing.T, c captured) string {
t.Helper()
raw, ok := c.body["response_format"]
if !ok {
return ""
}
var rf struct{ Type string }
if err := json.Unmarshal(raw, &rf); err != nil {
t.Fatalf("response_format: %v", err)
}
return rf.Type
}
func temperature(t *testing.T, c captured) (float64, bool) {
t.Helper()
raw, ok := c.body["temperature"]
if !ok {
return 0, false
}
var f float64
if err := json.Unmarshal(raw, &f); err != nil {
t.Fatalf("temperature: %v", err)
}
return f, true
}
func TestAnthropicPreset(t *testing.T) {
c := generate(t, llm.Settings{Provider: "anthropic", Model: "claude-sonnet-5", Temperature: 0.7}, testKey, plainRequest())
if c.url != "https://api.anthropic.com/v1/messages" {
t.Fatalf("URL = %q", c.url)
}
if got := c.header.Get("X-Api-Key"); got != testKey {
t.Fatalf("x-api-key = %q", got)
}
if got := c.header.Get("Authorization"); got != "" {
t.Fatalf("Authorization = %q, want none", got)
}
if got, ok := temperature(t, c); !ok || got != 0.7 {
t.Fatalf("temperature = %v, %v; want 0.7", got, ok)
}
if string(c.body["max_tokens"]) != "4096" {
t.Fatalf("max_tokens = %s", c.body["max_tokens"])
}
}
func TestAnthropicPresetAcceptsItsOwnBaseURL(t *testing.T) {
c := generate(t, llm.Settings{Provider: "anthropic", BaseURL: "https://api.anthropic.com", Model: "claude-sonnet-5"}, testKey, plainRequest())
if c.url != "https://api.anthropic.com/v1/messages" {
t.Fatalf("URL = %q", c.url)
}
}
func TestOpenAIPreset(t *testing.T) {
c := generate(t, llm.Settings{Provider: "openai", Model: "gpt-4o", Temperature: 0.4}, testKey, schemaRequest())
if c.url != "https://api.openai.com/v1/chat/completions" {
t.Fatalf("URL = %q", c.url)
}
if got := c.header.Get("Authorization"); got != "Bearer "+testKey {
t.Fatalf("Authorization = %q", got)
}
if got := responseFormatType(t, c); got != "json_schema" {
t.Fatalf("response_format type = %q, want json_schema", got)
}
if c.header.Get("HTTP-Referer") != "" || c.header.Get("X-Title") != "" {
t.Fatal("openai sends OpenRouter headers")
}
}
func TestOpenRouterPreset(t *testing.T) {
s := llm.Settings{Provider: "openrouter", Model: "openai/gpt-4o", PublicURL: "https://meals.example.com/"}
c := generate(t, s, testKey, schemaRequest())
if c.url != "https://openrouter.ai/api/v1/chat/completions" {
t.Fatalf("URL = %q", c.url)
}
if got := c.header.Get("Authorization"); got != "Bearer "+testKey {
t.Fatalf("Authorization = %q", got)
}
if got := c.header.Get("HTTP-Referer"); got != "https://meals.example.com/" {
t.Fatalf("HTTP-Referer = %q", got)
}
if got := c.header.Get("X-Title"); got != "MealQuest" {
t.Fatalf("X-Title = %q", got)
}
if got := responseFormatType(t, c); got != "json_schema" {
t.Fatalf("default response_format type = %q, want json_schema", got)
}
}
func TestOpenRouterFallbackModeIsJSONObject(t *testing.T) {
p, _ := Lookup("openrouter")
s := llm.Settings{
Provider: "openrouter",
Model: "meta-llama/llama-3.3-70b-instruct",
PublicURL: "https://meals.example.com",
StructuredOutput: string(p.FallbackStructuredOutput),
}
c := generate(t, s, testKey, schemaRequest())
if got := responseFormatType(t, c); got != "json_object" {
t.Fatalf("response_format type = %q, want json_object", got)
}
}
func TestXAIPreset(t *testing.T) {
c := generate(t, llm.Settings{Provider: "xai", Model: "grok-4", Temperature: 0.6}, testKey, schemaRequest())
if c.url != "https://api.x.ai/v1/chat/completions" {
t.Fatalf("URL = %q", c.url)
}
if got := c.header.Get("Authorization"); got != "Bearer "+testKey {
t.Fatalf("Authorization = %q", got)
}
for _, h := range []string{"X-Api-Key", "HTTP-Referer", "X-Title"} {
if got := c.header.Get(h); got != "" {
t.Fatalf("%s = %q, want none", h, got)
}
}
if got := responseFormatType(t, c); got != "json_schema" {
t.Fatalf("default response_format type = %q, want json_schema", got)
}
if got, ok := temperature(t, c); !ok || got != 0.6 {
t.Fatalf("temperature = %v, %v; want 0.6", got, ok)
}
}
func TestXAIFallbackModeIsJSONObject(t *testing.T) {
p, _ := Lookup("xai")
s := llm.Settings{Provider: "xai", Model: "grok-4", StructuredOutput: string(p.FallbackStructuredOutput)}
c := generate(t, s, testKey, schemaRequest())
if got := responseFormatType(t, c); got != "json_object" {
t.Fatalf("response_format type = %q, want json_object", got)
}
}
func TestXAIPresetAcceptsItsOwnBaseURL(t *testing.T) {
c := generate(t, llm.Settings{Provider: "xai", BaseURL: "https://api.x.ai/v1", Model: "grok-4"}, testKey, plainRequest())
if c.url != "https://api.x.ai/v1/chat/completions" {
t.Fatalf("URL = %q", c.url)
}
}
func TestOllamaPreset(t *testing.T) {
s := llm.Settings{Provider: "ollama", BaseURL: "http://ollama.lan:11434", Model: "llama3.2", Temperature: 0.5}
c := generate(t, s, "", plainRequest())
if c.url != "http://ollama.lan:11434/api/chat" {
t.Fatalf("URL = %q", c.url)
}
if got := c.header.Get("Authorization"); got != "" {
t.Fatalf("Authorization = %q, want none", got)
}
var opts struct{ Temperature *float64 }
if err := json.Unmarshal(c.body["options"], &opts); err != nil || opts.Temperature == nil || *opts.Temperature != 0.5 {
t.Fatalf("options = %s", c.body["options"])
}
}
func TestOpenAICompatiblePreset(t *testing.T) {
base := llm.Settings{Provider: "openai_compatible", BaseURL: "http://llm.lan:8000/v1", Model: "qwen2.5"}
c := generate(t, base, "", schemaRequest())
if c.url != "http://llm.lan:8000/v1/chat/completions" {
t.Fatalf("URL = %q", c.url)
}
if got := c.header.Get("Authorization"); got != "" {
t.Fatalf("Authorization without key = %q, want none", got)
}
if got := responseFormatType(t, c); got != "json_schema" {
t.Fatalf("default response_format type = %q, want json_schema", got)
}
c = generate(t, base, testKey, plainRequest())
if got := c.header.Get("Authorization"); got != "Bearer "+testKey {
t.Fatalf("Authorization with key = %q", got)
}
for mode, want := range map[string]string{"json_schema": "json_schema", "json_object": "json_object", "prompt_only": ""} {
s := base
s.StructuredOutput = mode
if got := responseFormatType(t, generate(t, s, "", schemaRequest())); got != want {
t.Errorf("mode %s: response_format type = %q, want %q", mode, got, want)
}
}
}
func TestTemperatureOmittedForRejectingModels(t *testing.T) {
cases := map[string][]string{
"openai": {"o1", "o1-mini", "o1-2024-12-17", "o3", "o3-mini"},
"openai_compatible": {"o1", "o1-mini", "o3-mini", "openai/o1"},
"openrouter": {"openai/o1", "openai/o1-mini", "openai/o3", "openai/o3-mini", "openai/o1:online"},
}
for provider, models := range cases {
for _, model := range models {
s := llm.Settings{Provider: provider, Model: model, Temperature: 0.8}
switch provider {
case "openai_compatible":
s.BaseURL = "http://llm.lan/v1"
case "openrouter":
s.PublicURL = "https://mealquest.example"
}
for _, req := range []llm.Request{plainRequest(), func() llm.Request { r := plainRequest(); r.Temperature = ptr(0.2); return r }()} {
if got, ok := temperature(t, generate(t, s, testKey, req)); ok {
t.Errorf("%s %s: temperature %v sent, want omitted", provider, model, got)
}
}
}
}
}
func TestTemperatureSentForOtherModels(t *testing.T) {
cases := map[string][]string{
"openai": {"gpt-4o", "gpt-4o-mini", "chatgpt-o1-like", "o123", "o3custom", "o1mini", "openai/o1"},
"openrouter": {"openai/gpt-4o", "openai/o123", "openai/o3custom", "acme/o1", "o1-mini"},
}
for provider, models := range cases {
for _, model := range models {
s := llm.Settings{Provider: provider, Model: model, Temperature: 0.8}
if provider == "openrouter" {
s.PublicURL = "https://mealquest.example"
}
if got, ok := temperature(t, generate(t, s, testKey, plainRequest())); !ok || got != 0.8 {
t.Errorf("%s %s: temperature = %v, %v; want 0.8", provider, model, got, ok)
}
}
}
// A request's own temperature wins over the household's.
req := plainRequest()
req.Temperature = ptr(0.1)
c := generate(t, llm.Settings{Provider: "openai", Model: "gpt-4o", Temperature: 0.8}, testKey, req)
if got, ok := temperature(t, c); !ok || got != 0.1 {
t.Fatalf("temperature = %v, %v; want 0.1", got, ok)
}
}
// A request is validated before the temperature quirk clears its
// temperature, so a negative one is an error and nothing is sent.
func TestNegativeTemperatureRejectedForRejectingModels(t *testing.T) {
rt := &recordingTransport{}
orig := http.DefaultTransport
http.DefaultTransport = rt
t.Cleanup(func() { http.DefaultTransport = orig })
for _, s := range []llm.Settings{
{Provider: "openai", Model: "o1"},
{Provider: "openrouter", Model: "openai/o1", PublicURL: "https://mealquest.example"},
} {
p, err := FromSettings(s, testKey)
if err != nil {
t.Fatalf("%s: FromSettings: %v", s.Model, err)
}
req := plainRequest()
req.Temperature = ptr(-0.5)
if _, err := p.Generate(context.Background(), req); err == nil || errors.Is(err, llm.ErrProvider) || !strings.Contains(err.Error(), "temperature") {
t.Errorf("%s: Generate = %v, want a temperature validation error", s.Model, err)
}
}
if len(rt.reqs) != 0 {
t.Fatalf("sent %d requests, want 0", len(rt.reqs))
}
}
func TestTemperatureQuirkPresets(t *testing.T) {
want := map[string]bool{"openai": true, "openrouter": true, "openai_compatible": true}
for name, p := range all {
if got := len(p.OmitTemperature) > 0; got != want[name] {
t.Errorf("%s: lists temperature quirk = %v, want %v", name, got, want[name])
}
}
}
func TestPresetTable(t *testing.T) {
cases := []struct {
name string
adapter Adapter
baseURL string
key KeyPolicy
}{
{"anthropic", AdapterAnthropic, "https://api.anthropic.com", KeyRequired},
{"openai", AdapterOpenAI, "https://api.openai.com/v1", KeyRequired},
{"openrouter", AdapterOpenAI, "https://openrouter.ai/api/v1", KeyRequired},
{"xai", AdapterOpenAI, "https://api.x.ai/v1", KeyRequired},
{"ollama", AdapterOllama, "", KeyNone},
{"openai_compatible", AdapterOpenAI, "", KeyOptional},
}
if len(all) != len(cases) {
t.Fatalf("%d presets, want %d", len(all), len(cases))
}
for _, c := range cases {
p, ok := Lookup(c.name)
if !ok {
t.Fatalf("no preset %q", c.name)
}
if p.Name != c.name || p.Adapter != c.adapter || p.BaseURL != c.baseURL || p.Key != c.key {
t.Errorf("%s = {%s %s %q %d}", c.name, p.Name, p.Adapter, p.BaseURL, p.Key)
}
}
}
func TestConfigErrors(t *testing.T) {
sentinels := []error{
llm.ErrUnknownProvider, llm.ErrMissingKey, llm.ErrUnexpectedKey, llm.ErrInvalidKey, llm.ErrMissingBaseURL,
llm.ErrMissingModel, llm.ErrMissingPublicURL, llm.ErrInvalidSetting,
}
cases := []struct {
name string
s llm.Settings
key string
want error
}{
{"unknown provider", llm.Settings{Provider: "gemini", Model: "m"}, testKey, llm.ErrUnknownProvider},
{"anthropic no key", llm.Settings{Provider: "anthropic", Model: "m"}, "", llm.ErrMissingKey},
{"openai no key", llm.Settings{Provider: "openai", Model: "m"}, "", llm.ErrMissingKey},
{"openrouter no key", llm.Settings{Provider: "openrouter", Model: "m", PublicURL: "https://x.example"}, "", llm.ErrMissingKey},
{"xai no key", llm.Settings{Provider: "xai", Model: "grok-4"}, "", llm.ErrMissingKey},
{"ollama with key", llm.Settings{Provider: "ollama", BaseURL: "http://o:11434", Model: "m"}, testKey, llm.ErrUnexpectedKey},
{"anthropic key with newline", llm.Settings{Provider: "anthropic", Model: "m"}, testKey + "\n", llm.ErrInvalidKey},
{"openai key with newline", llm.Settings{Provider: "openai", Model: "m"}, testKey + "\n", llm.ErrInvalidKey},
{"openrouter key with CR LF", llm.Settings{Provider: "openrouter", Model: "m", PublicURL: "https://x.example"}, testKey + "\r\nX-Injected: 1", llm.ErrInvalidKey},
{"xai key with newline", llm.Settings{Provider: "xai", Model: "grok-4"}, testKey + "\n", llm.ErrInvalidKey},
{"openai_compatible key with NUL", llm.Settings{Provider: "openai_compatible", BaseURL: "http://l/v1", Model: "m"}, "\x00" + testKey, llm.ErrInvalidKey},
{"ollama no base URL", llm.Settings{Provider: "ollama", Model: "m"}, "", llm.ErrMissingBaseURL},
{"openai_compatible no base URL", llm.Settings{Provider: "openai_compatible", Model: "m"}, "", llm.ErrMissingBaseURL},
{"anthropic no model", llm.Settings{Provider: "anthropic"}, testKey, llm.ErrMissingModel},
{"openai no model", llm.Settings{Provider: "openai"}, testKey, llm.ErrMissingModel},
{"openrouter no model", llm.Settings{Provider: "openrouter", PublicURL: "https://x.example"}, testKey, llm.ErrMissingModel},
{"ollama no model", llm.Settings{Provider: "ollama", BaseURL: "http://o:11434"}, "", llm.ErrMissingModel},
{"openai_compatible no model", llm.Settings{Provider: "openai_compatible", BaseURL: "http://l/v1"}, "", llm.ErrMissingModel},
{"xai no model", llm.Settings{Provider: "xai"}, testKey, llm.ErrMissingModel},
{"openrouter no public URL", llm.Settings{Provider: "openrouter", Model: "m"}, testKey, llm.ErrMissingPublicURL},
{"fixed base URL overridden", llm.Settings{Provider: "openai", BaseURL: "https://evil.example/v1", Model: "m"}, testKey, llm.ErrInvalidSetting},
{"xai base URL overridden", llm.Settings{Provider: "xai", BaseURL: "https://api.openai.com/v1", Model: "grok-4"}, testKey, llm.ErrInvalidSetting},
{"xai prompt_only", llm.Settings{Provider: "xai", Model: "grok-4", StructuredOutput: "prompt_only"}, testKey, llm.ErrInvalidSetting},
{"openai json_object", llm.Settings{Provider: "openai", Model: "m", StructuredOutput: "json_object"}, testKey, llm.ErrInvalidSetting},
{"anthropic mode", llm.Settings{Provider: "anthropic", Model: "m", StructuredOutput: "json_schema"}, testKey, llm.ErrInvalidSetting},
{"unknown mode", llm.Settings{Provider: "openai_compatible", BaseURL: "http://l/v1", Model: "m", StructuredOutput: "xml"}, "", llm.ErrInvalidSetting},
{"negative temperature", llm.Settings{Provider: "openai", Model: "m", Temperature: -0.1}, testKey, llm.ErrInvalidSetting},
{"bad base URL", llm.Settings{Provider: "ollama", BaseURL: "ftp://o", Model: "m"}, "", llm.ErrInvalidSetting},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
p, err := FromSettings(c.s, c.key)
if p != nil || !errors.Is(err, c.want) {
t.Fatalf("FromSettings = %v, %v; want %v", p, err, c.want)
}
for _, other := range sentinels {
if other != c.want && errors.Is(err, other) {
t.Fatalf("error %v also matches %v", err, other)
}
}
if c.key != "" && strings.Contains(err.Error(), c.key) {
t.Fatalf("error leaks the key: %v", err)
}
})
}
}
func TestReadyProviderForEveryPreset(t *testing.T) {
for _, s := range []struct {
s llm.Settings
key string
}{
{llm.Settings{Provider: "anthropic", Model: "m"}, testKey},
{llm.Settings{Provider: "openai", Model: "m"}, testKey},
{llm.Settings{Provider: "openrouter", Model: "m", PublicURL: "https://x.example"}, testKey},
{llm.Settings{Provider: "xai", Model: "m"}, testKey},
{llm.Settings{Provider: "ollama", BaseURL: "http://o:11434", Model: "m"}, ""},
{llm.Settings{Provider: "openai_compatible", BaseURL: "http://l/v1", Model: "m"}, ""},
} {
if p, err := FromSettings(s.s, s.key); err != nil || p == nil {
t.Errorf("%s: FromSettings = %v, %v", s.s.Provider, p, err)
}
}
}
// A key that matches a value the request carries, or text the error spells
// anyway, never appears in a validation error, and nothing is sent.
func TestGenerateValidationErrorsNeverLeakKey(t *testing.T) {
rt := &recordingTransport{}
orig := http.DefaultTransport
http.DefaultTransport = rt
t.Cleanup(func() { http.DefaultTransport = orig })
for _, key := range []string{"-0.5", "temperature", "llm", "negative"} {
p, err := FromSettings(llm.Settings{Provider: "openai_compatible", BaseURL: "http://l/v1", Model: "m"}, key)
if err != nil {
t.Fatalf("%s: FromSettings: %v", key, err)
}
req := plainRequest()
req.Temperature = ptr(-0.5)
_, err = p.Generate(context.Background(), req)
if err == nil || errors.Is(err, llm.ErrProvider) {
t.Fatalf("%s: Generate = %v, want a validation error", key, err)
}
for e := error(err); e != nil; e = errors.Unwrap(e) {
if s := e.Error() + fmt.Sprintf("%#v", e); strings.Contains(s, key) {
t.Fatalf("%s: error leaks the key: %s", key, s)
}
}
}
if len(rt.reqs) != 0 {
t.Fatalf("sent %d requests, want 0", len(rt.reqs))
}
}
// A configuration error keeps its sentinel but never holds the key, even
// when the key is also a setting's value or text the error spells anyway.
func TestConfigErrorsNeverLeakKey(t *testing.T) {
cases := []struct {
name string
s llm.Settings
key string
want error
}{
{"unknown provider named by key", llm.Settings{Provider: "llm", Model: "m"}, "llm", llm.ErrUnknownProvider},
{"no base URL", llm.Settings{Provider: "openai_compatible", Model: "m"}, "base URL", llm.ErrMissingBaseURL},
{"no model", llm.Settings{Provider: "openai_compatible", BaseURL: "http://l/v1"}, "model", llm.ErrMissingModel},
{"key as model, no public URL", llm.Settings{Provider: "openrouter", Model: "missing"}, "missing", llm.ErrMissingPublicURL},
{"key as base URL", llm.Settings{Provider: "openai", BaseURL: "llm", Model: "m"}, "llm", llm.ErrInvalidSetting},
{"key as mode", llm.Settings{Provider: "openai_compatible", BaseURL: "http://l/v1", Model: "m", StructuredOutput: "xml"}, "xml", llm.ErrInvalidSetting},
{"key as temperature", llm.Settings{Provider: "openai", Model: "m", Temperature: -0.1}, "-0.1", llm.ErrInvalidSetting},
{"key in bad base URL", llm.Settings{Provider: "openai_compatible", BaseURL: "ftp://k3y", Model: "m"}, "k3y", llm.ErrInvalidSetting},
{"ollama with key", llm.Settings{Provider: "ollama", BaseURL: "http://o:11434", Model: "m"}, "key", llm.ErrUnexpectedKey},
{"key with newline", llm.Settings{Provider: "openai", Model: "m"}, "sk-test\n", llm.ErrInvalidKey},
{"xai key as preset name", llm.Settings{Provider: "xai"}, "xai", llm.ErrMissingModel},
{"xai key as missing-model text", llm.Settings{Provider: "xai"}, "model is required", llm.ErrMissingModel},
{"xai key as fixed-URL text", llm.Settings{Provider: "xai", BaseURL: "https://x.example/v1", Model: "m"}, "fixed", llm.ErrInvalidSetting},
{"xai key as structured-output text", llm.Settings{Provider: "xai", Model: "m", StructuredOutput: "prompt_only"}, "structured output", llm.ErrInvalidSetting},
{"key with newline as text", llm.Settings{Provider: "openai", Model: "m"}, "header\n", llm.ErrInvalidKey},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
p, err := FromSettings(c.s, c.key)
if p != nil || !errors.Is(err, c.want) {
t.Fatalf("FromSettings = %v, %v; want %v", p, err, c.want)
}
for e := error(err); e != nil; e = errors.Unwrap(e) {
if s := e.Error() + fmt.Sprintf("%#v", e); strings.Contains(s, c.key) {
t.Fatalf("error leaks the key: %s", s)
}
}
})
}
}
// scriptedTransport answers each request with the next scripted status and
// body, echoing the key in every failure body, and counts the requests.
type scriptedTransport struct {
statuses []int
key string
sent int
}
func (st *scriptedTransport) RoundTrip(r *http.Request) (*http.Response, error) {
if st.sent == len(st.statuses) {
return nil, fmt.Errorf("unscripted request %d", st.sent+1)
}
status := st.statuses[st.sent]
st.sent++
header := http.Header{"Content-Type": {"application/json"}}
body := `{"error":{"message":"bad key ` + st.key + `"}}`
switch status {
case http.StatusOK:
body = `{"model":"m","choices":[{"message":{"content":"tacos"},"finish_reason":"stop"}]}`
case http.StatusTooManyRequests:
header.Set("Retry-After", "1")
}
return &http.Response{
StatusCode: status,
Header: header,
Body: io.NopCloser(strings.NewReader(body)),
Request: r,
}, nil
}
// useTransport makes the adapters send through rt for the rest of the test.
func useTransport(t *testing.T, rt http.RoundTripper) {
t.Helper()
orig := http.DefaultTransport
http.DefaultTransport = rt
t.Cleanup(func() { http.DefaultTransport = orig })
}
// The Provider FromSettings returns retries a rate-limited Generate.
func TestFromSettingsRetriesRateLimited(t *testing.T) {
st := &scriptedTransport{statuses: []int{http.StatusTooManyRequests, http.StatusOK}, key: testKey}
useTransport(t, st)
p, err := FromSettings(llm.Settings{Provider: "openai", Model: "m"}, testKey)
if err != nil {
t.Fatalf("FromSettings: %v", err)
}
resp, err := p.Generate(context.Background(), plainRequest())
if err != nil {
t.Fatalf("Generate: %v", err)
}
if resp.Text != "tacos" {
t.Errorf("Text = %q, want %q", resp.Text, "tacos")
}
if st.sent != 2 {
t.Errorf("sent %d requests, want 2", st.sent)
}
}
// When every retry fails, the error FromSettings's Provider returns keeps
// its sentinel and never holds the key the vendor echoed.
func TestFromSettingsRetryErrorNeverLeaksKey(t *testing.T) {
st := &scriptedTransport{statuses: []int{http.StatusTooManyRequests, http.StatusTooManyRequests, http.StatusServiceUnavailable}, key: testKey}
useTransport(t, st)
p, err := FromSettings(llm.Settings{Provider: "openai", Model: "m"}, testKey)
if err != nil {
t.Fatalf("FromSettings: %v", err)
}
_, err = p.Generate(context.Background(), plainRequest())
if !errors.Is(err, llm.ErrProvider) {
t.Fatalf("Generate = %v, want %v", err, llm.ErrProvider)
}
if st.sent != llm.MaxRetries+1 {
t.Errorf("sent %d requests, want %d", st.sent, llm.MaxRetries+1)
}
for e := error(err); e != nil; e = errors.Unwrap(e) {
if s := e.Error() + fmt.Sprintf("%#v", e); strings.Contains(s, testKey) {
t.Fatalf("error leaks the key: %s", s)
}
}
}
+107
View File
@@ -0,0 +1,107 @@
package llm
import (
"context"
"errors"
"math/rand/v2"
"time"
)
const (
// DefaultTimeout bounds each Generate attempt whose Request sets no
// Timeout.
DefaultTimeout = 60 * time.Second
// MaxRetries is how many times a failed Generate is retried, so a call
// makes at most MaxRetries+1 attempts.
MaxRetries = 2
// MaxRetryAfter caps the wait a rate-limited vendor may ask for.
MaxRetryAfter = 20 * time.Second
// backoffBase is the first retry's backoff before jitter; each later
// retry doubles it.
backoffBase = time.Second
)
// Retrying is a Provider that retries transient Generate failures of the
// Provider it wraps. It retries ErrRateLimited, ErrTimeout and a 5xx
// ErrProvider at most MaxRetries times, and nothing else. A rate-limited
// retry waits for the vendor's RetryAfter, capped at MaxRetryAfter; every
// other retry, and a rate limit that names no wait, waits a jittered
// exponential backoff. A retry whose wait would not end before the
// caller's context deadline is not made: the last error is returned at
// once. ListModels and Verify pass straight through.
type Retrying struct {
Provider
// now is the clock; tests replace it.
now func() time.Time
// sleep waits d or until ctx is done; tests replace it.
sleep func(ctx context.Context, d time.Duration) error
// jitter returns a random duration in [0, n) for n > 0; tests replace
// it.
jitter func(n time.Duration) time.Duration
}
// NewRetrying returns inner wrapped with retries and the default Generate
// timeout.
func NewRetrying(inner Provider) *Retrying {
return &Retrying{
Provider: inner,
now: time.Now,
sleep: sleep,
jitter: rand.N[time.Duration],
}
}
// Generate sends req to the wrapped Provider, retrying transient failures.
// A req.Timeout of 0 becomes DefaultTimeout; either bounds each attempt,
// not the whole call, which ctx bounds.
func (r *Retrying) Generate(ctx context.Context, req Request) (Response, error) {
if req.Timeout == 0 {
req.Timeout = DefaultTimeout
}
for retry := 0; ; retry++ {
resp, err := r.Provider.Generate(ctx, req)
if err == nil || retry == MaxRetries || ctx.Err() != nil {
return resp, err
}
wait, ok := r.wait(err, retry)
if !ok {
return resp, err
}
if deadline, has := ctx.Deadline(); has && !r.now().Add(wait).Before(deadline) {
return resp, err
}
if serr := r.sleep(ctx, wait); serr != nil {
return resp, err
}
}
}
// wait returns how long to wait before retrying after err, the retry-th
// retry counting from 0, and false when err is not retried.
func (r *Retrying) wait(err error, retry int) (time.Duration, bool) {
var rl *RateLimitedError
if errors.As(err, &rl) && rl.RetryAfter > 0 {
return min(rl.RetryAfter, MaxRetryAfter), true
}
var pe *ProviderError
switch {
case errors.Is(err, ErrRateLimited), errors.Is(err, ErrTimeout):
case errors.As(err, &pe) && pe.Status() >= 500 && pe.Status() <= 599:
default:
return 0, false
}
half := (backoffBase << retry) / 2
return half + r.jitter(half), true
}
// sleep waits d or until ctx is done, returning ctx's error in that case.
func sleep(ctx context.Context, d time.Duration) error {
t := time.NewTimer(d)
defer t.Stop()
select {
case <-t.C:
return nil
case <-ctx.Done():
return ctx.Err()
}
}
+271
View File
@@ -0,0 +1,271 @@
package llm
import (
"context"
"errors"
"fmt"
"slices"
"testing"
"time"
)
// fakeClock is a clock whose sleeps return at once, advance it and are
// recorded.
type fakeClock struct {
t time.Time
sleeps []time.Duration
}
func (c *fakeClock) now() time.Time { return c.t }
func (c *fakeClock) sleep(ctx context.Context, d time.Duration) error {
if err := ctx.Err(); err != nil {
return err
}
c.sleeps = append(c.sleeps, d)
c.t = c.t.Add(d)
return nil
}
// newTestRetrying returns a Retrying over a fresh Fake with a fake clock
// at the real current time, so real context deadlines compare sensibly,
// and jitter that always returns its maximum.
func newTestRetrying() (*Retrying, *Fake, *fakeClock) {
f := &Fake{}
c := &fakeClock{t: time.Now()}
r := NewRetrying(f)
r.now = c.now
r.sleep = c.sleep
r.jitter = func(n time.Duration) time.Duration { return n - 1 }
return r, f, c
}
var retryReq = Request{Messages: []Message{{Role: RoleUser, Content: "plan dinners"}}}
func ok(text string) GenerateResult { return GenerateResult{Response: Response{Text: text}} }
func fail(err error) GenerateResult { return GenerateResult{Err: err} }
func assertConsumed(t *testing.T, f *Fake, attempts int) {
t.Helper()
if g, _, _ := f.Remaining(); g != 0 {
t.Errorf("%d scripted Generate results unused", g)
}
if n := len(f.Requests()); n != attempts {
t.Errorf("made %d attempts, want %d", n, attempts)
}
}
func TestRetryingRateLimitedThenSuccess(t *testing.T) {
r, f, c := newTestRetrying()
f.QueueGenerate(fail(NewRateLimitedError(3*time.Second, "")), ok("dinner"))
resp, err := r.Generate(context.Background(), retryReq)
if err != nil || resp.Text != "dinner" {
t.Fatalf("Generate = %+v, %v", resp, err)
}
if want := []time.Duration{3 * time.Second}; !slices.Equal(c.sleeps, want) {
t.Errorf("sleeps = %v, want %v", c.sleeps, want)
}
assertConsumed(t, f, 2)
}
func TestRetryingRateLimitedWaitIsCapped(t *testing.T) {
r, f, c := newTestRetrying()
f.QueueGenerate(fail(NewRateLimitedError(time.Hour, "")), ok("dinner"))
if _, err := r.Generate(context.Background(), retryReq); err != nil {
t.Fatal(err)
}
if want := []time.Duration{MaxRetryAfter}; !slices.Equal(c.sleeps, want) {
t.Errorf("sleeps = %v, want %v", c.sleeps, want)
}
assertConsumed(t, f, 2)
}
func TestRetryingRateLimitedWithoutRetryAfterBacksOff(t *testing.T) {
r, f, c := newTestRetrying()
f.QueueGenerate(fail(NewRateLimitedError(0, "")), ok("dinner"))
if _, err := r.Generate(context.Background(), retryReq); err != nil {
t.Fatal(err)
}
if want := []time.Duration{time.Second - 1}; !slices.Equal(c.sleeps, want) {
t.Errorf("sleeps = %v, want %v", c.sleeps, want)
}
}
func TestRetrying5xxTwiceThenSuccess(t *testing.T) {
r, f, c := newTestRetrying()
f.QueueGenerate(
fail(NewProviderError(503, []byte("busy"), "")),
// Adapters wrap the error they return.
fail(fmt.Errorf("openai: generate: %w", NewProviderError(503, []byte("busy"), ""))),
ok("dinner"),
)
resp, err := r.Generate(context.Background(), retryReq)
if err != nil || resp.Text != "dinner" {
t.Fatalf("Generate = %+v, %v", resp, err)
}
// Max jitter: [0.5 s, 1 s) then [1 s, 2 s).
if want := []time.Duration{time.Second - 1, 2*time.Second - 1}; !slices.Equal(c.sleeps, want) {
t.Errorf("sleeps = %v, want %v", c.sleeps, want)
}
assertConsumed(t, f, 3)
}
func TestRetryingBackoffJitterLowerBound(t *testing.T) {
r, f, c := newTestRetrying()
r.jitter = func(time.Duration) time.Duration { return 0 }
f.QueueGenerate(fail(ErrTimeout), fail(ErrTimeout), ok("dinner"))
if _, err := r.Generate(context.Background(), retryReq); err != nil {
t.Fatal(err)
}
if want := []time.Duration{500 * time.Millisecond, time.Second}; !slices.Equal(c.sleeps, want) {
t.Errorf("sleeps = %v, want %v", c.sleeps, want)
}
}
func TestRetrying5xxThreeTimesFails(t *testing.T) {
r, f, c := newTestRetrying()
last := NewProviderError(503, []byte("third"), "")
f.QueueGenerate(
fail(NewProviderError(503, []byte("first"), "")),
fail(NewProviderError(503, []byte("second"), "")),
fail(last),
)
_, err := r.Generate(context.Background(), retryReq)
if err != last {
t.Fatalf("err = %v, want the last attempt's error", err)
}
if len(c.sleeps) != MaxRetries {
t.Errorf("slept %d times, want %d", len(c.sleeps), MaxRetries)
}
assertConsumed(t, f, 3)
}
func TestRetryingDoesNotRetryPermanentErrors(t *testing.T) {
for name, err := range map[string]error{
"401 auth": ErrAuth,
"400": NewProviderError(400, []byte("bad request"), ""),
"404": NewProviderError(404, []byte("no such model"), ""),
"bad output": ErrBadOutput,
"other": errors.New("llm: request has no messages"),
} {
t.Run(name, func(t *testing.T) {
r, f, c := newTestRetrying()
f.QueueGenerate(fail(err))
if _, got := r.Generate(context.Background(), retryReq); got != err {
t.Fatalf("err = %v, want %v", got, err)
}
if len(c.sleeps) != 0 {
t.Errorf("slept %v", c.sleeps)
}
assertConsumed(t, f, 1)
})
}
}
func TestRetryingDeadlineShorterThanRetryAfter(t *testing.T) {
r, f, c := newTestRetrying()
rl := NewRateLimitedError(10*time.Second, "")
f.QueueGenerate(fail(rl))
ctx, cancel := context.WithDeadline(context.Background(), c.t.Add(5*time.Second))
defer cancel()
if _, err := r.Generate(ctx, retryReq); err != rl {
t.Fatalf("err = %v, want the rate-limited error", err)
}
if len(c.sleeps) != 0 {
t.Errorf("slept %v", c.sleeps)
}
assertConsumed(t, f, 1)
}
func TestRetryingDeadlineShorterThanBackoff(t *testing.T) {
r, f, c := newTestRetrying()
f.QueueGenerate(fail(ErrTimeout))
// The first backoff with maximum jitter is just under 1 s.
ctx, cancel := context.WithDeadline(context.Background(), c.t.Add(time.Second-1))
defer cancel()
if _, err := r.Generate(ctx, retryReq); !errors.Is(err, ErrTimeout) {
t.Fatalf("err = %v, want ErrTimeout", err)
}
if len(c.sleeps) != 0 {
t.Errorf("slept %v", c.sleeps)
}
assertConsumed(t, f, 1)
}
func TestRetryingRetriesWithinDeadline(t *testing.T) {
r, f, c := newTestRetrying()
f.QueueGenerate(fail(NewRateLimitedError(3*time.Second, "")), ok("dinner"))
ctx, cancel := context.WithDeadline(context.Background(), c.t.Add(time.Minute))
defer cancel()
if _, err := r.Generate(ctx, retryReq); err != nil {
t.Fatal(err)
}
assertConsumed(t, f, 2)
}
func TestRetryingStopsWhenContextDone(t *testing.T) {
r, f, _ := newTestRetrying()
ctx, cancel := context.WithCancel(context.Background())
cancel()
f.QueueGenerate(fail(ErrTimeout))
if _, err := r.Generate(ctx, retryReq); !errors.Is(err, ErrTimeout) {
t.Fatalf("err = %v, want ErrTimeout", err)
}
assertConsumed(t, f, 1)
}
func TestRetryingDefaultTimeout(t *testing.T) {
r, f, _ := newTestRetrying()
f.QueueGenerate(ok("a"), ok("b"))
custom := retryReq
custom.Timeout = 5 * time.Second
for _, req := range []Request{retryReq, custom} {
if _, err := r.Generate(context.Background(), req); err != nil {
t.Fatal(err)
}
}
got := f.Requests()
if got[0].Timeout != DefaultTimeout || got[1].Timeout != 5*time.Second {
t.Fatalf("timeouts = %v, %v; want %v, 5s", got[0].Timeout, got[1].Timeout, DefaultTimeout)
}
}
func TestRetryingPassesThroughListModelsAndVerify(t *testing.T) {
r, f, _ := newTestRetrying()
f.QueueListModels(ListModelsResult{Err: ErrTimeout})
f.QueueVerify(ErrTimeout)
if _, err := r.ListModels(context.Background()); !errors.Is(err, ErrTimeout) {
t.Fatalf("ListModels err = %v", err)
}
if err := r.Verify(context.Background()); !errors.Is(err, ErrTimeout) {
t.Fatalf("Verify err = %v", err)
}
if _, l, v := f.Remaining(); l != 0 || v != 0 {
t.Fatalf("Remaining = %d, %d", l, v)
}
}
func TestSleepStopsWhenContextDone(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
if err := sleep(ctx, time.Hour); !errors.Is(err, context.Canceled) {
t.Fatalf("sleep err = %v", err)
}
if err := sleep(context.Background(), time.Millisecond); err != nil {
t.Fatalf("sleep err = %v", err)
}
}
+68
View File
@@ -0,0 +1,68 @@
package llm
import "errors"
// Settings is the household's AI configuration, the input to
// presets.FromSettings. The caller fills it from the db.AiSetting row and
// the service config; this package does not import internal/db.
//
// llm.Settings{
// Provider: row.Provider,
// BaseURL: row.BaseUrl,
// Model: row.Model,
// Temperature: row.Temperature,
// PublicURL: cfg.PublicURL.String(), // MEALQUEST_PUBLIC_URL
// }
//
// Settings never holds the API key: the caller decrypts it and passes it
// to presets.FromSettings separately.
type Settings struct {
// Provider names the preset: "anthropic", "openai", "openrouter",
// "xai", "ollama" or "openai_compatible".
Provider string
// BaseURL is the API root. It is required for ollama and
// openai_compatible; the other presets fix their own, and a non-empty
// value must equal it.
BaseURL string
// Model is the model id Generate requests. It is required.
Model string
// Temperature is the household's sampling temperature. It is sent with
// every request that sets none, except to models that reject it.
Temperature float64
// PublicURL is MEALQUEST_PUBLIC_URL. OpenRouter requires it and
// receives it as the HTTP-Referer header; other presets ignore it.
PublicURL string
// StructuredOutput selects how an OpenAI-compatible preset asks for
// JSON: "json_schema", "json_object" or "prompt_only". Empty selects
// the preset's default, json_schema. Each preset allows only the modes
// it lists; anthropic and ollama allow none, so it must be empty.
StructuredOutput string
}
// Configuration errors from presets.FromSettings. Each is distinct and
// matches with errors.Is.
var (
// ErrUnknownProvider means Settings.Provider names no preset.
ErrUnknownProvider = errors.New("llm: unknown provider")
// ErrMissingKey means the preset requires an API key and none was
// given.
ErrMissingKey = errors.New("llm: API key is required")
// ErrUnexpectedKey means an API key was given to a preset that sends
// none.
ErrUnexpectedKey = errors.New("llm: provider takes no API key")
// ErrInvalidKey means the API key is not a valid HTTP header value:
// it has a control character such as CR or LF.
ErrInvalidKey = errors.New("llm: API key is not a valid header value")
// ErrMissingBaseURL means the preset requires a base URL and none was
// given.
ErrMissingBaseURL = errors.New("llm: base URL is required")
// ErrMissingModel means Settings.Model is empty.
ErrMissingModel = errors.New("llm: model is required")
// ErrMissingPublicURL means the preset requires Settings.PublicURL and
// it is empty.
ErrMissingPublicURL = errors.New("llm: public URL is required")
// ErrInvalidSetting means a setting has a value the preset does not
// accept, such as a base URL other than the preset's, an unsupported
// structured output mode or a negative temperature.
ErrInvalidSetting = errors.New("llm: invalid setting")
)
+181 -4
View File
@@ -175,8 +175,8 @@ func TestMigrateEmbeddedMigrations(t *testing.T) {
if err := Migrate(context.Background(), s.Writer, migrations.FS); err != nil {
t.Fatal(err)
}
if got := versions(migrationRows(t, s)); got != "0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status" {
t.Errorf("versions = %q, want 0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status", got)
if got := versions(migrationRows(t, s)); got != "0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai" {
t.Errorf("versions = %q, want 0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai", got)
}
}
@@ -196,8 +196,8 @@ func TestMigrateBootstrapThenEmbeddedInitApplies(t *testing.T) {
if err := Migrate(ctx, s.Writer, migrations.FS); err != nil {
t.Fatal(err)
}
if got := versions(migrationRows(t, s)); got != "0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status" {
t.Errorf("versions = %q, want 0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status", got)
if got := versions(migrationRows(t, s)); got != "0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai" {
t.Errorf("versions = %q, want 0000_bootstrap,0001_init,0002_household_settings_updated_by,0003_jobs_status,0004_ai_provider_presets,0005_ai_provider_xai", got)
}
if !tableExists(t, s, "dinners") {
t.Error("table from 0001_init was not created")
@@ -309,3 +309,180 @@ func TestMigrateJobsStatusMapsLegacyRows(t *testing.T) {
t.Error("jobs_new left behind")
}
}
// TestMigrateAIProviderPresetsKeepsRowsAndReferences upgrades a database
// holding ai_settings and ai_runs rows, with suggestion_batches and
// week_plans referencing the run, and checks 0004 carries every row over,
// keeps the references enforced and accepts the new providers.
func TestMigrateAIProviderPresetsKeepsRowsAndReferences(t *testing.T) {
ctx := context.Background()
s := openTestStore(t)
before := fstest.MapFS{}
for _, name := range []string{"0000_bootstrap.sql", "0001_init.sql", "0002_household_settings_updated_by.sql", "0003_jobs_status.sql"} {
body, err := fs.ReadFile(migrations.FS, name)
if err != nil {
t.Fatal(err)
}
before[name] = &fstest.MapFile{Data: body}
}
if err := Migrate(ctx, s.Writer, before); err != nil {
t.Fatal(err)
}
for _, stmt := range []string{
`INSERT INTO ai_settings (id, provider, base_url, model, temperature, system_prompt, api_key_ct, api_key_last4, key_verified_at, created_at, updated_at)
VALUES (1, 'anthropic', 'https://api.anthropic.com', 'claude', 0.7, 'sp', x'0102', 'abcd', 'v', 'c', 'u')`,
`INSERT INTO ai_runs (id, skill, skill_version, provider, model, temperature, status, latency_ms, tokens_in, tokens_out, error, request_redacted, response_redacted, created_at, updated_at)
VALUES ('run-1', 'suggest', '1', 'ollama', 'llama', 0.5, 'ok', 10, 20, 30, 'e', 'req', 'resp', 'c', 'u')`,
`INSERT INTO suggestion_batches (id, trigger, status, ai_run_id, created_at, updated_at)
VALUES ('batch-1', 'manual', 'done', 'run-1', 'c', 'u')`,
`INSERT INTO week_plans (id, week_start, status, ai_run_id, created_at, updated_at)
VALUES ('plan-1', '2026-10-04', 'draft', 'run-1', 'c', 'u')`,
} {
if _, err := s.Writer.Exec(stmt); err != nil {
t.Fatalf("seed: %v", err)
}
}
if err := Migrate(ctx, s.Writer, migrations.FS); err != nil {
t.Fatal(err)
}
var provider, baseURL, model, systemPrompt, last4, verifiedAt, createdAt, updatedAt string
var temperature float64
var keyCT []byte
if err := s.Reader.QueryRow(`SELECT provider, base_url, model, temperature, system_prompt, api_key_ct, api_key_last4, key_verified_at, created_at, updated_at FROM ai_settings WHERE id = 1`).
Scan(&provider, &baseURL, &model, &temperature, &systemPrompt, &keyCT, &last4, &verifiedAt, &createdAt, &updatedAt); err != nil {
t.Fatalf("read ai_settings: %v", err)
}
settings := []any{provider, baseURL, model, temperature, systemPrompt, string(keyCT), last4, verifiedAt, createdAt, updatedAt}
if want := []any{"anthropic", "https://api.anthropic.com", "claude", 0.7, "sp", "\x01\x02", "abcd", "v", "c", "u"}; !slices.Equal(settings, want) {
t.Errorf("ai_settings = %v, want %v", settings, want)
}
var skill, skillVersion, status, errText, request, response string
var latency, tokensIn, tokensOut int
if err := s.Reader.QueryRow(`SELECT skill, skill_version, provider, model, temperature, status, latency_ms, tokens_in, tokens_out, error, request_redacted, response_redacted, created_at, updated_at FROM ai_runs WHERE id = 'run-1'`).
Scan(&skill, &skillVersion, &provider, &model, &temperature, &status, &latency, &tokensIn, &tokensOut, &errText, &request, &response, &createdAt, &updatedAt); err != nil {
t.Fatalf("read ai_runs: %v", err)
}
run := []any{skill, skillVersion, provider, model, temperature, status, latency, tokensIn, tokensOut, errText, request, response, createdAt, updatedAt}
if want := []any{"suggest", "1", "ollama", "llama", 0.5, "ok", 10, 20, 30, "e", "req", "resp", "c", "u"}; !slices.Equal(run, want) {
t.Errorf("ai_runs = %v, want %v", run, want)
}
rows, err := s.Reader.Query("PRAGMA foreign_key_check")
if err != nil {
t.Fatal(err)
}
if rows.Next() {
t.Error("foreign_key_check reported violations after 0004")
}
if err := rows.Close(); err != nil {
t.Fatal(err)
}
if _, err := s.Writer.Exec(`DELETE FROM ai_runs WHERE id = 'run-1'`); err == nil || !strings.Contains(err.Error(), "FOREIGN KEY constraint failed") {
t.Errorf("deleting a referenced ai_run: error %v, want FOREIGN KEY constraint failure", err)
}
if _, err := s.Writer.Exec(`UPDATE ai_settings SET provider = 'openrouter' WHERE id = 1`); err != nil {
t.Errorf("ai_settings.provider openrouter rejected: %v", err)
}
if _, err := s.Writer.Exec(`UPDATE ai_runs SET provider = 'openai' WHERE id = 'run-1'`); err != nil {
t.Errorf("ai_runs.provider openai rejected: %v", err)
}
for _, table := range []string{"ai_settings_copy", "ai_runs_copy"} {
if tableExists(t, s, table) {
t.Errorf("%s left behind", table)
}
}
}
// TestMigrateAIProviderXAIKeepsRowsAndReferences upgrades a database at 0004
// holding ai_settings and ai_runs rows with 0004's providers, with
// suggestion_batches and week_plans referencing the run, and checks 0005
// carries every row over, keeps the references enforced, accepts xai and
// still rejects an unknown provider.
func TestMigrateAIProviderXAIKeepsRowsAndReferences(t *testing.T) {
ctx := context.Background()
s := openTestStore(t)
before := fstest.MapFS{}
for _, name := range []string{"0000_bootstrap.sql", "0001_init.sql", "0002_household_settings_updated_by.sql", "0003_jobs_status.sql", "0004_ai_provider_presets.sql"} {
body, err := fs.ReadFile(migrations.FS, name)
if err != nil {
t.Fatal(err)
}
before[name] = &fstest.MapFile{Data: body}
}
if err := Migrate(ctx, s.Writer, before); err != nil {
t.Fatal(err)
}
for _, stmt := range []string{
`INSERT INTO ai_settings (id, provider, base_url, model, temperature, system_prompt, api_key_ct, api_key_last4, key_verified_at, created_at, updated_at)
VALUES (1, 'openrouter', 'https://openrouter.ai/api/v1', 'openai/gpt-4o', 0.7, 'sp', x'0102', 'abcd', 'v', 'c', 'u')`,
`INSERT INTO ai_runs (id, skill, skill_version, provider, model, temperature, status, latency_ms, tokens_in, tokens_out, error, request_redacted, response_redacted, created_at, updated_at)
VALUES ('run-1', 'suggest', '1', 'openai', 'gpt-4o', 0.5, 'ok', 10, 20, 30, 'e', 'req', 'resp', 'c', 'u')`,
`INSERT INTO suggestion_batches (id, trigger, status, ai_run_id, created_at, updated_at)
VALUES ('batch-1', 'manual', 'done', 'run-1', 'c', 'u')`,
`INSERT INTO week_plans (id, week_start, status, ai_run_id, created_at, updated_at)
VALUES ('plan-1', '2026-10-04', 'draft', 'run-1', 'c', 'u')`,
} {
if _, err := s.Writer.Exec(stmt); err != nil {
t.Fatalf("seed: %v", err)
}
}
if err := Migrate(ctx, s.Writer, migrations.FS); err != nil {
t.Fatal(err)
}
var provider, baseURL, model, systemPrompt, last4, verifiedAt, createdAt, updatedAt string
var temperature float64
var keyCT []byte
if err := s.Reader.QueryRow(`SELECT provider, base_url, model, temperature, system_prompt, api_key_ct, api_key_last4, key_verified_at, created_at, updated_at FROM ai_settings WHERE id = 1`).
Scan(&provider, &baseURL, &model, &temperature, &systemPrompt, &keyCT, &last4, &verifiedAt, &createdAt, &updatedAt); err != nil {
t.Fatalf("read ai_settings: %v", err)
}
settings := []any{provider, baseURL, model, temperature, systemPrompt, string(keyCT), last4, verifiedAt, createdAt, updatedAt}
if want := []any{"openrouter", "https://openrouter.ai/api/v1", "openai/gpt-4o", 0.7, "sp", "\x01\x02", "abcd", "v", "c", "u"}; !slices.Equal(settings, want) {
t.Errorf("ai_settings = %v, want %v", settings, want)
}
var skill, skillVersion, status, errText, request, response string
var latency, tokensIn, tokensOut int
if err := s.Reader.QueryRow(`SELECT skill, skill_version, provider, model, temperature, status, latency_ms, tokens_in, tokens_out, error, request_redacted, response_redacted, created_at, updated_at FROM ai_runs WHERE id = 'run-1'`).
Scan(&skill, &skillVersion, &provider, &model, &temperature, &status, &latency, &tokensIn, &tokensOut, &errText, &request, &response, &createdAt, &updatedAt); err != nil {
t.Fatalf("read ai_runs: %v", err)
}
run := []any{skill, skillVersion, provider, model, temperature, status, latency, tokensIn, tokensOut, errText, request, response, createdAt, updatedAt}
if want := []any{"suggest", "1", "openai", "gpt-4o", 0.5, "ok", 10, 20, 30, "e", "req", "resp", "c", "u"}; !slices.Equal(run, want) {
t.Errorf("ai_runs = %v, want %v", run, want)
}
rows, err := s.Reader.Query("PRAGMA foreign_key_check")
if err != nil {
t.Fatal(err)
}
if rows.Next() {
t.Error("foreign_key_check reported violations after 0005")
}
if err := rows.Close(); err != nil {
t.Fatal(err)
}
if _, err := s.Writer.Exec(`DELETE FROM ai_runs WHERE id = 'run-1'`); err == nil || !strings.Contains(err.Error(), "FOREIGN KEY constraint failed") {
t.Errorf("deleting a referenced ai_run: error %v, want FOREIGN KEY constraint failure", err)
}
for _, table := range []string{"ai_settings", "ai_runs"} {
if _, err := s.Writer.Exec(`UPDATE ` + table + ` SET provider = 'xai'`); err != nil {
t.Errorf("%s.provider xai rejected: %v", table, err)
}
if _, err := s.Writer.Exec(`UPDATE ` + table + ` SET provider = 'grok'`); err == nil || !strings.Contains(err.Error(), "CHECK constraint failed") {
t.Errorf("%s.provider grok: error %v, want CHECK constraint failure", table, err)
}
}
for _, table := range []string{"ai_settings_copy", "ai_runs_copy"} {
if tableExists(t, s, table) {
t.Errorf("%s left behind", table)
}
}
}
+19 -2
View File
@@ -305,7 +305,7 @@ func TestSchemaEnumChecksRejectInvalidValues(t *testing.T) {
{"household_settings", "plan_draft_dow", 5.5},
{"household_settings", "suggestions_dow", 0.5},
{"household_settings", "chat_autosend_dow", 3.5},
{"ai_runs", "provider", "openai"},
{"ai_runs", "provider", "gemini"},
{"ai_runs", "status", "done"},
{"dinners", "status", "deck"},
{"dinners", "origin", "ai"},
@@ -322,7 +322,7 @@ func TestSchemaEnumChecksRejectInvalidValues(t *testing.T) {
{"rotation_reviews", "decision", "promote"},
{"shopping_lists", "status", "frozen"},
{"shopping_items", "needed", 2},
{"ai_settings", "provider", "openai"},
{"ai_settings", "provider", "gemini"},
{"recipe_search_settings", "provider", "google"},
{"chat_settings", "last_test_status", "pending"},
{"notification_prefs", "event", "weekly_digest"},
@@ -360,6 +360,23 @@ func TestSchemaRotationReviewEventAllowed(t *testing.T) {
}
}
// TestSchemaAIProvidersAllowed checks every provider preset can be stored in
// both ai_settings and ai_runs.
func TestSchemaAIProvidersAllowed(t *testing.T) {
s := seededStore(t)
for _, provider := range []string{"openai_compatible", "anthropic", "ollama", "openai", "openrouter", "xai"} {
for _, table := range []string{"ai_settings", "ai_runs"} {
inRolledBackTx(t, s, func(tx *sql.Tx) {
row := testRow(table, 1)
row["provider"] = provider
if err := insertSchemaRow(tx, table, row); err != nil {
t.Errorf("%s.provider %s rejected: %v", table, provider, err)
}
})
}
}
}
func TestSchemaDinnerStatuses(t *testing.T) {
s := seededStore(t)
for _, status := range []string{"suggested", "quest_log", "tried", "regular", "archived"} {
+68
View File
@@ -0,0 +1,68 @@
-- 0004_ai_provider_presets: ai_settings.provider and ai_runs.provider become
-- openai_compatible, anthropic, ollama, openai, openrouter, the presets in
-- internal/ai/llm/presets. Every 0001 value is kept, so existing rows copy
-- unchanged. SQLite cannot change a CHECK in place, so both tables are
-- rebuilt: copy the rows aside, drop the table, create it again with the
-- wider CHECK, copy the rows back and drop the copy. Neither table has
-- indexes or triggers.
--
-- suggestion_batches.ai_run_id and week_plans.ai_run_id reference ai_runs,
-- so the rebuild cannot use 0003's create-new-then-rename: dropping ai_runs
-- counts every referencing row as a foreign key violation, and only inserting
-- the matching ai_runs rows under the name ai_runs clears them. Deferring
-- foreign keys holds the check until commit, after the rows are back. The
-- pragma lasts only for this migration's transaction.
PRAGMA defer_foreign_keys = ON;
CREATE TABLE ai_settings_copy AS SELECT * FROM ai_settings;
DROP TABLE ai_settings;
CREATE TABLE ai_settings (
id INTEGER PRIMARY KEY CHECK (id = 1),
provider TEXT NOT NULL CHECK (provider IN ('openai_compatible', 'anthropic', 'ollama', 'openai', 'openrouter')),
base_url TEXT NOT NULL,
model TEXT NOT NULL,
temperature REAL NOT NULL,
system_prompt TEXT,
api_key_ct BLOB,
api_key_last4 TEXT,
key_verified_at TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL
);
INSERT INTO ai_settings (id, provider, base_url, model, temperature, system_prompt, api_key_ct, api_key_last4, key_verified_at, created_at, updated_at)
SELECT id, provider, base_url, model, temperature, system_prompt, api_key_ct, api_key_last4, key_verified_at, created_at, updated_at
FROM ai_settings_copy;
DROP TABLE ai_settings_copy;
CREATE TABLE ai_runs_copy AS SELECT * FROM ai_runs;
DROP TABLE ai_runs;
CREATE TABLE ai_runs (
id TEXT NOT NULL PRIMARY KEY,
skill TEXT NOT NULL,
skill_version TEXT NOT NULL,
provider TEXT NOT NULL CHECK (provider IN ('openai_compatible', 'anthropic', 'ollama', 'openai', 'openrouter')),
model TEXT NOT NULL,
temperature REAL NOT NULL,
status TEXT NOT NULL CHECK (status IN ('running', 'ok', 'error')),
latency_ms INTEGER,
tokens_in INTEGER,
tokens_out INTEGER,
error TEXT,
request_redacted TEXT NOT NULL,
response_redacted TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL
);
INSERT INTO ai_runs (id, skill, skill_version, provider, model, temperature, status, latency_ms, tokens_in, tokens_out, error, request_redacted, response_redacted, created_at, updated_at)
SELECT id, skill, skill_version, provider, model, temperature, status, latency_ms, tokens_in, tokens_out, error, request_redacted, response_redacted, created_at, updated_at
FROM ai_runs_copy;
DROP TABLE ai_runs_copy;
+64
View File
@@ -0,0 +1,64 @@
-- 0005_ai_provider_xai: ai_settings.provider and ai_runs.provider also
-- accept xai, the xAI Grok preset in internal/ai/llm/presets. Every 0004
-- value is kept, so existing rows copy unchanged. The rebuild is 0004's:
-- copy the rows aside, drop the table, create it again with the wider CHECK,
-- copy the rows back and drop the copy. Neither table has indexes or
-- triggers.
--
-- suggestion_batches.ai_run_id and week_plans.ai_run_id reference ai_runs,
-- so foreign keys are deferred until commit, after the ai_runs rows are
-- back. The pragma lasts only for this migration's transaction.
PRAGMA defer_foreign_keys = ON;
CREATE TABLE ai_settings_copy AS SELECT * FROM ai_settings;
DROP TABLE ai_settings;
CREATE TABLE ai_settings (
id INTEGER PRIMARY KEY CHECK (id = 1),
provider TEXT NOT NULL CHECK (provider IN ('openai_compatible', 'anthropic', 'ollama', 'openai', 'openrouter', 'xai')),
base_url TEXT NOT NULL,
model TEXT NOT NULL,
temperature REAL NOT NULL,
system_prompt TEXT,
api_key_ct BLOB,
api_key_last4 TEXT,
key_verified_at TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL
);
INSERT INTO ai_settings (id, provider, base_url, model, temperature, system_prompt, api_key_ct, api_key_last4, key_verified_at, created_at, updated_at)
SELECT id, provider, base_url, model, temperature, system_prompt, api_key_ct, api_key_last4, key_verified_at, created_at, updated_at
FROM ai_settings_copy;
DROP TABLE ai_settings_copy;
CREATE TABLE ai_runs_copy AS SELECT * FROM ai_runs;
DROP TABLE ai_runs;
CREATE TABLE ai_runs (
id TEXT NOT NULL PRIMARY KEY,
skill TEXT NOT NULL,
skill_version TEXT NOT NULL,
provider TEXT NOT NULL CHECK (provider IN ('openai_compatible', 'anthropic', 'ollama', 'openai', 'openrouter', 'xai')),
model TEXT NOT NULL,
temperature REAL NOT NULL,
status TEXT NOT NULL CHECK (status IN ('running', 'ok', 'error')),
latency_ms INTEGER,
tokens_in INTEGER,
tokens_out INTEGER,
error TEXT,
request_redacted TEXT NOT NULL,
response_redacted TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL
);
INSERT INTO ai_runs (id, skill, skill_version, provider, model, temperature, status, latency_ms, tokens_in, tokens_out, error, request_redacted, response_redacted, created_at, updated_at)
SELECT id, skill, skill_version, provider, model, temperature, status, latency_ms, tokens_in, tokens_out, error, request_redacted, response_redacted, created_at, updated_at
FROM ai_runs_copy;
DROP TABLE ai_runs_copy;