283 lines
8.9 KiB
Go
283 lines
8.9 KiB
Go
package harness
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
|
|
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm"
|
|
)
|
|
|
|
func reply(js string) llm.GenerateResult {
|
|
return llm.GenerateResult{Response: llm.Response{Text: js, JSON: json.RawMessage(js)}}
|
|
}
|
|
|
|
// testConfig returns a Config whose provider is f, recording into a fresh
|
|
// migrated store.
|
|
func testConfig(t *testing.T, f *llm.Fake) Config {
|
|
t.Helper()
|
|
return Config{
|
|
Settings: llm.Settings{Provider: "anthropic", Model: "claude-test", Temperature: 0.7},
|
|
NewProvider: func(llm.Settings, string) (llm.Provider, error) { return f, nil },
|
|
Store: emptyHousehold(t),
|
|
Guard: testGuard(t, DefaultMaxCalls),
|
|
}
|
|
}
|
|
|
|
func testGuard(t *testing.T, maxCalls int) *Guard {
|
|
t.Helper()
|
|
g, err := NewGuard(maxCalls)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return g
|
|
}
|
|
|
|
func run(t *testing.T, f *llm.Fake) (echoOutput, error) {
|
|
t.Helper()
|
|
return Run(context.Background(), testConfig(t, f), echoSkill{}, "week-1", echoInput{Word: "taco"})
|
|
}
|
|
|
|
func assertCalls(t *testing.T, f *llm.Fake, want int) []llm.Request {
|
|
t.Helper()
|
|
reqs := f.Requests()
|
|
if len(reqs) != want {
|
|
t.Fatalf("Generate called %d times, want %d", len(reqs), want)
|
|
}
|
|
if g, _, _ := f.Remaining(); g != 0 {
|
|
t.Fatalf("%d scripted Generate results unused", g)
|
|
}
|
|
return reqs
|
|
}
|
|
|
|
func TestRunValidFirstTime(t *testing.T) {
|
|
var f llm.Fake
|
|
f.QueueGenerate(reply(`{"echo":" taco "}`))
|
|
out, err := run(t, &f)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if out.Echo != "taco" {
|
|
t.Fatalf("Echo = %q, want post-processed %q", out.Echo, "taco")
|
|
}
|
|
req := assertCalls(t, &f, 1)[0]
|
|
want, err := Render("", echoSkill{}, echoInput{Word: "taco"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if req.System != want.System || len(req.Messages) != 1 ||
|
|
req.Messages[0] != (llm.Message{Role: llm.RoleUser, Content: "Repeat: taco"}) {
|
|
t.Fatalf("request prompt = %+v", req)
|
|
}
|
|
if req.Temperature == nil || *req.Temperature != 0.7 {
|
|
t.Fatalf("Temperature = %v, want 0.7", req.Temperature)
|
|
}
|
|
if req.Schema == nil || req.Schema.Name != "echo" || string(req.Schema.Schema) != string(echoSkill{}.Schema().Schema) {
|
|
t.Fatalf("Schema = %+v", req.Schema)
|
|
}
|
|
}
|
|
|
|
func TestRunInvalidThenValid(t *testing.T) {
|
|
var f llm.Fake
|
|
f.QueueGenerate(reply(`{"echo":"TACO"}`), reply(`{"echo":"taco"}`))
|
|
out, err := run(t, &f)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if out.Echo != "taco" {
|
|
t.Fatalf("Echo = %q", out.Echo)
|
|
}
|
|
reqs := assertCalls(t, &f, 2)
|
|
msgs := reqs[1].Messages
|
|
if len(msgs) != 3 || msgs[0] != reqs[0].Messages[0] ||
|
|
msgs[1] != (llm.Message{Role: llm.RoleAssistant, Content: `{"echo":"TACO"}`}) || msgs[2].Role != llm.RoleUser {
|
|
t.Fatalf("repair messages = %+v", msgs)
|
|
}
|
|
if !strings.Contains(msgs[2].Content, "echo must be lower case") {
|
|
t.Fatalf("repair message %q lacks the validator error", msgs[2].Content)
|
|
}
|
|
if reqs[1].System != reqs[0].System || reqs[1].Schema == nil || *reqs[1].Temperature != 0.7 {
|
|
t.Fatalf("repair request lost system, schema or temperature: %+v", reqs[1])
|
|
}
|
|
}
|
|
|
|
func TestRunPostProcessedOutputIsValidated(t *testing.T) {
|
|
var f llm.Fake
|
|
f.QueueGenerate(reply(`{"echo":" "}`), reply(`{"echo":"taco"}`))
|
|
out, err := run(t, &f)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if out.Echo != "taco" {
|
|
t.Fatalf("Echo = %q", out.Echo)
|
|
}
|
|
msgs := assertCalls(t, &f, 2)[1].Messages
|
|
if len(msgs) != 3 || msgs[1] != (llm.Message{Role: llm.RoleAssistant, Content: `{"echo":" "}`}) ||
|
|
!strings.Contains(msgs[2].Content, "echo must not be empty") {
|
|
t.Fatalf("repair messages = %+v", msgs)
|
|
}
|
|
}
|
|
|
|
func TestRunWhitespaceTwiceIsBadOutput(t *testing.T) {
|
|
var f llm.Fake
|
|
f.QueueGenerate(reply(`{"echo":" "}`), reply(`{"echo":" "}`))
|
|
_, err := run(t, &f)
|
|
var bad *BadOutputError
|
|
if !errors.Is(err, llm.ErrBadOutput) || !errors.As(err, &bad) {
|
|
t.Fatalf("err = %v, want *BadOutputError", err)
|
|
}
|
|
if !slices.Equal(bad.First, []string{"echo must not be empty"}) ||
|
|
!slices.Equal(bad.Repair, []string{"echo must not be empty"}) {
|
|
t.Fatalf("BadOutputError = %+v", bad)
|
|
}
|
|
assertCalls(t, &f, 2)
|
|
}
|
|
|
|
func TestRunRepairQuotesJSONOverText(t *testing.T) {
|
|
var f llm.Fake
|
|
f.QueueGenerate(
|
|
llm.GenerateResult{Response: llm.Response{Text: "Here you go!", JSON: json.RawMessage(`{"echo":"TACO"}`)}},
|
|
reply(`{"echo":"taco"}`),
|
|
)
|
|
if out, err := run(t, &f); err != nil || out.Echo != "taco" {
|
|
t.Fatalf("Run = %+v, %v", out, err)
|
|
}
|
|
msgs := assertCalls(t, &f, 2)[1].Messages
|
|
if len(msgs) != 3 || msgs[1] != (llm.Message{Role: llm.RoleAssistant, Content: `{"echo":"TACO"}`}) {
|
|
t.Fatalf("repair messages = %+v, want the failed JSON as the assistant reply", msgs)
|
|
}
|
|
}
|
|
|
|
func TestRunDecodeFailureIsRepaired(t *testing.T) {
|
|
for name, first := range map[string]llm.GenerateResult{
|
|
"unknown field": reply(`{"echo":"taco","extra":1}`),
|
|
"wrong type": reply(`{"echo":5}`),
|
|
"trailing data": reply(`{"echo":"taco"}{}`),
|
|
"no JSON": {Response: llm.Response{Text: "taco"}},
|
|
"adapter refuse": {Response: llm.Response{Text: "nope"}, Err: llm.ErrBadOutput},
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
var f llm.Fake
|
|
f.QueueGenerate(first, reply(`{"echo":"taco"}`))
|
|
if out, err := run(t, &f); err != nil || out.Echo != "taco" {
|
|
t.Fatalf("Run = %+v, %v", out, err)
|
|
}
|
|
assertCalls(t, &f, 2)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRunInvalidTwice(t *testing.T) {
|
|
var f llm.Fake
|
|
f.QueueGenerate(reply(`{"echo":"TACO"}`), reply(`{"echo":""}`))
|
|
_, err := run(t, &f)
|
|
if !errors.Is(err, llm.ErrBadOutput) {
|
|
t.Fatalf("err = %v, want ErrBadOutput", err)
|
|
}
|
|
var bad *BadOutputError
|
|
if !errors.As(err, &bad) {
|
|
t.Fatalf("err = %T, want *BadOutputError", err)
|
|
}
|
|
if bad.Skill != "echo" || bad.Version != 1 ||
|
|
!slices.Equal(bad.First, []string{"echo must be lower case"}) ||
|
|
!slices.Equal(bad.Repair, []string{"echo must not be empty"}) {
|
|
t.Fatalf("BadOutputError = %+v", bad)
|
|
}
|
|
for _, want := range []string{"echo must be lower case", "echo must not be empty"} {
|
|
if !strings.Contains(err.Error(), want) {
|
|
t.Fatalf("error %q lacks %q", err, want)
|
|
}
|
|
}
|
|
assertCalls(t, &f, 2)
|
|
}
|
|
|
|
func TestRunProviderErrorPassesThrough(t *testing.T) {
|
|
for name, perr := range map[string]error{
|
|
"auth": llm.ErrAuth,
|
|
"rate limited": &llm.RateLimitedError{},
|
|
"timeout": llm.ErrTimeout,
|
|
"provider": llm.NewProviderError(500, []byte("boom"), ""),
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
var f llm.Fake
|
|
f.QueueGenerate(llm.GenerateResult{Err: perr})
|
|
_, err := run(t, &f)
|
|
if err != perr {
|
|
t.Fatalf("err = %v, want the provider error unchanged", err)
|
|
}
|
|
assertCalls(t, &f, 1)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRunProviderErrorOnRepairPassesThrough(t *testing.T) {
|
|
var f llm.Fake
|
|
f.QueueGenerate(reply(`{"echo":"TACO"}`), llm.GenerateResult{Err: llm.ErrTimeout})
|
|
if _, err := run(t, &f); err != llm.ErrTimeout {
|
|
t.Fatalf("err = %v, want ErrTimeout unchanged", err)
|
|
}
|
|
assertCalls(t, &f, 2)
|
|
}
|
|
|
|
func TestRunRenderError(t *testing.T) {
|
|
var f llm.Fake
|
|
cfg := testConfig(t, &f)
|
|
_, err := Run(context.Background(), cfg, echoSkill{}, "week-1", echoInput{})
|
|
if err == nil || !strings.Contains(err.Error(), "render") {
|
|
t.Fatalf("err = %v, want a render error", err)
|
|
}
|
|
assertCalls(t, &f, 0)
|
|
rows := runRows(t, cfg.Store)
|
|
if len(rows) != 1 {
|
|
t.Fatalf("render error wrote %d ai_runs rows, want 1", len(rows))
|
|
}
|
|
r := rows[0]
|
|
if r.Status != string(StatusBadOutput) || r.Error.String != err.Error() ||
|
|
r.Request != "[]" || r.Response.String != "[]" || r.TokensIn.Valid || r.TokensOut.Valid {
|
|
t.Fatalf("row = %+v, want bad_output with the render error and no turns", r)
|
|
}
|
|
}
|
|
|
|
func TestRunProviderBuildError(t *testing.T) {
|
|
var f llm.Fake
|
|
cfg := testConfig(t, &f)
|
|
cfg.NewProvider = func(llm.Settings, string) (llm.Provider, error) { return nil, llm.ErrMissingKey }
|
|
_, err := Run(context.Background(), cfg, echoSkill{}, "week-1", echoInput{Word: "taco"})
|
|
if !errors.Is(err, llm.ErrMissingKey) {
|
|
t.Fatalf("err = %v, want ErrMissingKey", err)
|
|
}
|
|
rows := runRows(t, cfg.Store)
|
|
if len(rows) != 1 || rows[0].Status != string(StatusProviderError) ||
|
|
rows[0].Error.String != err.Error() || rows[0].Request != "[]" {
|
|
t.Fatalf("rows = %+v, want one provider_error row with no turns", rows)
|
|
}
|
|
}
|
|
|
|
func TestRunConfigError(t *testing.T) {
|
|
var f llm.Fake
|
|
full := testConfig(t, &f)
|
|
for name, edit := range map[string]func(*Config){
|
|
"provider constructor": func(c *Config) { c.NewProvider = nil },
|
|
"provider name": func(c *Config) { c.Settings.Provider = "" },
|
|
"model": func(c *Config) { c.Settings.Model = "" },
|
|
"store": func(c *Config) { c.Store = nil },
|
|
"guard": func(c *Config) { c.Guard = nil },
|
|
} {
|
|
t.Run(name, func(t *testing.T) {
|
|
cfg := full
|
|
edit(&cfg)
|
|
_, err := Run(context.Background(), cfg, echoSkill{}, "week-1", echoInput{Word: "taco"})
|
|
if err == nil || !strings.Contains(err.Error(), "config has no "+name) {
|
|
t.Fatalf("err = %v, want missing %s", err, name)
|
|
}
|
|
})
|
|
}
|
|
assertCalls(t, &f, 0)
|
|
if rows := runRows(t, full.Store); len(rows) != 0 {
|
|
t.Fatalf("config errors wrote %d ai_runs rows, want none", len(rows))
|
|
}
|
|
}
|