Build the run's provider from the key it redacts; record render failures
Review findings on b0cf6fb:
1. Config took a ready-made Provider and a separate, optional APIKey used
only for redaction, so a provider holding a decrypted key could run
with APIKey empty and a reply quoting that key unlabelled was stored.
Config now holds the household's llm.Settings, the decrypted APIKey and
a NewProvider constructor (presets.FromSettings in production). Run
builds the provider from exactly that key and redacts the same key, so
the two cannot differ. ProviderName, Model and Temperature come from
Settings. A new test runs presets.FromSettings against a server that
echoes the bearer key unlabelled in the reply text, reply JSON and
model name, across a repair turn, and asserts the stored row never
holds it; with redaction disabled the test fails.
2. A render failure wrote no ai_runs row. Every run with a valid Config
now writes one: a render failure is bad_output and a provider that
cannot be built is provider_error, each with the error text and empty
request and response lists. TestRunRenderError now expects exactly one
such row. A Config error (no store, provider name or model) still
writes none, since the row cannot be written without them.
Redaction rules, the four statuses for provider-reaching runs (one row,
both turns) and Purge are unchanged.
Maestro task 53e382d8-1db3-4555-a31d-e44cf55cc98d.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -6,11 +6,14 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm"
|
||||
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm/presets"
|
||||
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/db"
|
||||
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/store"
|
||||
)
|
||||
@@ -212,7 +215,7 @@ func TestRunRecordsCancelledRun(t *testing.T) {
|
||||
func TestRunFailsWhenRowCannotBeWritten(t *testing.T) {
|
||||
var f llm.Fake
|
||||
cfg := testConfig(t, &f)
|
||||
cfg.ProviderName = "gemini" // refused by the provider CHECK
|
||||
cfg.Settings.Provider = "gemini" // refused by the provider CHECK
|
||||
f.QueueGenerate(reply(`{"echo":"taco"}`))
|
||||
_, err := Run(context.Background(), cfg, echoSkill{}, echoInput{Word: "taco"})
|
||||
if err == nil || !strings.Contains(err.Error(), "record ai_runs row") {
|
||||
@@ -266,6 +269,12 @@ func TestRunRowsHoldNoSecrets(t *testing.T) {
|
||||
f.QueueGenerate(script...)
|
||||
cfg := testConfig(t, &f)
|
||||
cfg.APIKey = seededKey
|
||||
cfg.NewProvider = func(_ llm.Settings, key string) (llm.Provider, error) {
|
||||
if key != seededKey {
|
||||
t.Errorf("provider built with key %q, want the seeded key", key)
|
||||
}
|
||||
return &f, nil
|
||||
}
|
||||
cfg.Persona = "Our kitchen. " + leaky
|
||||
_, _ = Run(context.Background(), cfg, echoSkill{}, echoInput{Word: "tacos " + seededWebhook})
|
||||
assertCalls(t, &f, len(script))
|
||||
@@ -289,6 +298,63 @@ func TestRunRowsHoldNoSecrets(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunRowsHoldNoProviderKey builds the provider the production way,
|
||||
// presets.FromSettings, against a server that quotes the bearer key it
|
||||
// receives with no Authorization label: in the reply text, the reply JSON
|
||||
// and the model name. The repair turn then sends the quoted key back.
|
||||
func TestRunRowsHoldNoProviderKey(t *testing.T) {
|
||||
turn := 0
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
key, ok := strings.CutPrefix(r.Header.Get("Authorization"), "Bearer ")
|
||||
if !ok || key == "" {
|
||||
t.Errorf("Authorization = %q, want a bearer key", r.Header.Get("Authorization"))
|
||||
}
|
||||
turn++
|
||||
echo := key // lower case, so usable
|
||||
if turn == 1 {
|
||||
echo = "Your key is " + key // upper case, so repaired
|
||||
}
|
||||
content, err := json.Marshal(map[string]string{"echo": echo})
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err := json.NewEncoder(w).Encode(map[string]any{
|
||||
"model": "model-" + key,
|
||||
"choices": []any{map[string]any{"finish_reason": "stop", "message": map[string]any{"role": "assistant", "content": string(content)}}},
|
||||
"usage": map[string]int{"prompt_tokens": 10, "completion_tokens": 5},
|
||||
}); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
cfg := Config{
|
||||
Settings: llm.Settings{Provider: "openai_compatible", BaseURL: srv.URL, Model: "local-test", Temperature: 0.7},
|
||||
APIKey: seededKey,
|
||||
NewProvider: presets.FromSettings,
|
||||
Store: emptyHousehold(t),
|
||||
}
|
||||
out, err := Run(context.Background(), cfg, echoSkill{}, echoInput{Word: "taco"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out.Echo != seededKey || turn != 2 {
|
||||
t.Fatalf("Echo = %q after %d turns, want the provider's key after 2", out.Echo, turn)
|
||||
}
|
||||
rows := runRows(t, cfg.Store)
|
||||
if len(rows) != 1 || rows[0].Status != string(StatusRepaired) {
|
||||
t.Fatalf("rows = %+v, want one repaired row", rows)
|
||||
}
|
||||
text := rows[0].text()
|
||||
if strings.Contains(text, seededKey) {
|
||||
t.Fatalf("stored row holds the provider's key:\n%s", text)
|
||||
}
|
||||
if !strings.Contains(text, "Your key is "+redactionMarker) || !strings.Contains(text, "model-"+redactionMarker) {
|
||||
t.Fatalf("stored row lacks the redacted quotes:\n%s", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedact(t *testing.T) {
|
||||
const key = "sk-abc123"
|
||||
for _, tc := range []struct{ in, want string }{
|
||||
|
||||
Reference in New Issue
Block a user