Build the run's provider from the key it redacts; record render failures
Review findings on b0cf6fb:
1. Config took a ready-made Provider and a separate, optional APIKey used
only for redaction, so a provider holding a decrypted key could run
with APIKey empty and a reply quoting that key unlabelled was stored.
Config now holds the household's llm.Settings, the decrypted APIKey and
a NewProvider constructor (presets.FromSettings in production). Run
builds the provider from exactly that key and redacts the same key, so
the two cannot differ. ProviderName, Model and Temperature come from
Settings. A new test runs presets.FromSettings against a server that
echoes the bearer key unlabelled in the reply text, reply JSON and
model name, across a repair turn, and asserts the stored row never
holds it; with redaction disabled the test fails.
2. A render failure wrote no ai_runs row. Every run with a valid Config
now writes one: a render failure is bad_output and a provider that
cannot be built is provider_error, each with the error text and empty
request and response lists. TestRunRenderError now expects exactly one
such row. A Config error (no store, provider name or model) still
writes none, since the row cannot be written without them.
Redaction rules, the four statuses for provider-reaching runs (one row,
both turns) and Purge are unchanged.
Maestro task 53e382d8-1db3-4555-a31d-e44cf55cc98d.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -26,10 +26,11 @@ const (
|
||||
// StatusRepaired means the first reply was not usable and the reply to
|
||||
// the repair turn was.
|
||||
StatusRepaired RunStatus = "repaired"
|
||||
// StatusBadOutput means neither reply was usable, or post-processing
|
||||
// a valid reply failed.
|
||||
// StatusBadOutput means neither reply was usable, post-processing a
|
||||
// valid reply failed, or the skill's prompt could not be rendered.
|
||||
StatusBadOutput RunStatus = "bad_output"
|
||||
// StatusProviderError means the provider failed on either turn.
|
||||
// StatusProviderError means the provider could not be built or failed
|
||||
// on either turn.
|
||||
StatusProviderError RunStatus = "provider_error"
|
||||
)
|
||||
|
||||
@@ -140,11 +141,12 @@ func (l *runLog) add(apiKey string, req llm.Request, resp llm.Response, err erro
|
||||
// save writes the run's ai_runs row. It is written even when ctx is done,
|
||||
// so a cancelled run is recorded too.
|
||||
func (l *runLog) save(ctx context.Context, cfg Config, skill string, version int, status RunStatus, runErr error) error {
|
||||
request, err := marshalRedacted(l.requests, cfg.APIKey)
|
||||
// A run that failed before its first turn stores an empty list.
|
||||
request, err := marshalRedacted(nonNil(l.requests), cfg.APIKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
response, err := marshalRedacted(l.responses, cfg.APIKey)
|
||||
response, err := marshalRedacted(nonNil(l.responses), cfg.APIKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -160,9 +162,9 @@ func (l *runLog) save(ctx context.Context, cfg Config, skill string, version int
|
||||
ID: store.NewID(),
|
||||
Skill: skill,
|
||||
SkillVersion: strconv.Itoa(version),
|
||||
Provider: cfg.ProviderName,
|
||||
Model: cfg.Model,
|
||||
Temperature: cfg.Temperature,
|
||||
Provider: cfg.Settings.Provider,
|
||||
Model: cfg.Settings.Model,
|
||||
Temperature: cfg.Settings.Temperature,
|
||||
Status: string(status),
|
||||
LatencyMs: sql.NullInt64{Int64: now.Sub(l.start.Time).Milliseconds(), Valid: true},
|
||||
TokensIn: reported(l.tokensIn),
|
||||
@@ -175,6 +177,14 @@ func (l *runLog) save(ctx context.Context, cfg Config, skill string, version int
|
||||
})
|
||||
}
|
||||
|
||||
// nonNil is s, or an empty slice when s is nil, so it encodes as [].
|
||||
func nonNil[T any](s []T) []T {
|
||||
if s == nil {
|
||||
return []T{}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// reported is n as a column value, NULL when no turn reported a count.
|
||||
func reported(n int64) sql.NullInt64 {
|
||||
return sql.NullInt64{Int64: n, Valid: n > 0}
|
||||
|
||||
Reference in New Issue
Block a user