Record every harness run in ai_runs with redaction and purge
harness.Run now writes one ai_runs row per run that reaches the provider, covering the repair turn: skill, version, provider, model, temperature, status (ok, repaired, bad_output, provider_error), latency, summed token counts when reported, error text and the per-turn request and response bodies. Every stored string is redacted first: the decrypted API key, Authorization and API-key header values, Google Chat webhook URLs and URLs with key-like query or fragment parameters or user info. JSON is decoded and its strings redacted, so escapes cannot hide a credential. A row that cannot be written fails the run. Purge deletes rows created before a cutoff (RunRetention is 30 days), clearing suggestion_batches and week_plans references in the same transaction. Migration 0006 rebuilds ai_runs with the new status CHECK and a created_at index; a pre-harness running or error row stops it. Maestro task 53e382d8-1db3-4555-a31d-e44cf55cc98d. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -4,7 +4,8 @@
|
||||
// for output matching the skill's schema, decodes and validates the reply,
|
||||
// and on failure sends exactly one repair turn quoting the problems. If the
|
||||
// repair fails too the run fails with llm.ErrBadOutput; there is no silent
|
||||
// fallback (D-007).
|
||||
// fallback (D-007). Every run that reaches the provider is recorded in
|
||||
// ai_runs (see Run and Purge).
|
||||
package harness
|
||||
|
||||
import (
|
||||
@@ -17,6 +18,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm"
|
||||
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/store"
|
||||
)
|
||||
|
||||
// Skill is one versioned AI task with input In and structured output Out.
|
||||
@@ -50,6 +52,30 @@ type Config struct {
|
||||
// Temperature is the household's sampling temperature, sent with
|
||||
// every request.
|
||||
Temperature float64
|
||||
// ProviderName is ai_settings.provider and Model is ai_settings.model,
|
||||
// both recorded in ai_runs. Neither may be empty.
|
||||
ProviderName string
|
||||
Model string
|
||||
// APIKey is the decrypted API key Provider sends, or "" when it sends
|
||||
// none. It is used only to remove the key from the ai_runs row.
|
||||
APIKey string
|
||||
// Store receives the run's ai_runs row. It must not be nil.
|
||||
Store *store.Store
|
||||
}
|
||||
|
||||
// validate reports a Config missing what every run needs.
|
||||
func (c Config) validate() error {
|
||||
switch {
|
||||
case c.Provider == nil:
|
||||
return errors.New("config has no provider")
|
||||
case c.ProviderName == "":
|
||||
return errors.New("config has no provider name")
|
||||
case c.Model == "":
|
||||
return errors.New("config has no model")
|
||||
case c.Store == nil:
|
||||
return errors.New("config has no store")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// BadOutputError is a run whose reply and repair reply were both unusable.
|
||||
@@ -74,8 +100,17 @@ func (e *BadOutputError) Unwrap() error { return llm.ErrBadOutput }
|
||||
// Run runs skill s on in with cfg and returns its validated, post-processed
|
||||
// output. A provider error other than llm.ErrBadOutput is returned
|
||||
// unchanged and never repaired.
|
||||
//
|
||||
// Every run that sends a request writes one ai_runs row when it ends,
|
||||
// covering both turns, with its RunStatus and the request and response
|
||||
// bodies redacted (see redact). A Config or render error fails the run
|
||||
// before any request, and writes no row. When the row cannot be written
|
||||
// the run fails with that error too, even if its output was usable.
|
||||
func Run[In, Out any](ctx context.Context, cfg Config, s Skill[In, Out], in In) (Out, error) {
|
||||
var zero Out
|
||||
if err := cfg.validate(); err != nil {
|
||||
return zero, fmt.Errorf("harness: skill %s v%d: %w", s.Name(), s.Version(), err)
|
||||
}
|
||||
prompt, err := Render(cfg.Persona, s, in)
|
||||
if err != nil {
|
||||
return zero, fmt.Errorf("harness: skill %s v%d: render: %w", s.Name(), s.Version(), err)
|
||||
@@ -89,33 +124,51 @@ func Run[In, Out any](ctx context.Context, cfg Config, s Skill[In, Out], in In)
|
||||
Schema: &schema,
|
||||
}
|
||||
|
||||
out, reply, first, err := attempt(ctx, cfg.Provider, s, req)
|
||||
log := &runLog{start: store.Now()}
|
||||
out, status, err := turns(ctx, cfg, s, req, log)
|
||||
if saveErr := log.save(ctx, cfg, s.Name(), s.Version(), status, err); saveErr != nil {
|
||||
return zero, errors.Join(err, fmt.Errorf("harness: skill %s v%d: record ai_runs row: %w", s.Name(), s.Version(), saveErr))
|
||||
}
|
||||
if err != nil {
|
||||
return zero, err
|
||||
}
|
||||
if len(first) > 0 {
|
||||
if reply != "" {
|
||||
req.Messages = append(req.Messages, llm.Message{Role: llm.RoleAssistant, Content: reply})
|
||||
}
|
||||
req.Messages = append(req.Messages, llm.Message{Role: llm.RoleUser, Content: repairMessage(schema.Name, first)})
|
||||
var second []string
|
||||
out, _, second, err = attempt(ctx, cfg.Provider, s, req)
|
||||
if err != nil {
|
||||
return zero, err
|
||||
}
|
||||
if len(second) > 0 {
|
||||
return zero, &BadOutputError{Skill: s.Name(), Version: s.Version(), First: first, Repair: second}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// attempt sends req once, then decodes, validates, post-processes and
|
||||
// validates the reply again. It returns the reply for the repair turn and
|
||||
// the problems found; err is set only for a provider or post-process
|
||||
// failure, neither of which is repairable.
|
||||
func attempt[In, Out any](ctx context.Context, p llm.Provider, s Skill[In, Out], req llm.Request) (out Out, reply string, problems []string, err error) {
|
||||
resp, err := p.Generate(ctx, req)
|
||||
// turns sends req and, when its reply is unusable, the repair turn,
|
||||
// recording both in log. It returns the run's status alongside its result.
|
||||
func turns[In, Out any](ctx context.Context, cfg Config, s Skill[In, Out], req llm.Request, log *runLog) (Out, RunStatus, error) {
|
||||
var zero Out
|
||||
out, reply, first, status, err := attempt(ctx, cfg, s, req, log)
|
||||
if err != nil {
|
||||
return zero, status, err
|
||||
}
|
||||
if len(first) == 0 {
|
||||
return out, StatusOK, nil
|
||||
}
|
||||
log.problems = first
|
||||
if reply != "" {
|
||||
req.Messages = append(req.Messages, llm.Message{Role: llm.RoleAssistant, Content: reply})
|
||||
}
|
||||
req.Messages = append(req.Messages, llm.Message{Role: llm.RoleUser, Content: repairMessage(req.Schema.Name, first)})
|
||||
out, _, second, status, err := attempt(ctx, cfg, s, req, log)
|
||||
if err != nil {
|
||||
return zero, status, err
|
||||
}
|
||||
if len(second) > 0 {
|
||||
return zero, StatusBadOutput, &BadOutputError{Skill: s.Name(), Version: s.Version(), First: first, Repair: second}
|
||||
}
|
||||
return out, StatusRepaired, nil
|
||||
}
|
||||
|
||||
// attempt sends req once, records the turn in log, then decodes,
|
||||
// validates, post-processes and validates the reply again. It returns the
|
||||
// reply for the repair turn and the problems found; err is set only for a
|
||||
// provider or post-process failure, neither of which is repairable, and
|
||||
// status then says which.
|
||||
func attempt[In, Out any](ctx context.Context, cfg Config, s Skill[In, Out], req llm.Request, log *runLog) (out Out, reply string, problems []string, status RunStatus, err error) {
|
||||
resp, err := cfg.Provider.Generate(ctx, req)
|
||||
log.add(cfg.APIKey, req, resp, err)
|
||||
// The repair turn must quote what failed: the structured output when
|
||||
// there is any, otherwise the text.
|
||||
reply = string(resp.JSON)
|
||||
@@ -124,21 +177,21 @@ func attempt[In, Out any](ctx context.Context, p llm.Provider, s Skill[In, Out],
|
||||
}
|
||||
if err != nil {
|
||||
if errors.Is(err, llm.ErrBadOutput) {
|
||||
return out, reply, []string{err.Error()}, nil
|
||||
return out, reply, []string{err.Error()}, "", nil
|
||||
}
|
||||
return out, "", nil, err
|
||||
return out, "", nil, StatusProviderError, err
|
||||
}
|
||||
if err := decode(resp.JSON, &out); err != nil {
|
||||
return out, reply, []string{err.Error()}, nil
|
||||
return out, reply, []string{err.Error()}, "", nil
|
||||
}
|
||||
if problems := s.Validate(out); len(problems) > 0 {
|
||||
return out, reply, problems, nil
|
||||
return out, reply, problems, "", nil
|
||||
}
|
||||
out, err = s.PostProcess(out)
|
||||
if err != nil {
|
||||
return out, "", nil, fmt.Errorf("harness: skill %s v%d: post-process: %w", s.Name(), s.Version(), err)
|
||||
return out, "", nil, StatusBadOutput, fmt.Errorf("harness: skill %s v%d: post-process: %w", s.Name(), s.Version(), err)
|
||||
}
|
||||
return out, reply, s.Validate(out), nil
|
||||
return out, reply, s.Validate(out), "", nil
|
||||
}
|
||||
|
||||
// decode strictly decodes data into out: one JSON value, no unknown fields.
|
||||
|
||||
Reference in New Issue
Block a user