Record every harness run in ai_runs with redaction and purge
harness.Run now writes one ai_runs row per run that reaches the provider, covering the repair turn: skill, version, provider, model, temperature, status (ok, repaired, bad_output, provider_error), latency, summed token counts when reported, error text and the per-turn request and response bodies. Every stored string is redacted first: the decrypted API key, Authorization and API-key header values, Google Chat webhook URLs and URLs with key-like query or fragment parameters or user info. JSON is decoded and its strings redacted, so escapes cannot hide a credential. A row that cannot be written fails the run. Purge deletes rows created before a cutoff (RunRetention is 30 days), clearing suggestion_batches and week_plans references in the same transaction. Migration 0006 rebuilds ai_runs with the new status CHECK and a created_at index; a pre-harness running or error row stops it. Maestro task 53e382d8-1db3-4555-a31d-e44cf55cc98d. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -15,9 +15,15 @@ func reply(js string) llm.GenerateResult {
|
||||
return llm.GenerateResult{Response: llm.Response{Text: js, JSON: json.RawMessage(js)}}
|
||||
}
|
||||
|
||||
// testConfig returns a Config for f recording into a fresh migrated store.
|
||||
func testConfig(t *testing.T, f *llm.Fake) Config {
|
||||
t.Helper()
|
||||
return Config{Provider: f, Temperature: 0.7, ProviderName: "anthropic", Model: "claude-test", Store: emptyHousehold(t)}
|
||||
}
|
||||
|
||||
func run(t *testing.T, f *llm.Fake) (echoOutput, error) {
|
||||
t.Helper()
|
||||
return Run(context.Background(), Config{Provider: f, Temperature: 0.7}, echoSkill{}, echoInput{Word: "taco"})
|
||||
return Run(context.Background(), testConfig(t, f), echoSkill{}, echoInput{Word: "taco"})
|
||||
}
|
||||
|
||||
func assertCalls(t *testing.T, f *llm.Fake, want int) []llm.Request {
|
||||
@@ -203,9 +209,37 @@ func TestRunProviderErrorOnRepairPassesThrough(t *testing.T) {
|
||||
|
||||
func TestRunRenderError(t *testing.T) {
|
||||
var f llm.Fake
|
||||
_, err := Run(context.Background(), Config{Provider: &f}, echoSkill{}, echoInput{})
|
||||
if err == nil || errors.Is(err, llm.ErrBadOutput) {
|
||||
cfg := testConfig(t, &f)
|
||||
_, err := Run(context.Background(), cfg, echoSkill{}, echoInput{})
|
||||
if err == nil || !strings.Contains(err.Error(), "render") {
|
||||
t.Fatalf("err = %v, want a render error", err)
|
||||
}
|
||||
assertCalls(t, &f, 0)
|
||||
if rows := runRows(t, cfg.Store); len(rows) != 0 {
|
||||
t.Fatalf("render error wrote %d ai_runs rows, want none", len(rows))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunConfigError(t *testing.T) {
|
||||
var f llm.Fake
|
||||
full := testConfig(t, &f)
|
||||
for name, edit := range map[string]func(*Config){
|
||||
"provider": func(c *Config) { c.Provider = nil },
|
||||
"provider name": func(c *Config) { c.ProviderName = "" },
|
||||
"model": func(c *Config) { c.Model = "" },
|
||||
"store": func(c *Config) { c.Store = nil },
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
cfg := full
|
||||
edit(&cfg)
|
||||
_, err := Run(context.Background(), cfg, echoSkill{}, echoInput{Word: "taco"})
|
||||
if err == nil || !strings.Contains(err.Error(), "config has no "+name) {
|
||||
t.Fatalf("err = %v, want missing %s", err, name)
|
||||
}
|
||||
})
|
||||
}
|
||||
assertCalls(t, &f, 0)
|
||||
if rows := runRows(t, full.Store); len(rows) != 0 {
|
||||
t.Fatalf("config errors wrote %d ai_runs rows, want none", len(rows))
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user