Validate post-processed skill output and quote failed JSON in repair
CI / ci (push) Successful in 1m33s
CI / ci (pull_request) Successful in 1m32s

The harness now validates a skill's output again after PostProcess, so
output that post-processing makes invalid (e.g. a whitespace-only echo
trimmed to empty) enters the single repair turn instead of returning as
a success. The repair turn now quotes Response.JSON as the previous
assistant reply whenever structured output was returned, falling back to
Response.Text only when there is none.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-05 07:49:23 -04:00
co-authored by Claude Opus 5.5
parent 9ed5cb1192
commit 7be4879f6d
2 changed files with 65 additions and 12 deletions
+18 -12
View File
@@ -40,8 +40,9 @@ type Skill[In, Out any] interface {
// The problems are sent to the model in the repair turn, so each must
// say what to fix.
Validate(out Out) []string
// PostProcess finishes a valid out, e.g. normalising names. Its error
// fails the run without a repair turn.
// PostProcess finishes a valid out, e.g. normalising names. Its result
// is validated again; problems found then are repaired like any other.
// Its error fails the run without a repair turn.
PostProcess(out Out) (Out, error)
}
@@ -112,22 +113,20 @@ func Run[In, Out any](ctx context.Context, cfg Config, s Skill[In, Out], in In)
return zero, &BadOutputError{Skill: s.Name(), Version: s.Version(), First: first, Repair: second}
}
}
out, err = s.PostProcess(out)
if err != nil {
return zero, fmt.Errorf("harness: skill %s v%d: post-process: %w", s.Name(), s.Version(), err)
}
return out, nil
}
// attempt sends req once and decodes and validates the reply. It returns
// the reply text for the repair turn and the problems found; err is set
// only for a provider failure, which is not repairable.
// attempt sends req once, then decodes, validates, post-processes and
// validates the reply again. It returns the reply for the repair turn and
// the problems found; err is set only for a provider or post-process
// failure, neither of which is repairable.
func attempt[In, Out any](ctx context.Context, p llm.Provider, s Skill[In, Out], req llm.Request) (out Out, reply string, problems []string, err error) {
resp, err := p.Generate(ctx, req)
reply = resp.Text
// The repair turn must quote what failed: the structured output when
// there is any, otherwise the text.
reply = string(resp.JSON)
if reply == "" {
reply = string(resp.JSON)
reply = resp.Text
}
if err != nil {
if errors.Is(err, llm.ErrBadOutput) {
@@ -138,6 +137,13 @@ func attempt[In, Out any](ctx context.Context, p llm.Provider, s Skill[In, Out],
if err := decode(resp.JSON, &out); err != nil {
return out, reply, []string{err.Error()}, nil
}
if problems := s.Validate(out); len(problems) > 0 {
return out, reply, problems, nil
}
out, err = s.PostProcess(out)
if err != nil {
return out, "", nil, fmt.Errorf("harness: skill %s v%d: post-process: %w", s.Name(), s.Version(), err)
}
return out, reply, s.Validate(out), nil
}