Validate post-processed skill output and quote failed JSON in repair

The harness now validates a skill's output again after PostProcess, so
output that post-processing makes invalid (e.g. a whitespace-only echo
trimmed to empty) enters the single repair turn instead of returning as
a success. The repair turn now quotes Response.JSON as the previous
assistant reply whenever structured output was returned, falling back to
Response.Text only when there is none.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.gitignore

@@ -3,8 +3,6 @@ design/.DS_Store
 
 # Secrets and local environment
 .env*
-!deploy/.env.example
-deploy/.env
 deploy/secrets/
 
 # Build artifacts and dependencies

deploy/.env.example

@@ -1,80 +0,0 @@
-# MealQuest production environment. Copy to deploy/.env (never committed)
-# and replace the placeholders. The service fails fast: a missing or invalid
-# required variable stops it at boot with the variable named.
-#
-# Secrets are never set here. They live in deploy/secrets/ (never committed),
-# one value per file, owned by the container user and readable only by it:
-#
-#   mkdir -p deploy/secrets && chmod 700 deploy/secrets
-#   head -c 32 /dev/urandom | base64 > deploy/secrets/secret_key
-#   printf '%s\n' '<Authentik client secret>' > deploy/secrets/oidc_client_secret
-#   sudo chown 10001:10001 deploy/secrets/* && sudo chmod 600 deploy/secrets/*
-#
-# deploy/compose.yaml mounts them at /run/secrets/<name> and points the
-# *_FILE variables at them.
-
-# --- Compose -----------------------------------------------------------------
-
-# Host address the service is published on, as <address>:8090. Production
-# binds the LAN address 10.60.1.14 only; 127.0.0.1 suits a dev machine.
-MEALQUEST_BIND_ADDRESS=127.0.0.1
-
-# --- Service -----------------------------------------------------------------
-
-# Public origin the household opens. Fixes the OIDC redirect URI
-# (<url>/auth/callback) and the Secure cookies. Must be https:// (plain
-# http:// is accepted only for localhost, 127.0.0.1 and [::1]).
-MEALQUEST_PUBLIC_URL=https://mealquest.cubecraftlabs.com
-
-# How long a graceful shutdown may take before in-flight requests are cut.
-MEALQUEST_SHUTDOWN_TIMEOUT=10s
-
-# Optional. Comma-separated IPs/CIDRs of reverse proxies whose
-# X-Forwarded-For/-Proto headers are honoured. Production lists Traefik;
-# unset trusts no peer.
-MEALQUEST_TRUSTED_PROXY=10.60.1.33
-
-# IANA zone the household settings are created with on first boot. Required
-# until those settings exist; ignored afterwards (change the zone in
-# household settings instead). Confirm the household's zone before the first
-# deploy.
-MEALQUEST_DEFAULT_TZ=America/New_York
-
-# --- Authentik OIDC ----------------------------------------------------------
-
-# Issuer of the Authentik application "mealquest". Discovered at boot, so it
-# must be reachable from the container. Must be https://.
-MEALQUEST_OIDC_ISSUER=https://auth.cubecraftlabs.com/application/o/mealquest/
-MEALQUEST_OIDC_CLIENT_ID=mealquest
-# Comma-separated Authentik groups whose members may sign in.
-MEALQUEST_OIDC_ALLOWED_GROUPS=mealquest-admin
-
-# Optional session lifetimes. A session ends after IDLE unused and always MAX
-# after sign-in; MAX must be at least IDLE. Defaults: 720h and 4320h.
-MEALQUEST_SESSION_IDLE=720h
-MEALQUEST_SESSION_MAX=4320h
-
-# --- Set by deploy/compose.yaml; do not set here -----------------------------
-#
-# MEALQUEST_HTTP_ADDR=:8080
-# MEALQUEST_DATA_DIR=/data
-# MEALQUEST_SECRET_KEY_FILE=/run/secrets/secret_key
-#   Base64 of 32 random bytes; encrypts secrets at rest (AES-256-GCM).
-# MEALQUEST_OIDC_CLIENT_SECRET_FILE=/run/secrets/oidc_client_secret
-#   The Authentik provider's client secret.
-
-# --- Planned; not read by the service yet -------------------------------------
-#
-# Added to deploy/compose.yaml and this file by the features that use them.
-#
-# Web Push (Feature 19). Generate the key pair with cmd/vapidgen.
-# MEALQUEST_VAPID_PUBLIC_KEY=<base64url VAPID public key>
-# MEALQUEST_VAPID_PRIVATE_KEY_FILE=/run/secrets/vapid_private_key
-# MEALQUEST_VAPID_SUBJECT=mailto:<owner address>
-#
-# Google Calendar OAuth web client (Feature 16).
-# MEALQUEST_GOOGLE_CLIENT_ID=<client id>.apps.googleusercontent.com
-# MEALQUEST_GOOGLE_CLIENT_SECRET_FILE=/run/secrets/google_client_secret
-#
-# SearXNG recipe search sidecar on the compose network (Feature 07).
-# MEALQUEST_SEARXNG_URL=http://searxng:8080

deploy/compose.yaml

@@ -1,61 +0,0 @@
-# Production stack for MealQuest on 10.60.1.14 (D-018). Build and start on
-# the host with:
-#
-#   docker compose -f deploy/compose.yaml up -d --build
-#
-# Configuration comes from deploy/.env (copy deploy/.env.example) and secrets
-# from deploy/secrets/; neither is committed. Compose bind-mounts file secrets
-# with their host owner and mode, so each file in deploy/secrets/ must be
-# owned by the container user 10001:10001 with mode 0600.
-name: mealquest
-
-services:
-  mealquest:
-    build:
-      context: ..
-      dockerfile: Dockerfile
-    user: "10001:10001"
-    restart: unless-stopped
-    env_file:
-      - path: .env
-        required: true
-    # Fixed by this file's container shape (listen port, volume, secret
-    # mounts); deploy/.env must not set them.
-    environment:
-      MEALQUEST_HTTP_ADDR: ":8080"
-      MEALQUEST_DATA_DIR: /data
-      MEALQUEST_SECRET_KEY_FILE: /run/secrets/secret_key
-      MEALQUEST_OIDC_CLIENT_SECRET_FILE: /run/secrets/oidc_client_secret
-    secrets:
-      - secret_key
-      - oidc_client_secret
-    volumes:
-      - mealquest-data:/data
-    ports:
-      - "${MEALQUEST_BIND_ADDRESS:?set MEALQUEST_BIND_ADDRESS in deploy/.env}:8090:8080"
-    networks:
-      - mealquest
-    healthcheck:
-      test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/health"]
-      interval: 30s
-      timeout: 3s
-      start_period: 10s
-      retries: 3
-    security_opt:
-      - no-new-privileges:true
-    cap_drop:
-      - ALL
-
-secrets:
-  secret_key:
-    file: ./secrets/secret_key
-  oidc_client_secret:
-    file: ./secrets/oidc_client_secret
-
-volumes:
-  mealquest-data:
-    name: mealquest-data
-
-networks:
-  mealquest:
-    name: mealquest

internal/ai/harness/echo_test.go

@@ -0,0 +1,51 @@
+package harness
+
+import (
+	"encoding/json"
+	"errors"
+	"strings"
+
+	"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm"
+)
+
+// echoSkill is the trivial test skill: it asks the model to repeat a word.
+type echoSkill struct{}
+
+type echoInput struct{ Word string }
+
+type echoOutput struct {
+	Echo string `json:"echo"`
+}
+
+var _ Skill[echoInput, echoOutput] = echoSkill{}
+
+func (echoSkill) Name() string { return "echo" }
+
+func (echoSkill) Version() int { return 1 }
+
+func (echoSkill) Render(in echoInput) (Prompt, error) {
+	if in.Word == "" {
+		return Prompt{}, errors.New("no word to echo")
+	}
+	return Prompt{System: "You repeat words.", User: "Repeat: " + in.Word}, nil
+}
+
+func (echoSkill) Schema() llm.JSONSchema {
+	return llm.JSONSchema{Name: "echo", Schema: json.RawMessage(
+		`{"type":"object","properties":{"echo":{"type":"string"}},"required":["echo"],"additionalProperties":false}`)}
+}
+
+func (echoSkill) Validate(out echoOutput) []string {
+	if out.Echo == "" {
+		return []string{"echo must not be empty"}
+	}
+	if out.Echo != strings.ToLower(out.Echo) {
+		return []string{"echo must be lower case"}
+	}
+	return nil
+}
+
+func (echoSkill) PostProcess(out echoOutput) (echoOutput, error) {
+	out.Echo = strings.TrimSpace(out.Echo)
+	return out, nil
+}

internal/ai/harness/harness.go

@@ -0,0 +1,174 @@
+// Package harness runs skills: versioned, typed prompts whose structured
+// output Go validates. A run renders the skill's prompt, asks the provider
+// for output matching the skill's schema, decodes and validates the reply,
+// and on failure sends exactly one repair turn quoting the problems. If the
+// repair fails too the run fails with llm.ErrBadOutput; there is no silent
+// fallback (D-007).
+package harness
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+
+	"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm"
+)
+
+// Prompt is a rendered skill prompt. Prompt layering builds on it later.
+type Prompt struct {
+	// System is the system prompt; empty means none.
+	System string
+	// User is the user message. It must not be empty.
+	User string
+}
+
+// Skill is one versioned AI task with input In and structured output Out.
+type Skill[In, Out any] interface {
+	// Name identifies the skill, e.g. "side_quests".
+	Name() string
+	// Version is bumped whenever the prompt, schema or validation changes.
+	Version() int
+	// Render builds the prompt for in.
+	Render(in In) (Prompt, error)
+	// Schema is the JSON Schema the reply must match; Out decodes from it.
+	Schema() llm.JSONSchema
+	// Validate returns every problem with out, or none when it is usable.
+	// The problems are sent to the model in the repair turn, so each must
+	// say what to fix.
+	Validate(out Out) []string
+	// PostProcess finishes a valid out, e.g. normalising names. Its result
+	// is validated again; problems found then are repaired like any other.
+	// Its error fails the run without a repair turn.
+	PostProcess(out Out) (Out, error)
+}
+
+// Config is what a run needs from the household's AI settings.
+type Config struct {
+	// Provider answers the requests.
+	Provider llm.Provider
+	// Temperature is the household's sampling temperature, sent with
+	// every request.
+	Temperature float64
+}
+
+// BadOutputError is a run whose reply and repair reply were both unusable.
+// It matches llm.ErrBadOutput.
+type BadOutputError struct {
+	// Skill and Version identify the skill that ran.
+	Skill   string
+	Version int
+	// First lists the problems with the first reply, Repair those with
+	// the reply to the repair turn.
+	First  []string
+	Repair []string
+}
+
+func (e *BadOutputError) Error() string {
+	return fmt.Sprintf("harness: skill %s v%d: %s: first reply: %s; repair reply: %s",
+		e.Skill, e.Version, llm.ErrBadOutput, strings.Join(e.First, "; "), strings.Join(e.Repair, "; "))
+}
+
+func (e *BadOutputError) Unwrap() error { return llm.ErrBadOutput }
+
+// Run runs skill s on in with cfg and returns its validated, post-processed
+// output. A provider error other than llm.ErrBadOutput is returned
+// unchanged and never repaired.
+func Run[In, Out any](ctx context.Context, cfg Config, s Skill[In, Out], in In) (Out, error) {
+	var zero Out
+	prompt, err := s.Render(in)
+	if err != nil {
+		return zero, fmt.Errorf("harness: skill %s v%d: render: %w", s.Name(), s.Version(), err)
+	}
+	if prompt.User == "" {
+		return zero, fmt.Errorf("harness: skill %s v%d: render: empty user message", s.Name(), s.Version())
+	}
+	schema := s.Schema()
+	temperature := cfg.Temperature
+	req := llm.Request{
+		System:      prompt.System,
+		Messages:    []llm.Message{{Role: llm.RoleUser, Content: prompt.User}},
+		Temperature: &temperature,
+		Schema:      &schema,
+	}
+
+	out, reply, first, err := attempt(ctx, cfg.Provider, s, req)
+	if err != nil {
+		return zero, err
+	}
+	if len(first) > 0 {
+		if reply != "" {
+			req.Messages = append(req.Messages, llm.Message{Role: llm.RoleAssistant, Content: reply})
+		}
+		req.Messages = append(req.Messages, llm.Message{Role: llm.RoleUser, Content: repairMessage(schema.Name, first)})
+		var second []string
+		out, _, second, err = attempt(ctx, cfg.Provider, s, req)
+		if err != nil {
+			return zero, err
+		}
+		if len(second) > 0 {
+			return zero, &BadOutputError{Skill: s.Name(), Version: s.Version(), First: first, Repair: second}
+		}
+	}
+	return out, nil
+}
+
+// attempt sends req once, then decodes, validates, post-processes and
+// validates the reply again. It returns the reply for the repair turn and
+// the problems found; err is set only for a provider or post-process
+// failure, neither of which is repairable.
+func attempt[In, Out any](ctx context.Context, p llm.Provider, s Skill[In, Out], req llm.Request) (out Out, reply string, problems []string, err error) {
+	resp, err := p.Generate(ctx, req)
+	// The repair turn must quote what failed: the structured output when
+	// there is any, otherwise the text.
+	reply = string(resp.JSON)
+	if reply == "" {
+		reply = resp.Text
+	}
+	if err != nil {
+		if errors.Is(err, llm.ErrBadOutput) {
+			return out, reply, []string{err.Error()}, nil
+		}
+		return out, "", nil, err
+	}
+	if err := decode(resp.JSON, &out); err != nil {
+		return out, reply, []string{err.Error()}, nil
+	}
+	if problems := s.Validate(out); len(problems) > 0 {
+		return out, reply, problems, nil
+	}
+	out, err = s.PostProcess(out)
+	if err != nil {
+		return out, "", nil, fmt.Errorf("harness: skill %s v%d: post-process: %w", s.Name(), s.Version(), err)
+	}
+	return out, reply, s.Validate(out), nil
+}
+
+// decode strictly decodes data into out: one JSON value, no unknown fields.
+func decode(data json.RawMessage, out any) error {
+	if len(data) == 0 {
+		return errors.New("reply holds no JSON")
+	}
+	dec := json.NewDecoder(bytes.NewReader(data))
+	dec.DisallowUnknownFields()
+	if err := dec.Decode(out); err != nil {
+		return fmt.Errorf("reply does not decode: %v", err)
+	}
+	if err := dec.Decode(new(json.RawMessage)); err != io.EOF {
+		return errors.New("reply has data after the JSON value")
+	}
+	return nil
+}
+
+// repairMessage asks the model to fix problems in its last reply.
+func repairMessage(schemaName string, problems []string) string {
+	var b strings.Builder
+	fmt.Fprintf(&b, "Your previous reply was not usable. Reply again with JSON matching the %s schema, fixing these problems:\n", schemaName)
+	for _, p := range problems {
+		fmt.Fprintf(&b, "- %s\n", p)
+	}
+	return b.String()
+}

internal/ai/harness/harness_test.go

@@ -0,0 +1,207 @@
+package harness
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"slices"
+	"strings"
+	"testing"
+
+	"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/ai/llm"
+)
+
+func reply(js string) llm.GenerateResult {
+	return llm.GenerateResult{Response: llm.Response{Text: js, JSON: json.RawMessage(js)}}
+}
+
+func run(t *testing.T, f *llm.Fake) (echoOutput, error) {
+	t.Helper()
+	return Run(context.Background(), Config{Provider: f, Temperature: 0.7}, echoSkill{}, echoInput{Word: "taco"})
+}
+
+func assertCalls(t *testing.T, f *llm.Fake, want int) []llm.Request {
+	t.Helper()
+	reqs := f.Requests()
+	if len(reqs) != want {
+		t.Fatalf("Generate called %d times, want %d", len(reqs), want)
+	}
+	if g, _, _ := f.Remaining(); g != 0 {
+		t.Fatalf("%d scripted Generate results unused", g)
+	}
+	return reqs
+}
+
+func TestRunValidFirstTime(t *testing.T) {
+	var f llm.Fake
+	f.QueueGenerate(reply(`{"echo":" taco "}`))
+	out, err := run(t, &f)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if out.Echo != "taco" {
+		t.Fatalf("Echo = %q, want post-processed %q", out.Echo, "taco")
+	}
+	req := assertCalls(t, &f, 1)[0]
+	if req.System != "You repeat words." || len(req.Messages) != 1 ||
+		req.Messages[0] != (llm.Message{Role: llm.RoleUser, Content: "Repeat: taco"}) {
+		t.Fatalf("request prompt = %+v", req)
+	}
+	if req.Temperature == nil || *req.Temperature != 0.7 {
+		t.Fatalf("Temperature = %v, want 0.7", req.Temperature)
+	}
+	if req.Schema == nil || req.Schema.Name != "echo" || string(req.Schema.Schema) != string(echoSkill{}.Schema().Schema) {
+		t.Fatalf("Schema = %+v", req.Schema)
+	}
+}
+
+func TestRunInvalidThenValid(t *testing.T) {
+	var f llm.Fake
+	f.QueueGenerate(reply(`{"echo":"TACO"}`), reply(`{"echo":"taco"}`))
+	out, err := run(t, &f)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if out.Echo != "taco" {
+		t.Fatalf("Echo = %q", out.Echo)
+	}
+	reqs := assertCalls(t, &f, 2)
+	msgs := reqs[1].Messages
+	if len(msgs) != 3 || msgs[0] != reqs[0].Messages[0] ||
+		msgs[1] != (llm.Message{Role: llm.RoleAssistant, Content: `{"echo":"TACO"}`}) || msgs[2].Role != llm.RoleUser {
+		t.Fatalf("repair messages = %+v", msgs)
+	}
+	if !strings.Contains(msgs[2].Content, "echo must be lower case") {
+		t.Fatalf("repair message %q lacks the validator error", msgs[2].Content)
+	}
+	if reqs[1].System != reqs[0].System || reqs[1].Schema == nil || *reqs[1].Temperature != 0.7 {
+		t.Fatalf("repair request lost system, schema or temperature: %+v", reqs[1])
+	}
+}
+
+func TestRunPostProcessedOutputIsValidated(t *testing.T) {
+	var f llm.Fake
+	f.QueueGenerate(reply(`{"echo":"   "}`), reply(`{"echo":"taco"}`))
+	out, err := run(t, &f)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if out.Echo != "taco" {
+		t.Fatalf("Echo = %q", out.Echo)
+	}
+	msgs := assertCalls(t, &f, 2)[1].Messages
+	if len(msgs) != 3 || msgs[1] != (llm.Message{Role: llm.RoleAssistant, Content: `{"echo":"   "}`}) ||
+		!strings.Contains(msgs[2].Content, "echo must not be empty") {
+		t.Fatalf("repair messages = %+v", msgs)
+	}
+}
+
+func TestRunWhitespaceTwiceIsBadOutput(t *testing.T) {
+	var f llm.Fake
+	f.QueueGenerate(reply(`{"echo":"   "}`), reply(`{"echo":" "}`))
+	_, err := run(t, &f)
+	var bad *BadOutputError
+	if !errors.Is(err, llm.ErrBadOutput) || !errors.As(err, &bad) {
+		t.Fatalf("err = %v, want *BadOutputError", err)
+	}
+	if !slices.Equal(bad.First, []string{"echo must not be empty"}) ||
+		!slices.Equal(bad.Repair, []string{"echo must not be empty"}) {
+		t.Fatalf("BadOutputError = %+v", bad)
+	}
+	assertCalls(t, &f, 2)
+}
+
+func TestRunRepairQuotesJSONOverText(t *testing.T) {
+	var f llm.Fake
+	f.QueueGenerate(
+		llm.GenerateResult{Response: llm.Response{Text: "Here you go!", JSON: json.RawMessage(`{"echo":"TACO"}`)}},
+		reply(`{"echo":"taco"}`),
+	)
+	if out, err := run(t, &f); err != nil || out.Echo != "taco" {
+		t.Fatalf("Run = %+v, %v", out, err)
+	}
+	msgs := assertCalls(t, &f, 2)[1].Messages
+	if len(msgs) != 3 || msgs[1] != (llm.Message{Role: llm.RoleAssistant, Content: `{"echo":"TACO"}`}) {
+		t.Fatalf("repair messages = %+v, want the failed JSON as the assistant reply", msgs)
+	}
+}
+
+func TestRunDecodeFailureIsRepaired(t *testing.T) {
+	for name, first := range map[string]llm.GenerateResult{
+		"unknown field":  reply(`{"echo":"taco","extra":1}`),
+		"wrong type":     reply(`{"echo":5}`),
+		"trailing data":  reply(`{"echo":"taco"}{}`),
+		"no JSON":        {Response: llm.Response{Text: "taco"}},
+		"adapter refuse": {Response: llm.Response{Text: "nope"}, Err: llm.ErrBadOutput},
+	} {
+		t.Run(name, func(t *testing.T) {
+			var f llm.Fake
+			f.QueueGenerate(first, reply(`{"echo":"taco"}`))
+			if out, err := run(t, &f); err != nil || out.Echo != "taco" {
+				t.Fatalf("Run = %+v, %v", out, err)
+			}
+			assertCalls(t, &f, 2)
+		})
+	}
+}
+
+func TestRunInvalidTwice(t *testing.T) {
+	var f llm.Fake
+	f.QueueGenerate(reply(`{"echo":"TACO"}`), reply(`{"echo":""}`))
+	_, err := run(t, &f)
+	if !errors.Is(err, llm.ErrBadOutput) {
+		t.Fatalf("err = %v, want ErrBadOutput", err)
+	}
+	var bad *BadOutputError
+	if !errors.As(err, &bad) {
+		t.Fatalf("err = %T, want *BadOutputError", err)
+	}
+	if bad.Skill != "echo" || bad.Version != 1 ||
+		!slices.Equal(bad.First, []string{"echo must be lower case"}) ||
+		!slices.Equal(bad.Repair, []string{"echo must not be empty"}) {
+		t.Fatalf("BadOutputError = %+v", bad)
+	}
+	for _, want := range []string{"echo must be lower case", "echo must not be empty"} {
+		if !strings.Contains(err.Error(), want) {
+			t.Fatalf("error %q lacks %q", err, want)
+		}
+	}
+	assertCalls(t, &f, 2)
+}
+
+func TestRunProviderErrorPassesThrough(t *testing.T) {
+	for name, perr := range map[string]error{
+		"auth":         llm.ErrAuth,
+		"rate limited": &llm.RateLimitedError{},
+		"timeout":      llm.ErrTimeout,
+		"provider":     llm.NewProviderError(500, []byte("boom"), ""),
+	} {
+		t.Run(name, func(t *testing.T) {
+			var f llm.Fake
+			f.QueueGenerate(llm.GenerateResult{Err: perr})
+			_, err := run(t, &f)
+			if err != perr {
+				t.Fatalf("err = %v, want the provider error unchanged", err)
+			}
+			assertCalls(t, &f, 1)
+		})
+	}
+}
+
+func TestRunProviderErrorOnRepairPassesThrough(t *testing.T) {
+	var f llm.Fake
+	f.QueueGenerate(reply(`{"echo":"TACO"}`), llm.GenerateResult{Err: llm.ErrTimeout})
+	if _, err := run(t, &f); err != llm.ErrTimeout {
+		t.Fatalf("err = %v, want ErrTimeout unchanged", err)
+	}
+	assertCalls(t, &f, 2)
+}
+
+func TestRunRenderError(t *testing.T) {
+	var f llm.Fake
+	_, err := Run(context.Background(), Config{Provider: &f}, echoSkill{}, echoInput{})
+	if err == nil || errors.Is(err, llm.ErrBadOutput) {
+		t.Fatalf("err = %v, want a render error", err)
+	}
+	assertCalls(t, &f, 0)
+}