Feature 09: AI harness and skills framework (#62)

Co-authored-by: dev_agent <dev_agent@cubecraftlabs.com>

internal/ai/skills/prompts/checksums.txt

@@ -0,0 +1,4 @@
+# SHA-256 of every prompt template: <skill> v<version> <sha256>.
+# A recorded line never changes. To change a prompt, bump the skill's
+# version, rename its template to match and add a line for the new version.
+echo v1 ab5d7884f5c7a2261c06b0c9353bb9597070c001420bd4cbb6ec6272c093fdce

internal/ai/skills/prompts/echo.v1.tmpl

@@ -0,0 +1,7 @@
+{{/* echo is the harness's test skill: it asks the model to repeat a word. */}}
+{{define "instructions" -}}
+You repeat words. Reply with the word you are given, exactly as written but in lower case.
+{{- end}}
+{{define "user" -}}
+Repeat: {{.Word}}
+{{- end}}

internal/ai/skills/prompts/prompts.go

@@ -0,0 +1,102 @@
+// Package prompts holds the skills' prompt templates, embedded from this
+// directory. Each skill version has one text/template file named
+// <skill>.v<version>.tmpl that defines two templates: "instructions", the
+// skill's layer of the system prompt, and "user", the user message. Both
+// execute with the data the skill derives from its input.
+//
+// A template never changes under a released version: checksums.txt records
+// the SHA-256 of every template, and a test fails when a template no longer
+// matches it. Changing a prompt means bumping the skill's version, which
+// renames its template file.
+package prompts
+
+import (
+	"bytes"
+	"embed"
+	"fmt"
+	"io/fs"
+	"regexp"
+	"strconv"
+	"strings"
+	"text/template"
+)
+
+//go:embed *.tmpl
+var files embed.FS
+
+// fileName matches a template file name and captures the skill name and
+// version.
+var fileName = regexp.MustCompile(`^([a-z][a-z0-9_]*)\.v([1-9][0-9]*)\.tmpl$`)
+
+// templates maps a template file name to its parsed template. A bad
+// template stops the service at start.
+var templates = mustParse(files)
+
+func mustParse(fsys fs.FS) map[string]*template.Template {
+	t, err := parse(fsys)
+	if err != nil {
+		panic(err)
+	}
+	return t
+}
+
+// parse parses every template in fsys, each on its own so that every skill
+// can define "instructions" and "user".
+func parse(fsys fs.FS) (map[string]*template.Template, error) {
+	names, err := fs.Glob(fsys, "*.tmpl")
+	if err != nil {
+		return nil, err
+	}
+	out := make(map[string]*template.Template, len(names))
+	for _, name := range names {
+		if !fileName.MatchString(name) {
+			return nil, fmt.Errorf("prompts: %s: file name is not <skill>.v<version>.tmpl", name)
+		}
+		t, err := template.New(name).Option("missingkey=error").ParseFS(fsys, name)
+		if err != nil {
+			return nil, fmt.Errorf("prompts: %w", err)
+		}
+		for _, part := range []string{"instructions", "user"} {
+			if t.Lookup(part) == nil {
+				return nil, fmt.Errorf("prompts: %s: does not define %q", name, part)
+			}
+		}
+		out[name] = t
+	}
+	return out, nil
+}
+
+// File is the name of the template file for version of skill.
+func File(skill string, version int) string {
+	return skill + ".v" + strconv.Itoa(version) + ".tmpl"
+}
+
+// Render executes the template for version of skill with data and returns
+// its instructions and user message, trimmed of surrounding white space.
+// Either being empty is an error.
+func Render(skill string, version int, data any) (instructions, user string, err error) {
+	name := File(skill, version)
+	t, ok := templates[name]
+	if !ok {
+		return "", "", fmt.Errorf("prompts: no template %s for skill %s v%d", name, skill, version)
+	}
+	if instructions, err = execute(t, "instructions", data); err != nil {
+		return "", "", err
+	}
+	if user, err = execute(t, "user", data); err != nil {
+		return "", "", err
+	}
+	return instructions, user, nil
+}
+
+func execute(t *template.Template, part string, data any) (string, error) {
+	var b bytes.Buffer
+	if err := t.ExecuteTemplate(&b, part, data); err != nil {
+		return "", fmt.Errorf("prompts: %w", err)
+	}
+	s := strings.TrimSpace(b.String())
+	if s == "" {
+		return "", fmt.Errorf("prompts: %s: %q rendered empty", t.Name(), part)
+	}
+	return s, nil
+}

internal/ai/skills/prompts/prompts_test.go

@@ -0,0 +1,197 @@
+package prompts
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io/fs"
+	"maps"
+	"os"
+	"slices"
+	"strings"
+	"testing"
+	"testing/fstest"
+)
+
+// checkChecksums compares every template in fsys with the checksums in
+// manifest and returns each mismatch, naming the skill and version.
+func checkChecksums(fsys fs.FS, manifest string) []string {
+	var problems []string
+	recorded := map[string]string{} // template file name -> checksum
+	for i, line := range strings.Split(manifest, "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		f := strings.Fields(line)
+		if len(f) != 3 || !fileName.MatchString(f[0]+"."+f[1]+".tmpl") {
+			problems = append(problems, fmt.Sprintf("checksums.txt line %d: want \"<skill> v<version> <sha256>\", got %q", i+1, line))
+			continue
+		}
+		name := f[0] + "." + f[1] + ".tmpl"
+		if _, dup := recorded[name]; dup {
+			problems = append(problems, fmt.Sprintf("skill %s %s: checksums.txt records it twice", f[0], f[1]))
+			continue
+		}
+		recorded[name] = f[2]
+	}
+
+	names, err := fs.Glob(fsys, "*.tmpl")
+	if err != nil {
+		return append(problems, err.Error())
+	}
+	for _, name := range names {
+		m := fileName.FindStringSubmatch(name)
+		if m == nil {
+			problems = append(problems, fmt.Sprintf("%s: file name is not <skill>.v<version>.tmpl", name))
+			continue
+		}
+		data, err := fs.ReadFile(fsys, name)
+		if err != nil {
+			problems = append(problems, err.Error())
+			continue
+		}
+		sum := sha256.Sum256(data)
+		got := hex.EncodeToString(sum[:])
+		want, ok := recorded[name]
+		delete(recorded, name)
+		switch {
+		case !ok:
+			problems = append(problems, fmt.Sprintf("skill %s v%s: %s has no recorded checksum; add \"%s v%s %s\" to checksums.txt",
+				m[1], m[2], name, m[1], m[2], got))
+		case got != want:
+			problems = append(problems, fmt.Sprintf("skill %s v%s: %s changed without a version bump (sha256 %s, recorded %s); bump the skill's version and rename its template",
+				m[1], m[2], name, got, want))
+		}
+	}
+	for _, name := range slices.Sorted(maps.Keys(recorded)) {
+		m := fileName.FindStringSubmatch(name)
+		problems = append(problems, fmt.Sprintf("skill %s v%s: checksums.txt records %s but there is no such template; remove the line", m[1], m[2], name))
+	}
+	return problems
+}
+
+func TestTemplateChecksums(t *testing.T) {
+	manifest, err := os.ReadFile("checksums.txt")
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, p := range checkChecksums(files, string(manifest)) {
+		t.Error(p)
+	}
+}
+
+const echoV1 = `{{define "instructions"}}Repeat words.{{end}}{{define "user"}}Repeat: {{.Word}}{{end}}`
+
+func sum(s string) string {
+	h := sha256.Sum256([]byte(s))
+	return hex.EncodeToString(h[:])
+}
+
+func TestCheckChecksums(t *testing.T) {
+	changed := echoV1 + "\n"
+	for name, tc := range map[string]struct {
+		fsys     fstest.MapFS
+		manifest string
+		want     []string
+	}{
+		"match": {
+			fsys:     fstest.MapFS{"echo.v1.tmpl": {Data: []byte(echoV1)}},
+			manifest: "# comment\necho v1 " + sum(echoV1) + "\n",
+		},
+		"changed without bump": {
+			fsys:     fstest.MapFS{"echo.v1.tmpl": {Data: []byte(changed)}},
+			manifest: "echo v1 " + sum(echoV1),
+			want: []string{"skill echo v1: echo.v1.tmpl changed without a version bump (sha256 " + sum(changed) +
+				", recorded " + sum(echoV1) + "); bump the skill's version and rename its template"},
+		},
+		"bumped but not recorded": {
+			fsys:     fstest.MapFS{"echo.v2.tmpl": {Data: []byte(changed)}},
+			manifest: "echo v1 " + sum(echoV1),
+			want: []string{
+				`skill echo v2: echo.v2.tmpl has no recorded checksum; add "echo v2 ` + sum(changed) + `" to checksums.txt`,
+				"skill echo v1: checksums.txt records echo.v1.tmpl but there is no such template; remove the line",
+			},
+		},
+		"duplicate and malformed": {
+			fsys:     fstest.MapFS{"echo.v1.tmpl": {Data: []byte(echoV1)}},
+			manifest: "echo v1 " + sum(echoV1) + "\necho v1 " + sum(echoV1) + "\necho 1 abc\necho v01 abc",
+			want: []string{
+				"skill echo v1: checksums.txt records it twice",
+				`checksums.txt line 3: want "<skill> v<version> <sha256>", got "echo 1 abc"`,
+				`checksums.txt line 4: want "<skill> v<version> <sha256>", got "echo v01 abc"`,
+			},
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			if got := checkChecksums(tc.fsys, tc.manifest); !slices.Equal(got, tc.want) {
+				t.Fatalf("problems =\n%s\nwant\n%s", strings.Join(got, "\n"), strings.Join(tc.want, "\n"))
+			}
+		})
+	}
+}
+
+func TestRender(t *testing.T) {
+	instructions, user, err := Render("echo", 1, struct{ Word string }{"Taco"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if instructions != "You repeat words. Reply with the word you are given, exactly as written but in lower case." {
+		t.Fatalf("instructions = %q", instructions)
+	}
+	if user != "Repeat: Taco" {
+		t.Fatalf("user = %q", user)
+	}
+}
+
+func TestRenderErrors(t *testing.T) {
+	for name, tc := range map[string]struct {
+		skill   string
+		version int
+		data    any
+		want    string
+	}{
+		"unknown version": {"echo", 2, struct{ Word string }{"x"}, "no template echo.v2.tmpl for skill echo v2"},
+		"unknown skill":   {"nope", 1, nil, "no template nope.v1.tmpl"},
+		"missing field":   {"echo", 1, struct{}{}, "can't evaluate field Word"},
+		"missing key":     {"echo", 1, map[string]string{}, "map has no entry for key"},
+	} {
+		t.Run(name, func(t *testing.T) {
+			_, _, err := Render(tc.skill, tc.version, tc.data)
+			if err == nil || !strings.Contains(err.Error(), tc.want) {
+				t.Fatalf("err = %v, want one containing %q", err, tc.want)
+			}
+		})
+	}
+}
+
+func TestRenderedEmptyIsError(t *testing.T) {
+	ts, err := parse(fstest.MapFS{"blank.v1.tmpl": {Data: []byte(`{{define "instructions"}}x{{end}}{{define "user"}} {{.}} {{end}}`)}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = execute(ts["blank.v1.tmpl"], "user", "\n")
+	if err == nil || !strings.Contains(err.Error(), `blank.v1.tmpl: "user" rendered empty`) {
+		t.Fatalf("err = %v, want rendered empty", err)
+	}
+}
+
+func TestParseRejects(t *testing.T) {
+	for name, tc := range map[string]struct {
+		fsys fstest.MapFS
+		want string
+	}{
+		"bad name":        {fstest.MapFS{"Echo.tmpl": {Data: []byte(echoV1)}}, "file name is not"},
+		"version zero":    {fstest.MapFS{"echo.v0.tmpl": {Data: []byte(echoV1)}}, "file name is not"},
+		"no instructions": {fstest.MapFS{"echo.v1.tmpl": {Data: []byte(`{{define "user"}}u{{end}}`)}}, `does not define "instructions"`},
+		"no user":         {fstest.MapFS{"echo.v1.tmpl": {Data: []byte(`{{define "instructions"}}i{{end}}`)}}, `does not define "user"`},
+		"syntax":          {fstest.MapFS{"echo.v1.tmpl": {Data: []byte(`{{define "user"}}`)}}, "unexpected EOF"},
+	} {
+		t.Run(name, func(t *testing.T) {
+			_, err := parse(tc.fsys)
+			if err == nil || !strings.Contains(err.Error(), tc.want) {
+				t.Fatalf("err = %v, want one containing %q", err, tc.want)
+			}
+		})
+	}
+}