Add async job runner and GET /api/v1/jobs/{id}

internal/jobs persists jobs in the jobs table, which is also the queue: a
bounded pool of workers claims queued jobs oldest first. Handlers are
registered per kind before Start with typed payloads and results, and can
publish partial results to the row while they run. Only typed results and
*jobs.Failure messages are stored; any other handler error is logged and
stored as "internal error", so dependency error text cannot surface a
secret. Start fails jobs a previous process left running with
"interrupted by restart" before any worker runs.

GET /api/v1/jobs/{id} returns {id, kind, status, result, error} inside the
protected /api/v1 group; an unknown id is a 404 not_found.

Migration 0003_jobs_status rebuilds jobs with status queued, running,
succeeded, failed, mapping legacy pending and done rows.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/jobs/jobs.go

@@ -0,0 +1,335 @@
+// Package jobs runs long work in the background. A job is a row in the jobs
+// table: Enqueue inserts it queued, and a fixed pool of workers claims queued
+// jobs oldest first, so the table itself is the queue and a job enqueued
+// before a restart still runs after it.
+//
+// Each job kind has one handler, registered before Start. A handler takes a
+// typed payload and returns a typed result, which the runner stores as JSON;
+// it may publish partial results while it runs. Only typed data and
+// *Failure messages are stored, so no handler error text, which may quote a
+// URL, token or response body from a dependency, can reach the table or the
+// API.
+package jobs
+
+import (
+	"bytes"
+	"context"
+	"database/sql"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log/slog"
+	"sync"
+
+	"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/db"
+	"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/store"
+)
+
+// Status is the state of a job.
+type Status string
+
+// Job states. A job moves queued → running → succeeded or failed.
+const (
+	StatusQueued    Status = "queued"
+	StatusRunning   Status = "running"
+	StatusSucceeded Status = "succeeded"
+	StatusFailed    Status = "failed"
+)
+
+// Stored error messages the runner writes itself.
+const (
+	// MessageInterrupted is the error of a job a previous process left running.
+	MessageInterrupted = "interrupted by restart"
+	// MessageInternal is the error of a job whose handler failed with an error
+	// that is not a *Failure. The handler's error is logged, not stored.
+	MessageInternal = "internal error"
+)
+
+// ErrNotFound is returned by Get for an unknown job id.
+var ErrNotFound = errors.New("jobs: job not found")
+
+// Failure is a handler error whose Message is safe to show the household and
+// is stored as the job's error. Every other handler error is stored as
+// MessageInternal.
+type Failure struct {
+	Message string
+}
+
+func (f *Failure) Error() string { return f.Message }
+
+// Handler runs one job with its payload. It may call publish any number of
+// times with a partial result, which Get returns while the job runs; its
+// return value is the final result. ctx is cancelled when the runner stops.
+type Handler[P, R any] func(ctx context.Context, payload P, publish func(partial R) error) (R, error)
+
+// Job is a job as reported by GET /api/v1/jobs/{id}. Result is the latest
+// partial or final result, and is null until the handler publishes one.
+// Error is set only on a failed job.
+type Job struct {
+	ID     string          `json:"id"`
+	Kind   string          `json:"kind"`
+	Status Status          `json:"status"`
+	Result json.RawMessage `json:"result"`
+	Error  *string         `json:"error"`
+}
+
+// runFunc is a registered Handler with its payload and results as JSON.
+type runFunc func(ctx context.Context, payload string, publish func(json.RawMessage) error) (json.RawMessage, error)
+
+// Runner owns the job handlers and the worker pool.
+type Runner struct {
+	st      *store.Store
+	logger  *slog.Logger
+	workers int
+	// wake holds up to one token per worker; Enqueue adds one so an idle
+	// worker looks for the new job.
+	wake chan struct{}
+
+	mu       sync.Mutex
+	handlers map[string]runFunc
+	started  bool
+
+	wg sync.WaitGroup
+}
+
+// NewRunner returns a runner over st with workers concurrent workers.
+func NewRunner(st *store.Store, logger *slog.Logger, workers int) (*Runner, error) {
+	if st == nil {
+		return nil, errors.New("jobs: nil store")
+	}
+	if logger == nil {
+		return nil, errors.New("jobs: nil logger")
+	}
+	if workers < 1 {
+		return nil, fmt.Errorf("jobs: %d workers, want at least 1", workers)
+	}
+	return &Runner{
+		st:       st,
+		logger:   logger,
+		workers:  workers,
+		wake:     make(chan struct{}, workers),
+		handlers: make(map[string]runFunc),
+	}, nil
+}
+
+// Kind enqueues jobs of one registered kind.
+type Kind[P any] struct {
+	r    *Runner
+	name string
+}
+
+// Register makes h the handler for kind and returns the Kind that enqueues
+// its jobs. It must be called before Start, once per kind.
+func Register[P, R any](r *Runner, kind string, h Handler[P, R]) (*Kind[P], error) {
+	if kind == "" {
+		return nil, errors.New("jobs: register: empty kind")
+	}
+	if h == nil {
+		return nil, fmt.Errorf("jobs: register %q: nil handler", kind)
+	}
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	if r.started {
+		return nil, fmt.Errorf("jobs: register %q: runner already started", kind)
+	}
+	if _, ok := r.handlers[kind]; ok {
+		return nil, fmt.Errorf("jobs: register %q: kind already registered", kind)
+	}
+	r.handlers[kind] = func(ctx context.Context, payload string, publish func(json.RawMessage) error) (json.RawMessage, error) {
+		var p P
+		dec := json.NewDecoder(bytes.NewReader([]byte(payload)))
+		dec.DisallowUnknownFields()
+		if err := dec.Decode(&p); err != nil {
+			return nil, fmt.Errorf("decode payload: %w", err)
+		}
+		result, err := h(ctx, p, func(partial R) error {
+			data, err := json.Marshal(partial)
+			if err != nil {
+				return fmt.Errorf("jobs: encode partial result: %w", err)
+			}
+			return publish(data)
+		})
+		if err != nil {
+			return nil, err
+		}
+		data, err := json.Marshal(result)
+		if err != nil {
+			return nil, fmt.Errorf("encode result: %w", err)
+		}
+		return data, nil
+	}
+	return &Kind[P]{r: r, name: kind}, nil
+}
+
+// Enqueue stores a queued job with payload and returns its id. A job
+// enqueued before Start waits for it.
+func (k *Kind[P]) Enqueue(ctx context.Context, payload P) (string, error) {
+	data, err := json.Marshal(payload)
+	if err != nil {
+		return "", fmt.Errorf("jobs: enqueue %s: encode payload: %w", k.name, err)
+	}
+	id := store.NewID()
+	now := store.Now().String()
+	if err := db.New(k.r.st.Writer).CreateJob(ctx, db.CreateJobParams{
+		ID:        id,
+		Kind:      k.name,
+		Payload:   string(data),
+		CreatedAt: now,
+		UpdatedAt: now,
+	}); err != nil {
+		return "", fmt.Errorf("jobs: enqueue %s: %w", k.name, err)
+	}
+	select {
+	case k.r.wake <- struct{}{}:
+	default: // every worker already has a token and will look again
+	}
+	return id, nil
+}
+
+// Start fails every job a previous process left running with
+// MessageInterrupted, then starts the workers. They stop when ctx is cancelled;
+// Wait waits for them. A job whose handler returns an error after ctx is
+// cancelled is left running, so the next Start fails it as interrupted.
+func (r *Runner) Start(ctx context.Context) error {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	if r.started {
+		return errors.New("jobs: runner already started")
+	}
+	now := store.Now().String()
+	n, err := db.New(r.st.Writer).FailRunningJobs(ctx, db.FailRunningJobsParams{
+		Error: sql.NullString{String: MessageInterrupted, Valid: true},
+		Now:   sql.NullString{String: now, Valid: true},
+	})
+	if err != nil {
+		return fmt.Errorf("jobs: fail interrupted jobs: %w", err)
+	}
+	if n > 0 {
+		r.logger.WarnContext(ctx, "failed jobs interrupted by restart", "count", n)
+	}
+	r.started = true
+	for range r.workers {
+		r.wg.Add(1)
+		go r.work(ctx)
+	}
+	return nil
+}
+
+// Wait blocks until every worker has stopped.
+func (r *Runner) Wait() {
+	r.wg.Wait()
+}
+
+// Get returns the job with id, or ErrNotFound.
+func (r *Runner) Get(ctx context.Context, id string) (Job, error) {
+	row, err := db.New(r.st.Reader).GetJob(ctx, id)
+	if errors.Is(err, sql.ErrNoRows) {
+		return Job{}, ErrNotFound
+	}
+	if err != nil {
+		return Job{}, fmt.Errorf("jobs: read job: %w", err)
+	}
+	job := Job{ID: row.ID, Kind: row.Kind, Status: Status(row.Status)}
+	if row.Result.Valid {
+		job.Result = json.RawMessage(row.Result.String)
+	}
+	if row.Error.Valid {
+		job.Error = &row.Error.String
+	}
+	return job, nil
+}
+
+// work claims and runs queued jobs until ctx is cancelled, waiting for a
+// wake token whenever none is queued.
+func (r *Runner) work(ctx context.Context) {
+	defer r.wg.Done()
+	q := db.New(r.st.Writer)
+	for ctx.Err() == nil {
+		job, err := q.ClaimNextJob(ctx, sql.NullString{String: store.Now().String(), Valid: true})
+		if err == nil {
+			r.run(ctx, job)
+			continue
+		}
+		if !errors.Is(err, sql.ErrNoRows) && ctx.Err() == nil {
+			r.logger.ErrorContext(ctx, "claim job", "error", err)
+		}
+		select {
+		case <-r.wake:
+		case <-ctx.Done():
+		}
+	}
+}
+
+// run runs a claimed job to completion. The final write is made even when
+// ctx has been cancelled, so a handler that finishes during shutdown is
+// recorded.
+func (r *Runner) run(ctx context.Context, job db.Job) {
+	write := context.WithoutCancel(ctx)
+	logger := r.logger.With("job_id", job.ID, "kind", job.Kind)
+	q := db.New(r.st.Writer)
+
+	h, ok := r.handlers[job.Kind] // read-only once started
+	if !ok {
+		logger.ErrorContext(ctx, "no handler for job kind")
+		r.fail(write, logger, job.ID, "no handler for job kind "+job.Kind)
+		return
+	}
+
+	publish := func(partial json.RawMessage) error {
+		n, err := q.PublishJobResult(ctx, db.PublishJobResultParams{
+			Result:    sql.NullString{String: string(partial), Valid: true},
+			UpdatedAt: store.Now().String(),
+			ID:        job.ID,
+		})
+		if err != nil {
+			return fmt.Errorf("jobs: publish result: %w", err)
+		}
+		if n != 1 {
+			return fmt.Errorf("jobs: publish result: job %s is not running", job.ID)
+		}
+		return nil
+	}
+
+	result, err := h(ctx, job.Payload, publish)
+	if err != nil {
+		if ctx.Err() != nil {
+			logger.WarnContext(write, "job interrupted by shutdown; it fails at the next start", "error", err)
+			return
+		}
+		var failure *Failure
+		message := MessageInternal
+		if errors.As(err, &failure) {
+			message = failure.Message
+		} else {
+			logger.ErrorContext(ctx, "job failed", "error", err)
+		}
+		r.fail(write, logger, job.ID, message)
+		return
+	}
+
+	n, err := q.SucceedJob(write, db.SucceedJobParams{
+		Result: sql.NullString{String: string(result), Valid: true},
+		Now:    sql.NullString{String: store.Now().String(), Valid: true},
+		ID:     job.ID,
+	})
+	if err == nil && n != 1 {
+		err = errors.New("job is not running")
+	}
+	if err != nil {
+		logger.ErrorContext(write, "record job success", "error", err)
+	}
+}
+
+func (r *Runner) fail(ctx context.Context, logger *slog.Logger, id, message string) {
+	n, err := db.New(r.st.Writer).FailJob(ctx, db.FailJobParams{
+		Error: sql.NullString{String: message, Valid: true},
+		Now:   sql.NullString{String: store.Now().String(), Valid: true},
+		ID:    id,
+	})
+	if err == nil && n != 1 {
+		err = errors.New("job is not running")
+	}
+	if err != nil {
+		logger.ErrorContext(ctx, "record job failure", "error", err)
+	}
+}

internal/jobs/jobs_test.go

@@ -0,0 +1,404 @@
+package jobs
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"log/slog"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/store"
+	"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/migrations"
+)
+
+// sentinelSecret stands in for a credential a handler holds while it runs.
+const sentinelSecret = "sk-sentinel-5f2c9a"
+
+type countPayload struct {
+	Steps int `json:"steps"`
+}
+
+type countResult struct {
+	Done int `json:"done"`
+}
+
+func migratedStore(t *testing.T) *store.Store {
+	t.Helper()
+	ctx := context.Background()
+	st, err := store.Open(ctx, t.TempDir())
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() {
+		if err := st.Close(); err != nil {
+			t.Error(err)
+		}
+	})
+	if err := store.Migrate(ctx, st.Writer, migrations.FS); err != nil {
+		t.Fatal(err)
+	}
+	return st
+}
+
+func newRunner(t *testing.T, st *store.Store, workers int) *Runner {
+	t.Helper()
+	r, err := NewRunner(st, slog.New(slog.NewTextHandler(io.Discard, nil)), workers)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return r
+}
+
+// start starts r and stops it when the test ends. The returned function
+// stops it early and waits for its workers.
+func start(t *testing.T, r *Runner) (stop func()) {
+	t.Helper()
+	ctx, cancel := context.WithCancel(context.Background())
+	if err := r.Start(ctx); err != nil {
+		cancel()
+		t.Fatal(err)
+	}
+	stop = func() {
+		cancel()
+		r.Wait()
+	}
+	t.Cleanup(stop)
+	return stop
+}
+
+func register[P, R any](t *testing.T, r *Runner, kind string, h Handler[P, R]) *Kind[P] {
+	t.Helper()
+	k, err := Register(r, kind, h)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return k
+}
+
+func enqueue[P any](t *testing.T, k *Kind[P], payload P) string {
+	t.Helper()
+	id, err := k.Enqueue(context.Background(), payload)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return id
+}
+
+// waitFor polls Get until done accepts the job, failing after five seconds.
+func waitFor(t *testing.T, r *Runner, id string, done func(Job) bool) Job {
+	t.Helper()
+	deadline := time.Now().Add(5 * time.Second)
+	for {
+		job, err := r.Get(context.Background(), id)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if done(job) {
+			return job
+		}
+		if time.Now().After(deadline) {
+			t.Fatalf("job %s never reached the expected state; last %+v (result %s)", id, job, job.Result)
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+}
+
+func finished(j Job) bool { return j.Status == StatusSucceeded || j.Status == StatusFailed }
+
+// rawRow returns every column of the job's row joined, for leak checks.
+func rawRow(t *testing.T, st *store.Store, id string) string {
+	t.Helper()
+	var cols [10]any
+	ptrs := make([]any, len(cols))
+	for i := range cols {
+		ptrs[i] = &cols[i]
+	}
+	if err := st.Reader.QueryRow("SELECT * FROM jobs WHERE id = ?", id).Scan(ptrs...); err != nil {
+		t.Fatal(err)
+	}
+	return fmt.Sprint(cols[:]...)
+}
+
+func TestJobSucceeds(t *testing.T) {
+	t.Parallel()
+	st := migratedStore(t)
+	r := newRunner(t, st, 2)
+	count := register(t, r, "count", func(_ context.Context, p countPayload, _ func(countResult) error) (countResult, error) {
+		return countResult{Done: p.Steps}, nil
+	})
+	start(t, r)
+
+	id := enqueue(t, count, countPayload{Steps: 3})
+	job := waitFor(t, r, id, finished)
+	if job.ID != id || job.Kind != "count" || job.Status != StatusSucceeded || job.Error != nil {
+		t.Fatalf("job = %+v, want succeeded count job %s with no error", job, id)
+	}
+	if string(job.Result) != `{"done":3}` {
+		t.Errorf("result = %s, want {\"done\":3}", job.Result)
+	}
+
+	var startedAt, finishedAt, createdAt, updatedAt, payload string
+	if err := st.Reader.QueryRow("SELECT started_at, finished_at, created_at, updated_at, payload FROM jobs WHERE id = ?", id).
+		Scan(&startedAt, &finishedAt, &createdAt, &updatedAt, &payload); err != nil {
+		t.Fatal(err)
+	}
+	for name, v := range map[string]string{"started_at": startedAt, "finished_at": finishedAt, "created_at": createdAt, "updated_at": updatedAt} {
+		var ts store.Timestamp
+		if err := ts.Scan(v); err != nil {
+			t.Errorf("%s = %q: %v", name, v, err)
+		}
+	}
+	if payload != `{"steps":3}` {
+		t.Errorf("payload = %s, want {\"steps\":3}", payload)
+	}
+}
+
+func TestJobFailsWithFailureMessage(t *testing.T) {
+	t.Parallel()
+	r := newRunner(t, migratedStore(t), 1)
+	k := register(t, r, "suggest", func(context.Context, countPayload, func(countResult) error) (countResult, error) {
+		return countResult{}, fmt.Errorf("suggest: %w", &Failure{Message: "add some Regulars first"})
+	})
+	start(t, r)
+
+	job := waitFor(t, r, enqueue(t, k, countPayload{}), finished)
+	if job.Status != StatusFailed || job.Error == nil || *job.Error != "add some Regulars first" {
+		t.Fatalf("job = %+v, want failed with the Failure message", job)
+	}
+	if job.Result != nil {
+		t.Errorf("result = %s, want none", job.Result)
+	}
+}
+
+// credential holds a secret the way config.Secret does: unexported, so
+// encoding/json cannot see it.
+type credential struct{ value string }
+
+func (credential) String() string { return "[REDACTED]" }
+
+type leakyResult struct {
+	Dinners []string   `json:"dinners"`
+	Key     credential `json:"key"`
+	Token   string     `json:"-"`
+	apiKey  string
+}
+
+// TestSecretsNeverStored proves a handler holding a secret cannot surface it
+// through its typed result, its partial results or its error text.
+func TestSecretsNeverStored(t *testing.T) {
+	t.Parallel()
+	st := migratedStore(t)
+	r := newRunner(t, st, 1)
+	secretResult := leakyResult{Dinners: []string{"Tacos"}, Key: credential{sentinelSecret}, Token: sentinelSecret, apiKey: sentinelSecret}
+	ok := register(t, r, "typed", func(_ context.Context, _ countPayload, publish func(leakyResult) error) (leakyResult, error) {
+		if err := publish(secretResult); err != nil {
+			return leakyResult{}, err
+		}
+		return secretResult, nil
+	})
+	failing := register(t, r, "leaky-error", func(context.Context, countPayload, func(leakyResult) error) (leakyResult, error) {
+		return leakyResult{}, fmt.Errorf("GET https://api.example.com/v1?key=%s: 401 %s", sentinelSecret, sentinelSecret)
+	})
+	start(t, r)
+
+	typed := waitFor(t, r, enqueue(t, ok, countPayload{}), finished)
+	if typed.Status != StatusSucceeded || string(typed.Result) != `{"dinners":["Tacos"],"key":{}}` {
+		t.Errorf("typed job = %+v (result %s), want succeeded with only the declared data", typed, typed.Result)
+	}
+	failed := waitFor(t, r, enqueue(t, failing, countPayload{}), finished)
+	if failed.Status != StatusFailed || failed.Error == nil || *failed.Error != MessageInternal {
+		t.Errorf("leaky-error job = %+v, want failed with %q", failed, MessageInternal)
+	}
+	for _, id := range []string{typed.ID, failed.ID} {
+		if row := rawRow(t, st, id); strings.Contains(row, sentinelSecret) {
+			t.Errorf("job %s row contains the secret: %s", id, row)
+		}
+	}
+}
+
+func TestProgressiveResultsVisibleMidRun(t *testing.T) {
+	t.Parallel()
+	r := newRunner(t, migratedStore(t), 1)
+	release := make(chan struct{})
+	k := register(t, r, "draft", func(ctx context.Context, p countPayload, publish func(countResult) error) (countResult, error) {
+		for i := 1; i < p.Steps; i++ {
+			if err := publish(countResult{Done: i}); err != nil {
+				return countResult{}, err
+			}
+			select {
+			case <-release:
+			case <-ctx.Done():
+				return countResult{}, ctx.Err()
+			}
+		}
+		return countResult{Done: p.Steps}, nil
+	})
+	start(t, r)
+
+	id := enqueue(t, k, countPayload{Steps: 3})
+	for step := 1; step < 3; step++ {
+		want := fmt.Sprintf(`{"done":%d}`, step)
+		job := waitFor(t, r, id, func(j Job) bool { return string(j.Result) == want })
+		if job.Status != StatusRunning || job.Error != nil {
+			t.Fatalf("mid-run job = %+v, want running with no error", job)
+		}
+		release <- struct{}{}
+	}
+	job := waitFor(t, r, id, finished)
+	if job.Status != StatusSucceeded || string(job.Result) != `{"done":3}` {
+		t.Fatalf("job = %+v (result %s), want succeeded with {\"done\":3}", job, job.Result)
+	}
+}
+
+func TestRestartFailsInterruptedJobsAndRunsQueuedOnes(t *testing.T) {
+	t.Parallel()
+	st := migratedStore(t)
+
+	first := newRunner(t, st, 1)
+	started := make(chan struct{})
+	k := register(t, first, "slow", func(ctx context.Context, _ countPayload, publish func(countResult) error) (countResult, error) {
+		if err := publish(countResult{Done: 1}); err != nil {
+			return countResult{}, err
+		}
+		close(started)
+		<-ctx.Done()
+		return countResult{}, ctx.Err()
+	})
+	stop := start(t, first)
+	interrupted := enqueue(t, k, countPayload{})
+	<-started
+	waiting := enqueue(t, k, countPayload{}) // the only worker is busy
+	stop()
+
+	if job := waitFor(t, first, interrupted, func(Job) bool { return true }); job.Status != StatusRunning {
+		t.Fatalf("after shutdown job = %+v, want left running", job)
+	}
+
+	second := newRunner(t, st, 1)
+	register(t, second, "slow", func(context.Context, countPayload, func(countResult) error) (countResult, error) {
+		return countResult{Done: 9}, nil
+	})
+	start(t, second)
+
+	job := waitFor(t, second, interrupted, finished)
+	if job.Status != StatusFailed || job.Error == nil || *job.Error != MessageInterrupted {
+		t.Errorf("interrupted job = %+v, want failed with %q", job, MessageInterrupted)
+	}
+	if string(job.Result) != `{"done":1}` {
+		t.Errorf("interrupted job result = %s, want the last partial result kept", job.Result)
+	}
+	if job := waitFor(t, second, waiting, finished); job.Status != StatusSucceeded {
+		t.Errorf("queued job = %+v, want run by the new runner", job)
+	}
+}
+
+func TestWorkerPoolIsBounded(t *testing.T) {
+	t.Parallel()
+	r := newRunner(t, migratedStore(t), 2)
+	var running, peak atomic.Int32
+	release := make(chan struct{})
+	k := register(t, r, "busy", func(context.Context, countPayload, func(countResult) error) (countResult, error) {
+		n := running.Add(1)
+		for {
+			p := peak.Load()
+			if n <= p || peak.CompareAndSwap(p, n) {
+				break
+			}
+		}
+		<-release
+		running.Add(-1)
+		return countResult{}, nil
+	})
+	start(t, r)
+
+	ids := make([]string, 5)
+	for i := range ids {
+		ids[i] = enqueue(t, k, countPayload{})
+	}
+	deadline := time.Now().Add(5 * time.Second)
+	for running.Load() < 2 {
+		if time.Now().After(deadline) {
+			t.Fatal("two workers never ran at once")
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+	close(release)
+	for _, id := range ids {
+		if job := waitFor(t, r, id, finished); job.Status != StatusSucceeded {
+			t.Errorf("job = %+v, want succeeded", job)
+		}
+	}
+	if got := peak.Load(); got != 2 {
+		t.Errorf("peak concurrent jobs = %d, want 2", got)
+	}
+}
+
+func TestJobWithoutHandlerFails(t *testing.T) {
+	t.Parallel()
+	st := migratedStore(t)
+	old := newRunner(t, st, 1)
+	k := register(t, old, "retired", func(context.Context, countPayload, func(countResult) error) (countResult, error) {
+		return countResult{}, nil
+	})
+	id := enqueue(t, k, countPayload{}) // never started
+
+	r := newRunner(t, st, 1)
+	start(t, r)
+	job := waitFor(t, r, id, finished)
+	if job.Status != StatusFailed || job.Error == nil || *job.Error != "no handler for job kind retired" {
+		t.Errorf("job = %+v, want failed for a missing handler", job)
+	}
+}
+
+func TestGetUnknownID(t *testing.T) {
+	t.Parallel()
+	r := newRunner(t, migratedStore(t), 1)
+	if _, err := r.Get(context.Background(), store.NewID()); !errors.Is(err, ErrNotFound) {
+		t.Errorf("Get unknown id error = %v, want ErrNotFound", err)
+	}
+}
+
+func TestRegisterRefusesBadRegistrations(t *testing.T) {
+	t.Parallel()
+	r := newRunner(t, migratedStore(t), 1)
+	h := func(context.Context, countPayload, func(countResult) error) (countResult, error) {
+		return countResult{}, nil
+	}
+	register(t, r, "count", h)
+
+	if _, err := Register(r, "count", h); err == nil {
+		t.Error("duplicate kind registered")
+	}
+	if _, err := Register(r, "", h); err == nil {
+		t.Error("empty kind registered")
+	}
+	if _, err := Register[countPayload, countResult](r, "nil", nil); err == nil {
+		t.Error("nil handler registered")
+	}
+	start(t, r)
+	if _, err := Register(r, "late", h); err == nil {
+		t.Error("kind registered after Start")
+	}
+	if err := r.Start(context.Background()); err == nil {
+		t.Error("second Start succeeded")
+	}
+}
+
+func TestNewRunnerRefusesBadConfig(t *testing.T) {
+	t.Parallel()
+	st := migratedStore(t)
+	logger := slog.New(slog.NewTextHandler(io.Discard, nil))
+	if _, err := NewRunner(st, logger, 0); err == nil {
+		t.Error("zero workers accepted")
+	}
+	if _, err := NewRunner(nil, logger, 1); err == nil {
+		t.Error("nil store accepted")
+	}
+	if _, err := NewRunner(st, nil, 1); err == nil {
+		t.Error("nil logger accepted")
+	}
+}