Register a built-in noop job kind and store only failure codes
The server now registers the framework's built-in noop kind through
registerJobKinds before the runner starts, so production can run a job; a
noop job succeeds with {"ok":true}. Each feature's kinds are registered in
the same function.
Handlers report failure only with a *jobs.Failure code their kind declared
at registration. The runner stores and returns only that code, or
internal_error, interrupted or unknown_kind, and logs only the job id, kind,
code and a fixed reason: handler error text is never stored, returned or
logged. Results implement jobs.Public, whose PublicJSON names exactly the
data that is stored and returned.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+92
-45
@@ -4,11 +4,12 @@
|
||||
// before a restart still runs after it.
|
||||
//
|
||||
// Each job kind has one handler, registered before Start. A handler takes a
|
||||
// typed payload and returns a typed result, which the runner stores as JSON;
|
||||
// it may publish partial results while it runs. Only typed data and
|
||||
// *Failure messages are stored, so no handler error text, which may quote a
|
||||
// URL, token or response body from a dependency, can reach the table or the
|
||||
// API.
|
||||
// typed payload and returns a result implementing Public; it may publish
|
||||
// partial results while it runs. The runner stores only what a result's
|
||||
// PublicJSON returns and, for a failed job, only a failure code: either a
|
||||
// *Failure code the kind declared at registration or one of the runner's own
|
||||
// codes. Handler error text, which may quote a URL, token or response body
|
||||
// from a dependency, is never stored, returned by the API or logged.
|
||||
package jobs
|
||||
|
||||
import (
|
||||
@@ -19,6 +20,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"regexp"
|
||||
"sync"
|
||||
|
||||
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/db"
|
||||
@@ -36,31 +38,46 @@ const (
|
||||
StatusFailed Status = "failed"
|
||||
)
|
||||
|
||||
// Stored error messages the runner writes itself.
|
||||
// Failure codes the runner stores itself. A kind cannot declare them.
|
||||
const (
|
||||
// MessageInterrupted is the error of a job a previous process left running.
|
||||
MessageInterrupted = "interrupted by restart"
|
||||
// MessageInternal is the error of a job whose handler failed with an error
|
||||
// that is not a *Failure. The handler's error is logged, not stored.
|
||||
MessageInternal = "internal error"
|
||||
// CodeInterrupted is the error of a job a previous process left running.
|
||||
CodeInterrupted = "interrupted"
|
||||
// CodeInternal is the error of a job whose handler failed with anything
|
||||
// but a *Failure with a code its kind declared, or whose payload or
|
||||
// result could not be encoded. The underlying error is neither stored
|
||||
// nor logged.
|
||||
CodeInternal = "internal_error"
|
||||
// CodeUnknownKind is the error of a job whose kind has no handler.
|
||||
CodeUnknownKind = "unknown_kind"
|
||||
)
|
||||
|
||||
// codePattern is the form of every failure code: lower-case snake case.
|
||||
var codePattern = regexp.MustCompile(`^[a-z][a-z0-9_]{0,63}$`)
|
||||
|
||||
// ErrNotFound is returned by Get for an unknown job id.
|
||||
var ErrNotFound = errors.New("jobs: job not found")
|
||||
|
||||
// Failure is a handler error whose Message is safe to show the household and
|
||||
// is stored as the job's error. Every other handler error is stored as
|
||||
// MessageInternal.
|
||||
// Failure is a handler error reporting a failure code the household may
|
||||
// see. Code must be one the job's kind declared at registration; the runner
|
||||
// stores it as the job's error. Any other handler error, including a Failure
|
||||
// with an undeclared code, is stored as CodeInternal.
|
||||
type Failure struct {
|
||||
Message string
|
||||
Code string
|
||||
}
|
||||
|
||||
func (f *Failure) Error() string { return f.Message }
|
||||
func (f *Failure) Error() string { return "jobs: failure " + f.Code }
|
||||
|
||||
// Public is a job result. PublicJSON returns the JSON the runner stores and
|
||||
// GET /api/v1/jobs/{id} returns, so a result type states exactly which of its
|
||||
// data is public; nothing else of the value is ever encoded.
|
||||
type Public interface {
|
||||
PublicJSON() ([]byte, error)
|
||||
}
|
||||
|
||||
// Handler runs one job with its payload. It may call publish any number of
|
||||
// times with a partial result, which Get returns while the job runs; its
|
||||
// return value is the final result. ctx is cancelled when the runner stops.
|
||||
type Handler[P, R any] func(ctx context.Context, payload P, publish func(partial R) error) (R, error)
|
||||
type Handler[P any, R Public] func(ctx context.Context, payload P, publish func(partial R) error) (R, error)
|
||||
|
||||
// Job is a job as reported by GET /api/v1/jobs/{id}. Result is the latest
|
||||
// partial or final result, and is null until the handler publishes one.
|
||||
@@ -73,8 +90,10 @@ type Job struct {
|
||||
Error *string `json:"error"`
|
||||
}
|
||||
|
||||
// runFunc is a registered Handler with its payload and results as JSON.
|
||||
type runFunc func(ctx context.Context, payload string, publish func(json.RawMessage) error) (json.RawMessage, error)
|
||||
// runFunc is a registered Handler with its payload and results as JSON. It
|
||||
// returns the failure code to store, or "" on success; the logged reason
|
||||
// never quotes an error.
|
||||
type runFunc func(ctx context.Context, payload string, publish func(json.RawMessage) error) (result json.RawMessage, code, reason string)
|
||||
|
||||
// Runner owns the job handlers and the worker pool.
|
||||
type Runner struct {
|
||||
@@ -119,14 +138,25 @@ type Kind[P any] struct {
|
||||
}
|
||||
|
||||
// Register makes h the handler for kind and returns the Kind that enqueues
|
||||
// its jobs. It must be called before Start, once per kind.
|
||||
func Register[P, R any](r *Runner, kind string, h Handler[P, R]) (*Kind[P], error) {
|
||||
// its jobs. failureCodes are the *Failure codes h may report. It must be
|
||||
// called before Start, once per kind.
|
||||
func Register[P any, R Public](r *Runner, kind string, h Handler[P, R], failureCodes ...string) (*Kind[P], error) {
|
||||
if kind == "" {
|
||||
return nil, errors.New("jobs: register: empty kind")
|
||||
}
|
||||
if h == nil {
|
||||
return nil, fmt.Errorf("jobs: register %q: nil handler", kind)
|
||||
}
|
||||
declared := make(map[string]bool, len(failureCodes))
|
||||
for _, code := range failureCodes {
|
||||
if !codePattern.MatchString(code) {
|
||||
return nil, fmt.Errorf("jobs: register %q: failure code %q is not lower-case snake case", kind, code)
|
||||
}
|
||||
if code == CodeInterrupted || code == CodeInternal || code == CodeUnknownKind {
|
||||
return nil, fmt.Errorf("jobs: register %q: failure code %q is reserved for the runner", kind, code)
|
||||
}
|
||||
declared[code] = true
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.started {
|
||||
@@ -135,32 +165,49 @@ func Register[P, R any](r *Runner, kind string, h Handler[P, R]) (*Kind[P], erro
|
||||
if _, ok := r.handlers[kind]; ok {
|
||||
return nil, fmt.Errorf("jobs: register %q: kind already registered", kind)
|
||||
}
|
||||
r.handlers[kind] = func(ctx context.Context, payload string, publish func(json.RawMessage) error) (json.RawMessage, error) {
|
||||
r.handlers[kind] = func(ctx context.Context, payload string, publish func(json.RawMessage) error) (json.RawMessage, string, string) {
|
||||
var p P
|
||||
dec := json.NewDecoder(bytes.NewReader([]byte(payload)))
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(&p); err != nil {
|
||||
return nil, fmt.Errorf("decode payload: %w", err)
|
||||
return nil, CodeInternal, "payload does not decode"
|
||||
}
|
||||
result, err := h(ctx, p, func(partial R) error {
|
||||
data, err := json.Marshal(partial)
|
||||
data, err := publicJSON(partial)
|
||||
if err != nil {
|
||||
return fmt.Errorf("jobs: encode partial result: %w", err)
|
||||
return err
|
||||
}
|
||||
return publish(data)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
var failure *Failure
|
||||
if !errors.As(err, &failure) {
|
||||
return nil, CodeInternal, "handler error"
|
||||
}
|
||||
if !declared[failure.Code] {
|
||||
return nil, CodeInternal, "undeclared failure code"
|
||||
}
|
||||
return nil, failure.Code, "handler failure"
|
||||
}
|
||||
data, err := json.Marshal(result)
|
||||
data, err := publicJSON(result)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("encode result: %w", err)
|
||||
return nil, CodeInternal, "result does not encode"
|
||||
}
|
||||
return data, nil
|
||||
return data, "", ""
|
||||
}
|
||||
return &Kind[P]{r: r, name: kind}, nil
|
||||
}
|
||||
|
||||
// publicJSON returns v's PublicJSON, refusing anything that is not one JSON
|
||||
// value. Its error never quotes PublicJSON's error.
|
||||
func publicJSON(v Public) (json.RawMessage, error) {
|
||||
data, err := v.PublicJSON()
|
||||
if err != nil || !json.Valid(data) {
|
||||
return nil, errors.New("jobs: result has no valid public JSON")
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// Enqueue stores a queued job with payload and returns its id. A job
|
||||
// enqueued before Start waits for it.
|
||||
func (k *Kind[P]) Enqueue(ctx context.Context, payload P) (string, error) {
|
||||
@@ -187,7 +234,7 @@ func (k *Kind[P]) Enqueue(ctx context.Context, payload P) (string, error) {
|
||||
}
|
||||
|
||||
// Start fails every job a previous process left running with
|
||||
// MessageInterrupted, then starts the workers. They stop when ctx is cancelled;
|
||||
// CodeInterrupted, then starts the workers. They stop when ctx is cancelled;
|
||||
// Wait waits for them. A job whose handler returns an error after ctx is
|
||||
// cancelled is left running, so the next Start fails it as interrupted.
|
||||
func (r *Runner) Start(ctx context.Context) error {
|
||||
@@ -198,7 +245,7 @@ func (r *Runner) Start(ctx context.Context) error {
|
||||
}
|
||||
now := store.Now().String()
|
||||
n, err := db.New(r.st.Writer).FailRunningJobs(ctx, db.FailRunningJobsParams{
|
||||
Error: sql.NullString{String: MessageInterrupted, Valid: true},
|
||||
Error: sql.NullString{String: CodeInterrupted, Valid: true},
|
||||
Now: sql.NullString{String: now, Valid: true},
|
||||
})
|
||||
if err != nil {
|
||||
@@ -270,8 +317,8 @@ func (r *Runner) run(ctx context.Context, job db.Job) {
|
||||
|
||||
h, ok := r.handlers[job.Kind] // read-only once started
|
||||
if !ok {
|
||||
logger.ErrorContext(ctx, "no handler for job kind")
|
||||
r.fail(write, logger, job.ID, "no handler for job kind "+job.Kind)
|
||||
logger.ErrorContext(ctx, "job failed", "code", CodeUnknownKind, "reason", "no handler registered for the kind")
|
||||
r.fail(write, logger, job.ID, CodeUnknownKind)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -290,20 +337,18 @@ func (r *Runner) run(ctx context.Context, job db.Job) {
|
||||
return nil
|
||||
}
|
||||
|
||||
result, err := h(ctx, job.Payload, publish)
|
||||
if err != nil {
|
||||
result, code, reason := h(ctx, job.Payload, publish)
|
||||
if code != "" {
|
||||
if ctx.Err() != nil {
|
||||
logger.WarnContext(write, "job interrupted by shutdown; it fails at the next start", "error", err)
|
||||
logger.WarnContext(write, "job interrupted by shutdown; it fails at the next start")
|
||||
return
|
||||
}
|
||||
var failure *Failure
|
||||
message := MessageInternal
|
||||
if errors.As(err, &failure) {
|
||||
message = failure.Message
|
||||
} else {
|
||||
logger.ErrorContext(ctx, "job failed", "error", err)
|
||||
level := slog.LevelInfo
|
||||
if code == CodeInternal {
|
||||
level = slog.LevelError
|
||||
}
|
||||
r.fail(write, logger, job.ID, message)
|
||||
logger.Log(ctx, level, "job failed", "code", code, "reason", reason)
|
||||
r.fail(write, logger, job.ID, code)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -320,9 +365,11 @@ func (r *Runner) run(ctx context.Context, job db.Job) {
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Runner) fail(ctx context.Context, logger *slog.Logger, id, message string) {
|
||||
// fail records the job as failed with code. The record errors it logs come
|
||||
// from the database, never from a handler.
|
||||
func (r *Runner) fail(ctx context.Context, logger *slog.Logger, id, code string) {
|
||||
n, err := db.New(r.st.Writer).FailJob(ctx, db.FailJobParams{
|
||||
Error: sql.NullString{String: message, Valid: true},
|
||||
Error: sql.NullString{String: code, Valid: true},
|
||||
Now: sql.NullString{String: store.Now().String(), Valid: true},
|
||||
ID: id,
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user