notify package and durable outbox worker (Feature 19) (#161)
CI / ci (push) Successful in 5m18s

Co-authored-by: orchestrator-agent <20+orchestrator-agent@code.cubecraftcreations.com>
This commit was merged in pull request #161.
This commit is contained in:
2026-10-09 13:05:39 -04:00
committed by approver_agent
parent c8e5829a77
commit 104396da55
7 changed files with 1574 additions and 5 deletions
+15 -5
View File
@@ -24,6 +24,7 @@ import (
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/gcal" "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/gcal"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/household" "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/household"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/jobs" "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/jobs"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/notify"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/plan" "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/plan"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/push" "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/push"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/recipes" "code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/recipes"
@@ -144,6 +145,11 @@ func run(cfg config.Config, logger *slog.Logger) error {
return err return err
} }
notifier, err := notify.NewService(db, logger)
if err != nil {
return err
}
sessions, err := auth.NewSessionManager(db, cfg.SessionIdle, cfg.SessionMax) sessions, err := auth.NewSessionManager(db, cfg.SessionIdle, cfg.SessionMax)
if err != nil { if err != nil {
return err return err
@@ -166,14 +172,18 @@ func run(cfg config.Config, logger *slog.Logger) error {
} }
// Start fails the jobs a previous process left running before any // Start fails the jobs a previous process left running before any
// worker runs. The workers stop, and are waited for, before the // worker runs. The job workers and the notification worker stop, and
// database closes. // are waited for, before the database closes.
jobsCtx, stopJobs := context.WithCancel(ctx) workersCtx, stopWorkers := context.WithCancel(ctx)
defer func() { defer func() {
stopJobs() stopWorkers()
runner.Wait() runner.Wait()
notifier.Wait()
}() }()
if err := runner.Start(jobsCtx); err != nil { if err := runner.Start(workersCtx); err != nil {
return err
}
if err := notifier.Start(workersCtx); err != nil {
return err return err
} }
// With those jobs failed, a dinner still pending with no grounding job // With those jobs failed, a dinner still pending with no grounding job
+202
View File
@@ -0,0 +1,202 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.31.1
// source: notifications.sql
package db
import (
"context"
"database/sql"
)
const claimDueNotification = `-- name: ClaimDueNotification :one
UPDATE notification_outbox
SET next_attempt_at = ?1, updated_at = ?2
WHERE id = (
SELECT id FROM notification_outbox
WHERE status = 'pending' AND next_attempt_at <= ?2
ORDER BY next_attempt_at, id
LIMIT 1
)
RETURNING id, event_key, channel, payload, status, attempts, next_attempt_at, last_error, created_at, updated_at
`
type ClaimDueNotificationParams struct {
LeaseUntil sql.NullString
Now string
}
// Leases the pending notification due longest ago by moving its
// next_attempt_at to lease_until, and returns it. A claimed row is not due
// again until the lease ends, so no other claim takes it while it is being
// delivered. Returns sql.ErrNoRows when none is due.
func (q *Queries) ClaimDueNotification(ctx context.Context, arg ClaimDueNotificationParams) (NotificationOutbox, error) {
row := q.db.QueryRowContext(ctx, claimDueNotification, arg.LeaseUntil, arg.Now)
var i NotificationOutbox
err := row.Scan(
&i.ID,
&i.EventKey,
&i.Channel,
&i.Payload,
&i.Status,
&i.Attempts,
&i.NextAttemptAt,
&i.LastError,
&i.CreatedAt,
&i.UpdatedAt,
)
return i, err
}
const failNotification = `-- name: FailNotification :execrows
UPDATE notification_outbox
SET status = 'failed', attempts = attempts + ?1, next_attempt_at = NULL, last_error = ?2, updated_at = ?3
WHERE id = ?4 AND status = 'pending' AND next_attempt_at = ?5
`
type FailNotificationParams struct {
Attempted int64
LastError sql.NullString
Now string
ID string
LeaseUntil sql.NullString
}
// Gives up on a notification. attempts counts the delivery attempt that
// failed, if one was made. The lease must still be the claim's.
func (q *Queries) FailNotification(ctx context.Context, arg FailNotificationParams) (int64, error) {
result, err := q.db.ExecContext(ctx, failNotification,
arg.Attempted,
arg.LastError,
arg.Now,
arg.ID,
arg.LeaseUntil,
)
if err != nil {
return 0, err
}
return result.RowsAffected()
}
const getNotificationPref = `-- name: GetNotificationPref :one
SELECT event, push, chat, created_at, updated_at FROM notification_prefs
WHERE event = ?
`
func (q *Queries) GetNotificationPref(ctx context.Context, event string) (NotificationPref, error) {
row := q.db.QueryRowContext(ctx, getNotificationPref, event)
var i NotificationPref
err := row.Scan(
&i.Event,
&i.Push,
&i.Chat,
&i.CreatedAt,
&i.UpdatedAt,
)
return i, err
}
const insertNotification = `-- name: InsertNotification :execrows
INSERT INTO notification_outbox (
id, event_key, channel, payload, status, attempts, next_attempt_at, created_at, updated_at
) VALUES (
?1, ?2, ?3, ?4, 'pending', 0, ?5, ?6, ?6
)
ON CONFLICT (event_key, channel) DO NOTHING
`
type InsertNotificationParams struct {
ID string
EventKey string
Channel string
Payload string
NextAttemptAt sql.NullString
Now string
}
// Stores a pending notification for one channel, due at once. A row already
// stored for the event key and channel, in any status, is left unchanged and
// no row is inserted.
func (q *Queries) InsertNotification(ctx context.Context, arg InsertNotificationParams) (int64, error) {
result, err := q.db.ExecContext(ctx, insertNotification,
arg.ID,
arg.EventKey,
arg.Channel,
arg.Payload,
arg.NextAttemptAt,
arg.Now,
)
if err != nil {
return 0, err
}
return result.RowsAffected()
}
const markNotificationSent = `-- name: MarkNotificationSent :execrows
UPDATE notification_outbox
SET status = 'sent', attempts = attempts + 1, next_attempt_at = NULL, updated_at = ?1
WHERE id = ?2 AND status = 'pending' AND next_attempt_at = ?3
`
type MarkNotificationSentParams struct {
Now string
ID string
LeaseUntil sql.NullString
}
// Records a delivered notification; updated_at is when it was sent. The
// lease must still be the claim's.
func (q *Queries) MarkNotificationSent(ctx context.Context, arg MarkNotificationSentParams) (int64, error) {
result, err := q.db.ExecContext(ctx, markNotificationSent, arg.Now, arg.ID, arg.LeaseUntil)
if err != nil {
return 0, err
}
return result.RowsAffected()
}
const nextNotificationDue = `-- name: NextNotificationDue :one
SELECT next_attempt_at FROM notification_outbox
WHERE status = 'pending'
ORDER BY next_attempt_at
LIMIT 1
`
// Returns when the next pending notification is due. Returns sql.ErrNoRows
// when none is pending.
func (q *Queries) NextNotificationDue(ctx context.Context) (sql.NullString, error) {
row := q.db.QueryRowContext(ctx, nextNotificationDue)
var next_attempt_at sql.NullString
err := row.Scan(&next_attempt_at)
return next_attempt_at, err
}
const retryNotification = `-- name: RetryNotification :execrows
UPDATE notification_outbox
SET attempts = attempts + 1, next_attempt_at = ?1, last_error = ?2, updated_at = ?3
WHERE id = ?4 AND status = 'pending' AND next_attempt_at = ?5
`
type RetryNotificationParams struct {
NextAttemptAt sql.NullString
LastError sql.NullString
Now string
ID string
LeaseUntil sql.NullString
}
// Records a failed delivery and when to try again. The lease must still be
// the claim's.
func (q *Queries) RetryNotification(ctx context.Context, arg RetryNotificationParams) (int64, error) {
result, err := q.db.ExecContext(ctx, retryNotification,
arg.NextAttemptAt,
arg.LastError,
arg.Now,
arg.ID,
arg.LeaseUntil,
)
if err != nil {
return 0, err
}
return result.RowsAffected()
}
+22
View File
@@ -12,6 +12,11 @@ import (
type Querier interface { type Querier interface {
// An unknown id returns sql.ErrNoRows. // An unknown id returns sql.ErrNoRows.
ArchiveDinner(ctx context.Context, arg ArchiveDinnerParams) (Dinner, error) ArchiveDinner(ctx context.Context, arg ArchiveDinnerParams) (Dinner, error)
// Leases the pending notification due longest ago by moving its
// next_attempt_at to lease_until, and returns it. A claimed row is not due
// again until the lease ends, so no other claim takes it while it is being
// delivered. Returns sql.ErrNoRows when none is due.
ClaimDueNotification(ctx context.Context, arg ClaimDueNotificationParams) (NotificationOutbox, error)
// Marks the oldest queued job running and returns it. Returns sql.ErrNoRows // Marks the oldest queued job running and returns it. Returns sql.ErrNoRows
// when no job is queued. // when no job is queued.
ClaimNextJob(ctx context.Context, now sql.NullString) (Job, error) ClaimNextJob(ctx context.Context, now sql.NullString) (Job, error)
@@ -68,6 +73,9 @@ type Querier interface {
DeleteShoppingGroup(ctx context.Context, id string) error DeleteShoppingGroup(ctx context.Context, id string) error
DeleteShoppingItem(ctx context.Context, id string) error DeleteShoppingItem(ctx context.Context, id string) error
FailJob(ctx context.Context, arg FailJobParams) (int64, error) FailJob(ctx context.Context, arg FailJobParams) (int64, error)
// Gives up on a notification. attempts counts the delivery attempt that
// failed, if one was made. The lease must still be the claim's.
FailNotification(ctx context.Context, arg FailNotificationParams) (int64, error)
// Fails every unarchived dinner still pending whose grounding job for its // Fails every unarchived dinner still pending whose grounding job for its
// current grounding_revision is no longer queued or running: the process // current grounding_revision is no longer queued or running: the process
// stopped before the job, or before its handler recorded a result. An // stopped before the job, or before its handler recorded a result. An
@@ -112,6 +120,7 @@ type Querier interface {
// The newest batch, whatever its status. Returns sql.ErrNoRows when there // The newest batch, whatever its status. Returns sql.ErrNoRows when there
// is none. // is none.
GetLatestSuggestionBatch(ctx context.Context) (SuggestionBatch, error) GetLatestSuggestionBatch(ctx context.Context) (SuggestionBatch, error)
GetNotificationPref(ctx context.Context, event string) (NotificationPref, error)
// The pending Rotation check of dinner_id. MarkTried makes one only for a // The pending Rotation check of dinner_id. MarkTried makes one only for a
// dinner in the Quest log, which it moves to tried, so a dinner has at // dinner in the Quest log, which it moves to tried, so a dinner has at
// most one. Returns sql.ErrNoRows when there is none. // most one. Returns sql.ErrNoRows when there is none.
@@ -153,6 +162,10 @@ type Querier interface {
// them only when a member edited them while it was grounding. Like // them only when a member edited them while it was grounding. Like
// GroundDinner it clears the pasted recipe_url: the recipe replaces the link. // GroundDinner it clears the pasted recipe_url: the recipe replaces the link.
GroundPendingDinner(ctx context.Context, arg GroundPendingDinnerParams) (int64, error) GroundPendingDinner(ctx context.Context, arg GroundPendingDinnerParams) (int64, error)
// Stores a pending notification for one channel, due at once. A row already
// stored for the event key and channel, in any status, is left unchanged and
// no row is inserted.
InsertNotification(ctx context.Context, arg InsertNotificationParams) (int64, error)
// The pending suggestions whose dinner is still suggested, oldest first, // The pending suggestions whose dinner is still suggested, oldest first,
// with their dinners. // with their dinners.
ListDeck(ctx context.Context) ([]ListDeckRow, error) ListDeck(ctx context.Context) ([]ListDeckRow, error)
@@ -211,6 +224,9 @@ type Querier interface {
// Moves a dinner still in the Quest log to tried; a dinner with any other // Moves a dinner still in the Quest log to tried; a dinner with any other
// status is left as it is (0 rows). // status is left as it is (0 rows).
MarkDinnerTried(ctx context.Context, arg MarkDinnerTriedParams) (int64, error) MarkDinnerTried(ctx context.Context, arg MarkDinnerTriedParams) (int64, error)
// Records a delivered notification; updated_at is when it was sent. The
// lease must still be the claim's.
MarkNotificationSent(ctx context.Context, arg MarkNotificationSentParams) (int64, error)
// Records an edit of plan id, raising its revision: a synced plan becomes // Records an edit of plan id, raising its revision: a synced plan becomes
// changed, any other keeps its status. // changed, any other keeps its status.
MarkWeekPlanEdited(ctx context.Context, arg MarkWeekPlanEditedParams) error MarkWeekPlanEdited(ctx context.Context, arg MarkWeekPlanEditedParams) error
@@ -230,6 +246,9 @@ type Querier interface {
// by a Rotation check decision, recording updated_by; its origin is kept. // by a Rotation check decision, recording updated_by; its origin is kept.
// A dinner no longer tried is left as it is (0 rows). // A dinner no longer tried is left as it is (0 rows).
MoveTriedDinner(ctx context.Context, arg MoveTriedDinnerParams) (int64, error) MoveTriedDinner(ctx context.Context, arg MoveTriedDinnerParams) (int64, error)
// Returns when the next pending notification is due. Returns sql.ErrNoRows
// when none is pending.
NextNotificationDue(ctx context.Context) (sql.NullString, error)
// Stores a partial result of a running job. // Stores a partial result of a running job.
PublishJobResult(ctx context.Context, arg PublishJobResultParams) (int64, error) PublishJobResult(ctx context.Context, arg PublishJobResultParams) (int64, error)
// Plans dinner_id on date of plan_id. An existing day keeps its calendar // Plans dinner_id on date of plan_id. An existing day keeps its calendar
@@ -241,6 +260,9 @@ type Querier interface {
RedraftWeekPlan(ctx context.Context, arg RedraftWeekPlanParams) error RedraftWeekPlan(ctx context.Context, arg RedraftWeekPlanParams) error
// Records a generation of list id from plan plan_id. // Records a generation of list id from plan plan_id.
RegenerateShoppingList(ctx context.Context, arg RegenerateShoppingListParams) error RegenerateShoppingList(ctx context.Context, arg RegenerateShoppingListParams) error
// Records a failed delivery and when to try again. The lease must still be
// the claim's.
RetryNotification(ctx context.Context, arg RetryNotificationParams) (int64, error)
// An unknown id returns sql.ErrNoRows. // An unknown id returns sql.ErrNoRows.
SetDinnerFavorite(ctx context.Context, arg SetDinnerFavoriteParams) (Dinner, error) SetDinnerFavorite(ctx context.Context, arg SetDinnerFavoriteParams) (Dinner, error)
// Sets the chosen calendar; affects no row when nothing is connected. // Sets the chosen calendar; affects no row when nothing is connected.
+248
View File
@@ -0,0 +1,248 @@
// Package notify decides who is told what and delivers it reliably. It is
// the only package that reads notification_prefs to choose channels.
//
// Emit is the one way to notify the household: it reads the preference row
// of the event's type and, for each channel switched on, stores one pending
// row in notification_outbox keyed by (event key, channel). The event key is
// the caller's idempotency key, so emitting the same event twice stores
// nothing the second time and never touches a row already stored, whatever
// its status.
//
// The worker started by Start delivers due rows through the Sender
// registered for their channel. A delivered row is marked sent. A failed
// delivery is retried after a backoff of one minute doubling up to one hour,
// until 24 hours after the row was created; the delivery that fails at or
// after that point marks it failed, with the error in last_error.
package notify
import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"log/slog"
"net/url"
"strings"
"sync"
"time"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/db"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/store"
)
// EventType is what a notification is about. Each has one row in
// notification_prefs saying which channels it goes to.
type EventType string
// The event types, as notification_prefs.event names them.
const (
PlanReady EventType = "plan_ready"
ListReady EventType = "list_ready"
DayOfReminder EventType = "day_of_reminder"
NewSuggestions EventType = "new_suggestions"
RotationReview EventType = "rotation_review"
)
// eventTypes is every EventType.
var eventTypes = map[EventType]bool{
PlanReady: true, ListReady: true, DayOfReminder: true, NewSuggestions: true, RotationReview: true,
}
// Channel is a way of telling the household, as notification_outbox.channel
// names it.
type Channel string
// The channels.
const (
ChannelPush Channel = "push"
ChannelChat Channel = "chat"
)
// Event is something to tell the household.
type Event struct {
// Type is what the event is about.
Type EventType
// Key identifies this event: it starts with Type and a colon, as in
// "plan_ready:2026-10-12". An event emitted again with the same key is
// ignored.
Key string
// Title is the notification's headline; it must not be blank.
Title string
// Body is the notification's text; it may be empty.
Body string
// URL is the path-absolute, same-origin path a tap on the notification
// opens, such as "/plan".
URL string
// Week is the Monday of the week the event concerns, for the senders'
// message builders; zero when the event concerns no week.
Week store.LocalDate
}
// Payload is the JSON stored in notification_outbox.payload and handed to
// a Sender: the one shape every channel delivers from. Its title, body, tag
// and url are the notice the service worker shows (web/src/push/notification.ts),
// which ignores the other fields.
type Payload struct {
Event EventType `json:"event"`
Title string `json:"title"`
Body string `json:"body"`
// URL is a path-absolute, same-origin path. A sender that needs an
// absolute URL resolves it against the public URL.
URL string `json:"url"`
// Tag is the event key, so a notification shown again for the same
// event replaces the first.
Tag string `json:"tag"`
// Week is the YYYY-MM-DD Monday of the event's week, or absent.
Week string `json:"week,omitempty"`
}
// ErrInvalidEvent is returned by Emit for an event it refuses; nothing is
// stored.
var ErrInvalidEvent = errors.New("notify: invalid event")
// Service emits notifications and runs the worker that delivers them.
type Service struct {
st *store.Store
logger *slog.Logger
// now is the clock rows are created, claimed and retried on; tests
// replace it.
now func() time.Time
// wake holds up to one token; Emit adds one so the worker looks for
// the rows it stored.
wake chan struct{}
mu sync.Mutex
senders map[Channel]Sender
started bool
wg sync.WaitGroup
}
// NewService returns a service over st with no senders registered.
func NewService(st *store.Store, logger *slog.Logger) (*Service, error) {
if st == nil {
return nil, errors.New("notify: nil store")
}
if logger == nil {
return nil, errors.New("notify: nil logger")
}
return &Service{
st: st,
logger: logger,
now: time.Now,
wake: make(chan struct{}, 1),
senders: make(map[Channel]Sender),
}, nil
}
// Emit stores one pending notification of ev for each channel its type's
// preference switches on, and wakes the worker. A channel already holding a
// row with ev.Key is left as it is. An invalid event is ErrInvalidEvent and
// stores nothing; a type with no preference row is an error.
func (s *Service) Emit(ctx context.Context, ev Event) error {
if err := validate(ev); err != nil {
return err
}
payload := Payload{Event: ev.Type, Title: ev.Title, Body: ev.Body, URL: ev.URL, Tag: ev.Key}
if !ev.Week.IsZero() {
payload.Week = ev.Week.String()
}
data, err := json.Marshal(payload)
if err != nil {
return fmt.Errorf("notify: encode payload of %s: %w", ev.Key, err)
}
var queued []Channel
err = s.st.WithTx(ctx, func(tx *sql.Tx) error {
q := db.New(tx)
pref, err := q.GetNotificationPref(ctx, string(ev.Type))
if errors.Is(err, sql.ErrNoRows) {
return fmt.Errorf("notify: no notification preference for %s", ev.Type)
}
if err != nil {
return fmt.Errorf("notify: read notification preference for %s: %w", ev.Type, err)
}
now := timestamp(s.now())
for _, ch := range enabledChannels(pref) {
n, err := q.InsertNotification(ctx, db.InsertNotificationParams{
ID: store.NewID(),
EventKey: ev.Key,
Channel: string(ch),
Payload: string(data),
NextAttemptAt: sql.NullString{String: now, Valid: true},
Now: now,
})
if err != nil {
return fmt.Errorf("notify: store %s notification %s: %w", ch, ev.Key, err)
}
if n == 1 {
queued = append(queued, ch)
}
}
return nil
})
if err != nil {
return err
}
s.logger.InfoContext(ctx, "notification emitted", "event_key", ev.Key, "queued", queued)
if len(queued) > 0 {
s.wakeWorker()
}
return nil
}
// enabledChannels returns the channels pref switches on.
func enabledChannels(pref db.NotificationPref) []Channel {
var chs []Channel
if pref.Push == 1 {
chs = append(chs, ChannelPush)
}
if pref.Chat == 1 {
chs = append(chs, ChannelChat)
}
return chs
}
// validate reports why ev cannot be emitted, wrapping ErrInvalidEvent.
func validate(ev Event) error {
if !eventTypes[ev.Type] {
return fmt.Errorf("%w: unknown type %q", ErrInvalidEvent, ev.Type)
}
prefix := string(ev.Type) + ":"
if !strings.HasPrefix(ev.Key, prefix) || len(ev.Key) == len(prefix) {
return fmt.Errorf("%w: key %q does not start with %q and an identifier", ErrInvalidEvent, ev.Key, prefix)
}
if strings.TrimSpace(ev.Title) == "" {
return fmt.Errorf("%w: %s has a blank title", ErrInvalidEvent, ev.Key)
}
if !isAppPath(ev.URL) {
return fmt.Errorf("%w: %s url %q is not a path-absolute same-origin path", ErrInvalidEvent, ev.Key, ev.URL)
}
return nil
}
// isAppPath reports whether raw is a path on the app's own origin, as the
// service worker requires: it starts with one slash and names no scheme or
// host, however it is parsed.
func isAppPath(raw string) bool {
if !strings.HasPrefix(raw, "/") || strings.HasPrefix(raw, "//") || strings.ContainsAny(raw, "\\\t\r\n") {
return false
}
u, err := url.Parse(raw)
return err == nil && u.Scheme == "" && u.Host == "" && !strings.HasPrefix(u.Path, "//")
}
// timestamp returns t in the stored UTC RFC 3339 form.
func timestamp(t time.Time) string {
return store.Timestamp{Time: t.UTC().Truncate(time.Millisecond)}.String()
}
// parseTimestamp parses a stored timestamp.
func parseTimestamp(s string) (time.Time, error) {
var ts store.Timestamp
if err := ts.Scan(s); err != nil {
return time.Time{}, err
}
return ts.Time, nil
}
+730
View File
@@ -0,0 +1,730 @@
package notify
import (
"context"
"database/sql"
"errors"
"fmt"
"io"
"log/slog"
"sync"
"testing"
"time"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/db"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/household"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/store"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/migrations"
)
// start is the fake clock's first instant.
var start = time.Date(2026, 10, 9, 15, 0, 0, 0, time.UTC)
// clock is a controllable clock.
type clock struct {
mu sync.Mutex
t time.Time
}
func (c *clock) now() time.Time {
c.mu.Lock()
defer c.mu.Unlock()
return c.t
}
func (c *clock) set(t time.Time) {
c.mu.Lock()
defer c.mu.Unlock()
c.t = t
}
// fakeSender records the messages it is given and fails while fail is set.
type fakeSender struct {
mu sync.Mutex
fail error
got []Message
}
func (f *fakeSender) Send(_ context.Context, m Message) error {
f.mu.Lock()
defer f.mu.Unlock()
f.got = append(f.got, m)
return f.fail
}
func (f *fakeSender) messages() []Message {
f.mu.Lock()
defer f.mu.Unlock()
return append([]Message(nil), f.got...)
}
func newTestService(t *testing.T) (*Service, *store.Store, *clock) {
t.Helper()
ctx := context.Background()
st, err := store.Open(ctx, t.TempDir())
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
if err := st.Close(); err != nil {
t.Error(err)
}
})
if err := store.Migrate(ctx, st.Writer, migrations.FS); err != nil {
t.Fatal(err)
}
if err := household.SeedNotificationPrefs(ctx, st); err != nil {
t.Fatal(err)
}
svc, err := NewService(st, slog.New(slog.NewTextHandler(io.Discard, nil)))
if err != nil {
t.Fatal(err)
}
c := &clock{t: start}
svc.now = c.now
return svc, st, c
}
func setPref(t *testing.T, st *store.Store, ev EventType, push, chat bool) {
t.Helper()
res, err := st.Writer.Exec(`UPDATE notification_prefs SET push = ?, chat = ? WHERE event = ?`, push, chat, string(ev))
if err != nil {
t.Fatal(err)
}
if n, _ := res.RowsAffected(); n != 1 {
t.Fatalf("no notification preference for %s", ev)
}
}
// outbox returns the stored rows of key by channel.
func outbox(t *testing.T, st *store.Store, key string) map[Channel]db.NotificationOutbox {
t.Helper()
rs, err := st.Reader.Query(`SELECT id, event_key, channel, payload, status, attempts, next_attempt_at, last_error, created_at, updated_at
FROM notification_outbox WHERE event_key = ?`, key)
if err != nil {
t.Fatal(err)
}
defer rs.Close()
got := map[Channel]db.NotificationOutbox{}
for rs.Next() {
var r db.NotificationOutbox
if err := rs.Scan(&r.ID, &r.EventKey, &r.Channel, &r.Payload, &r.Status, &r.Attempts, &r.NextAttemptAt, &r.LastError, &r.CreatedAt, &r.UpdatedAt); err != nil {
t.Fatal(err)
}
if _, dup := got[Channel(r.Channel)]; dup {
t.Fatalf("two %s rows for %s", r.Channel, key)
}
got[Channel(r.Channel)] = r
}
if err := rs.Err(); err != nil {
t.Fatal(err)
}
return got
}
func onlyRow(t *testing.T, st *store.Store, key string, ch Channel) db.NotificationOutbox {
t.Helper()
rows := outbox(t, st, key)
r, ok := rows[ch]
if !ok || len(rows) != 1 {
t.Fatalf("rows of %s = %v, want one %s row", key, rows, ch)
}
return r
}
func planReady(key string) Event {
return Event{
Type: PlanReady,
Key: key,
Title: "This week's plan is ready",
Body: "Five dinners on the Draft board.",
URL: "/plan",
Week: mustDate("2026-10-12"),
}
}
func mustDate(s string) store.LocalDate {
d, err := store.ParseLocalDate(s)
if err != nil {
panic(err)
}
return d
}
func TestEmitRespectsPrefsMatrix(t *testing.T) {
ctx := context.Background()
for _, tc := range []struct {
push, chat bool
want []Channel
}{
{push: false, chat: false, want: nil},
{push: true, chat: false, want: []Channel{ChannelPush}},
{push: false, chat: true, want: []Channel{ChannelChat}},
{push: true, chat: true, want: []Channel{ChannelPush, ChannelChat}},
} {
t.Run(fmt.Sprintf("push=%v,chat=%v", tc.push, tc.chat), func(t *testing.T) {
svc, st, _ := newTestService(t)
for ev := range eventTypes {
setPref(t, st, ev, tc.push, tc.chat)
key := string(ev) + ":2026-10-12"
if err := svc.Emit(ctx, Event{Type: ev, Key: key, Title: "Title", Body: "Body", URL: "/"}); err != nil {
t.Fatal(err)
}
rows := outbox(t, st, key)
if len(rows) != len(tc.want) {
t.Fatalf("%s: %d rows, want channels %v", ev, len(rows), tc.want)
}
for _, ch := range tc.want {
r, ok := rows[ch]
if !ok {
t.Fatalf("%s: no %s row", ev, ch)
}
now := timestamp(start)
if r.Status != "pending" || r.Attempts != 0 || r.NextAttemptAt.String != now || r.CreatedAt != now || r.LastError.Valid {
t.Errorf("%s %s row = %+v, want pending, due now", ev, ch, r)
}
}
}
})
}
}
func TestEmitStoresPayload(t *testing.T) {
svc, st, _ := newTestService(t)
setPref(t, st, PlanReady, true, true)
if err := svc.Emit(context.Background(), planReady("plan_ready:2026-10-12")); err != nil {
t.Fatal(err)
}
want := `{"event":"plan_ready","title":"This week's plan is ready","body":"Five dinners on the Draft board.","url":"/plan","tag":"plan_ready:2026-10-12","week":"2026-10-12"}`
for ch, r := range outbox(t, st, "plan_ready:2026-10-12") {
if r.Payload != want {
t.Errorf("%s payload = %s, want %s", ch, r.Payload, want)
}
}
// An event about no week omits it.
if err := svc.Emit(context.Background(), Event{Type: NewSuggestions, Key: "new_suggestions:batch1", Title: "New Side quests", URL: "/suggestions"}); err != nil {
t.Fatal(err)
}
r := onlyRow(t, st, "new_suggestions:batch1", ChannelPush)
if want := `{"event":"new_suggestions","title":"New Side quests","body":"","url":"/suggestions","tag":"new_suggestions:batch1"}`; r.Payload != want {
t.Errorf("payload = %s, want %s", r.Payload, want)
}
}
func TestEmitIgnoresDuplicateKey(t *testing.T) {
ctx := context.Background()
svc, st, c := newTestService(t)
setPref(t, st, PlanReady, true, false)
const key = "plan_ready:2026-10-12"
if err := svc.Emit(ctx, planReady(key)); err != nil {
t.Fatal(err)
}
first := onlyRow(t, st, key, ChannelPush)
c.set(start.Add(time.Minute))
again := planReady(key)
again.Title = "A different title"
if err := svc.Emit(ctx, again); err != nil {
t.Fatalf("duplicate emit: %v", err)
}
if got := onlyRow(t, st, key, ChannelPush); got != first {
t.Fatalf("duplicate emit changed the row:\n got %+v\nwant %+v", got, first)
}
// Once sent, a duplicate still leaves the row as it is.
sender := &fakeSender{}
if err := svc.Register(ChannelPush, sender); err != nil {
t.Fatal(err)
}
if _, err := svc.deliverDue(ctx); err != nil {
t.Fatal(err)
}
sent := onlyRow(t, st, key, ChannelPush)
if sent.Status != "sent" {
t.Fatalf("status = %s, want sent", sent.Status)
}
c.set(start.Add(2 * time.Minute))
if err := svc.Emit(ctx, again); err != nil {
t.Fatalf("emit after sent: %v", err)
}
if got := onlyRow(t, st, key, ChannelPush); got != sent {
t.Fatalf("emit after sent changed the row:\n got %+v\nwant %+v", got, sent)
}
if _, err := svc.deliverDue(ctx); err != nil {
t.Fatal(err)
}
if n := len(sender.messages()); n != 1 {
t.Fatalf("%d deliveries, want 1", n)
}
}
func TestEmitRejectsInvalidEvents(t *testing.T) {
svc, st, _ := newTestService(t)
valid := planReady("plan_ready:2026-10-12")
for name, mutate := range map[string]func(*Event){
"unknown type": func(e *Event) { e.Type = "dinner_burnt" },
"empty key": func(e *Event) { e.Key = "" },
"key without id": func(e *Event) { e.Key = "plan_ready:" },
"key of another type": func(e *Event) { e.Key = "list_ready:2026-10-12" },
"blank title": func(e *Event) { e.Title = " " },
"empty url": func(e *Event) { e.URL = "" },
"relative url": func(e *Event) { e.URL = "plan" },
"absolute url": func(e *Event) { e.URL = "https://evil.example/plan" },
"protocol-relative": func(e *Event) { e.URL = "//evil.example/plan" },
"backslash url": func(e *Event) { e.URL = "/\\evil.example" },
"url with a tab": func(e *Event) { e.URL = "/\t/evil.example" },
"url with a bad query": func(e *Event) { e.URL = "/plan%zz" },
} {
t.Run(name, func(t *testing.T) {
ev := valid
mutate(&ev)
if err := svc.Emit(context.Background(), ev); !errors.Is(err, ErrInvalidEvent) {
t.Fatalf("Emit = %v, want ErrInvalidEvent", err)
}
})
}
var n int
if err := st.Reader.QueryRow(`SELECT COUNT(*) FROM notification_outbox`).Scan(&n); err != nil {
t.Fatal(err)
}
if n != 0 {
t.Fatalf("%d rows stored, want 0", n)
}
}
func TestEmitWithoutPrefIsAnError(t *testing.T) {
svc, st, _ := newTestService(t)
if _, err := st.Writer.Exec(`DELETE FROM notification_prefs WHERE event = 'plan_ready'`); err != nil {
t.Fatal(err)
}
err := svc.Emit(context.Background(), planReady("plan_ready:2026-10-12"))
if err == nil || errors.Is(err, ErrInvalidEvent) {
t.Fatalf("Emit = %v, want a missing-preference error", err)
}
}
func TestDeliveryMarkedSent(t *testing.T) {
ctx := context.Background()
svc, st, c := newTestService(t)
setPref(t, st, PlanReady, true, true)
push, chat := &fakeSender{}, &fakeSender{}
if err := svc.Register(ChannelPush, push); err != nil {
t.Fatal(err)
}
if err := svc.Register(ChannelChat, chat); err != nil {
t.Fatal(err)
}
const key = "plan_ready:2026-10-12"
if err := svc.Emit(ctx, planReady(key)); err != nil {
t.Fatal(err)
}
sentAt := start.Add(3 * time.Second)
c.set(sentAt)
wait, err := svc.deliverDue(ctx)
if err != nil {
t.Fatal(err)
}
if wait != -1 {
t.Errorf("wait = %v, want -1 with nothing pending", wait)
}
rows := outbox(t, st, key)
wantPayload := Payload{Event: PlanReady, Title: "This week's plan is ready", Body: "Five dinners on the Draft board.", URL: "/plan", Tag: key, Week: "2026-10-12"}
for ch, sender := range map[Channel]*fakeSender{ChannelPush: push, ChannelChat: chat} {
r := rows[ch]
if r.Status != "sent" || r.Attempts != 1 || r.NextAttemptAt.Valid || r.UpdatedAt != timestamp(sentAt) || r.LastError.Valid {
t.Errorf("%s row = %+v, want sent at %s after 1 attempt", ch, r, timestamp(sentAt))
}
got := sender.messages()
want := Message{ID: r.ID, EventKey: key, Channel: ch, Payload: wantPayload, Attempt: 1}
if len(got) != 1 || got[0] != want {
t.Errorf("%s sender got %+v, want [%+v]", ch, got, want)
}
}
}
func TestRetryBackoffSchedule(t *testing.T) {
ctx := context.Background()
svc, st, c := newTestService(t)
setPref(t, st, PlanReady, true, false)
sender := &fakeSender{fail: errors.New("push service unavailable")}
if err := svc.Register(ChannelPush, sender); err != nil {
t.Fatal(err)
}
const key = "plan_ready:2026-10-12"
if err := svc.Emit(ctx, planReady(key)); err != nil {
t.Fatal(err)
}
now := start
for i, wantDelay := range []time.Duration{
time.Minute, 2 * time.Minute, 4 * time.Minute, 8 * time.Minute, 16 * time.Minute,
32 * time.Minute, time.Hour, time.Hour, time.Hour,
} {
c.set(now)
wait, err := svc.deliverDue(ctx)
if err != nil {
t.Fatal(err)
}
r := onlyRow(t, st, key, ChannelPush)
wantNext := now.Add(wantDelay)
if r.Status != "pending" || r.Attempts != int64(i+1) || r.NextAttemptAt.String != timestamp(wantNext) ||
r.LastError.String != "push service unavailable" {
t.Fatalf("after failure %d row = %+v, want pending, attempts %d, next %s", i+1, r, i+1, timestamp(wantNext))
}
if wait != wantDelay {
t.Fatalf("after failure %d wait = %v, want %v", i+1, wait, wantDelay)
}
// Not due a moment before the retry time.
c.set(wantNext.Add(-time.Millisecond))
if _, err := svc.deliverDue(ctx); err != nil {
t.Fatal(err)
}
if n := len(sender.messages()); n != i+1 {
t.Fatalf("%d deliveries before the retry was due, want %d", n, i+1)
}
now = wantNext
}
for i, m := range sender.messages() {
if m.Attempt != int64(i+1) {
t.Errorf("delivery %d has Attempt %d", i+1, m.Attempt)
}
}
}
func TestRetryThenSent(t *testing.T) {
ctx := context.Background()
svc, st, c := newTestService(t)
setPref(t, st, ListReady, false, true)
sender := &fakeSender{fail: errors.New("chat returned 503")}
if err := svc.Register(ChannelChat, sender); err != nil {
t.Fatal(err)
}
const key = "list_ready:2026-10-12"
if err := svc.Emit(ctx, Event{Type: ListReady, Key: key, Title: "Supply run ready", URL: "/supply-run"}); err != nil {
t.Fatal(err)
}
if _, err := svc.deliverDue(ctx); err != nil {
t.Fatal(err)
}
sender.mu.Lock()
sender.fail = nil
sender.mu.Unlock()
c.set(start.Add(time.Minute))
if _, err := svc.deliverDue(ctx); err != nil {
t.Fatal(err)
}
r := onlyRow(t, st, key, ChannelChat)
if r.Status != "sent" || r.Attempts != 2 || r.UpdatedAt != timestamp(start.Add(time.Minute)) {
t.Fatalf("row = %+v, want sent on attempt 2", r)
}
}
func TestGiveUpAfter24Hours(t *testing.T) {
ctx := context.Background()
svc, st, c := newTestService(t)
setPref(t, st, DayOfReminder, true, false)
sender := &fakeSender{fail: errors.New("push service unavailable")}
if err := svc.Register(ChannelPush, sender); err != nil {
t.Fatal(err)
}
const key = "day_of_reminder:2026-10-09"
if err := svc.Emit(ctx, Event{Type: DayOfReminder, Key: key, Title: "Tonight: tacos", URL: "/"}); err != nil {
t.Fatal(err)
}
// Deliver each retry as it falls due until the row leaves pending.
deadline := start.Add(24 * time.Hour)
var times []time.Time
for range 100 {
r := onlyRow(t, st, key, ChannelPush)
if r.Status != "pending" {
break
}
due, err := parseTimestamp(r.NextAttemptAt.String)
if err != nil {
t.Fatal(err)
}
if due.After(deadline) {
t.Fatalf("retry due %s, after the 24 h deadline %s", due, deadline)
}
c.set(due)
times = append(times, due)
if _, err := svc.deliverDue(ctx); err != nil {
t.Fatal(err)
}
}
// Attempts at 0, 1, 3, 7, 15, 31 and 63 minutes, then hourly to 1383
// minutes (22 more): 29 in all. The row is due once more at the
// deadline, where it is failed without being sent.
const wantAttempts = 29
r := onlyRow(t, st, key, ChannelPush)
if r.Status != "failed" || r.Attempts != wantAttempts || r.NextAttemptAt.Valid ||
r.LastError.String != "not delivered within 24 hours of creation; last error: push service unavailable" ||
r.UpdatedAt != timestamp(deadline) {
t.Fatalf("row = %+v, want failed at %s after %d attempts", r, timestamp(deadline), wantAttempts)
}
if len(times) != wantAttempts+1 || !times[len(times)-1].Equal(deadline) || !times[len(times)-2].Equal(start.Add(1383*time.Minute)) {
t.Fatalf("due at %v, want %d attempts ending 1383 min after creation, then the deadline", times, wantAttempts)
}
if n := len(sender.messages()); n != wantAttempts {
t.Fatalf("%d deliveries, want %d", n, wantAttempts)
}
// A failed row is never delivered again.
c.set(deadline.Add(48 * time.Hour))
if _, err := svc.deliverDue(ctx); err != nil {
t.Fatal(err)
}
if n := len(sender.messages()); n != wantAttempts {
t.Fatalf("%d deliveries, want %d", n, wantAttempts)
}
}
// TestOverdueRowIsNeverDelivered covers a row first claimed 24 hours or
// more after its creation, as when the service was down: it is failed
// without calling the sender, whether the sender would succeed or fail.
func TestOverdueRowIsNeverDelivered(t *testing.T) {
for _, tc := range []struct {
name string
at time.Duration
fail error
}{
{"succeeding sender at 24 h", 24 * time.Hour, nil},
{"succeeding sender after 24 h", 25 * time.Hour, nil},
{"failing sender at 24 h", 24 * time.Hour, errors.New("gone")},
{"failing sender after 24 h", 25 * time.Hour, errors.New("gone")},
} {
t.Run(tc.name, func(t *testing.T) {
ctx := context.Background()
svc, st, c := newTestService(t)
setPref(t, st, PlanReady, true, false)
sender := &fakeSender{fail: tc.fail}
if err := svc.Register(ChannelPush, sender); err != nil {
t.Fatal(err)
}
const key = "plan_ready:2026-10-12"
if err := svc.Emit(ctx, planReady(key)); err != nil {
t.Fatal(err)
}
c.set(start.Add(tc.at))
if _, err := svc.deliverDue(ctx); err != nil {
t.Fatal(err)
}
if n := len(sender.messages()); n != 0 {
t.Fatalf("%d deliveries of an overdue row, want 0", n)
}
r := onlyRow(t, st, key, ChannelPush)
if r.Status != "failed" || r.Attempts != 0 || r.NextAttemptAt.Valid ||
r.LastError.String != "not delivered within 24 hours of creation" || r.UpdatedAt != timestamp(start.Add(tc.at)) {
t.Fatalf("row = %+v, want failed without an attempt", r)
}
})
}
}
// TestRowJustBeforeDeadlineIsDelivered pins the boundary from the other
// side: a row claimed just under 24 hours after its creation is sent.
func TestRowJustBeforeDeadlineIsDelivered(t *testing.T) {
ctx := context.Background()
svc, st, c := newTestService(t)
setPref(t, st, PlanReady, true, false)
sender := &fakeSender{}
if err := svc.Register(ChannelPush, sender); err != nil {
t.Fatal(err)
}
const key = "plan_ready:2026-10-12"
if err := svc.Emit(ctx, planReady(key)); err != nil {
t.Fatal(err)
}
c.set(start.Add(24*time.Hour - time.Millisecond))
if _, err := svc.deliverDue(ctx); err != nil {
t.Fatal(err)
}
if r := onlyRow(t, st, key, ChannelPush); r.Status != "sent" || r.Attempts != 1 || len(sender.messages()) != 1 {
t.Fatalf("row = %+v after %d deliveries, want sent once", r, len(sender.messages()))
}
}
func TestChannelWithoutSenderFails(t *testing.T) {
ctx := context.Background()
svc, st, _ := newTestService(t)
setPref(t, st, PlanReady, true, true)
push := &fakeSender{}
if err := svc.Register(ChannelPush, push); err != nil {
t.Fatal(err)
}
const key = "plan_ready:2026-10-12"
if err := svc.Emit(ctx, planReady(key)); err != nil {
t.Fatal(err)
}
if _, err := svc.deliverDue(ctx); err != nil {
t.Fatal(err)
}
rows := outbox(t, st, key)
if r := rows[ChannelChat]; r.Status != "failed" || r.Attempts != 0 || r.LastError.String != "no sender registered for channel chat" {
t.Fatalf("chat row = %+v, want failed with no sender", r)
}
if r := rows[ChannelPush]; r.Status != "sent" {
t.Fatalf("push row = %+v, want sent", r)
}
}
func TestLeasedRowIsNotClaimedAgain(t *testing.T) {
ctx := context.Background()
svc, st, c := newTestService(t)
setPref(t, st, PlanReady, true, false)
sender := &fakeSender{}
if err := svc.Register(ChannelPush, sender); err != nil {
t.Fatal(err)
}
const key = "plan_ready:2026-10-12"
if err := svc.Emit(ctx, planReady(key)); err != nil {
t.Fatal(err)
}
// Another tick claims the row and stops before recording the delivery.
lease := start.Add(leaseFor)
if _, err := db.New(st.Writer).ClaimDueNotification(ctx, db.ClaimDueNotificationParams{
LeaseUntil: sql.NullString{String: timestamp(lease), Valid: true}, Now: timestamp(start),
}); err != nil {
t.Fatal(err)
}
c.set(lease.Add(-time.Millisecond))
if _, err := svc.deliverDue(ctx); err != nil {
t.Fatal(err)
}
if n := len(sender.messages()); n != 0 {
t.Fatalf("%d deliveries of a leased row, want 0", n)
}
// Once the lease ends the row is delivered.
c.set(lease)
if _, err := svc.deliverDue(ctx); err != nil {
t.Fatal(err)
}
if r := onlyRow(t, st, key, ChannelPush); r.Status != "sent" || r.Attempts != 1 || len(sender.messages()) != 1 {
t.Fatalf("row = %+v after %d deliveries, want sent once", r, len(sender.messages()))
}
}
// blockingSender holds every Send until release is closed.
type blockingSender struct {
fakeSender
entered chan struct{}
release chan struct{}
}
func (b *blockingSender) Send(ctx context.Context, m Message) error {
b.entered <- struct{}{}
<-b.release
return b.fakeSender.Send(ctx, m)
}
func TestConcurrentTicksDeliverOnce(t *testing.T) {
ctx := context.Background()
svc, st, _ := newTestService(t)
setPref(t, st, PlanReady, true, false)
sender := &blockingSender{entered: make(chan struct{}, 10), release: make(chan struct{})}
if err := svc.Register(ChannelPush, sender); err != nil {
t.Fatal(err)
}
const key = "plan_ready:2026-10-12"
if err := svc.Emit(ctx, planReady(key)); err != nil {
t.Fatal(err)
}
var wg sync.WaitGroup
first := make(chan error, 1)
wg.Go(func() {
_, err := svc.deliverDue(ctx)
first <- err
})
<-sender.entered // the first tick holds the row
if _, err := svc.deliverDue(ctx); err != nil {
t.Fatal(err)
}
close(sender.release)
wg.Wait()
if err := <-first; err != nil {
t.Fatal(err)
}
if n := len(sender.messages()); n != 1 {
t.Fatalf("%d deliveries, want 1", n)
}
if r := onlyRow(t, st, key, ChannelPush); r.Status != "sent" {
t.Fatalf("row = %+v, want sent", r)
}
}
func TestRetryDelay(t *testing.T) {
for attempt, want := range map[int64]time.Duration{
1: time.Minute, 2: 2 * time.Minute, 3: 4 * time.Minute, 6: 32 * time.Minute,
7: time.Hour, 8: time.Hour, 1000: time.Hour,
} {
if got := retryDelay(attempt); got != want {
t.Errorf("retryDelay(%d) = %v, want %v", attempt, got, want)
}
}
}
func TestRegister(t *testing.T) {
svc, _, _ := newTestService(t)
if err := svc.Register("email", &fakeSender{}); err == nil {
t.Error("registered an unknown channel")
}
if err := svc.Register(ChannelPush, nil); err == nil {
t.Error("registered a nil sender")
}
if err := svc.Register(ChannelPush, &fakeSender{}); err != nil {
t.Fatal(err)
}
if err := svc.Register(ChannelPush, &fakeSender{}); err == nil {
t.Error("registered a second push sender")
}
ctx, cancel := context.WithCancel(context.Background())
if err := svc.Start(ctx); err != nil {
t.Fatal(err)
}
if err := svc.Register(ChannelChat, &fakeSender{}); err == nil {
t.Error("registered a sender after Start")
}
if err := svc.Start(ctx); err == nil {
t.Error("started twice")
}
cancel()
svc.Wait()
}
// TestWorkerDeliversEmittedEvent runs the worker on the real clock: Emit
// wakes it, it delivers, and it stops when its context is cancelled.
func TestWorkerDeliversEmittedEvent(t *testing.T) {
svc, st, _ := newTestService(t)
svc.now = time.Now
setPref(t, st, PlanReady, true, false)
sender := &fakeSender{}
if err := svc.Register(ChannelPush, sender); err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
if err := svc.Start(ctx); err != nil {
t.Fatal(err)
}
const key = "plan_ready:2026-10-12"
if err := svc.Emit(context.Background(), planReady(key)); err != nil {
t.Fatal(err)
}
deadline := time.Now().Add(5 * time.Second)
for onlyRow(t, st, key, ChannelPush).Status != "sent" {
if time.Now().After(deadline) {
t.Fatal("worker did not deliver the notification")
}
time.Sleep(10 * time.Millisecond)
}
cancel()
svc.Wait()
if n := len(sender.messages()); n != 1 {
t.Fatalf("%d deliveries, want 1", n)
}
}
+299
View File
@@ -0,0 +1,299 @@
package notify
import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"log/slog"
"time"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/db"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/jobs"
)
// Delivery timing.
const (
// retryFirst is the wait after a row's first failed delivery; each
// further failure doubles it, up to retryMax.
retryFirst = time.Minute
retryMax = time.Hour
// giveUpAfter is how long after its creation a row may be delivered.
// The last retry is due at that moment; a row claimed then or later is
// marked failed without being sent, and a delivery failing at or
// after it marks the row failed.
giveUpAfter = 24 * time.Hour
// sendTimeout bounds one Send.
sendTimeout = 30 * time.Second
// leaseFor is how long a claimed row is not due again. It outlasts
// sendTimeout and the final write, so a row is only claimed again when
// the process that claimed it stopped before recording the delivery.
leaseFor = 2 * time.Minute
// pollRetryMin and pollRetryMax bound the backoff after a database
// error while claiming rows.
pollRetryMin = time.Second
pollRetryMax = time.Minute
)
// Message is one notification for a Sender to deliver.
type Message struct {
// ID is the notification_outbox row id.
ID string
// EventKey is the event's idempotency key.
EventKey string
Channel Channel
Payload Payload
// Attempt is this delivery's number, from 1.
Attempt int64
}
// Sender delivers notifications on one channel. Send returns nil once the
// channel has accepted m, and an error otherwise; the worker retries it.
// The error's text is stored in notification_outbox.last_error and logged,
// so it must never carry a secret: no webhook URL, key or token (a
// *url.Error quotes its URL).
type Sender interface {
Send(ctx context.Context, m Message) error
}
// Register makes sender the Sender of ch. It must be called before Start,
// once per channel.
func (s *Service) Register(ch Channel, sender Sender) error {
if ch != ChannelPush && ch != ChannelChat {
return fmt.Errorf("notify: register: unknown channel %q", ch)
}
if sender == nil {
return fmt.Errorf("notify: register %s: nil sender", ch)
}
s.mu.Lock()
defer s.mu.Unlock()
if s.started {
return fmt.Errorf("notify: register %s: worker already started", ch)
}
if _, ok := s.senders[ch]; ok {
return fmt.Errorf("notify: register %s: channel already has a sender", ch)
}
s.senders[ch] = sender
return nil
}
// Start starts the worker. It stops when ctx is cancelled; Wait waits for
// it. A row whose delivery is cut short by ctx keeps its lease and is
// delivered again once the lease ends.
func (s *Service) Start(ctx context.Context) error {
s.mu.Lock()
defer s.mu.Unlock()
if s.started {
return errors.New("notify: worker already started")
}
s.started = true
s.wg.Add(1)
go s.work(ctx)
return nil
}
// Wait blocks until the worker has stopped.
func (s *Service) Wait() {
s.wg.Wait()
}
// wakeWorker gives the worker a token to look for due rows.
func (s *Service) wakeWorker() {
select {
case s.wake <- struct{}{}:
default: // the worker already has a token and will look again
}
}
// work delivers due rows until ctx is cancelled, then sleeps until the next
// pending row is due or Emit stores one. A database error is retried with
// backoff.
func (s *Service) work(ctx context.Context) {
defer s.wg.Done()
backoff := pollRetryMin
for ctx.Err() == nil {
wait, err := s.deliverDue(ctx)
if err != nil {
if ctx.Err() != nil {
return
}
s.logger.ErrorContext(ctx, "deliver notifications failed; retrying", "error", err, "retry_in", backoff)
wait, backoff = backoff, min(2*backoff, pollRetryMax)
} else {
backoff = pollRetryMin
}
var timeout <-chan time.Time
var timer *time.Timer
if wait >= 0 {
timer = time.NewTimer(wait)
timeout = timer.C
}
select {
case <-s.wake:
case <-timeout:
case <-ctx.Done():
}
if timer != nil {
timer.Stop()
}
}
}
// deliverDue delivers every row due now and returns how long until the next
// pending row is due, or -1 when none is pending.
func (s *Service) deliverDue(ctx context.Context) (time.Duration, error) {
q := db.New(s.st.Writer)
for ctx.Err() == nil {
now := s.now()
lease := sql.NullString{String: timestamp(now.Add(leaseFor)), Valid: true}
row, err := q.ClaimDueNotification(ctx, db.ClaimDueNotificationParams{LeaseUntil: lease, Now: timestamp(now)})
if errors.Is(err, sql.ErrNoRows) {
break
}
if err != nil {
return 0, fmt.Errorf("notify: claim notification: %w", err)
}
s.deliver(ctx, row)
}
if err := ctx.Err(); err != nil {
return 0, err
}
next, err := db.New(s.st.Reader).NextNotificationDue(ctx)
if errors.Is(err, sql.ErrNoRows) {
return -1, nil
}
if err != nil {
return 0, fmt.Errorf("notify: read next due notification: %w", err)
}
due, err := parseTimestamp(next.String)
if err != nil {
return 0, fmt.Errorf("notify: next due notification: %w", err)
}
return max(due.Sub(s.now()), 0), nil
}
// deliver sends a claimed row through its channel's sender and records the
// outcome. A row that cannot be sent at all, because its payload does not
// decode or its channel has no sender, is failed at once: retrying cannot
// help. A row 24 hours or more past its creation is failed without being
// sent: it is too stale to deliver.
func (s *Service) deliver(ctx context.Context, row db.NotificationOutbox) {
logger := s.logger.With("notification_id", row.ID, "event_key", row.EventKey, "channel", row.Channel)
var payload Payload
if err := json.Unmarshal([]byte(row.Payload), &payload); err != nil {
logger.ErrorContext(ctx, "notification failed", "reason", "payload does not decode")
s.giveUp(ctx, logger, row, 0, "payload does not decode")
return
}
s.mu.Lock()
sender, ok := s.senders[Channel(row.Channel)]
s.mu.Unlock()
if !ok {
reason := fmt.Sprintf("no sender registered for channel %s", row.Channel)
logger.ErrorContext(ctx, "notification failed", "reason", reason)
s.giveUp(ctx, logger, row, 0, reason)
return
}
created, err := parseTimestamp(row.CreatedAt)
if err != nil {
reason := fmt.Sprintf("created_at does not parse: %v", err)
logger.ErrorContext(ctx, "notification failed", "reason", reason)
s.giveUp(ctx, logger, row, 0, reason)
return
}
deadline := created.Add(giveUpAfter)
if !s.now().Before(deadline) {
reason := "not delivered within 24 hours of creation"
if row.LastError.Valid {
reason += "; last error: " + row.LastError.String
}
logger.ErrorContext(ctx, "notification failed; giving up", "reason", reason)
s.giveUp(ctx, logger, row, 0, reason)
return
}
attempt := row.Attempts + 1
sendCtx, cancel := context.WithTimeout(ctx, sendTimeout)
err = sender.Send(sendCtx, Message{
ID: row.ID, EventKey: row.EventKey, Channel: Channel(row.Channel), Payload: payload, Attempt: attempt,
})
cancel()
now := s.now()
if err == nil {
s.record(ctx, logger, "record notification sent", func(write context.Context) (int64, error) {
return db.New(s.st.Writer).MarkNotificationSent(write, db.MarkNotificationSentParams{
Now: timestamp(now), ID: row.ID, LeaseUntil: row.NextAttemptAt,
})
})
logger.InfoContext(ctx, "notification sent", "attempt", attempt)
return
}
if ctx.Err() != nil {
logger.WarnContext(context.WithoutCancel(ctx), "notification delivery interrupted by shutdown; it is retried when its lease ends")
return
}
if !now.Before(deadline) {
logger.ErrorContext(ctx, "notification failed; giving up", "attempt", attempt, "error", err)
s.giveUp(ctx, logger, row, 1, err.Error())
return
}
next := now.Add(retryDelay(attempt))
if next.After(deadline) {
next = deadline
}
logger.WarnContext(ctx, "notification delivery failed; retrying", "attempt", attempt, "error", err, "retry_at", timestamp(next))
s.record(ctx, logger, "record notification retry", func(write context.Context) (int64, error) {
return db.New(s.st.Writer).RetryNotification(write, db.RetryNotificationParams{
NextAttemptAt: sql.NullString{String: timestamp(next), Valid: true},
LastError: sql.NullString{String: err.Error(), Valid: true},
Now: timestamp(now),
ID: row.ID,
LeaseUntil: row.NextAttemptAt,
})
})
}
// giveUp marks a claimed row failed with reason. attempted is 1 when a
// delivery was attempted, else 0.
func (s *Service) giveUp(ctx context.Context, logger *slog.Logger, row db.NotificationOutbox, attempted int64, reason string) {
now := s.now()
s.record(ctx, logger, "record notification failed", func(write context.Context) (int64, error) {
return db.New(s.st.Writer).FailNotification(write, db.FailNotificationParams{
Attempted: attempted,
LastError: sql.NullString{String: reason, Valid: true},
Now: timestamp(now),
ID: row.ID,
LeaseUntil: row.NextAttemptAt,
})
})
}
// record makes a row's final write for one delivery, even when ctx has been
// cancelled, bounded by jobs.FinalWriteTimeout. A write that fails, or that
// finds the row no longer leased to this claim, is logged; the row keeps
// its lease and is delivered again once the lease ends.
func (s *Service) record(ctx context.Context, logger *slog.Logger, what string, write func(context.Context) (int64, error)) {
attempt, cancel := jobs.FinalWriteContext(ctx)
defer cancel()
n, err := write(attempt)
if err != nil {
logger.ErrorContext(context.WithoutCancel(ctx), what+" failed; it is delivered again when its lease ends", "error", err)
return
}
if n != 1 {
logger.ErrorContext(context.WithoutCancel(ctx), what, "error", "notification is no longer leased to this delivery")
}
}
// retryDelay is the wait after the attempt-th failed delivery: retryFirst,
// doubling with each failure, at most retryMax.
func retryDelay(attempt int64) time.Duration {
d := retryFirst
for i := int64(1); i < attempt && d < retryMax; i++ {
d *= 2
}
return min(d, retryMax)
}
+58
View File
@@ -0,0 +1,58 @@
-- name: GetNotificationPref :one
SELECT * FROM notification_prefs
WHERE event = ?;
-- name: InsertNotification :execrows
-- Stores a pending notification for one channel, due at once. A row already
-- stored for the event key and channel, in any status, is left unchanged and
-- no row is inserted.
INSERT INTO notification_outbox (
id, event_key, channel, payload, status, attempts, next_attempt_at, created_at, updated_at
) VALUES (
sqlc.arg(id), sqlc.arg(event_key), sqlc.arg(channel), sqlc.arg(payload), 'pending', 0, sqlc.arg(next_attempt_at), sqlc.arg(now), sqlc.arg(now)
)
ON CONFLICT (event_key, channel) DO NOTHING;
-- name: ClaimDueNotification :one
-- Leases the pending notification due longest ago by moving its
-- next_attempt_at to lease_until, and returns it. A claimed row is not due
-- again until the lease ends, so no other claim takes it while it is being
-- delivered. Returns sql.ErrNoRows when none is due.
UPDATE notification_outbox
SET next_attempt_at = sqlc.arg(lease_until), updated_at = sqlc.arg(now)
WHERE id = (
SELECT id FROM notification_outbox
WHERE status = 'pending' AND next_attempt_at <= sqlc.arg(now)
ORDER BY next_attempt_at, id
LIMIT 1
)
RETURNING *;
-- name: MarkNotificationSent :execrows
-- Records a delivered notification; updated_at is when it was sent. The
-- lease must still be the claim's.
UPDATE notification_outbox
SET status = 'sent', attempts = attempts + 1, next_attempt_at = NULL, updated_at = sqlc.arg(now)
WHERE id = sqlc.arg(id) AND status = 'pending' AND next_attempt_at = sqlc.arg(lease_until);
-- name: RetryNotification :execrows
-- Records a failed delivery and when to try again. The lease must still be
-- the claim's.
UPDATE notification_outbox
SET attempts = attempts + 1, next_attempt_at = sqlc.arg(next_attempt_at), last_error = sqlc.arg(last_error), updated_at = sqlc.arg(now)
WHERE id = sqlc.arg(id) AND status = 'pending' AND next_attempt_at = sqlc.arg(lease_until);
-- name: FailNotification :execrows
-- Gives up on a notification. attempts counts the delivery attempt that
-- failed, if one was made. The lease must still be the claim's.
UPDATE notification_outbox
SET status = 'failed', attempts = attempts + sqlc.arg(attempted), next_attempt_at = NULL, last_error = sqlc.arg(last_error), updated_at = sqlc.arg(now)
WHERE id = sqlc.arg(id) AND status = 'pending' AND next_attempt_at = sqlc.arg(lease_until);
-- name: NextNotificationDue :one
-- Returns when the next pending notification is due. Returns sql.ErrNoRows
-- when none is pending.
SELECT next_attempt_at FROM notification_outbox
WHERE status = 'pending'
ORDER BY next_attempt_at
LIMIT 1;