Files
MealQuest/internal/scheduler/scheduler.go
T
dev_agentandClaude Opus 5.5 bcd875c11f
CI / ci (push) Successful in 7m29s
CI / ci (pull_request) Successful in 7m44s
Retry a failed claim for its own period before later fire times
A failed claim was retried by recomputing the most recent fire time from
the new now, so a daily claim that failed on 7 October and recovered after
the 8 October fire claimed only 8 October: the 7 October period was never
claimed nor recorded as missed. The scheduler now keeps each failed
claim's fire time and claims that same period every ClaimRetry until it
succeeds, applies the catch-up rule to it then, and only afterwards
handles later fire times and settings changes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-09 19:02:09 -04:00

588 lines
18 KiB
Go

// Package scheduler runs the household's timed work: each registered
// schedule fires once per period at a household-local time read from the
// household settings.
//
// # Periods and the claim
//
// Every fire time belongs to one period, named by a period key derived in
// the household zone from the schedule's cadence (see Rule.PeriodKey): the
// local date "2026-10-09" for a daily rule, and the ISO week "2026-W41" of
// the fire date, or of the week after it, for a weekly rule. Before running,
// the scheduler inserts schedule_runs(schedule, period_key); the insert is
// the claim, and a period already claimed, by an earlier run, an earlier
// process or another instance, is skipped. A period therefore runs at most
// once: a Run that fails is logged and its period is not retried.
//
// # Catch-up
//
// When the scheduler starts, and whenever it wakes late, it looks only at
// each schedule's most recent fire time. If that time passed within
// CatchUpWindow and the period is unclaimed, the period runs once. If it
// passed longer ago, the period is claimed without running and logged as
// missed, so it can never run later, not after a restart and not after a
// settings change moves the fire time. Fire times before the most recent
// one are never looked at.
//
// # Claim failures
//
// A claim that fails (the database refused the insert) is never skipped:
// the scheduler logs it and keeps that period's fire time, and every
// ClaimRetry claims that same period again until the claim succeeds. Only
// then does it apply the catch-up rule to it, running it if its fire time
// is within CatchUpWindow of the successful claim and recording it as
// missed otherwise, and only then does it go on to later fire times, even
// when one has passed meanwhile. Each period is claimed by its own fire
// time, so the later period runs once as well.
//
// # Time zone and DST
//
// Fire times are wall-clock times in the household zone, never the process
// zone, turned into instants by household.LocalTime. Its rule is the
// scheduler's DST rule: a time that does not exist on a spring-forward day
// fires later by the length of the gap (02:30 fires at 03:30 local), and a
// time that occurs twice on a fall-back day fires once, at its first
// occurrence. Either way the period key is that local date's, so the
// period still runs exactly once.
//
// # Settings changes
//
// The scheduler subscribes to the household settings and records each change
// with the instant it was saved. A change takes effect at once without a
// restart: fire times up to that instant follow the old settings and later
// ones the new, even when the scheduler gets to the change late. So moving
// a fire time to earlier the same day does not fire the passed time, and a
// new fire time that was still ahead when the change was saved fires.
package scheduler
import (
"context"
"errors"
"fmt"
"log/slog"
"sync"
"time"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/db"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/household"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/store"
)
// Schedule names, as the schedule_runs.schedule CHECK constraint allows.
const (
SuggestionsWeekly = "suggestions_weekly"
PlanDraft = "plan_draft"
ShoppingChatAutosend = "shopping_chat_autosend"
DayOfReminder = "day_of_reminder"
RotationCheck = "rotation_check"
WeekRollover = "week_rollover"
AIRunsPurge = "ai_runs_purge"
)
// names are the schedule names Register accepts.
var names = map[string]bool{
SuggestionsWeekly: true,
PlanDraft: true,
ShoppingChatAutosend: true,
DayOfReminder: true,
RotationCheck: true,
WeekRollover: true,
AIRunsPurge: true,
}
// CatchUpWindow is how long after its fire time a period that has not run
// still runs. A period found later than that is recorded as missed.
const CatchUpWindow = 24 * time.Hour
// ClaimRetry is how long the scheduler waits before repeating a pass in
// which a claim failed.
const ClaimRetry = time.Minute
// maxWait bounds one sleep, so the scheduler checks the wall clock at least
// this often even if the host was suspended or its clock stepped.
const maxWait = time.Hour
// Rule is when a schedule fires: at household-local Time ("HH:MM") every
// day, or on Weekday only when Weekly is set. Build one with Daily, Weekly
// or WeeklyForNextWeek.
type Rule struct {
Weekly bool
Weekday time.Weekday
Time string
// NextWeek keys a weekly period by the ISO week after the fire date,
// for work done ahead for the coming week.
NextWeek bool
}
// Daily fires every day at hhmm; its period is the local date.
func Daily(hhmm string) Rule {
return Rule{Time: hhmm}
}
// Weekly fires on day at hhmm; its period is the ISO week of the fire date.
func Weekly(day time.Weekday, hhmm string) Rule {
return Rule{Weekly: true, Weekday: day, Time: hhmm}
}
// WeeklyForNextWeek fires on day at hhmm; its period is the ISO week after
// the fire date's.
func WeeklyForNextWeek(day time.Weekday, hhmm string) Rule {
return Rule{Weekly: true, Weekday: day, Time: hhmm, NextWeek: true}
}
// PeriodKey returns the period key of a fire on household-local date
// fireDate: the date ("2026-10-09") for a daily rule, the ISO week
// ("2026-W53") for a weekly one, and the ISO week after that ("2027-W01")
// when NextWeek is set. It panics if fireDate is the zero LocalDate.
func (r Rule) PeriodKey(fireDate store.LocalDate) string {
if !r.Weekly {
return fireDate.String()
}
if r.NextWeek {
return household.PeriodKey(addDays(fireDate, 7))
}
return household.PeriodKey(fireDate)
}
// validate reports whether r can fire.
func (r Rule) validate() error {
if r.Weekly && (r.Weekday < time.Sunday || r.Weekday > time.Saturday) {
return fmt.Errorf("invalid weekday %d (want 0-6, Sunday = 0)", r.Weekday)
}
if !r.Weekly && (r.Weekday != time.Sunday || r.NextWeek) {
return errors.New("a daily rule has no weekday or next week")
}
// Any date checks the time's form.
_, err := household.LocalTime(time.UTC, store.LocalDateOf(time.Unix(0, 0).UTC()), r.Time)
return err
}
// Period is one period of a schedule, as given to its Run.
type Period struct {
// Key is the period key claimed in schedule_runs.
Key string
// FireAt is the instant the period fired, in the household zone.
FireAt time.Time
// Date is the household-local date of FireAt.
Date store.LocalDate
// Week is the ISO Monday of the week Key names; for a daily rule, of
// the week containing Date.
Week store.LocalDate
}
// Schedule is registered work that runs once per period.
type Schedule struct {
// Name is one of the schedule name constants.
Name string
// Rule returns when the schedule fires under settings. It is called
// again after every settings change.
Rule func(settings household.Settings) Rule
// Run does the period's work. It runs on the scheduler's goroutine,
// one schedule at a time, so it must return promptly: long work is
// enqueued as a job. ctx is cancelled when the scheduler stops. An
// error is logged; the period stays claimed and is not retried.
Run func(ctx context.Context, p Period) error
}
// Settings is the household settings source: *household.SettingsService.
type Settings interface {
Get(ctx context.Context) (household.Settings, error)
Subscribe(fn func(household.Settings)) (unsubscribe func())
}
// Scheduler runs the registered schedules.
type Scheduler struct {
st *store.Store
settings Settings
logger *slog.Logger
// now is the clock and after the timer; tests replace both.
now func() time.Time
after func(d time.Duration) <-chan time.Time
// wake holds one token when the settings changed since the loop last
// looked; pending is the changes to apply, oldest first.
wake chan struct{}
pendingMu sync.Mutex
pending []change
mu sync.Mutex
schedules []Schedule
started bool
wg sync.WaitGroup
}
// New returns a scheduler over st reading the household settings from
// settings.
func New(st *store.Store, settings Settings, logger *slog.Logger) (*Scheduler, error) {
if st == nil {
return nil, errors.New("scheduler: nil store")
}
if settings == nil {
return nil, errors.New("scheduler: nil settings")
}
if logger == nil {
return nil, errors.New("scheduler: nil logger")
}
return &Scheduler{
st: st,
settings: settings,
logger: logger,
now: time.Now,
after: time.After,
wake: make(chan struct{}, 1),
}, nil
}
// Register adds sch. It must be called before Start, once per name.
func (s *Scheduler) Register(sch Schedule) error {
if !names[sch.Name] {
return fmt.Errorf("scheduler: register: unknown schedule %q", sch.Name)
}
if sch.Rule == nil {
return fmt.Errorf("scheduler: register %s: nil rule", sch.Name)
}
if sch.Run == nil {
return fmt.Errorf("scheduler: register %s: nil run", sch.Name)
}
s.mu.Lock()
defer s.mu.Unlock()
if s.started {
return fmt.Errorf("scheduler: register %s: already started", sch.Name)
}
for _, have := range s.schedules {
if have.Name == sch.Name {
return fmt.Errorf("scheduler: register %s: already registered", sch.Name)
}
}
s.schedules = append(s.schedules, sch)
return nil
}
// Start reads the settings and starts the scheduler, which first catches
// up on each schedule's most recent period and then fires schedules as
// their times come. Settings whose zone or rules are invalid are an error
// and nothing starts. The scheduler stops when ctx is cancelled; Wait waits
// for it.
func (s *Scheduler) Start(ctx context.Context) error {
s.mu.Lock()
defer s.mu.Unlock()
if s.started {
return errors.New("scheduler: already started")
}
// Subscribing first means a change saved after the read below still
// wakes the loop.
unsubscribe := s.settings.Subscribe(s.changed)
settings, err := s.settings.Get(ctx)
if err != nil {
unsubscribe()
return fmt.Errorf("scheduler: %w", err)
}
p, err := compile(s.schedules, settings)
if err != nil {
unsubscribe()
return err
}
s.started = true
s.wg.Add(1)
go func() {
defer s.wg.Done()
defer unsubscribe()
s.loop(ctx, p)
}()
return nil
}
// Wait blocks until the scheduler has stopped.
func (s *Scheduler) Wait() {
s.wg.Wait()
}
// change is a settings change and the instant it was saved.
type change struct {
at time.Time
settings household.Settings
}
// changed is the settings subscriber. It runs on the updating goroutine, so
// it only records the change and wakes the loop.
func (s *Scheduler) changed(settings household.Settings) {
s.pendingMu.Lock()
s.pending = append(s.pending, change{at: s.now(), settings: settings})
s.pendingMu.Unlock()
select {
case s.wake <- struct{}{}:
default: // the loop already has a token and reads pending then
}
}
// takeChanges returns and clears the pending changes.
func (s *Scheduler) takeChanges() []change {
s.pendingMu.Lock()
defer s.pendingMu.Unlock()
out := s.pending
s.pending = nil
return out
}
// plan is the schedules compiled against one version of the settings.
type plan struct {
loc *time.Location
items []item
}
type item struct {
sch Schedule
rule Rule
}
// compile builds the plan for settings.
func compile(schedules []Schedule, settings household.Settings) (*plan, error) {
loc, err := household.LoadZone(settings.Timezone)
if err != nil {
return nil, fmt.Errorf("scheduler: household zone: %w", err)
}
p := &plan{loc: loc, items: make([]item, len(schedules))}
for i, sch := range schedules {
rule := sch.Rule(settings)
if err := rule.validate(); err != nil {
return nil, fmt.Errorf("scheduler: %s: %w", sch.Name, err)
}
p.items[i] = item{sch: sch, rule: rule}
}
return p, nil
}
// loop fires schedules until ctx is cancelled. done is the instant up to
// which fire times have been handled; it starts at the zero time so the
// first pass catches up. retry is the periods whose claim failed, each
// with its own fire time. Each pass first claims retry again; while any
// claim there still fails, the pass stops and is repeated after
// ClaimRetry. Otherwise it applies the pending settings changes in order,
// handling fire times up to a change under the plan it replaces, then
// handles fire times up to now under the current plan, stopping after the
// first stretch in which a claim failed.
func (s *Scheduler) loop(ctx context.Context, p *plan) {
var done time.Time
var pending []change
var retry []target
for {
now := s.now()
pending = append(pending, s.takeChanges()...)
retry = s.retry(ctx, retry, now)
for len(retry) == 0 && len(pending) > 0 {
c := pending[0]
if c.at.After(done) {
if p != nil {
retry = s.fire(ctx, p, done, c.at, now)
}
done = c.at
}
p = s.compileChange(ctx, c.settings)
pending = pending[1:]
}
if len(retry) == 0 {
if p != nil {
retry = s.fire(ctx, p, done, now, now)
}
done = now
}
if ctx.Err() != nil {
return
}
var timeout <-chan time.Time
switch {
case len(retry) > 0:
timeout = s.after(ClaimRetry)
case p != nil:
if next, ok := p.next(now); ok {
timeout = s.after(min(next.Sub(now), maxWait))
}
}
select {
case <-ctx.Done():
return
case <-timeout:
case <-s.wake:
}
}
}
// compileChange returns the plan for changed settings, or nil when they
// are unusable.
func (s *Scheduler) compileChange(ctx context.Context, settings household.Settings) *plan {
p, err := compile(s.schedules, settings)
if err != nil {
// Settings are validated before they are saved, so this is a
// bug; firing at a guessed time would be worse than not firing
// until the settings are fixed.
s.logger.ErrorContext(ctx, "scheduler: settings unusable; no schedule fires until they change", "error", err)
return nil
}
return p
}
// target is one schedule's period, named by the fire time it belongs to.
type target struct {
it item
loc *time.Location
at time.Time
}
// fire handles, for every schedule, its most recent fire time at or before
// until if it is after done; now is the current instant. It returns the
// targets whose claim failed, to be claimed again.
func (s *Scheduler) fire(ctx context.Context, p *plan, done, until, now time.Time) []target {
var failed []target
for _, it := range p.items {
if ctx.Err() != nil {
return failed
}
at, due := fireBefore(it.rule, p.loc, until)
if !due || !at.After(done) {
continue
}
if t := (target{it: it, loc: p.loc, at: at}); !s.handle(ctx, t, now) {
failed = append(failed, t)
}
}
return failed
}
// retry handles targets again at now and returns those whose claim still
// failed.
func (s *Scheduler) retry(ctx context.Context, targets []target, now time.Time) []target {
var failed []target
for i, t := range targets {
if ctx.Err() != nil {
return append(failed, targets[i:]...)
}
if !s.handle(ctx, t, now) {
failed = append(failed, t)
}
}
return failed
}
// handle claims the period of t and runs it, or records it as missed when
// its fire time is more than CatchUpWindow before now. It reports false
// when the claim failed and the period is still unclaimed.
func (s *Scheduler) handle(ctx context.Context, t target, now time.Time) bool {
it, at := t.it, t.at
date := household.Today(at, t.loc)
period := Period{Key: it.rule.PeriodKey(date), FireAt: at, Date: date}
if it.rule.NextWeek {
period.Week = household.WeekStart(addDays(date, 7))
} else {
period.Week = household.WeekStart(date)
}
log := s.logger.With("schedule", it.sch.Name, "period", period.Key, "fire_at", at.Format(time.RFC3339))
claimed, err := s.claim(ctx, it.sch.Name, period.Key, now)
if err != nil {
log.ErrorContext(ctx, "scheduler: claim period failed; retrying", "error", err, "retry_in", ClaimRetry.String())
return false
}
if !claimed {
return true
}
if late := now.Sub(at); late > CatchUpWindow {
log.WarnContext(ctx, "scheduler: period missed; recorded without running", "late", late.String())
return true
}
if err := it.sch.Run(ctx, period); err != nil {
log.ErrorContext(ctx, "scheduler: run failed", "error", err)
return true
}
log.InfoContext(ctx, "scheduler: ran")
return true
}
// claim inserts schedule_runs(name, key) and reports whether this call
// inserted it.
func (s *Scheduler) claim(ctx context.Context, name, key string, now time.Time) (bool, error) {
n, err := db.New(s.st.Writer).ClaimScheduleRun(ctx, db.ClaimScheduleRunParams{
ID: store.NewID(),
Schedule: name,
PeriodKey: key,
CreatedAt: store.Timestamp{Time: now.UTC().Truncate(time.Millisecond)}.String(),
})
if err != nil {
return false, err
}
return n == 1, nil
}
// next returns the earliest fire time of any schedule after now, or false
// when there is no schedule.
func (p *plan) next(now time.Time) (time.Time, bool) {
var next time.Time
for _, it := range p.items {
at, ok := fireAfter(it.rule, p.loc, now)
if ok && (next.IsZero() || at.Before(next)) {
next = at
}
}
return next, !next.IsZero()
}
// window is how many days either side of a date fireBefore and fireAfter
// search: more than a week, so a weekly rule always has a fire time on
// each side.
const window = 8
// fireBefore returns the latest fire time of r in zone loc at or before t.
func fireBefore(r Rule, loc *time.Location, t time.Time) (time.Time, bool) {
var best time.Time
for _, at := range fireTimes(r, loc, t) {
if !at.After(t) && (best.IsZero() || at.After(best)) {
best = at
}
}
return best, !best.IsZero()
}
// fireAfter returns the earliest fire time of r in zone loc after t.
func fireAfter(r Rule, loc *time.Location, t time.Time) (time.Time, bool) {
var best time.Time
for _, at := range fireTimes(r, loc, t) {
if at.After(t) && (best.IsZero() || at.Before(best)) {
best = at
}
}
return best, !best.IsZero()
}
// fireTimes returns r's fire times on the local dates within window days
// of t's. A rule that fails to resolve was validated at compile, so it
// cannot.
func fireTimes(r Rule, loc *time.Location, t time.Time) []time.Time {
today := household.Today(t, loc)
var out []time.Time
for i := -window; i <= window; i++ {
d := addDays(today, i)
if r.Weekly && weekday(d) != r.Weekday {
continue
}
at, err := household.LocalTime(loc, d, r.Time)
if err != nil {
panic(fmt.Sprintf("scheduler: validated rule %+v: %v", r, err))
}
out = append(out, at)
}
return out
}
// addDays returns d plus n calendar days.
func addDays(d store.LocalDate, n int) store.LocalDate {
y, m, day := d.Date()
return store.LocalDateOf(time.Date(y, m, day+n, 0, 0, 0, 0, time.UTC))
}
// weekday returns the day of the week of d.
func weekday(d store.LocalDate) time.Weekday {
y, m, day := d.Date()
return time.Date(y, m, day, 0, 0, 0, 0, time.UTC).Weekday()
}