Retry job claims after a database error instead of waiting for a wake
A claim that failed with anything but sql.ErrNoRows parked the worker on the wake channel, so a job queued across a restart (which has no wake token) stayed queued until another enqueue or restart. The worker now retries such a claim with backoff from 50ms doubling to 5s, reset on any successful or empty claim, and stops promptly on shutdown. The claim error is logged by category only. The claim query is a Runner field so a test can inject one failing claim; TestClaimErrorIsRetried drops the wake token and asserts the queued job still runs. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -3,6 +3,7 @@ package jobs
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -14,6 +15,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/db"
|
||||
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/store"
|
||||
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/migrations"
|
||||
)
|
||||
@@ -528,3 +530,41 @@ func TestNoopSucceeds(t *testing.T) {
|
||||
t.Errorf("noop job = %+v (result %s), want succeeded with {\"ok\":true}", job, job.Result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClaimErrorIsRetried(t *testing.T) {
|
||||
st := migratedStore(t)
|
||||
logs := &logBuffer{}
|
||||
r := newRunnerLogging(t, st, 1, logs)
|
||||
k := register(t, r, "count", func(ctx context.Context, p countPayload, publish func(countResult) error) (countResult, error) {
|
||||
return countResult{Done: p.Steps}, nil
|
||||
})
|
||||
id := enqueue(t, k, countPayload{Steps: 3})
|
||||
// Drop the wake token, as for a job queued before a restart, so only
|
||||
// the retry can find the job.
|
||||
<-r.wake
|
||||
|
||||
var claims atomic.Int32
|
||||
claim := r.claim
|
||||
r.claim = func(ctx context.Context, now sql.NullString) (db.Job, error) {
|
||||
if claims.Add(1) == 1 {
|
||||
return db.Job{}, errors.New("database is locked " + sentinelSecret)
|
||||
}
|
||||
return claim(ctx, now)
|
||||
}
|
||||
start(t, r)
|
||||
|
||||
job := waitFor(t, r, id, finished)
|
||||
if job.Status != StatusSucceeded || string(job.Result) != `{"done":3}` {
|
||||
t.Fatalf("job = %+v (result %s), want succeeded with {\"done\":3}", job, job.Result)
|
||||
}
|
||||
if n := claims.Load(); n < 2 {
|
||||
t.Fatalf("claims = %d, want a retry after the error", n)
|
||||
}
|
||||
out := logs.String()
|
||||
if !strings.Contains(out, "claim job failed; retrying") {
|
||||
t.Fatalf("claim error not logged: %s", out)
|
||||
}
|
||||
if strings.Contains(out, sentinelSecret) {
|
||||
t.Fatalf("claim error text logged: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user