Retry a job's final status write until it lands
CI / ci (push) Successful in 2m58s
CI / ci (pull_request) Successful in 2m39s

A worker whose final SucceedJob or FailJob write failed logged it and
moved on, leaving the job running. For a suggestion batch whose own end
write had also failed, both rows stayed running, so orphan reconciliation
kept treating the batch as active and it blocked manual and replenish
batches until a restart. The runner now retries the final write with the
claim backoff until it lands; on shutdown a failed attempt is left for
the next Start to fail as interrupted.

Document that orphaned batches are reconciled each time a batch is
started, before the active batch is selected, not at boot.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-07 00:12:52 -04:00
co-authored by Claude Opus 5.5
parent a6643aaa94
commit d5cd47a775
4 changed files with 174 additions and 37 deletions
+33
View File
@@ -609,6 +609,39 @@ func TestClaimErrorIsRetried(t *testing.T) {
}
}
func TestFinalWriteErrorIsRetried(t *testing.T) {
st := migratedStore(t)
logs := &logBuffer{}
r := newRunnerLogging(t, st, 1, logs)
k := register(t, r, "count", func(ctx context.Context, p countPayload, publish func(countResult) error) (countResult, error) {
return countResult{Done: p.Steps}, nil
})
if _, err := st.Writer.ExecContext(t.Context(), `CREATE TRIGGER refuse_end BEFORE UPDATE OF status ON jobs WHEN NEW.status IN ('succeeded', 'failed') BEGIN SELECT RAISE(ABORT, 'end refused'); END`); err != nil {
t.Fatal(err)
}
start(t, r)
id := enqueue(t, k, countPayload{Steps: 3})
deadline := time.Now().Add(5 * time.Second)
for !strings.Contains(logs.String(), "record job success") {
if time.Now().After(deadline) {
t.Fatal("job success never recorded")
}
time.Sleep(5 * time.Millisecond)
}
if _, err := st.Writer.ExecContext(t.Context(), `DROP TRIGGER refuse_end`); err != nil {
t.Fatal(err)
}
job := waitFor(t, r, id, finished)
if job.Status != StatusSucceeded || string(job.Result) != `{"done":3}` {
t.Fatalf("job = %+v (result %s), want succeeded with {\"done\":3}", job, job.Result)
}
if out := logs.String(); !strings.Contains(out, "record job success failed; retrying") {
t.Fatalf("final write error not logged as retried: %s", out)
}
}
func TestRunnerEnqueueByKindName(t *testing.T) {
t.Parallel()
st := migratedStore(t)