Handle Google token failures and report Google Calendar status (Feature 16) (#128)

Co-authored-by: orchestrator-agent <20+orchestrator-agent@code.cubecraftcreations.com>

internal/gcal/calendars.go

@@ -6,7 +6,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io"
 	"net/http"
 	"net/url"
 	"time"
@@ -55,16 +54,28 @@ type Selection struct {
 // response body.
 type CalendarError struct {
 	msg string
+	// retryable is set when Google answered every try with a transient
+	// failure (HTTP 429 or 5xx).
+	retryable bool
 }
 
 func (e *CalendarError) Error() string { return e.msg }
 
+// Retryable reports whether Google was only briefly unable to answer, so
+// trying again later may succeed. Nothing was disconnected.
+func (e *CalendarError) Retryable() bool { return e.retryable }
+
 // Calendars lists the connected account's calendars it can push to: those
-// it is a writer or owner of, in Google's order.
+// it is a writer or owner of, in Google's order. A grant Google refuses is
+// ErrReconnect, recorded as reconnect required (recordRefusal).
 func (s *Service) Calendars(ctx context.Context) ([]Calendar, error) {
 	s.account.Lock()
 	defer s.account.Unlock()
-	return s.calendars(ctx)
+	cals, err := s.calendars(ctx)
+	if err != nil {
+		return nil, s.recordRefusal(ctx, err)
+	}
+	return cals, nil
 }
 
 // SelectCalendar makes id the calendar MealQuest pushes to, storing its id
@@ -72,13 +83,15 @@ func (s *Service) Calendars(ctx context.Context) ([]Calendar, error) {
 // else, including a calendar chosen earlier that has since been deleted or
 // is no longer writable, is ErrUnknownCalendar and leaves the stored
 // choice as it was. Choosing another calendar than the one chosen returns
-// every plan pushed to that one to not synced; see saveSelection.
+// every plan pushed to that one to not synced; see saveSelection. A grant
+// Google refuses is ErrReconnect, recorded as reconnect required
+// (recordRefusal).
 func (s *Service) SelectCalendar(ctx context.Context, id string) (Selection, error) {
 	s.account.Lock()
 	defer s.account.Unlock()
 	cals, err := s.calendars(ctx)
 	if err != nil {
-		return Selection{}, err
+		return Selection{}, s.recordRefusal(ctx, err)
 	}
 	for _, c := range cals {
 		if c.ID != id {
@@ -147,7 +160,7 @@ func forgetEvents(ctx context.Context, q *db.Queries, now string) error {
 // calendars lists the writable calendars, each once, with the stored refresh
 // token. The caller holds s.account.
 func (s *Service) calendars(ctx context.Context) ([]Calendar, error) {
-	row, err := s.connection(ctx)
+	row, err := s.grantedConnection(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -194,21 +207,54 @@ func (s *Service) connection(ctx context.Context) (db.IntegrationsGoogle, error)
 	return row, nil
 }
 
-// accessToken trades row's refresh token for an access token. A grant
-// Google reports as invalid_grant is ErrReconnect.
+// grantedConnection reads the connected account for a Calendar call that
+// uses the stored grant: listing, choosing and pushing. It decides the
+// precedence of their preconditions: nothing connected is ErrNotConnected,
+// then a grant already recorded as refused (no refresh token stored) is
+// ErrReconnect, before anything else a call checks, such as whether a
+// calendar is chosen. Only connecting again restores a refused grant, so
+// it is reported whatever the calendar choice, and the refused token is
+// never offered to Google.
+func (s *Service) grantedConnection(ctx context.Context) (db.IntegrationsGoogle, error) {
+	row, err := s.connection(ctx)
+	if err != nil {
+		return db.IntegrationsGoogle{}, err
+	}
+	if row.RefreshTokenCt == nil {
+		return db.IntegrationsGoogle{}, ErrReconnect
+	}
+	return row, nil
+}
+
+// accessToken trades row's refresh token, from grantedConnection, for an
+// access token, retrying a transient failure as send does. A grant Google
+// reports as invalid_grant is ErrReconnect.
 func (s *Service) accessToken(ctx context.Context, row db.IntegrationsGoogle) (string, error) {
 	refresh, err := s.box.Decrypt(row.RefreshTokenCt)
 	if err != nil {
 		return "", fmt.Errorf("gcal: open refresh token: %w", err)
 	}
-	token, err := s.oauth.TokenSource(context.WithValue(ctx, oauth2.HTTPClient, s.client),
-		&oauth2.Token{RefreshToken: string(refresh)}).Token()
-	if err != nil {
+	for attempt := 0; ; attempt++ {
+		if attempt > 0 {
+			if err := s.backoff(ctx, attempt); err != nil {
+				return "", err
+			}
+		}
+		token, err := s.oauth.TokenSource(context.WithValue(ctx, oauth2.HTTPClient, s.client),
+			&oauth2.Token{RefreshToken: string(refresh)}).Token()
+		if err == nil {
+			return token.AccessToken, nil
+		}
 		var retrieve *oauth2.RetrieveError
 		var transport *url.Error
 		switch {
 		case errors.As(err, &retrieve) && retrieve.ErrorCode == "invalid_grant":
 			return "", ErrReconnect
+		case errors.As(err, &retrieve) && transient(retrieve.Response.StatusCode) && attempt < maxRetries:
+			continue
+		case errors.As(err, &retrieve) && transient(retrieve.Response.StatusCode):
+			return "", &CalendarError{retryable: true,
+				msg: fmt.Sprintf("gcal: Google could not take the token refresh (HTTP %d) in %d tries", retrieve.Response.StatusCode, maxRetries+1)}
 		case errors.As(err, &retrieve):
 			return "", &CalendarError{msg: fmt.Sprintf("gcal: Google refused the token refresh (HTTP %d)", retrieve.Response.StatusCode)}
 		case errors.As(err, &transport):
@@ -219,7 +265,6 @@ func (s *Service) accessToken(ctx context.Context, row db.IntegrationsGoogle) (s
 			return "", &CalendarError{msg: "gcal: token refresh returned an unusable response"}
 		}
 	}
-	return token.AccessToken, nil
 }
 
 // calendarListPage is the part of a calendarList page MealQuest reads.
@@ -234,35 +279,29 @@ type calendarListPage struct {
 }
 
 // calendarPage fetches the calendarList page at pageToken ("" for the
-// first) with access. HTTP 401 means Google no longer accepts the grant.
+// first) with access. HTTP 401 means Google no longer accepts the grant:
+// the access token was just refreshed from it.
 func (s *Service) calendarPage(ctx context.Context, access, pageToken string) (calendarListPage, error) {
 	q := url.Values{"maxResults": {fmt.Sprint(calendarPageSize)}}
 	if pageToken != "" {
 		q.Set("pageToken", pageToken)
 	}
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.calendarAPI+"/users/me/calendarList?"+q.Encode(), nil)
-	if err != nil {
-		return calendarListPage{}, fmt.Errorf("gcal: build calendar list request: %w", err)
-	}
-	req.Header.Set("Authorization", "Bearer "+access)
-	resp, err := s.client.Do(req)
-	if err != nil {
-		var transport *url.Error
-		if errors.As(err, &transport) {
-			return calendarListPage{}, &CalendarError{msg: fmt.Sprintf("gcal: calendar list request failed: %s: %v", transport.Op, transport.Err)}
+	status, body, err := s.send(ctx, "calendar list", maxCalendarBody+1, func() (*http.Request, error) {
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.calendarAPI+"/users/me/calendarList?"+q.Encode(), nil)
+		if err != nil {
+			return nil, err
 		}
-		return calendarListPage{}, &CalendarError{msg: "gcal: calendar list request failed"}
-	}
-	defer resp.Body.Close()
-	body, err := io.ReadAll(io.LimitReader(resp.Body, maxCalendarBody+1))
+		req.Header.Set("Authorization", "Bearer "+access)
+		return req, nil
+	})
 	if err != nil {
-		return calendarListPage{}, &CalendarError{msg: "gcal: read calendar list failed"}
+		return calendarListPage{}, err
 	}
 	switch {
-	case resp.StatusCode == http.StatusUnauthorized:
+	case status == http.StatusUnauthorized:
 		return calendarListPage{}, ErrReconnect
-	case resp.StatusCode != http.StatusOK:
-		return calendarListPage{}, &CalendarError{msg: fmt.Sprintf("gcal: Google refused the calendar list (HTTP %d)", resp.StatusCode)}
+	case status != http.StatusOK:
+		return calendarListPage{}, &CalendarError{msg: fmt.Sprintf("gcal: Google refused the calendar list (HTTP %d)", status)}
 	case len(body) > maxCalendarBody:
 		return calendarListPage{}, &CalendarError{msg: fmt.Sprintf("gcal: calendar list page exceeds %d bytes", maxCalendarBody)}
 	}

internal/gcal/connect.go

@@ -122,6 +122,9 @@ type Service struct {
 	calendarAPI string
 	client      *http.Client
 	cookie      auth.FlowCookie
+	// retryDelay is the wait before the first retry of a transient
+	// failure; tests shorten it.
+	retryDelay time.Duration
 	// now is the clock; tests replace it.
 	now func() time.Time
 
@@ -170,6 +173,7 @@ func NewService(st *store.Store, box *secretbox.Box, cfg Config) (*Service, erro
 		calendarAPI: cfg.CalendarAPI,
 		client:      &http.Client{Timeout: googleTimeout, CheckRedirect: auth.CheckRedirect},
 		cookie:      auth.NewFlowCookie(connectCookieName, connectCookieKeyLabel, connectCookieTTL, cfg.SecretKey),
+		retryDelay:  firstRetryDelay,
 		now:         time.Now,
 		pending:     map[string]time.Time{},
 	}, nil
@@ -343,6 +347,9 @@ func (s *Service) store(ctx context.Context, sealed []byte) error {
 // MealQuest's access in their Google account) counts as revoked. Any other
 // revoke failure keeps the connection so the household can retry.
 //
+// A connection whose grant Google already refused (reconnect required, no
+// refresh token stored) has nothing to revoke and is deleted.
+//
 // No calendar is chosen once disconnected, so the same transaction that
 // deletes the connection forgets every pushed event (forgetEvents): every
 // synced plan becomes changed, and the next push after connecting again and
@@ -350,20 +357,19 @@ func (s *Service) store(ctx context.Context, sealed []byte) error {
 func (s *Service) Disconnect(ctx context.Context) error {
 	s.account.Lock()
 	defer s.account.Unlock()
-	row, err := db.New(s.st.Reader).GetIntegrationsGoogle(ctx)
-	if errors.Is(err, sql.ErrNoRows) {
-		return ErrNotConnected
-	}
+	row, err := s.connection(ctx)
 	if err != nil {
-		return fmt.Errorf("gcal: read Google integration: %w", err)
-	}
-	token, err := s.box.Decrypt(row.RefreshTokenCt)
-	if err != nil {
-		return fmt.Errorf("gcal: open refresh token: %w", err)
-	}
-	if err := s.revoke(ctx, string(token)); err != nil {
 		return err
 	}
+	if row.RefreshTokenCt != nil {
+		token, err := s.box.Decrypt(row.RefreshTokenCt)
+		if err != nil {
+			return fmt.Errorf("gcal: open refresh token: %w", err)
+		}
+		if err := s.revoke(ctx, string(token)); err != nil {
+			return err
+		}
+	}
 	return s.st.WithTx(ctx, func(tx *sql.Tx) error {
 		q := db.New(tx)
 		if err := q.DeleteIntegrationsGoogle(ctx); err != nil {

internal/gcal/connect_test.go

@@ -80,6 +80,7 @@ func newHarness(t *testing.T) *harness {
 	}
 	h := &harness{svc: svc, google: google, st: st, box: box, clock: time.Date(2026, 10, 7, 18, 30, 0, 0, time.UTC)}
 	gcal.SetClock(svc, func() time.Time { return h.clock })
+	gcal.SetRetryDelay(svc, 0)
 	return h
 }
 

internal/gcal/events.go

@@ -6,7 +6,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io"
 	"net/http"
 	"net/url"
 	"regexp"
@@ -81,10 +80,13 @@ type eventTime struct {
 // includes an event deleted in Calendar) is updated in place instead, so
 // writing an event again never adds a second one. It stops at the first
 // failure; the events before it stay written, and writing them all again
-// is safe. Nothing connected is ErrNotConnected, no calendar chosen is
-// ErrNoCalendar, a chosen calendar that was deleted or is no longer
+// is safe. Nothing connected is ErrNotConnected, a grant already recorded
+// as refused is ErrReconnect whatever calendar is chosen, no calendar
+// chosen is ErrNoCalendar, a chosen calendar that was deleted or is no longer
 // writable is ErrCalendarUnavailable, a grant Google no longer accepts is
-// ErrReconnect, and any other Google failure is a *CalendarError.
+// ErrReconnect (recorded as reconnect required, see recordRefusal), and any
+// other Google failure is a *CalendarError, retryable when Google answered
+// every try with HTTP 429 or 5xx.
 //
 // Once every event is written, PushEvents returns record(), which the
 // caller uses to store what it pushed. record runs under the same hold of
@@ -101,7 +103,15 @@ func (s *Service) PushEvents(ctx context.Context, events []Event, record func()
 	}
 	s.account.Lock()
 	defer s.account.Unlock()
-	row, err := s.connection(ctx)
+	return s.recordRefusal(ctx, s.pushEvents(ctx, events, record))
+}
+
+// pushEvents is PushEvents once the events are valid. The caller holds
+// s.account. A refused grant is reported before no calendar chosen
+// (grantedConnection): choosing a calendar cannot fix it, connecting again
+// can.
+func (s *Service) pushEvents(ctx context.Context, events []Event, record func() error) error {
+	row, err := s.grantedConnection(ctx)
 	if err != nil {
 		return err
 	}
@@ -148,43 +158,37 @@ func (s *Service) writeEvent(ctx context.Context, access, base string, e Event)
 	return err
 }
 
-// eventRequest sends an event insert (POST) or update (PUT) and returns
-// HTTP 200 or, for an insert, 409; every other answer is an error.
+// eventRequest sends an event insert (POST) or update (PUT), retrying a
+// transient failure (send), and returns HTTP 200 or, for an insert, 409;
+// every other answer is an error. An insert Google took but answered with
+// a transient failure is answered 409 when retried, and updated.
 func (s *Service) eventRequest(ctx context.Context, access, method, target string, body []byte) (int, error) {
 	what := "event insert"
 	if method == http.MethodPut {
 		what = "event update"
 	}
-	req, err := http.NewRequestWithContext(ctx, method, target, bytes.NewReader(body))
-	if err != nil {
-		return 0, fmt.Errorf("gcal: build %s request: %w", what, err)
-	}
-	req.Header.Set("Authorization", "Bearer "+access)
-	req.Header.Set("Content-Type", "application/json")
-	resp, err := s.client.Do(req)
-	if err != nil {
-		var transport *url.Error
-		if errors.As(err, &transport) {
-			// The URL is dropped: after a refused redirect it is the
-			// Google-supplied Location.
-			return 0, &CalendarError{msg: fmt.Sprintf("gcal: %s request failed: %s: %v", what, transport.Op, transport.Err)}
+	status, reply, err := s.send(ctx, what, maxEventBody, func() (*http.Request, error) {
+		req, err := http.NewRequestWithContext(ctx, method, target, bytes.NewReader(body))
+		if err != nil {
+			return nil, err
 		}
-		return 0, &CalendarError{msg: fmt.Sprintf("gcal: %s request failed", what)}
-	}
-	defer resp.Body.Close()
-	reply, err := io.ReadAll(io.LimitReader(resp.Body, maxEventBody))
+		req.Header.Set("Authorization", "Bearer "+access)
+		req.Header.Set("Content-Type", "application/json")
+		return req, nil
+	})
 	if err != nil {
-		return 0, &CalendarError{msg: fmt.Sprintf("gcal: read %s response failed", what)}
+		return 0, err
 	}
 	switch {
-	case resp.StatusCode == http.StatusOK:
+	case status == http.StatusOK:
 		return http.StatusOK, nil
-	case resp.StatusCode == http.StatusConflict && method == http.MethodPost:
+	case status == http.StatusConflict && method == http.MethodPost:
 		return http.StatusConflict, nil
-	case resp.StatusCode == http.StatusUnauthorized:
+	case status == http.StatusUnauthorized:
+		// The access token was just refreshed from the grant.
 		return 0, ErrReconnect
-	case resp.StatusCode == http.StatusNotFound && method == http.MethodPost,
-		resp.StatusCode == http.StatusForbidden && errorReason(reply) == "requiredAccessLevel":
+	case status == http.StatusNotFound && method == http.MethodPost,
+		status == http.StatusForbidden && errorReason(reply) == "requiredAccessLevel":
 		// An insert into a deleted calendar, or one the account no longer
 		// sees, is not found; a write to one it can only read is
 		// requiredAccessLevel. An update not found is the event missing
@@ -192,7 +196,7 @@ func (s *Service) eventRequest(ctx context.Context, access, method, target strin
 		// says nothing about the calendar.
 		return 0, ErrCalendarUnavailable
 	}
-	return 0, &CalendarError{msg: fmt.Sprintf("gcal: Google refused the %s (HTTP %d)", what, resp.StatusCode)}
+	return 0, &CalendarError{msg: fmt.Sprintf("gcal: Google refused the %s (HTTP %d)", what, status)}
 }
 
 // errorReason is the reason of the first error in a Calendar API error

internal/gcal/events_test.go

@@ -117,13 +117,13 @@ func TestPushEventsGoogleFailureIsRedacted(t *testing.T) {
 				}
 			}
 			// The fake's failure body quotes the access token.
-			h.google.FailEventWrite(n, http.StatusInternalServerError)
+			h.google.FailEventWrite(n, http.StatusBadRequest)
 			err := h.svc.PushEvents(context.Background(), []gcal.Event{event("abcde")}, noRecord)
 			var calendar *gcal.CalendarError
 			if !errors.As(err, &calendar) {
 				t.Fatalf("PushEvents = %v, want *CalendarError", err)
 			}
-			if !strings.Contains(err.Error(), "event "+name) || !strings.Contains(err.Error(), "HTTP 500") {
+			if !strings.Contains(err.Error(), "event "+name) || !strings.Contains(err.Error(), "HTTP 400") {
 				t.Errorf("error = %q, want the failed %s and its status", err, name)
 			}
 			assertNoSecret(t, "error", err.Error())

internal/gcal/export_test.go

@@ -4,3 +4,7 @@ import "time"
 
 // SetClock replaces s's clock.
 func SetClock(s *Service, now func() time.Time) { s.now = now }
+
+// SetRetryDelay replaces s's wait before the first retry of a transient
+// Google failure.
+func SetRetryDelay(s *Service, d time.Duration) { s.retryDelay = d }

internal/gcal/gcaltest/gcaltest.go

@@ -76,17 +76,22 @@ type Google struct {
 	calendars []Calendar
 	pageSize  int
 	// calendarStatus, when not 200, fails the calendarList endpoint with a
-	// body that quotes the access token.
+	// body that quotes the access token: every request when calendarFails
+	// is 0, else the next calendarFails requests.
 	calendarStatus int
+	calendarFails  int
 	calendarCalls  int
+	// rejectAccess makes the Calendar API answer HTTP 401 to every access
+	// token, while the grant stays live at the token endpoint.
+	rejectAccess bool
 	// events holds each calendar's events by id.
 	events map[string]map[string]Event
 	// inserts and updates count the events created and updated.
 	inserts, updates int
 	// eventWrites counts the requests that reached the events endpoints;
-	// failWrite, when not 0, fails the request that makes it that count
-	// with failStatus and a body that quotes the access token.
-	eventWrites, failWrite, failStatus int
+	// failWrite, when not 0, fails the requests that make it that count up
+	// to failLast with failStatus and a body that quotes the access token.
+	eventWrites, failWrite, failLast, failStatus int
 	// eventHold, when set, stops the next events request.
 	eventHold *hold
 }
@@ -257,7 +262,32 @@ func (g *Google) SetCalendars(pageSize int, cals ...Calendar) {
 func (g *Google) FailCalendarList(status int) {
 	g.mu.Lock()
 	defer g.mu.Unlock()
-	g.calendarStatus = status
+	g.calendarStatus, g.calendarFails = status, 0
+}
+
+// FailCalendarListNext makes the next n calendarList requests answer status
+// as FailCalendarList does, and later ones answer normally.
+func (g *Google) FailCalendarListNext(n, status int) {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+	g.calendarStatus, g.calendarFails = status, n
+}
+
+// RejectAccessToken makes the Calendar API answer HTTP 401 to every access
+// token while refreshing the grant still succeeds, as Google does when it
+// refuses the access token it just issued.
+func (g *Google) RejectAccessToken() {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+	g.rejectAccess = true
+}
+
+// AcceptAccessToken undoes RejectAccessToken, as Google does for the grant
+// of a later connection.
+func (g *Google) AcceptAccessToken() {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+	g.rejectAccess = false
 }
 
 // CalendarCalls is how many requests reached the calendarList endpoint.
@@ -315,9 +345,23 @@ func (g *Google) CancelEvent(calendar, id string) {
 // FailEventWrite makes the nth request to the events endpoints from now
 // (1 for the next) answer status with a body that quotes the access token.
 func (g *Google) FailEventWrite(n, status int) {
+	g.FailEventWrites(n, 1, status)
+}
+
+// FailEventWrites makes count requests to the events endpoints in a row,
+// from the nth from now (1 for the next), answer status with a body that
+// quotes the access token.
+func (g *Google) FailEventWrites(n, count, status int) {
 	g.mu.Lock()
 	defer g.mu.Unlock()
-	g.failWrite, g.failStatus = g.eventWrites+n, status
+	g.failWrite, g.failLast, g.failStatus = g.eventWrites+n, g.eventWrites+n+count-1, status
+}
+
+// EventRequests is how many requests reached the events endpoints.
+func (g *Google) EventRequests() int {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+	return g.eventWrites
 }
 
 // HoldEvents stops the next request to the events endpoints before it is
@@ -476,16 +520,22 @@ func (g *Google) calendarList(w http.ResponseWriter, r *http.Request) {
 	g.calendarHold = nil
 	g.wait(h)
 	auth := r.Header.Get("Authorization")
-	if auth != "Bearer "+AccessToken || !g.grantLive() {
+	if auth != "Bearer "+AccessToken || !g.grantLive() || g.rejectAccess {
 		w.Header().Set("Content-Type", "application/json")
 		w.WriteHeader(http.StatusUnauthorized)
 		fmt.Fprintf(w, `{"error":{"code":401,"message":"Invalid Credentials: %s"}}`, auth)
 		return
 	}
-	if g.calendarStatus != http.StatusOK {
+	if status := g.calendarStatus; status != http.StatusOK {
+		if g.calendarFails > 0 {
+			g.calendarFails--
+			if g.calendarFails == 0 {
+				g.calendarStatus = http.StatusOK
+			}
+		}
 		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(g.calendarStatus)
-		fmt.Fprintf(w, `{"error":{"code":%d,"message":"echo %s"}}`, g.calendarStatus, auth)
+		w.WriteHeader(status)
+		fmt.Fprintf(w, `{"error":{"code":%d,"message":"echo %s"}}`, status, auth)
 		return
 	}
 	start := 0
@@ -598,11 +648,11 @@ func (g *Google) eventCalendar(w http.ResponseWriter, r *http.Request) (string,
 	g.eventHold = nil
 	g.wait(h)
 	auth := r.Header.Get("Authorization")
-	if auth != "Bearer "+AccessToken || !g.grantLive() {
+	if auth != "Bearer "+AccessToken || !g.grantLive() || g.rejectAccess {
 		apiError(w, http.StatusUnauthorized, "authError", "Invalid Credentials: "+auth)
 		return "", false
 	}
-	if g.failWrite != 0 && g.eventWrites == g.failWrite {
+	if g.failWrite != 0 && g.eventWrites >= g.failWrite && g.eventWrites <= g.failLast {
 		apiError(w, g.failStatus, "backendError", "echo "+auth)
 		return "", false
 	}

internal/gcal/grant.go

@@ -0,0 +1,179 @@
+package gcal
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"time"
+
+	"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/db"
+	"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/store"
+)
+
+// ReconnectReason is the last_sync_error stored when Google refuses the
+// stored grant.
+const ReconnectReason = "Google access was revoked or expired — reconnect"
+
+const (
+	// maxRetries is how many times a request Google answers with a
+	// transient failure (HTTP 429 or 5xx) is sent again.
+	maxRetries = 2
+	// firstRetryDelay is the wait before the first retry; each later retry
+	// waits twice as long as the one before.
+	firstRetryDelay = 500 * time.Millisecond
+)
+
+// The states Status reports.
+const (
+	// StatusNotConnected: no Google account is connected.
+	StatusNotConnected = "not_connected"
+	// StatusConnected: a Google account is connected and its grant was last
+	// accepted.
+	StatusConnected = "connected"
+	// StatusReconnectRequired: Google refused the stored grant, so it was
+	// cleared; only connecting again restores Push to Calendar.
+	StatusReconnectRequired = "reconnect_required"
+)
+
+// Status is the Google Calendar connection as the household sees it. It
+// never holds a token.
+type Status struct {
+	State         string  `json:"status"`
+	CalendarID    *string `json:"calendar_id"`
+	CalendarName  *string `json:"calendar_name"`
+	ConnectedAt   *string `json:"connected_at"`
+	LastSyncError *string `json:"last_sync_error"`
+}
+
+// Status reports the connection: not connected, connected, or reconnect
+// required, with the chosen calendar, when it was connected and the last
+// sync error. It is one read of one row, so it takes no lock.
+func (s *Service) Status(ctx context.Context) (Status, error) {
+	row, err := s.connection(ctx)
+	if errors.Is(err, ErrNotConnected) {
+		return Status{State: StatusNotConnected}, nil
+	}
+	if err != nil {
+		return Status{}, err
+	}
+	st := Status{
+		State:         StatusConnected,
+		CalendarID:    nullable(row.CalendarID),
+		CalendarName:  nullable(row.CalendarName),
+		ConnectedAt:   &row.ConnectedAt,
+		LastSyncError: nullable(row.LastSyncError),
+	}
+	if row.RefreshTokenCt == nil {
+		st.State = StatusReconnectRequired
+	}
+	return st, nil
+}
+
+func nullable(v sql.NullString) *string {
+	if !v.Valid {
+		return nil
+	}
+	return &v.String
+}
+
+// recordRefusal returns err, first recording, when it is ErrReconnect, that
+// Google refuses the stored grant: the refresh token is cleared and
+// last_sync_error set to ReconnectReason, so Status reports reconnect
+// required and no request offers Google the refused token again.
+//
+// The chosen calendar and every stored calendar event id are kept, and
+// forgetEvents is deliberately not run: the events MealQuest pushed are
+// still on the chosen calendar, so a stored calendar_event_id still names
+// an event on the chosen calendar. Reconnecting keeps the choice (Complete),
+// and the next push updates those events in place. The caller holds
+// s.account, which every push and calendar choice also holds, so the
+// refusal is recorded between them, never during one.
+func (s *Service) recordRefusal(ctx context.Context, err error) error {
+	if !errors.Is(err, ErrReconnect) {
+		return err
+	}
+	cleared := s.st.WithTx(ctx, func(tx *sql.Tx) error {
+		n, err := db.New(tx).ClearIntegrationsGoogleToken(ctx, db.ClearIntegrationsGoogleTokenParams{
+			LastSyncError: sql.NullString{String: ReconnectReason, Valid: true},
+			UpdatedAt:     store.Timestamp{Time: s.now().UTC().Truncate(time.Millisecond)}.String(),
+		})
+		if err != nil {
+			return err
+		}
+		if n == 0 {
+			// The lock is held since the connection was read, so only a
+			// change outside this service gets here.
+			return ErrNotConnected
+		}
+		return nil
+	})
+	if cleared != nil {
+		return fmt.Errorf("gcal: record that Google refused the grant: %w", cleared)
+	}
+	return err
+}
+
+// transient reports whether Google's HTTP status is a failure worth
+// retrying: rate limited (429) or a server error (5xx).
+func transient(status int) bool {
+	return status == http.StatusTooManyRequests || status >= http.StatusInternalServerError
+}
+
+// backoff waits before retry number attempt (1 for the first), or until ctx
+// ends.
+func (s *Service) backoff(ctx context.Context, attempt int) error {
+	t := time.NewTimer(s.retryDelay << (attempt - 1))
+	defer t.Stop()
+	select {
+	case <-ctx.Done():
+		return fmt.Errorf("gcal: wait to retry: %w", ctx.Err())
+	case <-t.C:
+		return nil
+	}
+}
+
+// send sends the request newRequest builds, and sends it again, up to
+// maxRetries times, while Google answers it with a transient failure. It
+// returns the final answer's status and up to limit bytes of its body. A
+// transient failure on the last try is a retryable *CalendarError; what
+// names the request in errors, which never hold the URL, a token or
+// Google's response body.
+func (s *Service) send(ctx context.Context, what string, limit int64, newRequest func() (*http.Request, error)) (int, []byte, error) {
+	for attempt := 0; ; attempt++ {
+		if attempt > 0 {
+			if err := s.backoff(ctx, attempt); err != nil {
+				return 0, nil, err
+			}
+		}
+		req, err := newRequest()
+		if err != nil {
+			return 0, nil, fmt.Errorf("gcal: build %s request: %w", what, err)
+		}
+		resp, err := s.client.Do(req)
+		if err != nil {
+			var transport *url.Error
+			if errors.As(err, &transport) {
+				// The URL is dropped: after a refused redirect it is the
+				// Google-supplied Location.
+				return 0, nil, &CalendarError{msg: fmt.Sprintf("gcal: %s request failed: %s: %v", what, transport.Op, transport.Err)}
+			}
+			return 0, nil, &CalendarError{msg: fmt.Sprintf("gcal: %s request failed", what)}
+		}
+		body, err := io.ReadAll(io.LimitReader(resp.Body, limit))
+		resp.Body.Close()
+		if err != nil {
+			return 0, nil, &CalendarError{msg: fmt.Sprintf("gcal: read %s response failed", what)}
+		}
+		if !transient(resp.StatusCode) {
+			return resp.StatusCode, body, nil
+		}
+		if attempt == maxRetries {
+			return 0, nil, &CalendarError{retryable: true,
+				msg: fmt.Sprintf("gcal: Google could not take the %s (HTTP %d) in %d tries", what, resp.StatusCode, maxRetries+1)}
+		}
+	}
+}

internal/gcal/grant_test.go

@@ -0,0 +1,336 @@
+package gcal_test
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"reflect"
+	"testing"
+	"time"
+
+	"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/gcal"
+	"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/gcal/gcaltest"
+)
+
+// refusals are the ways Google refuses the stored grant: invalid_grant on
+// the token refresh, and HTTP 401 from the Calendar API to the access token
+// just refreshed.
+var refusals = map[string]func(*gcaltest.Google){
+	"invalid_grant":       (*gcaltest.Google).RevokeGrant,
+	"401 after a refresh": (*gcaltest.Google).RejectAccessToken,
+}
+
+// calls are the Calendar calls that use the stored grant.
+var calls = map[string]func(*harness) error{
+	"list calendars": func(h *harness) error {
+		_, err := h.svc.Calendars(context.Background())
+		return err
+	},
+	"choose a calendar": func(h *harness) error {
+		_, err := h.svc.SelectCalendar(context.Background(), family.ID)
+		return err
+	},
+	"push events": func(h *harness) error {
+		return h.svc.PushEvents(context.Background(), []gcal.Event{event("abcde")}, func() error {
+			return errors.New("record ran after a refused grant")
+		})
+	},
+}
+
+// TestRefusedGrantRequiresReconnect refuses the grant on every Calendar
+// call: the call fails with ErrReconnect, the refresh token is cleared,
+// last_sync_error says why, the chosen calendar is kept and Status reports
+// reconnect required.
+func TestRefusedGrantRequiresReconnect(t *testing.T) {
+	t.Parallel()
+	for refusal, refuse := range refusals {
+		for call, do := range calls {
+			t.Run(refusal+"/"+call, func(t *testing.T) {
+				t.Parallel()
+				h := chosen(t)
+				refuse(h.google)
+				h.clock = h.clock.Add(time.Hour)
+				err := do(h)
+				if !errors.Is(err, gcal.ErrReconnect) {
+					t.Fatalf("%s = %v, want ErrReconnect", call, err)
+				}
+				assertNoSecret(t, "error", err.Error())
+				row, ok := h.row(t)
+				switch {
+				case !ok:
+					t.Fatal("a refused grant forgot the connection")
+				case row.RefreshTokenCt != nil:
+					t.Error("refresh token kept after Google refused it")
+				case row.LastSyncError.String != "Google access was revoked or expired — reconnect":
+					t.Errorf("last_sync_error = %v", row.LastSyncError)
+				case row.CalendarID.String != dinners.ID || row.CalendarName.String != dinners.Summary:
+					t.Errorf("calendar = %v %v, want %s kept", row.CalendarID, row.CalendarName, dinners.ID)
+				case row.UpdatedAt != "2026-10-07T19:30:00.000Z":
+					t.Errorf("updated_at = %s, want the refusal's time", row.UpdatedAt)
+				}
+				st, err := h.svc.Status(context.Background())
+				if err != nil {
+					t.Fatal(err)
+				}
+				if st.State != gcal.StatusReconnectRequired || st.LastSyncError == nil || *st.LastSyncError != gcal.ReconnectReason {
+					t.Errorf("Status = %+v, want reconnect_required with the reason", st)
+				}
+
+				// The cleared grant is never offered to Google again, by
+				// any call.
+				tokens, cals, events := h.google.TokenCalls(), h.google.CalendarCalls(), h.google.EventRequests()
+				for again, do := range calls {
+					if err := do(h); !errors.Is(err, gcal.ErrReconnect) {
+						t.Errorf("%s after the refusal = %v, want ErrReconnect", again, err)
+					}
+				}
+				if h.google.TokenCalls() != tokens || h.google.CalendarCalls() != cals || h.google.EventRequests() != events {
+					t.Error("a call reached Google after the grant was refused")
+				}
+			})
+		}
+	}
+}
+
+// TestRefusedGrantBeforeACalendarIsChosen refuses the grant while no
+// calendar is chosen, on each call that can record it then (listing and
+// choosing): every Calendar call, push included, is ErrReconnect rather than
+// ErrNoCalendar, since only connecting again restores the grant. After
+// connecting again and choosing a calendar, a push succeeds.
+func TestRefusedGrantBeforeACalendarIsChosen(t *testing.T) {
+	t.Parallel()
+	for refusal, refuse := range refusals {
+		for _, call := range []string{"list calendars", "choose a calendar"} {
+			t.Run(refusal+"/"+call, func(t *testing.T) {
+				t.Parallel()
+				h := connected(t)
+				refuse(h.google)
+				if err := calls[call](h); !errors.Is(err, gcal.ErrReconnect) {
+					t.Fatalf("%s = %v, want ErrReconnect", call, err)
+				}
+				if row, _ := h.row(t); row.RefreshTokenCt != nil || row.CalendarID.Valid {
+					t.Fatalf("row = %+v, want the grant cleared and no calendar chosen", row)
+				}
+				tokens, cals, events := h.google.TokenCalls(), h.google.CalendarCalls(), h.google.EventRequests()
+				for again, do := range calls {
+					if err := do(h); !errors.Is(err, gcal.ErrReconnect) {
+						t.Errorf("%s after the refusal = %v, want ErrReconnect", again, err)
+					}
+				}
+				if h.google.TokenCalls() != tokens || h.google.CalendarCalls() != cals || h.google.EventRequests() != events {
+					t.Error("a call reached Google after the grant was refused")
+				}
+
+				h.google.AcceptAccessToken()
+				c := h.connect(t, session)
+				if _, err := h.callback(c, session, h.consented(c)); err != nil {
+					t.Fatalf("Complete: %v", err)
+				}
+				if _, err := h.svc.SelectCalendar(context.Background(), dinners.ID); err != nil {
+					t.Fatalf("SelectCalendar after connecting again: %v", err)
+				}
+				if err := h.svc.PushEvents(context.Background(), []gcal.Event{event("abcde")}, noRecord); err != nil {
+					t.Errorf("PushEvents after connecting again: %v", err)
+				}
+			})
+		}
+	}
+}
+
+// TestReconnectAfterRefusal connects again after Google refused the grant:
+// the new token is stored, the reason cleared and the calendar kept.
+func TestReconnectAfterRefusal(t *testing.T) {
+	t.Parallel()
+	h := chosen(t)
+	h.google.RevokeGrant()
+	if _, err := h.svc.Calendars(context.Background()); !errors.Is(err, gcal.ErrReconnect) {
+		t.Fatalf("Calendars = %v, want ErrReconnect", err)
+	}
+	c := h.connect(t, session)
+	if _, err := h.callback(c, session, h.consented(c)); err != nil {
+		t.Fatalf("Complete: %v", err)
+	}
+	st, err := h.svc.Status(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if st.State != gcal.StatusConnected || st.LastSyncError != nil || st.CalendarID == nil || *st.CalendarID != dinners.ID {
+		t.Errorf("Status = %+v, want connected to %s with no error", st, dinners.ID)
+	}
+	if err := h.svc.PushEvents(context.Background(), []gcal.Event{event("abcde")}, noRecord); err != nil {
+		t.Errorf("PushEvents after connecting again: %v", err)
+	}
+}
+
+// TestDisconnectAfterRefusal disconnects a connection whose grant Google
+// refused: there is nothing to revoke, and the connection is deleted.
+func TestDisconnectAfterRefusal(t *testing.T) {
+	t.Parallel()
+	h := chosen(t)
+	h.google.RevokeGrant()
+	if _, err := h.svc.Calendars(context.Background()); !errors.Is(err, gcal.ErrReconnect) {
+		t.Fatalf("Calendars = %v, want ErrReconnect", err)
+	}
+	if err := h.svc.Disconnect(context.Background()); err != nil {
+		t.Fatalf("Disconnect: %v", err)
+	}
+	if n := len(h.google.Revoked()); n != 0 {
+		t.Errorf("revoked %d tokens; a refused grant has none to revoke", n)
+	}
+	if _, ok := h.row(t); ok {
+		t.Error("connection kept after disconnecting")
+	}
+}
+
+// transients are the failures Google asks clients to retry.
+var transients = []int{http.StatusServiceUnavailable, http.StatusInternalServerError, http.StatusTooManyRequests}
+
+// TestTransientFailureIsRetried fails each Google request three times in a
+// row: it is tried three times (two retries), then fails as retryable, and
+// nothing is disconnected or cleared.
+func TestTransientFailureIsRetried(t *testing.T) {
+	t.Parallel()
+	for _, status := range transients {
+		for name, tc := range map[string]struct {
+			fail  func(*gcaltest.Google)
+			do    func(*harness) error
+			tries func(*gcaltest.Google) int
+		}{
+			"token refresh": {
+				fail:  func(g *gcaltest.Google) { g.FailToken(status) },
+				do:    calls["list calendars"],
+				tries: (*gcaltest.Google).TokenCalls,
+			},
+			"calendar list": {
+				fail:  func(g *gcaltest.Google) { g.FailCalendarList(status) },
+				do:    calls["choose a calendar"],
+				tries: (*gcaltest.Google).CalendarCalls,
+			},
+			"event write": {
+				fail:  func(g *gcaltest.Google) { g.FailEventWrites(1, 3, status) },
+				do:    calls["push events"],
+				tries: (*gcaltest.Google).EventRequests,
+			},
+		} {
+			t.Run(name+"/"+http.StatusText(status), func(t *testing.T) {
+				t.Parallel()
+				h := chosen(t)
+				before, _ := h.row(t)
+				tried := tc.tries(h.google)
+				tc.fail(h.google)
+				err := tc.do(h)
+				var calendar *gcal.CalendarError
+				if !errors.As(err, &calendar) || !calendar.Retryable() {
+					t.Fatalf("err = %v, want a retryable *CalendarError", err)
+				}
+				assertNoSecret(t, "error", err.Error())
+				if n := tc.tries(h.google) - tried; n != 3 {
+					t.Errorf("tried %d times, want 3", n)
+				}
+				after, ok := h.row(t)
+				if !ok || !reflect.DeepEqual(before, after) {
+					t.Errorf("connection = %+v, %v; want it unchanged %+v", after, ok, before)
+				}
+				if st, err := h.svc.Status(context.Background()); err != nil || st.State != gcal.StatusConnected {
+					t.Errorf("Status = %+v, %v; want connected", st, err)
+				}
+			})
+		}
+	}
+}
+
+// TestTransientFailureRecovers fails each Google request twice: the second
+// retry succeeds.
+func TestTransientFailureRecovers(t *testing.T) {
+	t.Parallel()
+	for name, tc := range map[string]struct {
+		fail func(*gcaltest.Google)
+		do   func(*harness) error
+	}{
+		"calendar list": {
+			fail: func(g *gcaltest.Google) { g.FailCalendarListNext(2, http.StatusServiceUnavailable) },
+			do:   calls["choose a calendar"],
+		},
+		"event write": {
+			fail: func(g *gcaltest.Google) { g.FailEventWrites(1, 2, http.StatusServiceUnavailable) },
+			do: func(h *harness) error {
+				return h.svc.PushEvents(context.Background(), []gcal.Event{event("abcde")}, noRecord)
+			},
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+			h := chosen(t)
+			tc.fail(h.google)
+			if err := tc.do(h); err != nil {
+				t.Fatalf("err = %v, want the second retry to succeed", err)
+			}
+		})
+	}
+}
+
+// TestTransientFailureBacksOff checks the waits between tries: the first
+// retry waits the retry delay, the second twice that.
+func TestTransientFailureBacksOff(t *testing.T) {
+	t.Parallel()
+	h := chosen(t)
+	gcal.SetRetryDelay(h.svc, 50*time.Millisecond)
+	h.google.FailCalendarList(http.StatusServiceUnavailable)
+	start := time.Now()
+	if _, err := h.svc.Calendars(context.Background()); err == nil {
+		t.Fatal("Calendars succeeded")
+	}
+	if took := time.Since(start); took < 150*time.Millisecond {
+		t.Errorf("three tries took %s, want at least 150ms of waits", took)
+	}
+}
+
+func TestStatus(t *testing.T) {
+	t.Parallel()
+	h := newHarness(t)
+	st, err := h.svc.Status(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := statusJSON(t, st); got != `{"status":"not_connected","calendar_id":null,"calendar_name":null,"connected_at":null,"last_sync_error":null}` {
+		t.Errorf("not connected: Status = %s", got)
+	}
+	c := h.connect(t, session)
+	if _, err := h.callback(c, session, h.consented(c)); err != nil {
+		t.Fatal(err)
+	}
+	st, err = h.svc.Status(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := statusJSON(t, st); got != `{"status":"connected","calendar_id":null,"calendar_name":null,"connected_at":"2026-10-07T18:30:00.000Z","last_sync_error":null}` {
+		t.Errorf("connected: Status = %s", got)
+	}
+	h.google.SetCalendars(0, dinners)
+	if _, err := h.svc.SelectCalendar(context.Background(), dinners.ID); err != nil {
+		t.Fatal(err)
+	}
+	h.google.RevokeGrant()
+	if _, err := h.svc.Calendars(context.Background()); !errors.Is(err, gcal.ErrReconnect) {
+		t.Fatalf("Calendars = %v, want ErrReconnect", err)
+	}
+	st, err = h.svc.Status(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := `{"status":"reconnect_required","calendar_id":"dinners@group.calendar.google.com","calendar_name":"Dinners",` +
+		`"connected_at":"2026-10-07T18:30:00.000Z","last_sync_error":"Google access was revoked or expired — reconnect"}`
+	if got := statusJSON(t, st); got != want {
+		t.Errorf("reconnect required: Status =\n%s\nwant\n%s", got, want)
+	}
+}
+
+func statusJSON(t *testing.T, st gcal.Status) string {
+	t.Helper()
+	b, err := json.Marshal(st)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return string(b)
+}