Handle Google token failures and report Google Calendar status
CI / ci (push) Successful in 4m29s
CI / ci (pull_request) Successful in 4m21s

When Google refuses the stored grant (invalid_grant on refresh, or HTTP
401 from the Calendar API to the access token just refreshed) on any
Calendar call - listing, choosing or pushing - the connection becomes
reconnect required under the account lock: the refresh token is cleared
and last_sync_error set to "Google access was revoked or expired —
reconnect". The call answers 409 google_reconnect_required, and later
calls answer the same without contacting Google until a reconnect or a
disconnect. The chosen calendar and stored calendar event ids are kept:
the pushed events are still on that calendar, so forgetEvents is not
run.

HTTP 429 and 5xx from the token endpoint or the Calendar API are retried
at most twice with backoff, then answered 503 google_unavailable,
retryable, without clearing or disconnecting anything.

Add GET /api/v1/integrations/google, reporting not_connected, connected
or reconnect_required with the chosen calendar and last_sync_error.

Migration 0015 rebuilds integrations_google so refresh_token_ct can be
cleared, with a CHECK that a row without a token carries the reason.

Maestro task b8d8da5a-0450-4571-963e-b3dfead24283.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-07 23:41:50 -04:00
co-authored by Claude Opus 5.5
parent 6262489376
commit 1c8092800a
22 changed files with 1059 additions and 149 deletions
+54 -30
View File
@@ -6,7 +6,6 @@ import (
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"time"
@@ -55,16 +54,28 @@ type Selection struct {
// response body.
type CalendarError struct {
msg string
// retryable is set when Google answered every try with a transient
// failure (HTTP 429 or 5xx).
retryable bool
}
func (e *CalendarError) Error() string { return e.msg }
// Retryable reports whether Google was only briefly unable to answer, so
// trying again later may succeed. Nothing was disconnected.
func (e *CalendarError) Retryable() bool { return e.retryable }
// Calendars lists the connected account's calendars it can push to: those
// it is a writer or owner of, in Google's order.
// it is a writer or owner of, in Google's order. A grant Google refuses is
// ErrReconnect, recorded as reconnect required (recordRefusal).
func (s *Service) Calendars(ctx context.Context) ([]Calendar, error) {
s.account.Lock()
defer s.account.Unlock()
return s.calendars(ctx)
cals, err := s.calendars(ctx)
if err != nil {
return nil, s.recordRefusal(ctx, err)
}
return cals, nil
}
// SelectCalendar makes id the calendar MealQuest pushes to, storing its id
@@ -72,13 +83,15 @@ func (s *Service) Calendars(ctx context.Context) ([]Calendar, error) {
// else, including a calendar chosen earlier that has since been deleted or
// is no longer writable, is ErrUnknownCalendar and leaves the stored
// choice as it was. Choosing another calendar than the one chosen returns
// every plan pushed to that one to not synced; see saveSelection.
// every plan pushed to that one to not synced; see saveSelection. A grant
// Google refuses is ErrReconnect, recorded as reconnect required
// (recordRefusal).
func (s *Service) SelectCalendar(ctx context.Context, id string) (Selection, error) {
s.account.Lock()
defer s.account.Unlock()
cals, err := s.calendars(ctx)
if err != nil {
return Selection{}, err
return Selection{}, s.recordRefusal(ctx, err)
}
for _, c := range cals {
if c.ID != id {
@@ -194,21 +207,39 @@ func (s *Service) connection(ctx context.Context) (db.IntegrationsGoogle, error)
return row, nil
}
// accessToken trades row's refresh token for an access token. A grant
// Google reports as invalid_grant is ErrReconnect.
// accessToken trades row's refresh token for an access token, retrying a
// transient failure as send does. A grant Google reports as invalid_grant,
// or one already recorded as refused (no refresh token stored), is
// ErrReconnect.
func (s *Service) accessToken(ctx context.Context, row db.IntegrationsGoogle) (string, error) {
if row.RefreshTokenCt == nil {
return "", ErrReconnect
}
refresh, err := s.box.Decrypt(row.RefreshTokenCt)
if err != nil {
return "", fmt.Errorf("gcal: open refresh token: %w", err)
}
token, err := s.oauth.TokenSource(context.WithValue(ctx, oauth2.HTTPClient, s.client),
&oauth2.Token{RefreshToken: string(refresh)}).Token()
if err != nil {
for attempt := 0; ; attempt++ {
if attempt > 0 {
if err := s.backoff(ctx, attempt); err != nil {
return "", err
}
}
token, err := s.oauth.TokenSource(context.WithValue(ctx, oauth2.HTTPClient, s.client),
&oauth2.Token{RefreshToken: string(refresh)}).Token()
if err == nil {
return token.AccessToken, nil
}
var retrieve *oauth2.RetrieveError
var transport *url.Error
switch {
case errors.As(err, &retrieve) && retrieve.ErrorCode == "invalid_grant":
return "", ErrReconnect
case errors.As(err, &retrieve) && transient(retrieve.Response.StatusCode) && attempt < maxRetries:
continue
case errors.As(err, &retrieve) && transient(retrieve.Response.StatusCode):
return "", &CalendarError{retryable: true,
msg: fmt.Sprintf("gcal: Google could not take the token refresh (HTTP %d) in %d tries", retrieve.Response.StatusCode, maxRetries+1)}
case errors.As(err, &retrieve):
return "", &CalendarError{msg: fmt.Sprintf("gcal: Google refused the token refresh (HTTP %d)", retrieve.Response.StatusCode)}
case errors.As(err, &transport):
@@ -219,7 +250,6 @@ func (s *Service) accessToken(ctx context.Context, row db.IntegrationsGoogle) (s
return "", &CalendarError{msg: "gcal: token refresh returned an unusable response"}
}
}
return token.AccessToken, nil
}
// calendarListPage is the part of a calendarList page MealQuest reads.
@@ -234,35 +264,29 @@ type calendarListPage struct {
}
// calendarPage fetches the calendarList page at pageToken ("" for the
// first) with access. HTTP 401 means Google no longer accepts the grant.
// first) with access. HTTP 401 means Google no longer accepts the grant:
// the access token was just refreshed from it.
func (s *Service) calendarPage(ctx context.Context, access, pageToken string) (calendarListPage, error) {
q := url.Values{"maxResults": {fmt.Sprint(calendarPageSize)}}
if pageToken != "" {
q.Set("pageToken", pageToken)
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.calendarAPI+"/users/me/calendarList?"+q.Encode(), nil)
if err != nil {
return calendarListPage{}, fmt.Errorf("gcal: build calendar list request: %w", err)
}
req.Header.Set("Authorization", "Bearer "+access)
resp, err := s.client.Do(req)
if err != nil {
var transport *url.Error
if errors.As(err, &transport) {
return calendarListPage{}, &CalendarError{msg: fmt.Sprintf("gcal: calendar list request failed: %s: %v", transport.Op, transport.Err)}
status, body, err := s.send(ctx, "calendar list", maxCalendarBody+1, func() (*http.Request, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.calendarAPI+"/users/me/calendarList?"+q.Encode(), nil)
if err != nil {
return nil, err
}
return calendarListPage{}, &CalendarError{msg: "gcal: calendar list request failed"}
}
defer resp.Body.Close()
body, err := io.ReadAll(io.LimitReader(resp.Body, maxCalendarBody+1))
req.Header.Set("Authorization", "Bearer "+access)
return req, nil
})
if err != nil {
return calendarListPage{}, &CalendarError{msg: "gcal: read calendar list failed"}
return calendarListPage{}, err
}
switch {
case resp.StatusCode == http.StatusUnauthorized:
case status == http.StatusUnauthorized:
return calendarListPage{}, ErrReconnect
case resp.StatusCode != http.StatusOK:
return calendarListPage{}, &CalendarError{msg: fmt.Sprintf("gcal: Google refused the calendar list (HTTP %d)", resp.StatusCode)}
case status != http.StatusOK:
return calendarListPage{}, &CalendarError{msg: fmt.Sprintf("gcal: Google refused the calendar list (HTTP %d)", status)}
case len(body) > maxCalendarBody:
return calendarListPage{}, &CalendarError{msg: fmt.Sprintf("gcal: calendar list page exceeds %d bytes", maxCalendarBody)}
}
+17 -11
View File
@@ -122,6 +122,9 @@ type Service struct {
calendarAPI string
client *http.Client
cookie auth.FlowCookie
// retryDelay is the wait before the first retry of a transient
// failure; tests shorten it.
retryDelay time.Duration
// now is the clock; tests replace it.
now func() time.Time
@@ -170,6 +173,7 @@ func NewService(st *store.Store, box *secretbox.Box, cfg Config) (*Service, erro
calendarAPI: cfg.CalendarAPI,
client: &http.Client{Timeout: googleTimeout, CheckRedirect: auth.CheckRedirect},
cookie: auth.NewFlowCookie(connectCookieName, connectCookieKeyLabel, connectCookieTTL, cfg.SecretKey),
retryDelay: firstRetryDelay,
now: time.Now,
pending: map[string]time.Time{},
}, nil
@@ -343,6 +347,9 @@ func (s *Service) store(ctx context.Context, sealed []byte) error {
// MealQuest's access in their Google account) counts as revoked. Any other
// revoke failure keeps the connection so the household can retry.
//
// A connection whose grant Google already refused (reconnect required, no
// refresh token stored) has nothing to revoke and is deleted.
//
// No calendar is chosen once disconnected, so the same transaction that
// deletes the connection forgets every pushed event (forgetEvents): every
// synced plan becomes changed, and the next push after connecting again and
@@ -350,20 +357,19 @@ func (s *Service) store(ctx context.Context, sealed []byte) error {
func (s *Service) Disconnect(ctx context.Context) error {
s.account.Lock()
defer s.account.Unlock()
row, err := db.New(s.st.Reader).GetIntegrationsGoogle(ctx)
if errors.Is(err, sql.ErrNoRows) {
return ErrNotConnected
}
row, err := s.connection(ctx)
if err != nil {
return fmt.Errorf("gcal: read Google integration: %w", err)
}
token, err := s.box.Decrypt(row.RefreshTokenCt)
if err != nil {
return fmt.Errorf("gcal: open refresh token: %w", err)
}
if err := s.revoke(ctx, string(token)); err != nil {
return err
}
if row.RefreshTokenCt != nil {
token, err := s.box.Decrypt(row.RefreshTokenCt)
if err != nil {
return fmt.Errorf("gcal: open refresh token: %w", err)
}
if err := s.revoke(ctx, string(token)); err != nil {
return err
}
}
return s.st.WithTx(ctx, func(tx *sql.Tx) error {
q := db.New(tx)
if err := q.DeleteIntegrationsGoogle(ctx); err != nil {
+1
View File
@@ -80,6 +80,7 @@ func newHarness(t *testing.T) *harness {
}
h := &harness{svc: svc, google: google, st: st, box: box, clock: time.Date(2026, 10, 7, 18, 30, 0, 0, time.UTC)}
gcal.SetClock(svc, func() time.Time { return h.clock })
gcal.SetRetryDelay(svc, 0)
return h
}
+29 -28
View File
@@ -6,7 +6,6 @@ import (
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"regexp"
@@ -84,7 +83,9 @@ type eventTime struct {
// is safe. Nothing connected is ErrNotConnected, no calendar chosen is
// ErrNoCalendar, a chosen calendar that was deleted or is no longer
// writable is ErrCalendarUnavailable, a grant Google no longer accepts is
// ErrReconnect, and any other Google failure is a *CalendarError.
// ErrReconnect (recorded as reconnect required, see recordRefusal), and any
// other Google failure is a *CalendarError, retryable when Google answered
// every try with HTTP 429 or 5xx.
//
// Once every event is written, PushEvents returns record(), which the
// caller uses to store what it pushed. record runs under the same hold of
@@ -101,6 +102,12 @@ func (s *Service) PushEvents(ctx context.Context, events []Event, record func()
}
s.account.Lock()
defer s.account.Unlock()
return s.recordRefusal(ctx, s.pushEvents(ctx, events, record))
}
// pushEvents is PushEvents once the events are valid. The caller holds
// s.account.
func (s *Service) pushEvents(ctx context.Context, events []Event, record func() error) error {
row, err := s.connection(ctx)
if err != nil {
return err
@@ -148,43 +155,37 @@ func (s *Service) writeEvent(ctx context.Context, access, base string, e Event)
return err
}
// eventRequest sends an event insert (POST) or update (PUT) and returns
// HTTP 200 or, for an insert, 409; every other answer is an error.
// eventRequest sends an event insert (POST) or update (PUT), retrying a
// transient failure (send), and returns HTTP 200 or, for an insert, 409;
// every other answer is an error. An insert Google took but answered with
// a transient failure is answered 409 when retried, and updated.
func (s *Service) eventRequest(ctx context.Context, access, method, target string, body []byte) (int, error) {
what := "event insert"
if method == http.MethodPut {
what = "event update"
}
req, err := http.NewRequestWithContext(ctx, method, target, bytes.NewReader(body))
if err != nil {
return 0, fmt.Errorf("gcal: build %s request: %w", what, err)
}
req.Header.Set("Authorization", "Bearer "+access)
req.Header.Set("Content-Type", "application/json")
resp, err := s.client.Do(req)
if err != nil {
var transport *url.Error
if errors.As(err, &transport) {
// The URL is dropped: after a refused redirect it is the
// Google-supplied Location.
return 0, &CalendarError{msg: fmt.Sprintf("gcal: %s request failed: %s: %v", what, transport.Op, transport.Err)}
status, reply, err := s.send(ctx, what, maxEventBody, func() (*http.Request, error) {
req, err := http.NewRequestWithContext(ctx, method, target, bytes.NewReader(body))
if err != nil {
return nil, err
}
return 0, &CalendarError{msg: fmt.Sprintf("gcal: %s request failed", what)}
}
defer resp.Body.Close()
reply, err := io.ReadAll(io.LimitReader(resp.Body, maxEventBody))
req.Header.Set("Authorization", "Bearer "+access)
req.Header.Set("Content-Type", "application/json")
return req, nil
})
if err != nil {
return 0, &CalendarError{msg: fmt.Sprintf("gcal: read %s response failed", what)}
return 0, err
}
switch {
case resp.StatusCode == http.StatusOK:
case status == http.StatusOK:
return http.StatusOK, nil
case resp.StatusCode == http.StatusConflict && method == http.MethodPost:
case status == http.StatusConflict && method == http.MethodPost:
return http.StatusConflict, nil
case resp.StatusCode == http.StatusUnauthorized:
case status == http.StatusUnauthorized:
// The access token was just refreshed from the grant.
return 0, ErrReconnect
case resp.StatusCode == http.StatusNotFound && method == http.MethodPost,
resp.StatusCode == http.StatusForbidden && errorReason(reply) == "requiredAccessLevel":
case status == http.StatusNotFound && method == http.MethodPost,
status == http.StatusForbidden && errorReason(reply) == "requiredAccessLevel":
// An insert into a deleted calendar, or one the account no longer
// sees, is not found; a write to one it can only read is
// requiredAccessLevel. An update not found is the event missing
@@ -192,7 +193,7 @@ func (s *Service) eventRequest(ctx context.Context, access, method, target strin
// says nothing about the calendar.
return 0, ErrCalendarUnavailable
}
return 0, &CalendarError{msg: fmt.Sprintf("gcal: Google refused the %s (HTTP %d)", what, resp.StatusCode)}
return 0, &CalendarError{msg: fmt.Sprintf("gcal: Google refused the %s (HTTP %d)", what, status)}
}
// errorReason is the reason of the first error in a Calendar API error
+2 -2
View File
@@ -117,13 +117,13 @@ func TestPushEventsGoogleFailureIsRedacted(t *testing.T) {
}
}
// The fake's failure body quotes the access token.
h.google.FailEventWrite(n, http.StatusInternalServerError)
h.google.FailEventWrite(n, http.StatusBadRequest)
err := h.svc.PushEvents(context.Background(), []gcal.Event{event("abcde")}, noRecord)
var calendar *gcal.CalendarError
if !errors.As(err, &calendar) {
t.Fatalf("PushEvents = %v, want *CalendarError", err)
}
if !strings.Contains(err.Error(), "event "+name) || !strings.Contains(err.Error(), "HTTP 500") {
if !strings.Contains(err.Error(), "event "+name) || !strings.Contains(err.Error(), "HTTP 400") {
t.Errorf("error = %q, want the failed %s and its status", err, name)
}
assertNoSecret(t, "error", err.Error())
+4
View File
@@ -4,3 +4,7 @@ import "time"
// SetClock replaces s's clock.
func SetClock(s *Service, now func() time.Time) { s.now = now }
// SetRetryDelay replaces s's wait before the first retry of a transient
// Google failure.
func SetRetryDelay(s *Service, d time.Duration) { s.retryDelay = d }
+54 -12
View File
@@ -76,17 +76,22 @@ type Google struct {
calendars []Calendar
pageSize int
// calendarStatus, when not 200, fails the calendarList endpoint with a
// body that quotes the access token.
// body that quotes the access token: every request when calendarFails
// is 0, else the next calendarFails requests.
calendarStatus int
calendarFails int
calendarCalls int
// rejectAccess makes the Calendar API answer HTTP 401 to every access
// token, while the grant stays live at the token endpoint.
rejectAccess bool
// events holds each calendar's events by id.
events map[string]map[string]Event
// inserts and updates count the events created and updated.
inserts, updates int
// eventWrites counts the requests that reached the events endpoints;
// failWrite, when not 0, fails the request that makes it that count
// with failStatus and a body that quotes the access token.
eventWrites, failWrite, failStatus int
// failWrite, when not 0, fails the requests that make it that count up
// to failLast with failStatus and a body that quotes the access token.
eventWrites, failWrite, failLast, failStatus int
// eventHold, when set, stops the next events request.
eventHold *hold
}
@@ -257,7 +262,24 @@ func (g *Google) SetCalendars(pageSize int, cals ...Calendar) {
func (g *Google) FailCalendarList(status int) {
g.mu.Lock()
defer g.mu.Unlock()
g.calendarStatus = status
g.calendarStatus, g.calendarFails = status, 0
}
// FailCalendarListNext makes the next n calendarList requests answer status
// as FailCalendarList does, and later ones answer normally.
func (g *Google) FailCalendarListNext(n, status int) {
g.mu.Lock()
defer g.mu.Unlock()
g.calendarStatus, g.calendarFails = status, n
}
// RejectAccessToken makes the Calendar API answer HTTP 401 to every access
// token while refreshing the grant still succeeds, as Google does when it
// refuses the access token it just issued.
func (g *Google) RejectAccessToken() {
g.mu.Lock()
defer g.mu.Unlock()
g.rejectAccess = true
}
// CalendarCalls is how many requests reached the calendarList endpoint.
@@ -315,9 +337,23 @@ func (g *Google) CancelEvent(calendar, id string) {
// FailEventWrite makes the nth request to the events endpoints from now
// (1 for the next) answer status with a body that quotes the access token.
func (g *Google) FailEventWrite(n, status int) {
g.FailEventWrites(n, 1, status)
}
// FailEventWrites makes count requests to the events endpoints in a row,
// from the nth from now (1 for the next), answer status with a body that
// quotes the access token.
func (g *Google) FailEventWrites(n, count, status int) {
g.mu.Lock()
defer g.mu.Unlock()
g.failWrite, g.failStatus = g.eventWrites+n, status
g.failWrite, g.failLast, g.failStatus = g.eventWrites+n, g.eventWrites+n+count-1, status
}
// EventRequests is how many requests reached the events endpoints.
func (g *Google) EventRequests() int {
g.mu.Lock()
defer g.mu.Unlock()
return g.eventWrites
}
// HoldEvents stops the next request to the events endpoints before it is
@@ -476,16 +512,22 @@ func (g *Google) calendarList(w http.ResponseWriter, r *http.Request) {
g.calendarHold = nil
g.wait(h)
auth := r.Header.Get("Authorization")
if auth != "Bearer "+AccessToken || !g.grantLive() {
if auth != "Bearer "+AccessToken || !g.grantLive() || g.rejectAccess {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusUnauthorized)
fmt.Fprintf(w, `{"error":{"code":401,"message":"Invalid Credentials: %s"}}`, auth)
return
}
if g.calendarStatus != http.StatusOK {
if status := g.calendarStatus; status != http.StatusOK {
if g.calendarFails > 0 {
g.calendarFails--
if g.calendarFails == 0 {
g.calendarStatus = http.StatusOK
}
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(g.calendarStatus)
fmt.Fprintf(w, `{"error":{"code":%d,"message":"echo %s"}}`, g.calendarStatus, auth)
w.WriteHeader(status)
fmt.Fprintf(w, `{"error":{"code":%d,"message":"echo %s"}}`, status, auth)
return
}
start := 0
@@ -598,11 +640,11 @@ func (g *Google) eventCalendar(w http.ResponseWriter, r *http.Request) (string,
g.eventHold = nil
g.wait(h)
auth := r.Header.Get("Authorization")
if auth != "Bearer "+AccessToken || !g.grantLive() {
if auth != "Bearer "+AccessToken || !g.grantLive() || g.rejectAccess {
apiError(w, http.StatusUnauthorized, "authError", "Invalid Credentials: "+auth)
return "", false
}
if g.failWrite != 0 && g.eventWrites == g.failWrite {
if g.failWrite != 0 && g.eventWrites >= g.failWrite && g.eventWrites <= g.failLast {
apiError(w, g.failStatus, "backendError", "echo "+auth)
return "", false
}
+179
View File
@@ -0,0 +1,179 @@
package gcal
import (
"context"
"database/sql"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"time"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/db"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/store"
)
// ReconnectReason is the last_sync_error stored when Google refuses the
// stored grant.
const ReconnectReason = "Google access was revoked or expired — reconnect"
const (
// maxRetries is how many times a request Google answers with a
// transient failure (HTTP 429 or 5xx) is sent again.
maxRetries = 2
// firstRetryDelay is the wait before the first retry; each later retry
// waits twice as long as the one before.
firstRetryDelay = 500 * time.Millisecond
)
// The states Status reports.
const (
// StatusNotConnected: no Google account is connected.
StatusNotConnected = "not_connected"
// StatusConnected: a Google account is connected and its grant was last
// accepted.
StatusConnected = "connected"
// StatusReconnectRequired: Google refused the stored grant, so it was
// cleared; only connecting again restores Push to Calendar.
StatusReconnectRequired = "reconnect_required"
)
// Status is the Google Calendar connection as the household sees it. It
// never holds a token.
type Status struct {
State string `json:"status"`
CalendarID *string `json:"calendar_id"`
CalendarName *string `json:"calendar_name"`
ConnectedAt *string `json:"connected_at"`
LastSyncError *string `json:"last_sync_error"`
}
// Status reports the connection: not connected, connected, or reconnect
// required, with the chosen calendar, when it was connected and the last
// sync error. It is one read of one row, so it takes no lock.
func (s *Service) Status(ctx context.Context) (Status, error) {
row, err := s.connection(ctx)
if errors.Is(err, ErrNotConnected) {
return Status{State: StatusNotConnected}, nil
}
if err != nil {
return Status{}, err
}
st := Status{
State: StatusConnected,
CalendarID: nullable(row.CalendarID),
CalendarName: nullable(row.CalendarName),
ConnectedAt: &row.ConnectedAt,
LastSyncError: nullable(row.LastSyncError),
}
if row.RefreshTokenCt == nil {
st.State = StatusReconnectRequired
}
return st, nil
}
func nullable(v sql.NullString) *string {
if !v.Valid {
return nil
}
return &v.String
}
// recordRefusal returns err, first recording, when it is ErrReconnect, that
// Google refuses the stored grant: the refresh token is cleared and
// last_sync_error set to ReconnectReason, so Status reports reconnect
// required and no request offers Google the refused token again.
//
// The chosen calendar and every stored calendar event id are kept, and
// forgetEvents is deliberately not run: the events MealQuest pushed are
// still on the chosen calendar, so a stored calendar_event_id still names
// an event on the chosen calendar. Reconnecting keeps the choice (Complete),
// and the next push updates those events in place. The caller holds
// s.account, which every push and calendar choice also holds, so the
// refusal is recorded between them, never during one.
func (s *Service) recordRefusal(ctx context.Context, err error) error {
if !errors.Is(err, ErrReconnect) {
return err
}
cleared := s.st.WithTx(ctx, func(tx *sql.Tx) error {
n, err := db.New(tx).ClearIntegrationsGoogleToken(ctx, db.ClearIntegrationsGoogleTokenParams{
LastSyncError: sql.NullString{String: ReconnectReason, Valid: true},
UpdatedAt: store.Timestamp{Time: s.now().UTC().Truncate(time.Millisecond)}.String(),
})
if err != nil {
return err
}
if n == 0 {
// The lock is held since the connection was read, so only a
// change outside this service gets here.
return ErrNotConnected
}
return nil
})
if cleared != nil {
return fmt.Errorf("gcal: record that Google refused the grant: %w", cleared)
}
return err
}
// transient reports whether Google's HTTP status is a failure worth
// retrying: rate limited (429) or a server error (5xx).
func transient(status int) bool {
return status == http.StatusTooManyRequests || status >= http.StatusInternalServerError
}
// backoff waits before retry number attempt (1 for the first), or until ctx
// ends.
func (s *Service) backoff(ctx context.Context, attempt int) error {
t := time.NewTimer(s.retryDelay << (attempt - 1))
defer t.Stop()
select {
case <-ctx.Done():
return fmt.Errorf("gcal: wait to retry: %w", ctx.Err())
case <-t.C:
return nil
}
}
// send sends the request newRequest builds, and sends it again, up to
// maxRetries times, while Google answers it with a transient failure. It
// returns the final answer's status and up to limit bytes of its body. A
// transient failure on the last try is a retryable *CalendarError; what
// names the request in errors, which never hold the URL, a token or
// Google's response body.
func (s *Service) send(ctx context.Context, what string, limit int64, newRequest func() (*http.Request, error)) (int, []byte, error) {
for attempt := 0; ; attempt++ {
if attempt > 0 {
if err := s.backoff(ctx, attempt); err != nil {
return 0, nil, err
}
}
req, err := newRequest()
if err != nil {
return 0, nil, fmt.Errorf("gcal: build %s request: %w", what, err)
}
resp, err := s.client.Do(req)
if err != nil {
var transport *url.Error
if errors.As(err, &transport) {
// The URL is dropped: after a refused redirect it is the
// Google-supplied Location.
return 0, nil, &CalendarError{msg: fmt.Sprintf("gcal: %s request failed: %s: %v", what, transport.Op, transport.Err)}
}
return 0, nil, &CalendarError{msg: fmt.Sprintf("gcal: %s request failed", what)}
}
body, err := io.ReadAll(io.LimitReader(resp.Body, limit))
resp.Body.Close()
if err != nil {
return 0, nil, &CalendarError{msg: fmt.Sprintf("gcal: read %s response failed", what)}
}
if !transient(resp.StatusCode) {
return resp.StatusCode, body, nil
}
if attempt == maxRetries {
return 0, nil, &CalendarError{retryable: true,
msg: fmt.Sprintf("gcal: Google could not take the %s (HTTP %d) in %d tries", what, resp.StatusCode, maxRetries+1)}
}
}
}
+291
View File
@@ -0,0 +1,291 @@
package gcal_test
import (
"context"
"encoding/json"
"errors"
"net/http"
"reflect"
"testing"
"time"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/gcal"
"code.cubecraftlabs.com/Maniacs_Mansion/Meal-Quest/internal/gcal/gcaltest"
)
// refusals are the ways Google refuses the stored grant: invalid_grant on
// the token refresh, and HTTP 401 from the Calendar API to the access token
// just refreshed.
var refusals = map[string]func(*gcaltest.Google){
"invalid_grant": (*gcaltest.Google).RevokeGrant,
"401 after a refresh": (*gcaltest.Google).RejectAccessToken,
}
// calls are the Calendar calls that use the stored grant.
var calls = map[string]func(*harness) error{
"list calendars": func(h *harness) error {
_, err := h.svc.Calendars(context.Background())
return err
},
"choose a calendar": func(h *harness) error {
_, err := h.svc.SelectCalendar(context.Background(), family.ID)
return err
},
"push events": func(h *harness) error {
return h.svc.PushEvents(context.Background(), []gcal.Event{event("abcde")}, func() error {
return errors.New("record ran after a refused grant")
})
},
}
// TestRefusedGrantRequiresReconnect refuses the grant on every Calendar
// call: the call fails with ErrReconnect, the refresh token is cleared,
// last_sync_error says why, the chosen calendar is kept and Status reports
// reconnect required.
func TestRefusedGrantRequiresReconnect(t *testing.T) {
t.Parallel()
for refusal, refuse := range refusals {
for call, do := range calls {
t.Run(refusal+"/"+call, func(t *testing.T) {
t.Parallel()
h := chosen(t)
refuse(h.google)
h.clock = h.clock.Add(time.Hour)
err := do(h)
if !errors.Is(err, gcal.ErrReconnect) {
t.Fatalf("%s = %v, want ErrReconnect", call, err)
}
assertNoSecret(t, "error", err.Error())
row, ok := h.row(t)
switch {
case !ok:
t.Fatal("a refused grant forgot the connection")
case row.RefreshTokenCt != nil:
t.Error("refresh token kept after Google refused it")
case row.LastSyncError.String != "Google access was revoked or expired — reconnect":
t.Errorf("last_sync_error = %v", row.LastSyncError)
case row.CalendarID.String != dinners.ID || row.CalendarName.String != dinners.Summary:
t.Errorf("calendar = %v %v, want %s kept", row.CalendarID, row.CalendarName, dinners.ID)
case row.UpdatedAt != "2026-10-07T19:30:00.000Z":
t.Errorf("updated_at = %s, want the refusal's time", row.UpdatedAt)
}
st, err := h.svc.Status(context.Background())
if err != nil {
t.Fatal(err)
}
if st.State != gcal.StatusReconnectRequired || st.LastSyncError == nil || *st.LastSyncError != gcal.ReconnectReason {
t.Errorf("Status = %+v, want reconnect_required with the reason", st)
}
// The cleared grant is never offered to Google again, by
// any call.
tokens, cals, events := h.google.TokenCalls(), h.google.CalendarCalls(), h.google.EventRequests()
for again, do := range calls {
if err := do(h); !errors.Is(err, gcal.ErrReconnect) {
t.Errorf("%s after the refusal = %v, want ErrReconnect", again, err)
}
}
if h.google.TokenCalls() != tokens || h.google.CalendarCalls() != cals || h.google.EventRequests() != events {
t.Error("a call reached Google after the grant was refused")
}
})
}
}
}
// TestReconnectAfterRefusal connects again after Google refused the grant:
// the new token is stored, the reason cleared and the calendar kept.
func TestReconnectAfterRefusal(t *testing.T) {
t.Parallel()
h := chosen(t)
h.google.RevokeGrant()
if _, err := h.svc.Calendars(context.Background()); !errors.Is(err, gcal.ErrReconnect) {
t.Fatalf("Calendars = %v, want ErrReconnect", err)
}
c := h.connect(t, session)
if _, err := h.callback(c, session, h.consented(c)); err != nil {
t.Fatalf("Complete: %v", err)
}
st, err := h.svc.Status(context.Background())
if err != nil {
t.Fatal(err)
}
if st.State != gcal.StatusConnected || st.LastSyncError != nil || st.CalendarID == nil || *st.CalendarID != dinners.ID {
t.Errorf("Status = %+v, want connected to %s with no error", st, dinners.ID)
}
if err := h.svc.PushEvents(context.Background(), []gcal.Event{event("abcde")}, noRecord); err != nil {
t.Errorf("PushEvents after connecting again: %v", err)
}
}
// TestDisconnectAfterRefusal disconnects a connection whose grant Google
// refused: there is nothing to revoke, and the connection is deleted.
func TestDisconnectAfterRefusal(t *testing.T) {
t.Parallel()
h := chosen(t)
h.google.RevokeGrant()
if _, err := h.svc.Calendars(context.Background()); !errors.Is(err, gcal.ErrReconnect) {
t.Fatalf("Calendars = %v, want ErrReconnect", err)
}
if err := h.svc.Disconnect(context.Background()); err != nil {
t.Fatalf("Disconnect: %v", err)
}
if n := len(h.google.Revoked()); n != 0 {
t.Errorf("revoked %d tokens; a refused grant has none to revoke", n)
}
if _, ok := h.row(t); ok {
t.Error("connection kept after disconnecting")
}
}
// transients are the failures Google asks clients to retry.
var transients = []int{http.StatusServiceUnavailable, http.StatusInternalServerError, http.StatusTooManyRequests}
// TestTransientFailureIsRetried fails each Google request three times in a
// row: it is tried three times (two retries), then fails as retryable, and
// nothing is disconnected or cleared.
func TestTransientFailureIsRetried(t *testing.T) {
t.Parallel()
for _, status := range transients {
for name, tc := range map[string]struct {
fail func(*gcaltest.Google)
do func(*harness) error
tries func(*gcaltest.Google) int
}{
"token refresh": {
fail: func(g *gcaltest.Google) { g.FailToken(status) },
do: calls["list calendars"],
tries: (*gcaltest.Google).TokenCalls,
},
"calendar list": {
fail: func(g *gcaltest.Google) { g.FailCalendarList(status) },
do: calls["choose a calendar"],
tries: (*gcaltest.Google).CalendarCalls,
},
"event write": {
fail: func(g *gcaltest.Google) { g.FailEventWrites(1, 3, status) },
do: calls["push events"],
tries: (*gcaltest.Google).EventRequests,
},
} {
t.Run(name+"/"+http.StatusText(status), func(t *testing.T) {
t.Parallel()
h := chosen(t)
before, _ := h.row(t)
tried := tc.tries(h.google)
tc.fail(h.google)
err := tc.do(h)
var calendar *gcal.CalendarError
if !errors.As(err, &calendar) || !calendar.Retryable() {
t.Fatalf("err = %v, want a retryable *CalendarError", err)
}
assertNoSecret(t, "error", err.Error())
if n := tc.tries(h.google) - tried; n != 3 {
t.Errorf("tried %d times, want 3", n)
}
after, ok := h.row(t)
if !ok || !reflect.DeepEqual(before, after) {
t.Errorf("connection = %+v, %v; want it unchanged %+v", after, ok, before)
}
if st, err := h.svc.Status(context.Background()); err != nil || st.State != gcal.StatusConnected {
t.Errorf("Status = %+v, %v; want connected", st, err)
}
})
}
}
}
// TestTransientFailureRecovers fails each Google request twice: the second
// retry succeeds.
func TestTransientFailureRecovers(t *testing.T) {
t.Parallel()
for name, tc := range map[string]struct {
fail func(*gcaltest.Google)
do func(*harness) error
}{
"calendar list": {
fail: func(g *gcaltest.Google) { g.FailCalendarListNext(2, http.StatusServiceUnavailable) },
do: calls["choose a calendar"],
},
"event write": {
fail: func(g *gcaltest.Google) { g.FailEventWrites(1, 2, http.StatusServiceUnavailable) },
do: func(h *harness) error {
return h.svc.PushEvents(context.Background(), []gcal.Event{event("abcde")}, noRecord)
},
},
} {
t.Run(name, func(t *testing.T) {
t.Parallel()
h := chosen(t)
tc.fail(h.google)
if err := tc.do(h); err != nil {
t.Fatalf("err = %v, want the second retry to succeed", err)
}
})
}
}
// TestTransientFailureBacksOff checks the waits between tries: the first
// retry waits the retry delay, the second twice that.
func TestTransientFailureBacksOff(t *testing.T) {
t.Parallel()
h := chosen(t)
gcal.SetRetryDelay(h.svc, 50*time.Millisecond)
h.google.FailCalendarList(http.StatusServiceUnavailable)
start := time.Now()
if _, err := h.svc.Calendars(context.Background()); err == nil {
t.Fatal("Calendars succeeded")
}
if took := time.Since(start); took < 150*time.Millisecond {
t.Errorf("three tries took %s, want at least 150ms of waits", took)
}
}
func TestStatus(t *testing.T) {
t.Parallel()
h := newHarness(t)
st, err := h.svc.Status(context.Background())
if err != nil {
t.Fatal(err)
}
if got := statusJSON(t, st); got != `{"status":"not_connected","calendar_id":null,"calendar_name":null,"connected_at":null,"last_sync_error":null}` {
t.Errorf("not connected: Status = %s", got)
}
c := h.connect(t, session)
if _, err := h.callback(c, session, h.consented(c)); err != nil {
t.Fatal(err)
}
st, err = h.svc.Status(context.Background())
if err != nil {
t.Fatal(err)
}
if got := statusJSON(t, st); got != `{"status":"connected","calendar_id":null,"calendar_name":null,"connected_at":"2026-10-07T18:30:00.000Z","last_sync_error":null}` {
t.Errorf("connected: Status = %s", got)
}
h.google.SetCalendars(0, dinners)
if _, err := h.svc.SelectCalendar(context.Background(), dinners.ID); err != nil {
t.Fatal(err)
}
h.google.RevokeGrant()
if _, err := h.svc.Calendars(context.Background()); !errors.Is(err, gcal.ErrReconnect) {
t.Fatalf("Calendars = %v, want ErrReconnect", err)
}
st, err = h.svc.Status(context.Background())
if err != nil {
t.Fatal(err)
}
want := `{"status":"reconnect_required","calendar_id":"dinners@group.calendar.google.com","calendar_name":"Dinners",` +
`"connected_at":"2026-10-07T18:30:00.000Z","last_sync_error":"Google access was revoked or expired — reconnect"}`
if got := statusJSON(t, st); got != want {
t.Errorf("reconnect required: Status =\n%s\nwant\n%s", got, want)
}
}
func statusJSON(t *testing.T, st gcal.Status) string {
t.Helper()
b, err := json.Marshal(st)
if err != nil {
t.Fatal(err)
}
return string(b)
}