diff --git a/CHANGELOG.md b/CHANGELOG.md index eedce02fd..2486a6640 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- Fixed a leader announcing the loss of leadership only after trying to resign, which can take several seconds when the database is unreachable. It now announces the loss first, so maintenance services stop sooner. [PR #1600](https://github.com/riverqueue/river/pull/1600). + ## [0.51.2] - 2026-11-09 ### Fixed diff --git a/internal/leadership/elector.go b/internal/leadership/elector.go index 260e57453..aea68271b 100644 --- a/internal/leadership/elector.go +++ b/internal/leadership/elector.go @@ -490,7 +490,6 @@ func (e *Elector) handleLeadershipNotification(ctx context.Context, topic notifi // either a reelection interval, a forced resignation, or shutdown. func (e *Elector) runLeaderState(ctx context.Context, term leadershipTerm) error { defer e.clearPendingRequestResign() - defer e.publishLeadershipState(false) shouldResign := true @@ -501,6 +500,10 @@ func (e *Elector) runLeaderState(ctx context.Context, term leadershipTerm) error // // This doesn't use ctx because it runs *after* the ctx is done. defer func() { + // Publish the loss first so the client can stop maintenance. The + // resignation can take seconds when the database is unreachable, and + // another client may win the lease meanwhile. + e.publishLeadershipState(false) if shouldResign { e.attemptResignLoop(ctx, term) // will resign using WithoutCancel context, but ctx sent for logging } diff --git a/internal/leadership/elector_test.go b/internal/leadership/elector_test.go index 0dc7b79aa..ec34521d4 100644 --- a/internal/leadership/elector_test.go +++ b/internal/leadership/elector_test.go @@ -60,6 +60,7 @@ type leaderReelectExecutorMock struct { riverdriver.Executor LeaderAttemptReelectFunc func(ctx context.Context, params *riverdriver.LeaderReelectParams) (*riverdriver.Leader, error) + LeaderResignFunc func(ctx context.Context, params *riverdriver.LeaderResignParams) (bool, error) } func (m *leaderReelectExecutorMock) LeaderAttemptReelect(ctx context.Context, params *riverdriver.LeaderReelectParams) (*riverdriver.Leader, error) { @@ -70,6 +71,14 @@ func (m *leaderReelectExecutorMock) LeaderAttemptReelect(ctx context.Context, pa return m.LeaderAttemptReelectFunc(ctx, params) } +func (m *leaderReelectExecutorMock) LeaderResign(ctx context.Context, params *riverdriver.LeaderResignParams) (bool, error) { + if m.LeaderResignFunc == nil { + return m.Executor.LeaderResign(ctx, params) + } + + return m.LeaderResignFunc(ctx, params) +} + type localNowTimeGeneratorStub struct { now time.Time } @@ -321,6 +330,43 @@ func TestElectorRunLeaderState(t *testing.T) { return elector, exec } + t.Run("PublishesLostLeadershipBeforeResigning", func(t *testing.T) { + t.Parallel() + + elector, exec := setup(t) + initialNow := elector.Time.StubNow(time.Now().UTC()) + elector.publishLeadershipState(true) + + leader := testfactory.Leader(ctx, t, exec, &testfactory.LeaderOpts{ + ElectedAt: new(initialNow), + ExpiresAt: new(initialNow.Add(elector.leaderTTL())), + LeaderID: new(elector.config.ClientID), + }) + + // The client stops maintenance once it receives the published loss, + // which shouldn't wait for a resignation that can take seconds. + var leaderWhileResigning []bool + elector.exec = &leaderReelectExecutorMock{ + Executor: exec, + LeaderAttemptReelectFunc: func(ctx context.Context, params *riverdriver.LeaderReelectParams) (*riverdriver.Leader, error) { + elector.Time.StubNow(initialNow.Add(elector.leaderTTL())) + return nil, errors.New("reelection error") + }, + LeaderResignFunc: func(ctx context.Context, params *riverdriver.LeaderResignParams) (bool, error) { + elector.mu.Lock() + leaderWhileResigning = append(leaderWhileResigning, elector.isLeader) + elector.mu.Unlock() + return exec.LeaderResign(ctx, params) + }, + } + + runCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + + require.NoError(t, elector.runLeaderState(runCtx, newLeadershipTerm(elector.config.ClientID, leader.ElectedAt, initialNow, elector.leaderTTL()))) + require.Equal(t, []bool{false}, leaderWhileResigning) + }) + t.Run("ResignsCurrentTermAfterReelectErrorsExhaustTrust", func(t *testing.T) { t.Parallel()