Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]

### Fixed

- Fixed a leader announcing the loss of leadership only after trying to resign, which can take several seconds when the database is unreachable. It now announces the loss first, so maintenance services stop sooner. [PR #1600](https://github.com/riverqueue/river/pull/1600).

## [0.51.2] - 2026-11-09

### Fixed
Expand Down
5 changes: 4 additions & 1 deletion internal/leadership/elector.go
Original file line number Diff line number Diff line change
Expand Up @@ -490,7 +490,6 @@ func (e *Elector) handleLeadershipNotification(ctx context.Context, topic notifi
// either a reelection interval, a forced resignation, or shutdown.
func (e *Elector) runLeaderState(ctx context.Context, term leadershipTerm) error {
defer e.clearPendingRequestResign()
defer e.publishLeadershipState(false)

shouldResign := true

Expand All @@ -501,6 +500,10 @@ func (e *Elector) runLeaderState(ctx context.Context, term leadershipTerm) error
//
// This doesn't use ctx because it runs *after* the ctx is done.
defer func() {
// Publish the loss first so the client can stop maintenance. The
// resignation can take seconds when the database is unreachable, and
// another client may win the lease meanwhile.
e.publishLeadershipState(false)
if shouldResign {
e.attemptResignLoop(ctx, term) // will resign using WithoutCancel context, but ctx sent for logging
}
Expand Down
46 changes: 46 additions & 0 deletions internal/leadership/elector_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ type leaderReelectExecutorMock struct {
riverdriver.Executor

LeaderAttemptReelectFunc func(ctx context.Context, params *riverdriver.LeaderReelectParams) (*riverdriver.Leader, error)
LeaderResignFunc func(ctx context.Context, params *riverdriver.LeaderResignParams) (bool, error)
}

func (m *leaderReelectExecutorMock) LeaderAttemptReelect(ctx context.Context, params *riverdriver.LeaderReelectParams) (*riverdriver.Leader, error) {
Expand All @@ -70,6 +71,14 @@ func (m *leaderReelectExecutorMock) LeaderAttemptReelect(ctx context.Context, pa
return m.LeaderAttemptReelectFunc(ctx, params)
}

func (m *leaderReelectExecutorMock) LeaderResign(ctx context.Context, params *riverdriver.LeaderResignParams) (bool, error) {
if m.LeaderResignFunc == nil {
return m.Executor.LeaderResign(ctx, params)
}

return m.LeaderResignFunc(ctx, params)
}

type localNowTimeGeneratorStub struct {
now time.Time
}
Expand Down Expand Up @@ -321,6 +330,43 @@ func TestElectorRunLeaderState(t *testing.T) {
return elector, exec
}

t.Run("PublishesLostLeadershipBeforeResigning", func(t *testing.T) {
t.Parallel()

elector, exec := setup(t)
initialNow := elector.Time.StubNow(time.Now().UTC())
elector.publishLeadershipState(true)

leader := testfactory.Leader(ctx, t, exec, &testfactory.LeaderOpts{
ElectedAt: new(initialNow),
ExpiresAt: new(initialNow.Add(elector.leaderTTL())),
LeaderID: new(elector.config.ClientID),
})

// The client stops maintenance once it receives the published loss,
// which shouldn't wait for a resignation that can take seconds.
var leaderWhileResigning []bool
elector.exec = &leaderReelectExecutorMock{
Executor: exec,
LeaderAttemptReelectFunc: func(ctx context.Context, params *riverdriver.LeaderReelectParams) (*riverdriver.Leader, error) {
elector.Time.StubNow(initialNow.Add(elector.leaderTTL()))
return nil, errors.New("reelection error")
},
LeaderResignFunc: func(ctx context.Context, params *riverdriver.LeaderResignParams) (bool, error) {
elector.mu.Lock()
leaderWhileResigning = append(leaderWhileResigning, elector.isLeader)
elector.mu.Unlock()
return exec.LeaderResign(ctx, params)
},
}

runCtx, cancel := context.WithTimeout(ctx, time.Second)
defer cancel()

require.NoError(t, elector.runLeaderState(runCtx, newLeadershipTerm(elector.config.ClientID, leader.ElectedAt, initialNow, elector.leaderTTL())))
require.Equal(t, []bool{false}, leaderWhileResigning)
})

t.Run("ResignsCurrentTermAfterReelectErrorsExhaustTrust", func(t *testing.T) {
t.Parallel()

Expand Down