diff --git a/node/pkg/raft/raft.go b/node/pkg/raft/raft.go index 6b6102f4c..2965a01ea 100644 --- a/node/pkg/raft/raft.go +++ b/node/pkg/raft/raft.go @@ -40,6 +40,7 @@ func NewRaftNode( HeartbeatTimeout: HEARTBEAT_TIMEOUT, LeaderJobTimeout: leaderJobTimeout, + MissedHeartbeats: 0, } return r } @@ -129,6 +130,8 @@ func (r *Raft) handleHeartbeat(msg Message) error { r.Mutex.Lock() defer r.Mutex.Unlock() + r.MissedHeartbeats = 0 + currentRole := r.Role currentTerm := r.Term @@ -387,13 +390,22 @@ func (r *Raft) startElectionTimer() { } func (r *Raft) startElection(ctx context.Context) { - log.Debug().Msg("start election") r.Mutex.Lock() defer r.Mutex.Unlock() + + if r.MissedHeartbeats < MaxMissedHeartbeats { + r.MissedHeartbeats++ + log.Debug().Int("missed heartbeats", r.MissedHeartbeats).Msg("missed heartbeats") + r.startElectionTimer() + return + } + + log.Debug().Msg("start election") r.Term++ r.VotesReceived = 0 r.Role = Candidate r.VotedFor = r.GetHostId() + r.MissedHeartbeats = 0 r.startElectionTimer() diff --git a/node/pkg/raft/types.go b/node/pkg/raft/types.go index c64c32a8b..7c3cc7b8b 100644 --- a/node/pkg/raft/types.go +++ b/node/pkg/raft/types.go @@ -23,6 +23,8 @@ const ( Leader RoleType = "leader" Candidate RoleType = "candidate" Follower RoleType = "follower" + + MaxMissedHeartbeats = 3 ) type Message struct { @@ -67,4 +69,6 @@ type Raft struct { LeaderJobTicker *time.Ticker HandleCustomMessage func(context.Context, Message) error LeaderJob func(context.Context) error + + MissedHeartbeats int }