From fc671cb50e92b3196c9ab3586dca3daabf14b223 Mon Sep 17 00:00:00 2001
From: Paul Lorenz <paul.lorenz@netfoundry.io>
Date: Mon, 20 Nov 2023 15:31:59 -0500
Subject: [PATCH] Retry bootstrap members until bootstrapped. Fixes #1523

---
 controller/raft/raft.go | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/controller/raft/raft.go b/controller/raft/raft.go
index 7b12282dd..6eb939584 100644
--- a/controller/raft/raft.go
+++ b/controller/raft/raft.go
@@ -683,6 +683,7 @@ func (self *Controller) addConfiguredBootstrapMembers() error {
 
 		if id, addr, err := self.Mesh.GetPeerInfo(bootstrapMember, time.Second*5); err != nil {
 			pfxlog.Logger().WithError(err).Errorf("unable to get id for bootstrap member [%v]", bootstrapMember)
+			go self.retryBootstrapMember(bootstrapMember)
 		} else {
 			self.addBootstrapServer(raft.Server{
 				Suffrage: raft.Voter,
@@ -694,6 +695,34 @@ func (self *Controller) addConfiguredBootstrapMembers() error {
 	return nil
 }
 
+func (self *Controller) retryBootstrapMember(bootstrapMember string) {
+	ticker := time.NewTicker(6 * time.Second)
+	defer ticker.Stop()
+
+	for {
+		<-ticker.C
+
+		// If we've bootstrapped, exit out
+		if self.bootstrapped.Load() {
+			return
+		}
+
+		if id, addr, err := self.Mesh.GetPeerInfo(bootstrapMember, time.Second*5); err != nil {
+			pfxlog.Logger().WithError(err).Errorf("unable to get id for bootstrap member [%v]", bootstrapMember)
+		} else {
+			req := &cmd_pb.AddPeerRequest{
+				Addr:    string(addr),
+				Id:      string(id),
+				IsVoter: true,
+			}
+
+			if err = self.Join(req); err == nil {
+				return
+			}
+		}
+	}
+}
+
 // Join adds the given node to the raft cluster
 func (self *Controller) Join(req *cmd_pb.AddPeerRequest) error {
 	self.clusterLock.Lock()