Improve syncing process

api/api.go

@@ -149,7 +149,7 @@ type HelloResponse struct {
 	Version       string              `json:"version"`
 	CID           string              `json:"cid"`
 	Auth          *CORE.SuperuserAuth `json:"auth"`
-	Instances     uint16              `json:"instances_num"`
+	InstancesNum  int                 `json:"instances_num"`
 	MemoryUsage   uint64              `json:"mem_usage"`
 	SentinelWorks bool                `json:"sentinel_works"`
 }

sync/master/master.go

@@ -231,6 +231,8 @@ func helloHandler(w http.ResponseWriter, r *http.Request) {
 		CID:           genCID(),
 		SentinelWorks: CORE.IsSentinelActive(),
 		Auth:          auth,
+		InstancesNum:  len(CORE.GetInstanceIDList()),
+		MemoryUsage:   calculateMemUsage(),
 	}
 
 	if coreCompat == API.CORE_COMPAT_PARTIAL {
@@ -1160,3 +1162,15 @@ func renderClientInfo(client *ClientInfo) string {
 		client.Role, client.Version, client.Hostname, client.IP,
 	)
 }
+
+// calculateMemUsage calculates total memory usage by all instances
+func calculateMemUsage() uint64 {
+	var total uint64
+
+	for _, id := range CORE.GetInstanceIDList() {
+		rss, swap, _ := CORE.GetInstanceMemUsage(id)
+		total += rss + swap
+	}
+
+	return total
+}

sync/minion/minion.go

@@ -21,6 +21,7 @@ import (
 	"github.com/essentialkaos/ek/v13/mathutil"
 	"github.com/essentialkaos/ek/v13/pluralize"
 	"github.com/essentialkaos/ek/v13/req"
+	"github.com/essentialkaos/ek/v13/system"
 	"github.com/essentialkaos/ek/v13/timeutil"
 	"github.com/essentialkaos/ek/v13/version"
 
@@ -140,17 +141,21 @@ func sendHelloCommand() bool {
 		return false
 	}
 
+	if !checkForRequiredMemoryToSync(helloResponse.InstancesNum, helloResponse.MemoryUsage) {
+		return false
+	}
+
 	switch AUXI.GetCoreCompatibility(helloResponse.Version) {
 	case API.CORE_COMPAT_PARTIAL:
-		log.Warn("This client might be incompatible with master node")
+		log.Warn("This minion node might be incompatible with master node")
 	case API.CORE_COMPAT_ERROR:
-		log.Crit("This client is not compatible with master node")
+		log.Crit("This minion node is not compatible with master node")
 		return false
 	}
 
 	cid = helloResponse.CID
 
-	log.Info("Master (%s) return CID %s for this client", helloResponse.Version, cid)
+	log.Info("Master (%s) return CID %s for this minion node", helloResponse.Version, cid)
 
 	sentinelWorks = helloResponse.SentinelWorks
 
@@ -923,6 +928,35 @@ func sendRequest(method API.Method, reqData, respData any) error {
 	return nil
 }
 
+// checkForRequiredMemoryToSync checks if system has enough memory to sync
+func checkForRequiredMemoryToSync(instanceNum int, memoryUsage uint64) bool {
+	systemMem, err := system.GetMemUsage()
+
+	if err != nil {
+		log.Error("Can't check system memory usage for sync: %v", err)
+		return true
+	}
+
+	usageRatio := float64(memoryUsage) / float64(systemMem.MemFree)
+
+	if usageRatio >= 0.9 {
+		log.Crit(
+			"System has no enough free memory (%s is reqired, %s is free) to sync",
+			fmtutil.PrettySize(memoryUsage), fmtutil.PrettySize(systemMem.MemFree),
+		)
+		return false
+	}
+
+	if usageRatio >= 0.55 {
+		log.Warn(
+			"System has dangerously low amount of free memory (%s is reqired, %s is free) to sync, keep an eye on it.",
+			fmtutil.PrettySize(memoryUsage), fmtutil.PrettySize(systemMem.MemFree),
+		)
+	}
+
+	return true
+}
+
 // syncSentinelState syncs state of Sentinel with master
 func syncSentinelState(sentinelWorks bool) {
 	var err error