From 0151c54da512c5d49f3b2e13dff04afad242b12d Mon Sep 17 00:00:00 2001 From: Anton Novojilov Date: Fri, 17 Jan 2025 15:48:47 +0300 Subject: [PATCH] Improve syncing process --- api/api.go | 2 +- sync/master/master.go | 14 ++++++++++++++ sync/minion/minion.go | 40 +++++++++++++++++++++++++++++++++++++--- 3 files changed, 52 insertions(+), 4 deletions(-) diff --git a/api/api.go b/api/api.go index f657f43..85f3573 100644 --- a/api/api.go +++ b/api/api.go @@ -149,7 +149,7 @@ type HelloResponse struct { Version string `json:"version"` CID string `json:"cid"` Auth *CORE.SuperuserAuth `json:"auth"` - Instances uint16 `json:"instances_num"` + InstancesNum int `json:"instances_num"` MemoryUsage uint64 `json:"mem_usage"` SentinelWorks bool `json:"sentinel_works"` } diff --git a/sync/master/master.go b/sync/master/master.go index 312834c..03237af 100644 --- a/sync/master/master.go +++ b/sync/master/master.go @@ -231,6 +231,8 @@ func helloHandler(w http.ResponseWriter, r *http.Request) { CID: genCID(), SentinelWorks: CORE.IsSentinelActive(), Auth: auth, + InstancesNum: len(CORE.GetInstanceIDList()), + MemoryUsage: calculateMemUsage(), } if coreCompat == API.CORE_COMPAT_PARTIAL { @@ -1160,3 +1162,15 @@ func renderClientInfo(client *ClientInfo) string { client.Role, client.Version, client.Hostname, client.IP, ) } + +// calculateMemUsage calculates total memory usage by all instances +func calculateMemUsage() uint64 { + var total uint64 + + for _, id := range CORE.GetInstanceIDList() { + rss, swap, _ := CORE.GetInstanceMemUsage(id) + total += rss + swap + } + + return total +} diff --git a/sync/minion/minion.go b/sync/minion/minion.go index d30f493..ad0e833 100644 --- a/sync/minion/minion.go +++ b/sync/minion/minion.go @@ -21,6 +21,7 @@ import ( "github.com/essentialkaos/ek/v13/mathutil" "github.com/essentialkaos/ek/v13/pluralize" "github.com/essentialkaos/ek/v13/req" + "github.com/essentialkaos/ek/v13/system" "github.com/essentialkaos/ek/v13/timeutil" "github.com/essentialkaos/ek/v13/version" @@ -140,17 +141,21 @@ func sendHelloCommand() bool { return false } + if !checkForRequiredMemoryToSync(helloResponse.InstancesNum, helloResponse.MemoryUsage) { + return false + } + switch AUXI.GetCoreCompatibility(helloResponse.Version) { case API.CORE_COMPAT_PARTIAL: - log.Warn("This client might be incompatible with master node") + log.Warn("This minion node might be incompatible with master node") case API.CORE_COMPAT_ERROR: - log.Crit("This client is not compatible with master node") + log.Crit("This minion node is not compatible with master node") return false } cid = helloResponse.CID - log.Info("Master (%s) return CID %s for this client", helloResponse.Version, cid) + log.Info("Master (%s) return CID %s for this minion node", helloResponse.Version, cid) sentinelWorks = helloResponse.SentinelWorks @@ -923,6 +928,35 @@ func sendRequest(method API.Method, reqData, respData any) error { return nil } +// checkForRequiredMemoryToSync checks if system has enough memory to sync +func checkForRequiredMemoryToSync(instanceNum int, memoryUsage uint64) bool { + systemMem, err := system.GetMemUsage() + + if err != nil { + log.Error("Can't check system memory usage for sync: %v", err) + return true + } + + usageRatio := float64(memoryUsage) / float64(systemMem.MemFree) + + if usageRatio >= 0.9 { + log.Crit( + "System has no enough free memory (%s is reqired, %s is free) to sync", + fmtutil.PrettySize(memoryUsage), fmtutil.PrettySize(systemMem.MemFree), + ) + return false + } + + if usageRatio >= 0.55 { + log.Warn( + "System has dangerously low amount of free memory (%s is reqired, %s is free) to sync, keep an eye on it.", + fmtutil.PrettySize(memoryUsage), fmtutil.PrettySize(systemMem.MemFree), + ) + } + + return true +} + // syncSentinelState syncs state of Sentinel with master func syncSentinelState(sentinelWorks bool) { var err error