From 93b5db77170568a582be3bef4f18c7d1cf2c6d95 Mon Sep 17 00:00:00 2001 From: AleksandrMatsko Date: Tue, 10 Dec 2024 19:06:06 +0700 Subject: [PATCH 1/9] refactor: move notifier alive metric from heartbeat to notifierMetrics --- cmd/notifier/main.go | 2 +- metrics/heartbeat.go | 22 ------------------- metrics/notifier.go | 11 ++++++++++ notifier/selfstate/heartbeat/notifier.go | 17 +++++--------- notifier/selfstate/heartbeat/notifier_test.go | 5 +---- notifier/selfstate/selfstate.go | 19 ++++++++++------ notifier/selfstate/selfstate_test.go | 6 +---- 7 files changed, 31 insertions(+), 51 deletions(-) delete mode 100644 metrics/heartbeat.go diff --git a/cmd/notifier/main.go b/cmd/notifier/main.go index 37e1ac7f7..41043104e 100644 --- a/cmd/notifier/main.go +++ b/cmd/notifier/main.go @@ -119,7 +119,7 @@ func main() { // Start moira self state checker if config.Notifier.SelfState.getSettings().Enabled { - selfState := selfstate.NewSelfCheckWorker(logger, database, sender, config.Notifier.SelfState.getSettings(), metrics.ConfigureHeartBeatMetrics(telemetry.Metrics)) + selfState := selfstate.NewSelfCheckWorker(logger, database, sender, config.Notifier.SelfState.getSettings()) if err := selfState.Start(); err != nil { logger.Fatal(). Error(err). diff --git a/metrics/heartbeat.go b/metrics/heartbeat.go deleted file mode 100644 index 378e158cf..000000000 --- a/metrics/heartbeat.go +++ /dev/null @@ -1,22 +0,0 @@ -package metrics - -// HeartBeatMetrics is a collection of metrics used in hearbeats. -type HeartBeatMetrics struct { - notifierIsAlive Meter -} - -// ConfigureHeartBeatMetrics is notifier metrics configurator. -func ConfigureHeartBeatMetrics(registry Registry) *HeartBeatMetrics { - return &HeartBeatMetrics{ - notifierIsAlive: registry.NewMeter("", "alive"), - } -} - -// MarkNotifierIsAlive marks metric value. -func (hb HeartBeatMetrics) MarkNotifierIsAlive(isAlive bool) { - if isAlive { - hb.notifierIsAlive.Mark(1) - } - - hb.notifierIsAlive.Mark(0) -} diff --git a/metrics/notifier.go b/metrics/notifier.go index 8711ef11c..ebec0dddf 100644 --- a/metrics/notifier.go +++ b/metrics/notifier.go @@ -16,6 +16,7 @@ type NotifierMetrics struct { PlotsBuildDurationMs Histogram PlotsEvaluateTriggerDurationMs Histogram fetchNotificationsDurationMs Histogram + notifierIsAlive Meter } // ConfigureNotifierMetrics is notifier metrics configurator. @@ -33,6 +34,7 @@ func ConfigureNotifierMetrics(registry Registry, prefix string) *NotifierMetrics PlotsBuildDurationMs: registry.NewHistogram("plots", "build", "duration", "ms"), PlotsEvaluateTriggerDurationMs: registry.NewHistogram("plots", "evaluate", "trigger", "duration", "ms"), fetchNotificationsDurationMs: registry.NewHistogram("fetch", "notifications", "duration", "ms"), + notifierIsAlive: registry.NewMeter("", "alive"), } } @@ -66,3 +68,12 @@ func (metrics *NotifierMetrics) MarkSendersFailedMetrics(contactType string) { func (metrics *NotifierMetrics) MarkSendingFailed() { metrics.SendingFailed.Mark(1) } + +// MarkNotifierIsAlive marks metric value. +func (metrics *NotifierMetrics) MarkNotifierIsAlive(isAlive bool) { + if isAlive { + metrics.notifierIsAlive.Mark(1) + } + + metrics.notifierIsAlive.Mark(0) +} diff --git a/notifier/selfstate/heartbeat/notifier.go b/notifier/selfstate/heartbeat/notifier.go index 7b6877be4..0b418d0c9 100644 --- a/notifier/selfstate/heartbeat/notifier.go +++ b/notifier/selfstate/heartbeat/notifier.go @@ -3,37 +3,30 @@ package heartbeat import ( "fmt" - "github.com/moira-alert/moira/metrics" - "github.com/moira-alert/moira" ) type notifier struct { - db moira.Database - log moira.Logger - metrics *metrics.HeartBeatMetrics + db moira.Database + log moira.Logger } -func GetNotifier(logger moira.Logger, database moira.Database, metrics *metrics.HeartBeatMetrics) Heartbeater { +func GetNotifier(logger moira.Logger, database moira.Database) Heartbeater { return ¬ifier{ - db: database, - log: logger, - metrics: metrics, + db: database, + log: logger, } } func (check notifier) Check(int64) (int64, bool, error) { state, _ := check.db.GetNotifierState() if state != moira.SelfStateOK { - check.metrics.MarkNotifierIsAlive(false) - check.log.Error(). String("error", check.GetErrorMessage()). Msg("Notifier is not healthy") return 0, true, nil } - check.metrics.MarkNotifierIsAlive(true) check.log.Debug(). String("state", state). diff --git a/notifier/selfstate/heartbeat/notifier_test.go b/notifier/selfstate/heartbeat/notifier_test.go index 3d4976035..ba56a6d7d 100644 --- a/notifier/selfstate/heartbeat/notifier_test.go +++ b/notifier/selfstate/heartbeat/notifier_test.go @@ -4,8 +4,6 @@ import ( "testing" "time" - "github.com/moira-alert/moira/metrics" - "github.com/moira-alert/moira" mock_moira_alert "github.com/moira-alert/moira/mock/moira-alert" @@ -47,7 +45,6 @@ func TestNotifierState(t *testing.T) { func createNotifierStateTest(t *testing.T) *notifier { mockCtrl := gomock.NewController(t) logger, _ := logging.GetLogger("MetricDelay") - metric := metrics.ConfigureHeartBeatMetrics(metrics.NewDummyRegistry()) - return GetNotifier(logger, mock_moira_alert.NewMockDatabase(mockCtrl), metric).(*notifier) + return GetNotifier(logger, mock_moira_alert.NewMockDatabase(mockCtrl)).(*notifier) } diff --git a/notifier/selfstate/selfstate.go b/notifier/selfstate/selfstate.go index 99bba0ed0..27245cebc 100644 --- a/notifier/selfstate/selfstate.go +++ b/notifier/selfstate/selfstate.go @@ -3,8 +3,6 @@ package selfstate import ( "time" - "github.com/moira-alert/moira/metrics" - "github.com/moira-alert/moira/notifier/selfstate/heartbeat" "gopkg.in/tomb.v2" @@ -30,9 +28,15 @@ type SelfCheckWorker struct { } // NewSelfCheckWorker creates SelfCheckWorker. -func NewSelfCheckWorker(logger moira.Logger, database moira.Database, notifier notifier.Notifier, config Config, metrics *metrics.HeartBeatMetrics) *SelfCheckWorker { - heartbeats := createStandardHeartbeats(logger, database, config, metrics) - return &SelfCheckWorker{Logger: logger, Database: database, Notifier: notifier, Config: config, heartbeats: heartbeats} +func NewSelfCheckWorker(logger moira.Logger, database moira.Database, notifier notifier.Notifier, config Config) *SelfCheckWorker { + heartbeats := createStandardHeartbeats(logger, database, config) + return &SelfCheckWorker{ + Logger: logger, + Database: database, + Notifier: notifier, + Config: config, + heartbeats: heartbeats, + } } // Start self check worker. @@ -49,6 +53,7 @@ func (selfCheck *SelfCheckWorker) Start() error { selfCheck.Database.NewLock(selfStateLockName, selfStateLockTTL), selfCheck.selfStateChecker, ).Run(selfCheck.tomb.Dying()) + return nil }) @@ -61,7 +66,7 @@ func (selfCheck *SelfCheckWorker) Stop() error { return selfCheck.tomb.Wait() } -func createStandardHeartbeats(logger moira.Logger, database moira.Database, conf Config, metrics *metrics.HeartBeatMetrics) []heartbeat.Heartbeater { +func createStandardHeartbeats(logger moira.Logger, database moira.Database, conf Config) []heartbeat.Heartbeater { heartbeats := make([]heartbeat.Heartbeater, 0) if hb := heartbeat.GetDatabase(conf.RedisDisconnectDelaySeconds, logger, database); hb != nil { @@ -80,7 +85,7 @@ func createStandardHeartbeats(logger moira.Logger, database moira.Database, conf heartbeats = append(heartbeats, hb) } - if hb := heartbeat.GetNotifier(logger, database, metrics); hb != nil { + if hb := heartbeat.GetNotifier(logger, database); hb != nil { heartbeats = append(heartbeats, hb) } diff --git a/notifier/selfstate/selfstate_test.go b/notifier/selfstate/selfstate_test.go index 71859dca9..aa94d93fe 100644 --- a/notifier/selfstate/selfstate_test.go +++ b/notifier/selfstate/selfstate_test.go @@ -5,8 +5,6 @@ import ( "testing" "time" - "github.com/moira-alert/moira/metrics" - mock_heartbeat "github.com/moira-alert/moira/mock/heartbeat" "github.com/moira-alert/moira/notifier/selfstate/heartbeat" @@ -177,10 +175,8 @@ func configureWorker(t *testing.T, isStart bool) *selfCheckWorkerMock { database.EXPECT().NewLock(gomock.Any(), gomock.Any()).Return(lock) } - metric := &metrics.HeartBeatMetrics{} - return &selfCheckWorkerMock{ - selfCheckWorker: NewSelfCheckWorker(logger, database, notif, conf, metric), + selfCheckWorker: NewSelfCheckWorker(logger, database, notif, conf), database: database, notif: notif, conf: conf, From bbb6a9df31b2f4f1ee5015907c5ba076f16beee0 Mon Sep 17 00:00:00 2001 From: AleksandrMatsko Date: Wed, 11 Dec 2024 11:55:48 +0700 Subject: [PATCH 2/9] feat: alive watcher --- cmd/notifier/config.go | 4 +++ cmd/notifier/main.go | 8 +++++ notifier/alive_watcher.go | 61 +++++++++++++++++++++++++++++++++++++++ notifier/config.go | 1 + 4 files changed, 74 insertions(+) create mode 100644 notifier/alive_watcher.go diff --git a/cmd/notifier/config.go b/cmd/notifier/config.go index eb737a798..b9bc5a030 100644 --- a/cmd/notifier/config.go +++ b/cmd/notifier/config.go @@ -56,6 +56,8 @@ type notifierConfig struct { MaxFailAttemptToSendAvailable int `yaml:"max_fail_attempt_to_send_available"` // Specify log level by entities SetLogLevel setLogLevelConfig `yaml:"set_log_level"` + // CheckNotifierStateTimeout is the timeout between marking *.alive.count metric based on notifier state. + CheckNotifierStateTimeout string `yaml:"check_notifier_state_timeout"` } type selfStateConfig struct { @@ -116,6 +118,7 @@ func getDefault() config { Timezone: "UTC", ReadBatchSize: int(notifier.NotificationsLimitUnlimited), MaxFailAttemptToSendAvailable: 3, + CheckNotifierStateTimeout: "10s", }, Telemetry: cmd.TelemetryConfig{ Listen: ":8093", @@ -202,6 +205,7 @@ func (config *notifierConfig) getSettings(logger moira.Logger) notifier.Config { MaxFailAttemptToSendAvailable: config.MaxFailAttemptToSendAvailable, LogContactsToLevel: contacts, LogSubscriptionsToLevel: subscriptions, + CheckNotifierStateTimeout: to.Duration(config.CheckNotifierStateTimeout), } } diff --git a/cmd/notifier/main.go b/cmd/notifier/main.go index 41043104e..cc64f7db1 100644 --- a/cmd/notifier/main.go +++ b/cmd/notifier/main.go @@ -1,6 +1,7 @@ package main import ( + "context" "flag" "fmt" "os" @@ -99,6 +100,7 @@ func main() { } notifierMetrics := metrics.ConfigureNotifierMetrics(telemetry.Metrics, serviceName) + sender := notifier.NewNotifier( database, logger, @@ -156,6 +158,12 @@ func main() { fetchEventsWorker.Start() defer stopFetchEvents(fetchEventsWorker) + aliveWatcher := notifier.NewAliveWatcher(logger, database, notifierConfig, notifierMetrics) + ctx, cancel := context.WithCancel(context.Background()) + + aliveWatcher.Start(ctx) + defer cancel() + logger.Info(). String("moira_version", MoiraVersion). Msg("Moira Notifier Started") diff --git a/notifier/alive_watcher.go b/notifier/alive_watcher.go new file mode 100644 index 000000000..3bad07b1b --- /dev/null +++ b/notifier/alive_watcher.go @@ -0,0 +1,61 @@ +package notifier + +import ( + "context" + "time" + + "github.com/moira-alert/moira" + "github.com/moira-alert/moira/metrics" +) + +type AliveWatcher struct { + logger moira.Logger + database moira.Database + config Config + notifierMetrics *metrics.NotifierMetrics +} + +func NewAliveWatcher( + logger moira.Logger, + database moira.Database, + config Config, + notifierMetrics *metrics.NotifierMetrics, +) *AliveWatcher { + return &AliveWatcher{ + logger: logger, + database: database, + config: config, + notifierMetrics: notifierMetrics, + } +} + +func (watcher *AliveWatcher) Start(ctx context.Context) { + go watcher.stateChecker(ctx) +} + +func (watcher *AliveWatcher) stateChecker(ctx context.Context) { + watcher.logger.Info(). + Interface("check_timeout_seconds", watcher.config.CheckNotifierStateTimeout.Seconds()). + Msg("Moira Notifier alive watcher started") + + ticker := time.NewTicker(watcher.config.CheckNotifierStateTimeout) + + for { + select { + case <-ctx.Done(): + watcher.logger.Info().Msg("Moira Notifier alive watcher stopped") + return + case <-ticker.C: + watcher.checkNotifierState() + } + } +} + +func (watcher *AliveWatcher) checkNotifierState() { + state, _ := watcher.database.GetNotifierState() + if state != moira.SelfStateOK { + watcher.notifierMetrics.MarkNotifierIsAlive(false) + } + + watcher.notifierMetrics.MarkNotifierIsAlive(true) +} diff --git a/notifier/config.go b/notifier/config.go index 34c6e53d4..8e1abe66b 100644 --- a/notifier/config.go +++ b/notifier/config.go @@ -25,4 +25,5 @@ type Config struct { MaxFailAttemptToSendAvailable int LogContactsToLevel map[string]string LogSubscriptionsToLevel map[string]string + CheckNotifierStateTimeout time.Duration } From cf24dc6c22ba1eb4dd25d5b55b6aa3853f266d87 Mon Sep 17 00:00:00 2001 From: AleksandrMatsko Date: Wed, 11 Dec 2024 14:10:44 +0700 Subject: [PATCH 3/9] test: for alive watcher --- notifier/alive_watcher_test.go | 79 ++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 notifier/alive_watcher_test.go diff --git a/notifier/alive_watcher_test.go b/notifier/alive_watcher_test.go new file mode 100644 index 000000000..a59128568 --- /dev/null +++ b/notifier/alive_watcher_test.go @@ -0,0 +1,79 @@ +package notifier + +import ( + "errors" + "testing" + "time" + + "github.com/moira-alert/moira" + logging "github.com/moira-alert/moira/logging/zerolog_adapter" + "github.com/moira-alert/moira/metrics" + . "github.com/smartystreets/goconvey/convey" + "go.uber.org/mock/gomock" + + mock_moira_alert "github.com/moira-alert/moira/mock/moira-alert" + mock_metrics "github.com/moira-alert/moira/mock/moira-alert/metrics" +) + +func initAliveMeter(mockCtrl *gomock.Controller) (*mock_metrics.MockRegistry, *mock_metrics.MockMeter) { + mockRegistry := mock_metrics.NewMockRegistry(mockCtrl) + mockAliveMeter := mock_metrics.NewMockMeter(mockCtrl) + + mockRegistry.EXPECT().NewMeter(gomock.Any()).Times(5) + mockRegistry.EXPECT().NewHistogram(gomock.Any()).Times(3) + mockRegistry.EXPECT().NewMeter("", "alive").Return(mockAliveMeter) + + return mockRegistry, mockAliveMeter +} + +func TestAliveWatcher_checkNotifierState(t *testing.T) { + logger, _ = logging.GetLogger("test alive watcher") + + mockCtrl := gomock.NewController(t) + dataBase := mock_moira_alert.NewMockDatabase(mockCtrl) + defer mockCtrl.Finish() + + testConf := Config{ + CheckNotifierStateTimeout: time.Second * 10, + } + + mockRegistry, mockAliveMeter := initAliveMeter(mockCtrl) + testNotifierMetrics := metrics.ConfigureNotifierMetrics(mockRegistry, "") + + aliveWatcher := NewAliveWatcher(logger, dataBase, testConf, testNotifierMetrics) + + Convey("checkNotifierState", t, func() { + Convey("when OK", func() { + dataBase.EXPECT().GetNotifierState().Return(moira.SelfStateOK, nil) + mockAliveMeter.EXPECT().Mark(int64(1)) + + aliveWatcher.checkNotifierState() + }) + + Convey("when not OK state and no errors", func() { + notOKStates := []string{moira.SelfStateERROR, "err", "bad", "", "1"} + + for _, badState := range notOKStates { + dataBase.EXPECT().GetNotifierState().Return(badState, nil) + mockAliveMeter.EXPECT().Mark(int64(0)) + + aliveWatcher.checkNotifierState() + } + }) + + Convey("when not OK state and errors", func() { + notOKState := "" + givenErrors := []error{ + errors.New("one error"), + errors.New("another error"), + } + + for _, err := range givenErrors { + dataBase.EXPECT().GetNotifierState().Return(notOKState, err) + mockAliveMeter.EXPECT().Mark(int64(0)) + + aliveWatcher.checkNotifierState() + } + }) + }) +} From 6ca7722fd19bc241b5ea32ade80567a506fea795 Mon Sep 17 00:00:00 2001 From: AleksandrMatsko Date: Wed, 11 Dec 2024 14:29:52 +0700 Subject: [PATCH 4/9] fix: add return statement --- notifier/alive_watcher.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/notifier/alive_watcher.go b/notifier/alive_watcher.go index 3bad07b1b..82963ca69 100644 --- a/notifier/alive_watcher.go +++ b/notifier/alive_watcher.go @@ -2,6 +2,7 @@ package notifier import ( "context" + "fmt" "time" "github.com/moira-alert/moira" @@ -53,9 +54,13 @@ func (watcher *AliveWatcher) stateChecker(ctx context.Context) { func (watcher *AliveWatcher) checkNotifierState() { state, _ := watcher.database.GetNotifierState() + fmt.Println(state) if state != moira.SelfStateOK { watcher.notifierMetrics.MarkNotifierIsAlive(false) + fmt.Println("Marked as not alive") + return } watcher.notifierMetrics.MarkNotifierIsAlive(true) + fmt.Println("Marked as alive") } From 13c318e6d2a2b11ac4b19d8ed36416d9207354f4 Mon Sep 17 00:00:00 2001 From: AleksandrMatsko Date: Wed, 11 Dec 2024 15:39:22 +0700 Subject: [PATCH 5/9] test: for starting goroutine --- metrics/notifier.go | 5 +++- notifier/alive_watcher.go | 4 --- notifier/alive_watcher_test.go | 49 ++++++++++++++++++++++++++++------ 3 files changed, 45 insertions(+), 13 deletions(-) diff --git a/metrics/notifier.go b/metrics/notifier.go index ebec0dddf..3af294d63 100644 --- a/metrics/notifier.go +++ b/metrics/notifier.go @@ -1,6 +1,8 @@ package metrics -import "time" +import ( + "time" +) // NotifierMetrics is a collection of metrics used in notifier. type NotifierMetrics struct { @@ -73,6 +75,7 @@ func (metrics *NotifierMetrics) MarkSendingFailed() { func (metrics *NotifierMetrics) MarkNotifierIsAlive(isAlive bool) { if isAlive { metrics.notifierIsAlive.Mark(1) + return } metrics.notifierIsAlive.Mark(0) diff --git a/notifier/alive_watcher.go b/notifier/alive_watcher.go index 82963ca69..3ce57e632 100644 --- a/notifier/alive_watcher.go +++ b/notifier/alive_watcher.go @@ -2,7 +2,6 @@ package notifier import ( "context" - "fmt" "time" "github.com/moira-alert/moira" @@ -54,13 +53,10 @@ func (watcher *AliveWatcher) stateChecker(ctx context.Context) { func (watcher *AliveWatcher) checkNotifierState() { state, _ := watcher.database.GetNotifierState() - fmt.Println(state) if state != moira.SelfStateOK { watcher.notifierMetrics.MarkNotifierIsAlive(false) - fmt.Println("Marked as not alive") return } watcher.notifierMetrics.MarkNotifierIsAlive(true) - fmt.Println("Marked as alive") } diff --git a/notifier/alive_watcher_test.go b/notifier/alive_watcher_test.go index a59128568..40fc6c6c0 100644 --- a/notifier/alive_watcher_test.go +++ b/notifier/alive_watcher_test.go @@ -1,12 +1,12 @@ package notifier import ( + "context" "errors" "testing" "time" "github.com/moira-alert/moira" - logging "github.com/moira-alert/moira/logging/zerolog_adapter" "github.com/moira-alert/moira/metrics" . "github.com/smartystreets/goconvey/convey" "go.uber.org/mock/gomock" @@ -27,20 +27,15 @@ func initAliveMeter(mockCtrl *gomock.Controller) (*mock_metrics.MockRegistry, *m } func TestAliveWatcher_checkNotifierState(t *testing.T) { - logger, _ = logging.GetLogger("test alive watcher") - mockCtrl := gomock.NewController(t) - dataBase := mock_moira_alert.NewMockDatabase(mockCtrl) defer mockCtrl.Finish() - testConf := Config{ - CheckNotifierStateTimeout: time.Second * 10, - } + dataBase := mock_moira_alert.NewMockDatabase(mockCtrl) mockRegistry, mockAliveMeter := initAliveMeter(mockCtrl) testNotifierMetrics := metrics.ConfigureNotifierMetrics(mockRegistry, "") - aliveWatcher := NewAliveWatcher(logger, dataBase, testConf, testNotifierMetrics) + aliveWatcher := NewAliveWatcher(nil, dataBase, Config{}, testNotifierMetrics) Convey("checkNotifierState", t, func() { Convey("when OK", func() { @@ -77,3 +72,41 @@ func TestAliveWatcher_checkNotifierState(t *testing.T) { }) }) } + +func TestAliveWatcher_Start(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + logger := mock_moira_alert.NewMockLogger(mockCtrl) + eventsBuilder := mock_moira_alert.NewMockEventBuilder(mockCtrl) + logger.EXPECT().Info().Return(eventsBuilder).AnyTimes() + + dataBase := mock_moira_alert.NewMockDatabase(mockCtrl) + + testConf := Config{ + CheckNotifierStateTimeout: time.Second, + } + + mockRegistry, mockAliveMeter := initAliveMeter(mockCtrl) + testNotifierMetrics := metrics.ConfigureNotifierMetrics(mockRegistry, "") + + aliveWatcher := NewAliveWatcher(logger, dataBase, testConf, testNotifierMetrics) + + Convey("AliveWatcher stops on cancel", t, func() { + eventsBuilder.EXPECT().Interface("check_timeout_seconds", testConf.CheckNotifierStateTimeout.Seconds()).Return(eventsBuilder) + eventsBuilder.EXPECT().Msg("Moira Notifier alive watcher started") + + eventsBuilder.EXPECT().Msg("Moira Notifier alive watcher stopped") + + dataBase.EXPECT().GetNotifierState().Return(moira.SelfStateOK, nil).AnyTimes() + mockAliveMeter.EXPECT().Mark(int64(1)).AnyTimes() + + ctx, cancel := context.WithCancel(context.Background()) + aliveWatcher.Start(ctx) + + select { + case <-time.After(time.Second * 3): + cancel() + } + }) +} From 874069d480033031fa35e3cdbf03b4e5ee131a44 Mon Sep 17 00:00:00 2001 From: AleksandrMatsko Date: Wed, 11 Dec 2024 15:52:27 +0700 Subject: [PATCH 6/9] test: use sleep for waiting --- notifier/alive_watcher_test.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/notifier/alive_watcher_test.go b/notifier/alive_watcher_test.go index 40fc6c6c0..c8fad63ef 100644 --- a/notifier/alive_watcher_test.go +++ b/notifier/alive_watcher_test.go @@ -104,9 +104,7 @@ func TestAliveWatcher_Start(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) aliveWatcher.Start(ctx) - select { - case <-time.After(time.Second * 3): - cancel() - } + time.Sleep(time.Second * 3) + cancel() }) } From 9b37eceb96a2d11f80048e2912f3647255b27087 Mon Sep 17 00:00:00 2001 From: AleksandrMatsko Date: Wed, 11 Dec 2024 17:03:59 +0700 Subject: [PATCH 7/9] doc: add godocs --- notifier/alive_watcher.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/notifier/alive_watcher.go b/notifier/alive_watcher.go index 3ce57e632..9f50a1da5 100644 --- a/notifier/alive_watcher.go +++ b/notifier/alive_watcher.go @@ -8,6 +8,7 @@ import ( "github.com/moira-alert/moira/metrics" ) +// AliveWatcher is responsible for checking notifier state and marking notifier.alive metrics. type AliveWatcher struct { logger moira.Logger database moira.Database @@ -15,6 +16,7 @@ type AliveWatcher struct { notifierMetrics *metrics.NotifierMetrics } +// NewAliveWatcher is an initializer for AliveWatcher. func NewAliveWatcher( logger moira.Logger, database moira.Database, @@ -29,6 +31,8 @@ func NewAliveWatcher( } } +// Start starts the checking loop in separate goroutine. +// Use context.WithCancel, context.WithTimeout etc. to terminate check loop. func (watcher *AliveWatcher) Start(ctx context.Context) { go watcher.stateChecker(ctx) } From 57b699f5b7441f3346f4f28f4fa3bca2dbd3b2d3 Mon Sep 17 00:00:00 2001 From: AleksandrMatsko Date: Fri, 13 Dec 2024 11:53:11 +0700 Subject: [PATCH 8/9] refactor: use single field from config instead of entire config --- cmd/notifier/main.go | 7 ++++++- notifier/alive_watcher.go | 22 +++++++++++----------- notifier/alive_watcher_test.go | 14 ++++++++------ 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/cmd/notifier/main.go b/cmd/notifier/main.go index cc64f7db1..7c9ea395d 100644 --- a/cmd/notifier/main.go +++ b/cmd/notifier/main.go @@ -158,7 +158,12 @@ func main() { fetchEventsWorker.Start() defer stopFetchEvents(fetchEventsWorker) - aliveWatcher := notifier.NewAliveWatcher(logger, database, notifierConfig, notifierMetrics) + aliveWatcher := notifier.NewAliveWatcher( + logger, + database, + notifierConfig.CheckNotifierStateTimeout, + notifierMetrics, + ) ctx, cancel := context.WithCancel(context.Background()) aliveWatcher.Start(ctx) diff --git a/notifier/alive_watcher.go b/notifier/alive_watcher.go index 9f50a1da5..962e017b0 100644 --- a/notifier/alive_watcher.go +++ b/notifier/alive_watcher.go @@ -10,24 +10,24 @@ import ( // AliveWatcher is responsible for checking notifier state and marking notifier.alive metrics. type AliveWatcher struct { - logger moira.Logger - database moira.Database - config Config - notifierMetrics *metrics.NotifierMetrics + logger moira.Logger + database moira.Database + checkNotifierStateTimeout time.Duration + notifierMetrics *metrics.NotifierMetrics } // NewAliveWatcher is an initializer for AliveWatcher. func NewAliveWatcher( logger moira.Logger, database moira.Database, - config Config, + checkNotifierStateTimeout time.Duration, notifierMetrics *metrics.NotifierMetrics, ) *AliveWatcher { return &AliveWatcher{ - logger: logger, - database: database, - config: config, - notifierMetrics: notifierMetrics, + logger: logger, + database: database, + checkNotifierStateTimeout: checkNotifierStateTimeout, + notifierMetrics: notifierMetrics, } } @@ -39,10 +39,10 @@ func (watcher *AliveWatcher) Start(ctx context.Context) { func (watcher *AliveWatcher) stateChecker(ctx context.Context) { watcher.logger.Info(). - Interface("check_timeout_seconds", watcher.config.CheckNotifierStateTimeout.Seconds()). + Interface("check_timeout_seconds", watcher.checkNotifierStateTimeout.Seconds()). Msg("Moira Notifier alive watcher started") - ticker := time.NewTicker(watcher.config.CheckNotifierStateTimeout) + ticker := time.NewTicker(watcher.checkNotifierStateTimeout) for { select { diff --git a/notifier/alive_watcher_test.go b/notifier/alive_watcher_test.go index c8fad63ef..0fce9a874 100644 --- a/notifier/alive_watcher_test.go +++ b/notifier/alive_watcher_test.go @@ -35,7 +35,7 @@ func TestAliveWatcher_checkNotifierState(t *testing.T) { mockRegistry, mockAliveMeter := initAliveMeter(mockCtrl) testNotifierMetrics := metrics.ConfigureNotifierMetrics(mockRegistry, "") - aliveWatcher := NewAliveWatcher(nil, dataBase, Config{}, testNotifierMetrics) + aliveWatcher := NewAliveWatcher(nil, dataBase, 0, testNotifierMetrics) Convey("checkNotifierState", t, func() { Convey("when OK", func() { @@ -83,17 +83,19 @@ func TestAliveWatcher_Start(t *testing.T) { dataBase := mock_moira_alert.NewMockDatabase(mockCtrl) - testConf := Config{ - CheckNotifierStateTimeout: time.Second, - } + const ( + testCheckNotifierStateTimeout = time.Second + ) mockRegistry, mockAliveMeter := initAliveMeter(mockCtrl) testNotifierMetrics := metrics.ConfigureNotifierMetrics(mockRegistry, "") - aliveWatcher := NewAliveWatcher(logger, dataBase, testConf, testNotifierMetrics) + aliveWatcher := NewAliveWatcher(logger, dataBase, testCheckNotifierStateTimeout, testNotifierMetrics) Convey("AliveWatcher stops on cancel", t, func() { - eventsBuilder.EXPECT().Interface("check_timeout_seconds", testConf.CheckNotifierStateTimeout.Seconds()).Return(eventsBuilder) + eventsBuilder.EXPECT(). + Interface("check_timeout_seconds", testCheckNotifierStateTimeout.Seconds()). + Return(eventsBuilder) eventsBuilder.EXPECT().Msg("Moira Notifier alive watcher started") eventsBuilder.EXPECT().Msg("Moira Notifier alive watcher stopped") From 0a1fc71e6ae8cf46b1a33f5c4bee65696b5b941d Mon Sep 17 00:00:00 2001 From: AleksandrMatsko Date: Fri, 13 Dec 2024 12:01:55 +0700 Subject: [PATCH 9/9] test: prevent from flap --- notifier/alive_watcher_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notifier/alive_watcher_test.go b/notifier/alive_watcher_test.go index 0fce9a874..3ab25818b 100644 --- a/notifier/alive_watcher_test.go +++ b/notifier/alive_watcher_test.go @@ -97,7 +97,6 @@ func TestAliveWatcher_Start(t *testing.T) { Interface("check_timeout_seconds", testCheckNotifierStateTimeout.Seconds()). Return(eventsBuilder) eventsBuilder.EXPECT().Msg("Moira Notifier alive watcher started") - eventsBuilder.EXPECT().Msg("Moira Notifier alive watcher stopped") dataBase.EXPECT().GetNotifierState().Return(moira.SelfStateOK, nil).AnyTimes() @@ -108,5 +107,6 @@ func TestAliveWatcher_Start(t *testing.T) { time.Sleep(time.Second * 3) cancel() + time.Sleep(time.Millisecond) }) }