From dcedfa7ea1ff6ec810ee3bea7459157463a077b8 Mon Sep 17 00:00:00 2001 From: almostinf Date: Tue, 5 Nov 2024 16:29:32 +0300 Subject: [PATCH 1/6] add monitor --- api/controller/contact_test.go | 2 +- api/controller/emergency_contact_test.go | 6 +- api/dto/emergency_contact_test.go | 2 +- api/handler/contact_test.go | 2 +- api/handler/emergency_contact_test.go | 6 +- cmd/notifier/config.go | 231 +++++-- cmd/notifier/main.go | 26 +- database/redis/emergency_contact_test.go | 12 +- .../redis/reply/emergency_contacts_test.go | 4 +- datatypes/emergency_contact.go | 10 +- datatypes/emergency_contact_test.go | 6 +- go.mod | 17 +- go.sum | 25 +- local/notifier.yml | 49 +- mock/heartbeat/heartbeat.go | 63 +- notifier/config.go | 3 +- notifier/registrator.go | 12 +- notifier/selfstate/check.go | 120 ---- notifier/selfstate/config.go | 36 +- notifier/selfstate/config_test.go | 114 ---- notifier/selfstate/heartbeat/database.go | 4 +- notifier/selfstate/heartbeat/database_test.go | 8 +- notifier/selfstate/heartbeat/filter.go | 4 +- notifier/selfstate/heartbeat/filter_test.go | 8 +- notifier/selfstate/heartbeat/heartbeat.go | 19 +- .../selfstate/heartbeat/heartbeat_test.go | 17 - notifier/selfstate/heartbeat/local_checker.go | 4 +- .../selfstate/heartbeat/local_checker_test.go | 8 +- notifier/selfstate/heartbeat/notifier.go | 2 +- notifier/selfstate/heartbeat/notifier_test.go | 2 +- .../selfstate/heartbeat/remote_checker.go | 4 +- .../heartbeat/remote_checker_test.go | 11 +- notifier/selfstate/monitor/admin.go | 105 +++ notifier/selfstate/monitor/admin_test.go | 220 +++++++ notifier/selfstate/monitor/monitor.go | 310 +++++++++ notifier/selfstate/monitor/monitor_test.go | 616 ++++++++++++++++++ notifier/selfstate/monitor/user.go | 93 +++ notifier/selfstate/monitor/user_test.go | 162 +++++ notifier/selfstate/selfstate.go | 139 ++-- notifier/selfstate/selfstate_test.go | 189 ------ 40 files changed, 1974 insertions(+), 697 deletions(-) delete mode 100644 notifier/selfstate/check.go delete mode 100644 notifier/selfstate/config_test.go create mode 100644 notifier/selfstate/monitor/admin.go create mode 100644 notifier/selfstate/monitor/admin_test.go create mode 100644 notifier/selfstate/monitor/monitor.go create mode 100644 notifier/selfstate/monitor/monitor_test.go create mode 100644 notifier/selfstate/monitor/user.go create mode 100644 notifier/selfstate/monitor/user_test.go delete mode 100644 notifier/selfstate/selfstate_test.go diff --git a/api/controller/contact_test.go b/api/controller/contact_test.go index 8144f2cf4..7b2411448 100644 --- a/api/controller/contact_test.go +++ b/api/controller/contact_test.go @@ -890,7 +890,7 @@ func TestRemoveContact(t *testing.T) { Convey("With emergency contact", func() { emergencyContact := datatypes.EmergencyContact{ ContactID: contactID, - HeartbeatTypes: []datatypes.HeartbeatType{datatypes.HeartbeatNotifierOff}, + HeartbeatTypes: []datatypes.HeartbeatType{datatypes.HeartbeatNotifier}, } dataBase.EXPECT().GetTeamSubscriptionIDs(teamID).Return(make([]string, 0), nil) dataBase.EXPECT().GetEmergencyContact(contactID).Return(emergencyContact, nil) diff --git a/api/controller/emergency_contact_test.go b/api/controller/emergency_contact_test.go index b0c5c0d14..85b62cd31 100644 --- a/api/controller/emergency_contact_test.go +++ b/api/controller/emergency_contact_test.go @@ -21,7 +21,7 @@ var ( testEmergencyContact = datatypes.EmergencyContact{ ContactID: testContactID, - HeartbeatTypes: []datatypes.HeartbeatType{datatypes.HeartbeatNotifierOff}, + HeartbeatTypes: []datatypes.HeartbeatType{datatypes.HeartbeatNotifier}, } testEmergencyContact2 = datatypes.EmergencyContact{ ContactID: testContactID2, @@ -30,7 +30,7 @@ var ( testEmergencyContactDTO = dto.EmergencyContact{ ContactID: testContactID, - HeartbeatTypes: []datatypes.HeartbeatType{datatypes.HeartbeatNotifierOff}, + HeartbeatTypes: []datatypes.HeartbeatType{datatypes.HeartbeatNotifier}, } testEmergencyContact2DTO = dto.EmergencyContact{ ContactID: testContactID2, @@ -286,7 +286,7 @@ func TestUpdateEmergencyContact(t *testing.T) { Convey("With empty contact id", func() { emergencyContactDTO := dto.EmergencyContact{ - HeartbeatTypes: []datatypes.HeartbeatType{datatypes.HeartbeatNotifierOff}, + HeartbeatTypes: []datatypes.HeartbeatType{datatypes.HeartbeatNotifier}, } database.EXPECT().SaveEmergencyContact(testEmergencyContact).Return(nil) diff --git a/api/dto/emergency_contact_test.go b/api/dto/emergency_contact_test.go index 4e4790dfd..8d0725fae 100644 --- a/api/dto/emergency_contact_test.go +++ b/api/dto/emergency_contact_test.go @@ -12,7 +12,7 @@ var ( testEmergencyContact = datatypes.EmergencyContact{ ContactID: testContactID, - HeartbeatTypes: []datatypes.HeartbeatType{datatypes.HeartbeatNotifierOff}, + HeartbeatTypes: []datatypes.HeartbeatType{datatypes.HeartbeatNotifier}, } ) diff --git a/api/handler/contact_test.go b/api/handler/contact_test.go index 8369c504b..e307fb090 100644 --- a/api/handler/contact_test.go +++ b/api/handler/contact_test.go @@ -837,7 +837,7 @@ func TestRemoveContact(t *testing.T) { emergencyContact := datatypes.EmergencyContact{ ContactID: contactID, - HeartbeatTypes: []datatypes.HeartbeatType{datatypes.HeartbeatNotifierOff}, + HeartbeatTypes: []datatypes.HeartbeatType{datatypes.HeartbeatNotifier}, } mockDb.EXPECT().GetUserSubscriptionIDs(defaultLogin).Return([]string{"test"}, nil).Times(1) diff --git a/api/handler/emergency_contact_test.go b/api/handler/emergency_contact_test.go index d2ac2774a..ac4bd67b4 100644 --- a/api/handler/emergency_contact_test.go +++ b/api/handler/emergency_contact_test.go @@ -28,7 +28,7 @@ var ( testEmergencyContact = datatypes.EmergencyContact{ ContactID: testContactID, - HeartbeatTypes: []datatypes.HeartbeatType{datatypes.HeartbeatNotifierOff}, + HeartbeatTypes: []datatypes.HeartbeatType{datatypes.HeartbeatNotifier}, } testEmergencyContact2 = datatypes.EmergencyContact{ ContactID: testContactID2, @@ -247,7 +247,7 @@ func TestCreateEmergencyContact(t *testing.T) { Convey("Try to create emergency contact without contact id", func() { emergencyContact := datatypes.EmergencyContact{ - HeartbeatTypes: []datatypes.HeartbeatType{datatypes.HeartbeatNotifierOff}, + HeartbeatTypes: []datatypes.HeartbeatType{datatypes.HeartbeatNotifier}, } emergencyContactDTO := dto.EmergencyContact(emergencyContact) @@ -473,7 +473,7 @@ func TestUpdateEmergencyContact(t *testing.T) { Convey("Successfully update emergency contact without contact id in dto", func() { emergencyContact := datatypes.EmergencyContact{ - HeartbeatTypes: []datatypes.HeartbeatType{datatypes.HeartbeatNotifierOff}, + HeartbeatTypes: []datatypes.HeartbeatType{datatypes.HeartbeatNotifier}, } emergencyContactDTO := dto.EmergencyContact(emergencyContact) diff --git a/cmd/notifier/config.go b/cmd/notifier/config.go index eb737a798..6e0571270 100644 --- a/cmd/notifier/config.go +++ b/cmd/notifier/config.go @@ -10,6 +10,8 @@ import ( "github.com/moira-alert/moira/cmd" "github.com/moira-alert/moira/notifier" "github.com/moira-alert/moira/notifier/selfstate" + "github.com/moira-alert/moira/notifier/selfstate/heartbeat" + "github.com/moira-alert/moira/notifier/selfstate/monitor" ) type config struct { @@ -42,8 +44,8 @@ type notifierConfig struct { ReschedulingDelay string `yaml:"rescheduling_delay"` // Senders configuration section. See https://moira.readthedocs.io/en/latest/installation/configuration.html for more explanation Senders []map[string]interface{} `yaml:"senders"` - // Self state monitor configuration section. Note: No inner subscriptions is required. It's own notification mechanism will be used. - SelfState selfStateConfig `yaml:"moira_selfstate"` + // Selfstate monitor configuration section. Note: No inner subscriptions is required. It's own notification mechanism will be used. + Selfstate selfstateConfig `yaml:"moira_selfstate"` // Web-UI uri prefix for trigger links in notifications. For example: with 'http://localhost' every notification will contain link like 'http://localhost/trigger/triggerId' FrontURI string `yaml:"front_uri"` // Timezone to use to convert ticks. Default is UTC. See https://golang.org/pkg/time/#LoadLocation for more details. @@ -58,25 +60,140 @@ type notifierConfig struct { SetLogLevel setLogLevelConfig `yaml:"set_log_level"` } -type selfStateConfig struct { - // If true, Self state monitor will be enabled - Enabled bool `yaml:"enabled"` - // If true, Self state monitor will check remote checker status - RemoteTriggersEnabled bool `yaml:"remote_triggers_enabled"` - // Max Redis disconnect delay to send alert when reached - RedisDisconnectDelay string `yaml:"redis_disconect_delay"` - // Max Filter metrics receive delay to send alert when reached - LastMetricReceivedDelay string `yaml:"last_metric_received_delay"` - // Max Checker checks perform delay to send alert when reached - LastCheckDelay string `yaml:"last_check_delay"` - // Max Remote triggers Checker checks perform delay to send alert when reached - LastRemoteCheckDelay string `yaml:"last_remote_check_delay"` - // Contact list for Self state monitor alerts - Contacts []map[string]string `yaml:"contacts"` - // Self state monitor alerting interval - NoticeInterval string `yaml:"notice_interval"` - // Self state monitor check interval - CheckInterval string `yaml:"check_interval"` +type heartbeaterAlertConfig struct { + Name string `yaml:"name"` + Desc string `yaml:"desc"` +} + +type heartbeaterBaseConfig struct { + Enabled bool `yaml:"enabled"` + NeedTurnOffNotifier bool `yaml:"need_turn_off_notifier"` + + AlertCfg heartbeaterAlertConfig `yaml:"alert"` +} + +func (cfg heartbeaterBaseConfig) getSettings() heartbeat.HeartbeaterBaseConfig { + return heartbeat.HeartbeaterBaseConfig{ + Enabled: cfg.Enabled, + NeedTurnOffNotifier: cfg.NeedTurnOffNotifier, + + AlertCfg: heartbeat.AlertConfig{ + Name: cfg.AlertCfg.Name, + Desc: cfg.AlertCfg.Desc, + }, + } +} + +type databaseHeartbeaterConfig struct { + heartbeaterBaseConfig `yaml:",inline"` + + RedisDisconnectDelay string `yaml:"redis_disconnect_delay"` +} + +type filterHeartbeaterConfig struct { + heartbeaterBaseConfig `yaml:",inline"` + + MetricReceivedDelay string `yaml:"last_metric_received_delay"` +} + +type localCheckerHeartbeaterConfig struct { + heartbeaterBaseConfig `yaml:",inline"` + + LocalCheckDelay string `yaml:"last_check_delay"` +} + +type remoteCheckerHeartbeaterConfig struct { + heartbeaterBaseConfig `yaml:",inline"` + + RemoteCheckDelay string `yaml:"last_remote_check_delay"` +} + +type notifierHeartbeaterConfig struct { + heartbeaterBaseConfig `yaml:",inline"` +} + +type heartbeatsConfig struct { + DatabaseCfg databaseHeartbeaterConfig `yaml:"database"` + FilterCfg filterHeartbeaterConfig `yaml:"filter"` + LocalCheckerCfg localCheckerHeartbeaterConfig `yaml:"local_checker"` + RemoteCheckerCfg remoteCheckerHeartbeaterConfig `yaml:"remote_checker"` + NotifierCfg notifierHeartbeaterConfig `yaml:"notifier"` +} + +func (cfg heartbeatsConfig) getSettings() heartbeat.HeartbeatersConfig { + return heartbeat.HeartbeatersConfig{ + DatabaseCfg: heartbeat.DatabaseHeartbeaterConfig{ + HeartbeaterBaseConfig: cfg.DatabaseCfg.heartbeaterBaseConfig.getSettings(), + RedisDisconnectDelay: to.Duration(cfg.DatabaseCfg.RedisDisconnectDelay), + }, + FilterCfg: heartbeat.FilterHeartbeaterConfig{ + HeartbeaterBaseConfig: cfg.FilterCfg.heartbeaterBaseConfig.getSettings(), + MetricReceivedDelay: to.Duration(cfg.FilterCfg.MetricReceivedDelay), + }, + LocalCheckerCfg: heartbeat.LocalCheckerHeartbeaterConfig{ + HeartbeaterBaseConfig: cfg.LocalCheckerCfg.heartbeaterBaseConfig.getSettings(), + LocalCheckDelay: to.Duration(cfg.LocalCheckerCfg.LocalCheckDelay), + }, + RemoteCheckerCfg: heartbeat.RemoteCheckerHeartbeaterConfig{ + HeartbeaterBaseConfig: cfg.RemoteCheckerCfg.heartbeaterBaseConfig.getSettings(), + RemoteCheckDelay: to.Duration(cfg.RemoteCheckerCfg.RemoteCheckDelay), + }, + NotifierCfg: heartbeat.NotifierHeartbeaterConfig{ + HeartbeaterBaseConfig: cfg.NotifierCfg.heartbeaterBaseConfig.getSettings(), + }, + } +} + +type monitorBaseConfig struct { + Enabled bool `yaml:"enabled"` + HearbeatersCfg heartbeatsConfig `yaml:"heartbeaters"` + NoticeInterval string `yaml:"notice_interval"` + CheckInterval string `yaml:"check_interval"` +} + +type adminMonitorConfig struct { + monitorBaseConfig `yaml:",inline"` + + AdminContacts []map[string]string `yaml:"contacts"` +} + +type userMonitorConfig struct { + monitorBaseConfig `yaml:",inline"` +} + +type monitorConfig struct { + AdminCfg adminMonitorConfig `yaml:"admin"` + UserCfg userMonitorConfig `yaml:"user"` +} + +type selfstateConfig struct { + Enabled bool `yaml:"enabled"` + MonitorCfg monitorConfig `yaml:"monitor"` +} + +func (cfg *selfstateConfig) getSettings() selfstate.Config { + return selfstate.Config{ + Enabled: cfg.Enabled, + MonitorCfg: selfstate.MonitorConfig{ + AdminCfg: monitor.AdminMonitorConfig{ + MonitorBaseConfig: monitor.MonitorBaseConfig{ + Enabled: cfg.MonitorCfg.AdminCfg.Enabled, + HeartbeatersCfg: cfg.MonitorCfg.AdminCfg.HearbeatersCfg.getSettings(), + NoticeInterval: to.Duration(cfg.MonitorCfg.AdminCfg.NoticeInterval), + CheckInterval: to.Duration(cfg.MonitorCfg.AdminCfg.CheckInterval), + }, + AdminContacts: cfg.MonitorCfg.AdminCfg.AdminContacts, + }, + UserCfg: monitor.UserMonitorConfig{ + MonitorBaseConfig: monitor.MonitorBaseConfig{ + Enabled: cfg.MonitorCfg.UserCfg.Enabled, + HeartbeatersCfg: cfg.MonitorCfg.UserCfg.HearbeatersCfg.getSettings(), + NoticeInterval: to.Duration(cfg.MonitorCfg.UserCfg.NoticeInterval), + CheckInterval: to.Duration(cfg.MonitorCfg.UserCfg.CheckInterval), + }, + }, + }, + } } func getDefault() config { @@ -105,12 +222,50 @@ func getDefault() config { SenderTimeout: "10s", ResendingTimeout: "1:00", ReschedulingDelay: "60s", - SelfState: selfStateConfig{ - Enabled: false, - RedisDisconnectDelay: "30s", - LastMetricReceivedDelay: "60s", - LastCheckDelay: "60s", - NoticeInterval: "300s", + Selfstate: selfstateConfig{ + Enabled: false, + MonitorCfg: monitorConfig{ + AdminCfg: adminMonitorConfig{ + monitorBaseConfig: monitorBaseConfig{ + Enabled: false, + HearbeatersCfg: heartbeatsConfig{ + DatabaseCfg: databaseHeartbeaterConfig{ + RedisDisconnectDelay: "30s", + }, + FilterCfg: filterHeartbeaterConfig{ + MetricReceivedDelay: "60s", + }, + LocalCheckerCfg: localCheckerHeartbeaterConfig{ + LocalCheckDelay: "60s", + }, + RemoteCheckerCfg: remoteCheckerHeartbeaterConfig{ + RemoteCheckDelay: "300s", + }, + NotifierCfg: notifierHeartbeaterConfig{}, + }, + }, + }, + UserCfg: userMonitorConfig{ + monitorBaseConfig: monitorBaseConfig{ + Enabled: false, + HearbeatersCfg: heartbeatsConfig{ + DatabaseCfg: databaseHeartbeaterConfig{ + RedisDisconnectDelay: "30s", + }, + FilterCfg: filterHeartbeaterConfig{ + MetricReceivedDelay: "60s", + }, + LocalCheckerCfg: localCheckerHeartbeaterConfig{ + LocalCheckDelay: "60s", + }, + RemoteCheckerCfg: remoteCheckerHeartbeaterConfig{ + RemoteCheckDelay: "300s", + }, + NotifierCfg: notifierHeartbeaterConfig{}, + }, + }, + }, + }, }, FrontURI: "http://localhost", Timezone: "UTC", @@ -189,8 +344,7 @@ func (config *notifierConfig) getSettings(logger moira.Logger) notifier.Config { Msg("Found dynamic log rules in config for some contacts and subscriptions") return notifier.Config{ - SelfStateEnabled: config.SelfState.Enabled, - SelfStateContacts: config.SelfState.Contacts, + SelfstateEnabled: config.Selfstate.Enabled, SendingTimeout: to.Duration(config.SenderTimeout), ResendingTimeout: to.Duration(config.ResendingTimeout), ReschedulingDelay: to.Duration(config.ReschedulingDelay), @@ -213,22 +367,3 @@ func checkDateTimeFormat(format string) error { } return nil } - -func (config *selfStateConfig) getSettings() selfstate.Config { - // 10 sec is default check value - checkInterval := 10 * time.Second - if config.CheckInterval != "" { - checkInterval = to.Duration(config.CheckInterval) - } - - return selfstate.Config{ - Enabled: config.Enabled, - RedisDisconnectDelaySeconds: int64(to.Duration(config.RedisDisconnectDelay).Seconds()), - LastMetricReceivedDelaySeconds: int64(to.Duration(config.LastMetricReceivedDelay).Seconds()), - LastCheckDelaySeconds: int64(to.Duration(config.LastCheckDelay).Seconds()), - LastRemoteCheckDelaySeconds: int64(to.Duration(config.LastRemoteCheckDelay).Seconds()), - CheckInterval: checkInterval, - Contacts: config.Contacts, - NoticeIntervalSeconds: int64(to.Duration(config.NoticeInterval).Seconds()), - } -} diff --git a/cmd/notifier/main.go b/cmd/notifier/main.go index 37e1ac7f7..7eda85e05 100644 --- a/cmd/notifier/main.go +++ b/cmd/notifier/main.go @@ -117,17 +117,23 @@ func main() { Msg("Can not configure senders") } - // Start moira self state checker - if config.Notifier.SelfState.getSettings().Enabled { - selfState := selfstate.NewSelfCheckWorker(logger, database, sender, config.Notifier.SelfState.getSettings(), metrics.ConfigureHeartBeatMetrics(telemetry.Metrics)) - if err := selfState.Start(); err != nil { + selfstateCfg := config.Notifier.Selfstate.getSettings() + + // Start moira selfstate checker + if selfstateCfg.Enabled { + fmt.Println(selfstateCfg) + logger.Info().Msg("Selfstate enabled") + selfstateWorker, err := selfstate.NewSelfstateWorker(selfstateCfg, logger, database, sender, systemClock) + if err != nil { logger.Fatal(). Error(err). - Msg("SelfState failed") + Msg("Failed to create new selfstate worker") } - defer stopSelfStateChecker(selfState) + + selfstateWorker.Start() + defer stopSelfstateWorker(selfstateWorker) } else { - logger.Debug().Msg("Moira Self State Monitoring disabled") + logger.Debug().Msg("Moira Selfstate Monitoring disabled") } // Start moira notification fetcher @@ -181,10 +187,10 @@ func stopNotificationsFetcher(worker *notifications.FetchNotificationsWorker) { } } -func stopSelfStateChecker(checker *selfstate.SelfCheckWorker) { - if err := checker.Stop(); err != nil { +func stopSelfstateWorker(selfstateWorker selfstate.SelfstateWorker) { + if err := selfstateWorker.Stop(); err != nil { logger.Error(). Error(err). - Msg("Failed to stop self check worker") + Msg("Failed to stop selfstate worker") } } diff --git a/database/redis/emergency_contact_test.go b/database/redis/emergency_contact_test.go index 9da23eb0a..cd5a36993 100644 --- a/database/redis/emergency_contact_test.go +++ b/database/redis/emergency_contact_test.go @@ -18,12 +18,12 @@ var ( testEmergencyContact = datatypes.EmergencyContact{ ContactID: testContactID, - HeartbeatTypes: []datatypes.HeartbeatType{datatypes.HeartbeatNotifierOff}, + HeartbeatTypes: []datatypes.HeartbeatType{datatypes.HeartbeatNotifier}, } testEmergencyContact2 = datatypes.EmergencyContact{ ContactID: testContactID2, - HeartbeatTypes: []datatypes.HeartbeatType{datatypes.HeartbeatNotifierOff}, + HeartbeatTypes: []datatypes.HeartbeatType{datatypes.HeartbeatNotifier}, } testEmergencyContact3 = datatypes.EmergencyContact{ @@ -147,7 +147,7 @@ func TestGetHeartbeatTypeContactIDs(t *testing.T) { Convey("Test GetHeartbeatTypeContactIDs", t, func() { Convey("Without any emergency contacts by heartbeat type", func() { - emergencyContactIDs, err := database.GetHeartbeatTypeContactIDs(datatypes.HeartbeatNotifierOff) + emergencyContactIDs, err := database.GetHeartbeatTypeContactIDs(datatypes.HeartbeatNotifier) So(err, ShouldBeNil) So(emergencyContactIDs, ShouldBeEmpty) }) @@ -159,7 +159,7 @@ func TestGetHeartbeatTypeContactIDs(t *testing.T) { testEmergencyContact3, }) - emergencyContactIDs, err := database.GetHeartbeatTypeContactIDs(datatypes.HeartbeatNotifierOff) + emergencyContactIDs, err := database.GetHeartbeatTypeContactIDs(datatypes.HeartbeatNotifier) So(err, ShouldBeNil) assert.ElementsMatch(t, emergencyContactIDs, []string{ testContactID, @@ -197,7 +197,7 @@ func TestSaveEmergencyContact(t *testing.T) { So(err, ShouldBeNil) So(emergencyContacts, ShouldResemble, expectedEmergencyContacts) - emergencyContactIDs, err := database.GetHeartbeatTypeContactIDs(datatypes.HeartbeatNotifierOff) + emergencyContactIDs, err := database.GetHeartbeatTypeContactIDs(datatypes.HeartbeatNotifier) So(err, ShouldBeNil) So(emergencyContactIDs, ShouldResemble, expectedEmergencyContactIDs) }) @@ -230,7 +230,7 @@ func TestSaveEmergencyContacts(t *testing.T) { So(err, ShouldBeNil) assert.ElementsMatch(t, emergencyContacts, expectedEmergencyContacts) - emergencyContactIDs, err := database.GetHeartbeatTypeContactIDs(datatypes.HeartbeatNotifierOff) + emergencyContactIDs, err := database.GetHeartbeatTypeContactIDs(datatypes.HeartbeatNotifier) So(err, ShouldBeNil) assert.ElementsMatch(t, emergencyContactIDs, expectedEmergencyContactIDs) }) diff --git a/database/redis/reply/emergency_contacts_test.go b/database/redis/reply/emergency_contacts_test.go index 4916a3c94..15b981d23 100644 --- a/database/redis/reply/emergency_contacts_test.go +++ b/database/redis/reply/emergency_contacts_test.go @@ -10,14 +10,14 @@ import ( ) const ( - testEmergencyContactVal = `{"contact_id":"test-contact-id","heartbeat_types":["notifier_off"]}` + testEmergencyContactVal = `{"contact_id":"test-contact-id","heartbeat_types":["heartbeat_notifier"]}` testEmptyEmergencyContactVal = `{"contact_id":"","heartbeat_types":null}` ) var ( testEmergencyContact = datatypes.EmergencyContact{ ContactID: "test-contact-id", - HeartbeatTypes: []datatypes.HeartbeatType{datatypes.HeartbeatNotifierOff}, + HeartbeatTypes: []datatypes.HeartbeatType{datatypes.HeartbeatNotifier}, } testEmptyEmergencyContact = datatypes.EmergencyContact{} ) diff --git a/datatypes/emergency_contact.go b/datatypes/emergency_contact.go index deec88061..0a1bf21cc 100644 --- a/datatypes/emergency_contact.go +++ b/datatypes/emergency_contact.go @@ -4,14 +4,18 @@ package datatypes type HeartbeatType string const ( - HeartbeatTypeNotSet HeartbeatType = "type_not_set" - HeartbeatNotifierOff HeartbeatType = "notifier_off" + HeartbeatTypeNotSet HeartbeatType = "Heartbeat_type_not_set" + HeartbeatNotifier HeartbeatType = "heartbeat_notifier" + HeartbeatDatabase HeartbeatType = "heartbeat_database" + HeartbeatLocalChecker HeartbeatType = "heartbeat_local_checker" + HeartbeatRemoteChecker HeartbeatType = "heartbeat_remote_checker" + HeartbeatFilter HeartbeatType = "heartbeat_filter" ) // IsValid checks if such an heartbeat type exists. func (heartbeatType HeartbeatType) IsValid() bool { switch heartbeatType { - case HeartbeatNotifierOff: + case HeartbeatNotifier, HeartbeatDatabase, HeartbeatLocalChecker, HeartbeatRemoteChecker, HeartbeatFilter: return true default: return false diff --git a/datatypes/emergency_contact_test.go b/datatypes/emergency_contact_test.go index ec0389b32..4e9b85b43 100644 --- a/datatypes/emergency_contact_test.go +++ b/datatypes/emergency_contact_test.go @@ -10,7 +10,11 @@ func TestIsValidHeartbeatType(t *testing.T) { Convey("Test IsValid heartbeat type", t, func() { Convey("Test valid cases", func() { testcases := []HeartbeatType{ - HeartbeatNotifierOff, + HeartbeatNotifier, + HeartbeatDatabase, + HeartbeatLocalChecker, + HeartbeatRemoteChecker, + HeartbeatFilter, } for _, testcase := range testcases { diff --git a/go.mod b/go.mod index b309808bd..c34dc6d03 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/moira-alert/moira -go 1.22 +go 1.22.0 + +toolchain go1.22.2 require ( github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible @@ -46,7 +48,7 @@ require ( require github.com/prometheus/common v0.37.0 require ( - github.com/go-playground/validator/v10 v10.4.1 + github.com/go-playground/validator/v10 v10.22.1 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/mattermost/mattermost/server/public v0.1.1 github.com/mitchellh/mapstructure v1.5.0 @@ -159,7 +161,7 @@ require ( golang.org/x/text v0.16.0 // indirect gonum.org/v1/gonum v0.15.0 // indirect google.golang.org/appengine v1.6.8 // indirect - google.golang.org/protobuf v1.33.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect @@ -173,25 +175,26 @@ require ( github.com/blang/semver/v4 v4.0.0 // indirect github.com/dyatlov/go-opengraph/opengraph v0.0.0-20220524092352-606d7b1e5f8a // indirect github.com/fatih/color v1.16.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/go-openapi/jsonpointer v0.20.0 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/spec v0.20.9 // indirect github.com/go-openapi/swag v0.22.4 // indirect - github.com/go-playground/locales v0.13.0 // indirect - github.com/go-playground/universal-translator v0.17.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect github.com/hashicorp/go-hclog v1.6.3 // indirect github.com/hashicorp/go-plugin v1.6.0 // indirect github.com/hashicorp/yamux v0.1.1 // indirect github.com/huandu/xstrings v1.3.3 // indirect github.com/imdario/mergo v0.3.11 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/leodido/go-urn v1.2.0 // indirect + github.com/leodido/go-urn v1.4.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mitchellh/copystructure v1.0.0 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/mitchellh/reflectwalk v1.0.0 // indirect github.com/oklog/run v1.1.0 // indirect - github.com/rogpeppe/go-internal v1.10.0 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/shopspring/decimal v1.2.0 // indirect diff --git a/go.sum b/go.sum index a013bebd9..8bd0a4657 100644 --- a/go.sum +++ b/go.sum @@ -238,6 +238,8 @@ github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4 github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-asn1-ber/asn1-ber v1.5.5 h1:MNHlNMBDgEKD4TcKr36vQN68BA00aDfjIt3/bD50WnA= @@ -279,14 +281,18 @@ github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/ github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= -github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= -github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= +github.com/go-playground/validator/v10 v10.22.1 h1:40JcKH+bBNGFczGuoBYgX4I6m/i27HYW8P9FDk5PbgA= +github.com/go-playground/validator/v10 v10.22.1/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-redis/redis/v7 v7.4.0 h1:7obg6wUoj05T0EpY0o8B59S9w5yeMWql7sw2kwNW1x4= @@ -527,8 +533,9 @@ github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lomik/og-rek v0.0.0-20170411191824-628eefeb8d80 h1:KVyDGUXjVOdHQt24wIgY4ZdGFXHtQHLWw0L/MAK3Kb0= @@ -703,8 +710,8 @@ github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqn github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/cors v1.11.0 h1:0B9GE/r9Bc2UxRMMtymBkHTenPkHDv0CW4Y98GBY+po= github.com/rs/cors v1.11.0/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= @@ -1409,8 +1416,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= diff --git a/local/notifier.yml b/local/notifier.yml index 6a5cf57a7..e7097b4cd 100644 --- a/local/notifier.yml +++ b/local/notifier.yml @@ -30,15 +30,48 @@ notifier: sender_timeout: 10s resending_timeout: "1:00" rescheduling_delay: 60s - senders: [] + senders: + - sender_type: "webhook" + contact_type: "webhook" + url: "$${q}{contact_value}" moira_selfstate: - enabled: false - remote_triggers_enabled: false - redis_disconect_delay: 60s - last_metric_received_delay: 120s - last_check_delay: 120s - last_remote_check_delay: 300s - notice_interval: 300s + enabled: true + monitor: + admin: + enabled: true + heartbeaters: + database: + enabled: true + need_turn_off_notifier: true + alert: + name: Database Problems + redis_disconect_delay: 60s + notifier: + enabled: true + need_turn_off_notifier: true + alert: + name: Notifier Problems + notice_interval: 30s + check_interval: 5s + contacts: + - type: webhook + value: "http://localhost:8091" + user: + enabled: true + heartbeaters: + database: + enabled: true + need_turn_off_notifier: true + alert: + name: Database Problems + redis_disconect_delay: 60s + notifier: + enabled: true + need_turn_off_notifier: true + alert: + name: Notifier Problems + notice_interval: 10s + check_interval: 5s front_uri: http://localhost timezone: UTC date_time_format: "15:04 02.01.2006" diff --git a/mock/heartbeat/heartbeat.go b/mock/heartbeat/heartbeat.go index f0a6f6acf..83787c639 100644 --- a/mock/heartbeat/heartbeat.go +++ b/mock/heartbeat/heartbeat.go @@ -12,6 +12,8 @@ package mock_heartbeat import ( reflect "reflect" + datatypes "github.com/moira-alert/moira/datatypes" + heartbeat "github.com/moira-alert/moira/notifier/selfstate/heartbeat" gomock "go.uber.org/mock/gomock" ) @@ -38,48 +40,33 @@ func (m *MockHeartbeater) EXPECT() *MockHeartbeaterMockRecorder { return m.recorder } -// Check mocks base method. -func (m *MockHeartbeater) Check(arg0 int64) (int64, bool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Check", arg0) - ret0, _ := ret[0].(int64) - ret1, _ := ret[1].(bool) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// Check indicates an expected call of Check. -func (mr *MockHeartbeaterMockRecorder) Check(arg0 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Check", reflect.TypeOf((*MockHeartbeater)(nil).Check), arg0) -} - -// GetErrorMessage mocks base method. -func (m *MockHeartbeater) GetErrorMessage() string { +// AlertSettings mocks base method. +func (m *MockHeartbeater) AlertSettings() heartbeat.AlertConfig { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetErrorMessage") - ret0, _ := ret[0].(string) + ret := m.ctrl.Call(m, "AlertSettings") + ret0, _ := ret[0].(heartbeat.AlertConfig) return ret0 } -// GetErrorMessage indicates an expected call of GetErrorMessage. -func (mr *MockHeartbeaterMockRecorder) GetErrorMessage() *gomock.Call { +// AlertSettings indicates an expected call of AlertSettings. +func (mr *MockHeartbeaterMockRecorder) AlertSettings() *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetErrorMessage", reflect.TypeOf((*MockHeartbeater)(nil).GetErrorMessage)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AlertSettings", reflect.TypeOf((*MockHeartbeater)(nil).AlertSettings)) } -// NeedToCheckOthers mocks base method. -func (m *MockHeartbeater) NeedToCheckOthers() bool { +// Check mocks base method. +func (m *MockHeartbeater) Check() (heartbeat.State, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "NeedToCheckOthers") - ret0, _ := ret[0].(bool) - return ret0 + ret := m.ctrl.Call(m, "Check") + ret0, _ := ret[0].(heartbeat.State) + ret1, _ := ret[1].(error) + return ret0, ret1 } -// NeedToCheckOthers indicates an expected call of NeedToCheckOthers. -func (mr *MockHeartbeaterMockRecorder) NeedToCheckOthers() *gomock.Call { +// Check indicates an expected call of Check. +func (mr *MockHeartbeaterMockRecorder) Check() *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NeedToCheckOthers", reflect.TypeOf((*MockHeartbeater)(nil).NeedToCheckOthers)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Check", reflect.TypeOf((*MockHeartbeater)(nil).Check)) } // NeedTurnOffNotifier mocks base method. @@ -95,3 +82,17 @@ func (mr *MockHeartbeaterMockRecorder) NeedTurnOffNotifier() *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NeedTurnOffNotifier", reflect.TypeOf((*MockHeartbeater)(nil).NeedTurnOffNotifier)) } + +// Type mocks base method. +func (m *MockHeartbeater) Type() datatypes.HeartbeatType { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Type") + ret0, _ := ret[0].(datatypes.HeartbeatType) + return ret0 +} + +// Type indicates an expected call of Type. +func (mr *MockHeartbeaterMockRecorder) Type() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Type", reflect.TypeOf((*MockHeartbeater)(nil).Type)) +} diff --git a/notifier/config.go b/notifier/config.go index 34c6e53d4..1d86cd311 100644 --- a/notifier/config.go +++ b/notifier/config.go @@ -10,8 +10,7 @@ const NotificationsLimitUnlimited = int64(-1) // Config is sending settings including log settings. type Config struct { Enabled bool - SelfStateEnabled bool - SelfStateContacts []map[string]string + SelfstateEnabled bool SendingTimeout time.Duration ResendingTimeout time.Duration ReschedulingDelay time.Duration diff --git a/notifier/registrator.go b/notifier/registrator.go index d1c4bfebb..506d05501 100644 --- a/notifier/registrator.go +++ b/notifier/registrator.go @@ -28,7 +28,7 @@ const ( pushoverSender = "pushover" discordSender = "discord" scriptSender = "script" - selfStateSender = "selfstate" + selfstateSender = "selfstate" slackSender = "slack" telegramSender = "telegram" twilioSmsSender = "twilio sms" @@ -91,12 +91,12 @@ func (notifier *StandardNotifier) RegisterSenders(connector moira.Database) erro return err } } - if notifier.config.SelfStateEnabled { - selfStateSettings := map[string]interface{}{ - "sender_type": selfStateSender, - "contact_type": selfStateSender, + if notifier.config.SelfstateEnabled { + selfstateSettings := map[string]interface{}{ + "sender_type": selfstateSender, + "contact_type": selfstateSender, } - if err = notifier.RegisterSender(selfStateSettings, &selfstate.Sender{Database: connector}); err != nil { + if err = notifier.RegisterSender(selfstateSettings, &selfstate.Sender{Database: connector}); err != nil { notifier.logger.Warning(). Error(err). Msg("Failed to register selfstate sender") diff --git a/notifier/selfstate/check.go b/notifier/selfstate/check.go deleted file mode 100644 index 9b2ad2fe5..000000000 --- a/notifier/selfstate/check.go +++ /dev/null @@ -1,120 +0,0 @@ -package selfstate - -import ( - "encoding/json" - "sync" - "time" - - "github.com/moira-alert/moira" - "github.com/moira-alert/moira/notifier" -) - -func (selfCheck *SelfCheckWorker) selfStateChecker(stop <-chan struct{}) error { - selfCheck.Logger.Info().Msg("Moira Notifier Self State Monitor started") - - checkTicker := time.NewTicker(selfCheck.Config.CheckInterval) - defer checkTicker.Stop() - - nextSendErrorMessage := time.Now().Unix() - - for { - select { - case <-stop: - selfCheck.Logger.Info().Msg("Moira Notifier Self State Monitor stopped") - return nil - case <-checkTicker.C: - selfCheck.Logger.Debug(). - Int64("nextSendErrorMessage", nextSendErrorMessage). - Msg("call check") - - nextSendErrorMessage = selfCheck.check(time.Now().Unix(), nextSendErrorMessage) - } - } -} - -func (selfCheck *SelfCheckWorker) handleCheckServices(nowTS int64) []moira.NotificationEvent { - var events []moira.NotificationEvent - - for _, heartbeat := range selfCheck.heartbeats { - currentValue, hasErrors, err := heartbeat.Check(nowTS) - if err != nil { - selfCheck.Logger.Error(). - Error(err). - Msg("Heartbeat failed") - } - - if hasErrors { - events = append(events, generateNotificationEvent(heartbeat.GetErrorMessage(), currentValue)) - if heartbeat.NeedTurnOffNotifier() { - selfCheck.setNotifierState(moira.SelfStateERROR) - } - - if !heartbeat.NeedToCheckOthers() { - break - } - } - } - - return events -} - -func (selfCheck *SelfCheckWorker) sendNotification(events []moira.NotificationEvent, nowTS int64) int64 { - eventsJSON, _ := json.Marshal(events) - selfCheck.Logger.Error(). - Int("number_of_events", len(events)). - String("events_json", string(eventsJSON)). - Msg("Health check. Send package notification events") - selfCheck.sendErrorMessages(events) - return nowTS + selfCheck.Config.NoticeIntervalSeconds -} - -func (selfCheck *SelfCheckWorker) check(nowTS int64, nextSendErrorMessage int64) int64 { - events := selfCheck.handleCheckServices(nowTS) - if nextSendErrorMessage < nowTS && len(events) > 0 { - nextSendErrorMessage = selfCheck.sendNotification(events, nowTS) - } - - return nextSendErrorMessage -} - -func (selfCheck *SelfCheckWorker) sendErrorMessages(events []moira.NotificationEvent) { - var sendingWG sync.WaitGroup - - for _, adminContact := range selfCheck.Config.Contacts { - pkg := notifier.NotificationPackage{ - Contact: moira.ContactData{ - Type: adminContact["type"], - Value: adminContact["value"], - }, - Trigger: moira.TriggerData{ - Name: "Moira health check", - ErrorValue: float64(0), - }, - Events: events, - DontResend: true, - } - - selfCheck.Notifier.Send(&pkg, &sendingWG) - sendingWG.Wait() - } -} - -func generateNotificationEvent(message string, currentValue int64) moira.NotificationEvent { - val := float64(currentValue) - return moira.NotificationEvent{ - Timestamp: time.Now().Unix(), - OldState: moira.StateNODATA, - State: moira.StateERROR, - Metric: message, - Value: &val, - } -} - -func (selfCheck *SelfCheckWorker) setNotifierState(state string) { - err := selfCheck.Database.SetNotifierState(state) - if err != nil { - selfCheck.Logger.Error(). - Error(err). - Msg("Can't set notifier state") - } -} diff --git a/notifier/selfstate/config.go b/notifier/selfstate/config.go index ad25c7e7e..03a83960d 100644 --- a/notifier/selfstate/config.go +++ b/notifier/selfstate/config.go @@ -1,37 +1,15 @@ package selfstate import ( - "fmt" - "time" + "github.com/moira-alert/moira/notifier/selfstate/monitor" ) -// Config is representation of self state worker settings like moira admins contacts and threshold values for checked services. -type Config struct { - Enabled bool - RedisDisconnectDelaySeconds int64 - LastMetricReceivedDelaySeconds int64 - LastCheckDelaySeconds int64 - LastRemoteCheckDelaySeconds int64 - NoticeIntervalSeconds int64 - CheckInterval time.Duration - Contacts []map[string]string +type MonitorConfig struct { + UserCfg monitor.UserMonitorConfig + AdminCfg monitor.AdminMonitorConfig } -func (config *Config) checkConfig(senders map[string]bool) error { - if !config.Enabled { - return nil - } - if len(config.Contacts) < 1 { - return fmt.Errorf("contacts must be specified") - } - for _, adminContact := range config.Contacts { - if _, ok := senders[adminContact["type"]]; !ok { - return fmt.Errorf("unknown contact type [%s]", adminContact["type"]) - } - if adminContact["value"] == "" { - return fmt.Errorf("value for [%s] must be present", adminContact["type"]) - } - } - - return nil +type Config struct { + Enabled bool + MonitorCfg MonitorConfig } diff --git a/notifier/selfstate/config_test.go b/notifier/selfstate/config_test.go deleted file mode 100644 index 438ae3927..000000000 --- a/notifier/selfstate/config_test.go +++ /dev/null @@ -1,114 +0,0 @@ -package selfstate - -import ( - "fmt" - "testing" - - . "github.com/smartystreets/goconvey/convey" -) - -func TestConfigCheck(testing *testing.T) { - contactTypes := map[string]bool{ - "admin-mail": true, - } - - Convey("SelfCheck disabled", testing, func() { - config := Config{ - Enabled: false, - Contacts: []map[string]string{ - { - "type": "admin-mail", - "value": "admin@company.com", - }, - }, - } - - Convey("all data valid, should return nil error", func() { - actual := config.checkConfig(contactTypes) - So(actual, ShouldBeNil) - }) - - Convey("contacts empty, should return nil error", func() { - config.Contacts = []map[string]string{} - actual := config.checkConfig(contactTypes) - So(actual, ShouldBeNil) - }) - - Convey("admin sending type not registered, should return nil error", func() { - actual := config.checkConfig(make(map[string]bool)) - So(actual, ShouldBeNil) - }) - - Convey("admin sending contact empty, should return nil error", func() { - config.Contacts = []map[string]string{ - { - "type": "admin-mail", - "value": "", - }, - } - actual := config.checkConfig(make(map[string]bool)) - So(actual, ShouldBeNil) - }) - }) - - Convey("SelfCheck contacts empty, should return contacts must be specified error", testing, func() { - config := Config{ - Enabled: true, - } - actual := config.checkConfig(make(map[string]bool)) - So(actual, ShouldResemble, fmt.Errorf("contacts must be specified")) - }) - - Convey("Admin sending type not registered, should not pass check without admin contact type", testing, func() { - config := Config{ - Enabled: true, - Contacts: []map[string]string{ - { - "type": "admin-mail", - "value": "admin@company.com", - }, - }, - } - - actual := config.checkConfig(make(map[string]bool)) - So(actual, ShouldResemble, fmt.Errorf("unknown contact type [admin-mail]")) - }) - - Convey("Admin sending contact empty, should not pass check without admin contact", testing, func() { - config := Config{ - Enabled: true, - Contacts: []map[string]string{ - { - "type": "admin-mail", - "value": "", - }, - }, - } - - contactTypes := map[string]bool{ - "admin-mail": true, - } - - actual := config.checkConfig(contactTypes) - So(actual, ShouldResemble, fmt.Errorf("value for [admin-mail] must be present")) - }) - - Convey("Has registered valid admin contact, should pass check", testing, func() { - config := Config{ - Enabled: true, - Contacts: []map[string]string{ - { - "type": "admin-mail", - "value": "admin@company.com", - }, - }, - } - - contactTypes := map[string]bool{ - "admin-mail": true, - } - - actual := config.checkConfig(contactTypes) - So(actual, ShouldBeNil) - }) -} diff --git a/notifier/selfstate/heartbeat/database.go b/notifier/selfstate/heartbeat/database.go index 47b466ffa..d4abd19a5 100644 --- a/notifier/selfstate/heartbeat/database.go +++ b/notifier/selfstate/heartbeat/database.go @@ -15,7 +15,7 @@ var _ Heartbeater = (*databaseHeartbeater)(nil) type DatabaseHeartbeaterConfig struct { HeartbeaterBaseConfig - RedisDisconnectDelay time.Duration `validate:"required,gt=0"` + RedisDisconnectDelay time.Duration `validate:"required_if=Enabled true,gte=0"` } type databaseHeartbeater struct { @@ -60,7 +60,7 @@ func (heartbeater databaseHeartbeater) NeedTurnOffNotifier() bool { // Type is a function that returns the current heartbeat type. func (databaseHeartbeater) Type() datatypes.HeartbeatType { - return datatypes.HeartbeatTypeNotSet + return datatypes.HeartbeatDatabase } // AlertSettings is a function that returns the current settings for alerts. diff --git a/notifier/selfstate/heartbeat/database_test.go b/notifier/selfstate/heartbeat/database_test.go index 99312d892..382e23951 100644 --- a/notifier/selfstate/heartbeat/database_test.go +++ b/notifier/selfstate/heartbeat/database_test.go @@ -34,7 +34,11 @@ func TestNewDatabaseHeartbeater(t *testing.T) { }) Convey("Without redis disconnect delay", func() { - cfg := DatabaseHeartbeaterConfig{} + cfg := DatabaseHeartbeaterConfig{ + HeartbeaterBaseConfig: HeartbeaterBaseConfig{ + Enabled: true, + }, + } databaseHeartbeater, err := NewDatabaseHeartbeater(cfg, heartbeaterBase) So(errors.As(err, &validationErr), ShouldBeTrue) @@ -138,7 +142,7 @@ func TestDatabaseHeartbeaterType(t *testing.T) { So(err, ShouldBeNil) databaseHeartbeaterType := databaseHeartbeater.Type() - So(databaseHeartbeaterType, ShouldResemble, datatypes.HeartbeatTypeNotSet) + So(databaseHeartbeaterType, ShouldResemble, datatypes.HeartbeatDatabase) }) } diff --git a/notifier/selfstate/heartbeat/filter.go b/notifier/selfstate/heartbeat/filter.go index 73ff3fcc3..18bda5bac 100644 --- a/notifier/selfstate/heartbeat/filter.go +++ b/notifier/selfstate/heartbeat/filter.go @@ -19,7 +19,7 @@ var ( type FilterHeartbeaterConfig struct { HeartbeaterBaseConfig - MetricReceivedDelay time.Duration `validate:"required,gt=0"` + MetricReceivedDelay time.Duration `validate:"required_if=Enabled true,gte=00"` } type filterHeartbeater struct { @@ -74,7 +74,7 @@ func (heartbeater filterHeartbeater) NeedTurnOffNotifier() bool { // Type is a function that returns the current heartbeat type. func (filterHeartbeater) Type() datatypes.HeartbeatType { - return datatypes.HeartbeatTypeNotSet + return datatypes.HeartbeatFilter } // AlertSettings is a function that returns the current settings for alerts. diff --git a/notifier/selfstate/heartbeat/filter_test.go b/notifier/selfstate/heartbeat/filter_test.go index fe54dc894..e5f2676b8 100644 --- a/notifier/selfstate/heartbeat/filter_test.go +++ b/notifier/selfstate/heartbeat/filter_test.go @@ -32,7 +32,11 @@ func TestNewFilterHeartbeater(t *testing.T) { }) Convey("Without metric received delay", func() { - cfg := FilterHeartbeaterConfig{} + cfg := FilterHeartbeaterConfig{ + HeartbeaterBaseConfig: HeartbeaterBaseConfig{ + Enabled: true, + }, + } filterHeartbeater, err := NewFilterHeartbeater(cfg, heartbeaterBase) So(errors.As(err, &validationErr), ShouldBeTrue) @@ -180,7 +184,7 @@ func TestFilterHeartbeaterType(t *testing.T) { So(err, ShouldBeNil) filterHeartbeaterType := filterHeartbeater.Type() - So(filterHeartbeaterType, ShouldResemble, datatypes.HeartbeatTypeNotSet) + So(filterHeartbeaterType, ShouldResemble, datatypes.HeartbeatFilter) }) } diff --git a/notifier/selfstate/heartbeat/heartbeat.go b/notifier/selfstate/heartbeat/heartbeat.go index ef8d72852..9837179ac 100644 --- a/notifier/selfstate/heartbeat/heartbeat.go +++ b/notifier/selfstate/heartbeat/heartbeat.go @@ -15,9 +15,9 @@ const ( StateError State = "heartbeat_state_error" ) -// IsDegraded checks if the condition has degraded. -func (lastState State) IsDegraded(newState State) bool { - return lastState == StateOK && newState == StateError +// IsDegraded checks if the condition is still degraded. +func (State) IsDegraded(newState State) bool { + return newState == StateError } // IsRecovered checks if the condition has recovered. @@ -39,12 +39,12 @@ type HeartbeaterBaseConfig struct { NeedTurnOffNotifier bool NeedToCheckOthers bool - AlertCfg AlertConfig `validate:"required_if=Enabled true"` + AlertCfg AlertConfig } // AlertConfig contains the configuration of the alerts that heartbeater sends out. type AlertConfig struct { - Name string `validate:"required_if=Enabled true"` + Name string Desc string } @@ -71,3 +71,12 @@ func NewHeartbeaterBase( lastSuccessfulCheck: clock.NowUTC(), } } + +// HeartbeatersConfig is a structure that combines the configs of all heartbeaters within it. +type HeartbeatersConfig struct { + DatabaseCfg DatabaseHeartbeaterConfig + FilterCfg FilterHeartbeaterConfig + LocalCheckerCfg LocalCheckerHeartbeaterConfig + RemoteCheckerCfg RemoteCheckerHeartbeaterConfig + NotifierCfg NotifierHeartbeaterConfig +} diff --git a/notifier/selfstate/heartbeat/heartbeat_test.go b/notifier/selfstate/heartbeat/heartbeat_test.go index 539c44587..40a68fc26 100644 --- a/notifier/selfstate/heartbeat/heartbeat_test.go +++ b/notifier/selfstate/heartbeat/heartbeat_test.go @@ -104,23 +104,6 @@ func TestValidateHeartbeaterBaseConfig(t *testing.T) { So(err, ShouldBeNil) }) - Convey("With just enabled config", func() { - hbCfg := HeartbeaterBaseConfig{ - Enabled: true, - } - err := moira.ValidateStruct(hbCfg) - So(err, ShouldNotBeNil) - }) - - Convey("With enabled config and added alert config", func() { - hbCfg := HeartbeaterBaseConfig{ - Enabled: true, - AlertCfg: AlertConfig{}, - } - err := moira.ValidateStruct(hbCfg) - So(err, ShouldNotBeNil) - }) - Convey("With enabled config, added and filled alert config", func() { hbCfg := HeartbeaterBaseConfig{ Enabled: true, diff --git a/notifier/selfstate/heartbeat/local_checker.go b/notifier/selfstate/heartbeat/local_checker.go index 4ea80c113..35d562c28 100644 --- a/notifier/selfstate/heartbeat/local_checker.go +++ b/notifier/selfstate/heartbeat/local_checker.go @@ -15,7 +15,7 @@ var _ Heartbeater = (*localCheckerHeartbeater)(nil) type LocalCheckerHeartbeaterConfig struct { HeartbeaterBaseConfig - LocalCheckDelay time.Duration `validate:"required,gt=0"` + LocalCheckDelay time.Duration `validate:"required_if=Enabled true,gte=0"` } type localCheckerHeartbeater struct { @@ -70,7 +70,7 @@ func (heartbeater localCheckerHeartbeater) NeedTurnOffNotifier() bool { // Type is a function that returns the current heartbeat type. func (localCheckerHeartbeater) Type() datatypes.HeartbeatType { - return datatypes.HeartbeatTypeNotSet + return datatypes.HeartbeatLocalChecker } // AlertSettings is a function that returns the current settings for alerts. diff --git a/notifier/selfstate/heartbeat/local_checker_test.go b/notifier/selfstate/heartbeat/local_checker_test.go index 50eae03e8..a5c39ec2f 100644 --- a/notifier/selfstate/heartbeat/local_checker_test.go +++ b/notifier/selfstate/heartbeat/local_checker_test.go @@ -32,7 +32,11 @@ func TestNewLocalCheckerHeartbeater(t *testing.T) { }) Convey("Without local check delay", func() { - cfg := LocalCheckerHeartbeaterConfig{} + cfg := LocalCheckerHeartbeaterConfig{ + HeartbeaterBaseConfig: HeartbeaterBaseConfig{ + Enabled: true, + }, + } localCheckerHeartbeater, err := NewLocalCheckerHeartbeater(cfg, heartbeaterBase) So(errors.As(err, &validationErr), ShouldBeTrue) @@ -180,7 +184,7 @@ func TestLocalCheckerHeartbeaterType(t *testing.T) { So(err, ShouldBeNil) localCheckerHeartbeaterType := localCheckerHeartbeater.Type() - So(localCheckerHeartbeaterType, ShouldResemble, datatypes.HeartbeatTypeNotSet) + So(localCheckerHeartbeaterType, ShouldResemble, datatypes.HeartbeatLocalChecker) }) } diff --git a/notifier/selfstate/heartbeat/notifier.go b/notifier/selfstate/heartbeat/notifier.go index 9d26e45c5..90dff781e 100644 --- a/notifier/selfstate/heartbeat/notifier.go +++ b/notifier/selfstate/heartbeat/notifier.go @@ -48,7 +48,7 @@ func (heartbeater *notifierHeartbeater) NeedTurnOffNotifier() bool { // Type is a function that returns the current heartbeat type. func (notifierHeartbeater) Type() datatypes.HeartbeatType { - return datatypes.HeartbeatNotifierOff + return datatypes.HeartbeatNotifier } // AlertSettings is a function that returns the current settings for alerts. diff --git a/notifier/selfstate/heartbeat/notifier_test.go b/notifier/selfstate/heartbeat/notifier_test.go index 9b1767eb5..b03eed88e 100644 --- a/notifier/selfstate/heartbeat/notifier_test.go +++ b/notifier/selfstate/heartbeat/notifier_test.go @@ -93,7 +93,7 @@ func TestNotifierHeartbeaterType(t *testing.T) { So(err, ShouldBeNil) notifierHeartbeaterType := notifierHeartbeater.Type() - So(notifierHeartbeaterType, ShouldResemble, datatypes.HeartbeatNotifierOff) + So(notifierHeartbeaterType, ShouldResemble, datatypes.HeartbeatNotifier) }) } diff --git a/notifier/selfstate/heartbeat/remote_checker.go b/notifier/selfstate/heartbeat/remote_checker.go index 096342378..9300fc74c 100644 --- a/notifier/selfstate/heartbeat/remote_checker.go +++ b/notifier/selfstate/heartbeat/remote_checker.go @@ -19,7 +19,7 @@ var ( type RemoteCheckerHeartbeaterConfig struct { HeartbeaterBaseConfig - RemoteCheckDelay time.Duration `validate:"required,gt=0"` + RemoteCheckDelay time.Duration `validate:"required_if=Enabled true,gte=0"` } type remoteCheckerHeartbeater struct { @@ -74,7 +74,7 @@ func (heartbeater remoteCheckerHeartbeater) NeedTurnOffNotifier() bool { // Type is a function that returns the current heartbeat type. func (remoteCheckerHeartbeater) Type() datatypes.HeartbeatType { - return datatypes.HeartbeatTypeNotSet + return datatypes.HeartbeatRemoteChecker } // AlertSettings is a function that returns the current settings for alerts. diff --git a/notifier/selfstate/heartbeat/remote_checker_test.go b/notifier/selfstate/heartbeat/remote_checker_test.go index aac4dbaf9..339076335 100644 --- a/notifier/selfstate/heartbeat/remote_checker_test.go +++ b/notifier/selfstate/heartbeat/remote_checker_test.go @@ -23,6 +23,9 @@ func TestNewRemoteCheckerHeartbeater(t *testing.T) { Convey("Test NewRemoteCheckerHeartbeater", t, func() { Convey("With too low remote check delay", func() { cfg := RemoteCheckerHeartbeaterConfig{ + HeartbeaterBaseConfig: HeartbeaterBaseConfig{ + Enabled: true, + }, RemoteCheckDelay: -1, } @@ -32,7 +35,11 @@ func TestNewRemoteCheckerHeartbeater(t *testing.T) { }) Convey("Without remote check delay", func() { - cfg := RemoteCheckerHeartbeaterConfig{} + cfg := RemoteCheckerHeartbeaterConfig{ + HeartbeaterBaseConfig: HeartbeaterBaseConfig{ + Enabled: true, + }, + } remoteCheckerHeartbeater, err := NewRemoteCheckerHeartbeater(cfg, heartbeaterBase) So(errors.As(err, &validationErr), ShouldBeTrue) @@ -180,7 +187,7 @@ func TestRemoteCheckerHeartbeaterType(t *testing.T) { So(err, ShouldBeNil) remoteCheckerHeartbeaterType := remoteCheckerHeartbeater.Type() - So(remoteCheckerHeartbeaterType, ShouldResemble, datatypes.HeartbeatTypeNotSet) + So(remoteCheckerHeartbeaterType, ShouldResemble, datatypes.HeartbeatRemoteChecker) }) } diff --git a/notifier/selfstate/monitor/admin.go b/notifier/selfstate/monitor/admin.go new file mode 100644 index 000000000..65246cad4 --- /dev/null +++ b/notifier/selfstate/monitor/admin.go @@ -0,0 +1,105 @@ +package monitor + +import ( + "fmt" + "sync" + "time" + + "github.com/moira-alert/moira" + "github.com/moira-alert/moira/notifier" +) + +const ( + adminMonitorName = "Moira Admin Selfstate Monitoring" + adminMonitorLockName = "moira-admin-selfstate-monitor" + adminMonitorLockTTL = 15 * time.Second +) + +type AdminMonitorConfig struct { + MonitorBaseConfig + + AdminContacts []map[string]string `validate:"required_if=Enabled true"` +} + +func (cfg AdminMonitorConfig) validate(senders map[string]bool) error { + if err := moira.ValidateStruct(cfg); err != nil { + return err + } + + for _, contact := range cfg.AdminContacts { + contactType := contact["type"] + contactValue := contact["value"] + + if _, ok := senders[contactType]; !ok { + return fmt.Errorf("unknown contact type in admin config: [%s]", contactType) + } + + if contactValue == "" { + return fmt.Errorf("value for [%s] must be present", contactType) + } + } + + return nil +} + +type adminMonitor struct { + adminCfg AdminMonitorConfig + database moira.Database + notifier notifier.Notifier +} + +func NewForAdmin( + adminCfg AdminMonitorConfig, + logger moira.Logger, + database moira.Database, + clock moira.Clock, + notifier notifier.Notifier, +) (*monitor, error) { + if err := adminCfg.validate(notifier.GetSenders()); err != nil { + return nil, fmt.Errorf("admin config validation error: %w", err) + } + + adminMonitor := adminMonitor{ + adminCfg: adminCfg, + database: database, + notifier: notifier, + } + + cfg := monitorConfig{ + Name: adminMonitorName, + LockName: adminMonitorLockName, + LockTTL: adminMonitorLockTTL, + NoticeInterval: adminCfg.NoticeInterval, + CheckInterval: adminCfg.CheckInterval, + } + + heartbeaters := createHearbeaters(adminCfg.HeartbeatersCfg, logger, database, clock) + + return newMonitor( + cfg, + logger, + database, + clock, + notifier, + heartbeaters, + adminMonitor.sendNotifications, + ) +} + +func (am *adminMonitor) sendNotifications(pkgs []notifier.NotificationPackage) error { + sendingWG := &sync.WaitGroup{} + + for _, pkg := range pkgs { + for _, adminContact := range am.adminCfg.AdminContacts { + contact := moira.ContactData{ + Type: adminContact["type"], + Value: adminContact["value"], + } + pkg.Contact = contact + am.notifier.Send(&pkg, sendingWG) + sendingWG.Wait() + } + } + + return nil +} diff --git a/notifier/selfstate/monitor/admin_test.go b/notifier/selfstate/monitor/admin_test.go new file mode 100644 index 000000000..caa2c0688 --- /dev/null +++ b/notifier/selfstate/monitor/admin_test.go @@ -0,0 +1,220 @@ +package monitor + +import ( + "errors" + "fmt" + "sync" + "testing" + "time" + + "github.com/go-playground/validator/v10" + "github.com/moira-alert/moira" + mock_notifier "github.com/moira-alert/moira/mock/notifier" + "github.com/moira-alert/moira/notifier" + "github.com/moira-alert/moira/notifier/selfstate/heartbeat" + . "github.com/smartystreets/goconvey/convey" + "go.uber.org/mock/gomock" +) + +var defaultHeartbeatersConfig = heartbeat.HeartbeatersConfig{ + DatabaseCfg: heartbeat.DatabaseHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + Enabled: true, + }, + RedisDisconnectDelay: 1, + }, + FilterCfg: heartbeat.FilterHeartbeaterConfig{}, + LocalCheckerCfg: heartbeat.LocalCheckerHeartbeaterConfig{}, + RemoteCheckerCfg: heartbeat.RemoteCheckerHeartbeaterConfig{}, + NotifierCfg: heartbeat.NotifierHeartbeaterConfig{}, +} + +func TestValidateConfig(t *testing.T) { + senders := map[string]bool{ + "telegram": true, + } + + validationErr := validator.ValidationErrors{} + + Convey("Test Validate", t, func() { + Convey("With disabled admin and user selfchecks", func() { + cfg := AdminMonitorConfig{ + MonitorBaseConfig: MonitorBaseConfig{ + Enabled: false, + }, + } + + err := cfg.validate(senders) + So(err, ShouldBeNil) + }) + + Convey("Without heartbeats config", func() { + cfg := AdminMonitorConfig{ + MonitorBaseConfig: MonitorBaseConfig{ + Enabled: true, + }, + } + + err := cfg.validate(senders) + So(errors.As(err, &validationErr), ShouldBeTrue) + }) + + Convey("Without admin notice interval", func() { + cfg := AdminMonitorConfig{ + MonitorBaseConfig: MonitorBaseConfig{ + Enabled: true, + HeartbeatersCfg: defaultHeartbeatersConfig, + }, + } + + err := cfg.validate(senders) + So(errors.As(err, &validationErr), ShouldBeTrue) + }) + + Convey("Without admin check interval", func() { + cfg := AdminMonitorConfig{ + MonitorBaseConfig: MonitorBaseConfig{ + Enabled: true, + HeartbeatersCfg: defaultHeartbeatersConfig, + NoticeInterval: time.Minute, + }, + } + + err := cfg.validate(senders) + So(errors.As(err, &validationErr), ShouldBeTrue) + }) + + Convey("Without admin contacts", func() { + cfg := AdminMonitorConfig{ + MonitorBaseConfig: MonitorBaseConfig{ + Enabled: true, + HeartbeatersCfg: defaultHeartbeatersConfig, + NoticeInterval: time.Minute, + CheckInterval: time.Minute, + }, + } + + err := cfg.validate(senders) + So(errors.As(err, &validationErr), ShouldBeTrue) + }) + + Convey("With empty admin contacts", func() { + cfg := AdminMonitorConfig{ + MonitorBaseConfig: MonitorBaseConfig{ + Enabled: true, + HeartbeatersCfg: defaultHeartbeatersConfig, + NoticeInterval: time.Minute, + CheckInterval: time.Minute, + }, + AdminContacts: []map[string]string{}, + } + + err := cfg.validate(senders) + So(err, ShouldBeNil) + }) + + Convey("With unknown admin contact type", func() { + cfg := AdminMonitorConfig{ + MonitorBaseConfig: MonitorBaseConfig{ + Enabled: true, + HeartbeatersCfg: defaultHeartbeatersConfig, + NoticeInterval: time.Minute, + CheckInterval: time.Minute, + }, + AdminContacts: []map[string]string{ + { + "type": "test-contact-type", + }, + }, + } + + err := cfg.validate(senders) + So(err, ShouldResemble, fmt.Errorf("unknown contact type in admin config: [%s]", cfg.AdminContacts[0]["type"])) + }) + + Convey("Without admin contact value", func() { + cfg := AdminMonitorConfig{ + MonitorBaseConfig: MonitorBaseConfig{ + Enabled: true, + HeartbeatersCfg: defaultHeartbeatersConfig, + NoticeInterval: time.Minute, + CheckInterval: time.Minute, + }, + AdminContacts: []map[string]string{ + { + "type": "telegram", + }, + }, + } + + err := cfg.validate(senders) + So(err, ShouldResemble, fmt.Errorf("value for [%s] must be present", cfg.AdminContacts[0]["type"])) + }) + + Convey("With valid admin contact type and value", func() { + cfg := AdminMonitorConfig{ + MonitorBaseConfig: MonitorBaseConfig{ + Enabled: true, + HeartbeatersCfg: defaultHeartbeatersConfig, + NoticeInterval: time.Minute, + CheckInterval: time.Minute, + }, + AdminContacts: []map[string]string{ + { + "type": "telegram", + "value": "@webcamsmodel", + }, + }, + } + + err := cfg.validate(senders) + So(err, ShouldBeNil) + }) + }) +} + +func TestAdminSendNotifications(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + mockNotifier := mock_notifier.NewMockNotifier(mockCtrl) + + adminCfg := AdminMonitorConfig{ + AdminContacts: []map[string]string{ + { + "type": "telegram", + "value": "@webcamsmodel", + }, + }, + } + + sendingWG := &sync.WaitGroup{} + + adminMonitor := adminMonitor{ + notifier: mockNotifier, + adminCfg: adminCfg, + } + + Convey("Test sendNotifications", t, func() { + Convey("With empty notification packages", func() { + pkgs := []notifier.NotificationPackage{} + err := adminMonitor.sendNotifications(pkgs) + So(err, ShouldBeNil) + }) + + Convey("With correct sending notification packages", func() { + pkgs := []notifier.NotificationPackage{ + {}, + } + pkgWithContact := ¬ifier.NotificationPackage{ + Contact: moira.ContactData{ + Type: adminCfg.AdminContacts[0]["type"], + Value: adminCfg.AdminContacts[0]["value"], + }, + } + mockNotifier.EXPECT().Send(pkgWithContact, sendingWG).Times(1) + err := adminMonitor.sendNotifications(pkgs) + So(err, ShouldBeNil) + }) + }) +} diff --git a/notifier/selfstate/monitor/monitor.go b/notifier/selfstate/monitor/monitor.go new file mode 100644 index 000000000..265c5a925 --- /dev/null +++ b/notifier/selfstate/monitor/monitor.go @@ -0,0 +1,310 @@ +package monitor + +import ( + "fmt" + "time" + + "github.com/moira-alert/moira" + "github.com/moira-alert/moira/datatypes" + "github.com/moira-alert/moira/notifier" + "github.com/moira-alert/moira/notifier/selfstate/heartbeat" + w "github.com/moira-alert/moira/worker" + "gopkg.in/tomb.v2" +) + +var ( + okValue = 0.0 + errorValue = 1.0 + triggerErrorValue = 1.0 + + _ Monitor = (*monitor)(nil) +) + +type MonitorBaseConfig struct { + Enabled bool + HeartbeatersCfg heartbeat.HeartbeatersConfig `validate:"required_if=Enabled true"` + NoticeInterval time.Duration `validate:"required_if=Enabled true,gte=0"` + CheckInterval time.Duration `validate:"required_if=Enabled true,gte=0"` +} + +type hearbeatInfo struct { + lastAlertTime time.Time + lastCheckState heartbeat.State +} + +type monitorConfig struct { + Name string `validate:"required"` + LockName string `validate:"required"` + LockTTL time.Duration `validate:"required,gt=0"` + NoticeInterval time.Duration `validate:"required,gt=0"` + CheckInterval time.Duration `validate:"required,gt=0"` +} + +type Monitor interface { + Start() + Stop() error +} + +type monitor struct { + cfg monitorConfig + logger moira.Logger + database moira.Database + notifier notifier.Notifier + tomb tomb.Tomb + heartbeaters []heartbeat.Heartbeater + clock moira.Clock + heartbeatsInfo map[datatypes.HeartbeatType]*hearbeatInfo + sendNotifications func(pkgs []notifier.NotificationPackage) error +} + +func newMonitor( + cfg monitorConfig, + logger moira.Logger, + database moira.Database, + clock moira.Clock, + notifier notifier.Notifier, + heartbeaters []heartbeat.Heartbeater, + sendNotifications func(pkgs []notifier.NotificationPackage) error, +) (*monitor, error) { + if err := moira.ValidateStruct(cfg); err != nil { + return nil, fmt.Errorf("monitor configuration error: %w", err) + } + + hearbeatersInfo := make(map[datatypes.HeartbeatType]*hearbeatInfo, len(heartbeaters)) + for _, heartbeater := range heartbeaters { + hearbeatersInfo[heartbeater.Type()] = &hearbeatInfo{ + lastCheckState: heartbeat.StateOK, + } + } + + return &monitor{ + cfg: cfg, + logger: logger, + database: database, + notifier: notifier, + heartbeaters: heartbeaters, + clock: clock, + heartbeatsInfo: hearbeatersInfo, + sendNotifications: sendNotifications, + }, nil +} + +func createHearbeaters( + heartbeatersCfg heartbeat.HeartbeatersConfig, + logger moira.Logger, + database moira.Database, + clock moira.Clock, +) []heartbeat.Heartbeater { + hearbeaterBase := heartbeat.NewHeartbeaterBase(logger, database, clock) + + heartbeaters := make([]heartbeat.Heartbeater, 0) + + if heartbeatersCfg.DatabaseCfg.Enabled { + databaseHeartbeater, err := heartbeat.NewDatabaseHeartbeater(heartbeatersCfg.DatabaseCfg, hearbeaterBase) + if err != nil { + logger.Error(). + Error(err). + String("heartbeater", string(databaseHeartbeater.Type())). + Msg("Failed to create a new database heartbeater") + } else { + heartbeaters = append(heartbeaters, databaseHeartbeater) + } + } + + if heartbeatersCfg.FilterCfg.Enabled { + filterHeartbeater, err := heartbeat.NewFilterHeartbeater(heartbeatersCfg.FilterCfg, hearbeaterBase) + if err != nil { + logger.Error(). + Error(err). + String("heartbeater", string(filterHeartbeater.Type())). + Msg("Failed to create a new filter heartbeater") + } else { + heartbeaters = append(heartbeaters, filterHeartbeater) + } + } + + if heartbeatersCfg.LocalCheckerCfg.Enabled { + localCheckerHeartbeater, err := heartbeat.NewLocalCheckerHeartbeater(heartbeatersCfg.LocalCheckerCfg, hearbeaterBase) + if err != nil { + logger.Error(). + Error(err). + String("heartbeater", string(localCheckerHeartbeater.Type())). + Msg("Failed to create a new local checker heartbeater") + } else { + heartbeaters = append(heartbeaters, localCheckerHeartbeater) + } + } + + if heartbeatersCfg.RemoteCheckerCfg.Enabled { + remoteCheckerHeartbeater, err := heartbeat.NewRemoteCheckerHeartbeater(heartbeatersCfg.RemoteCheckerCfg, hearbeaterBase) + if err != nil { + logger.Error(). + Error(err). + String("heartbeater", string(remoteCheckerHeartbeater.Type())). + Msg("Failed to create a new remote checker heartbeater") + } else { + heartbeaters = append(heartbeaters, remoteCheckerHeartbeater) + } + } + + if heartbeatersCfg.NotifierCfg.Enabled { + notifierHeartbeater, err := heartbeat.NewNotifierHeartbeater(heartbeatersCfg.NotifierCfg, hearbeaterBase) + if err != nil { + logger.Error(). + Error(err). + String("heartbeater", string(notifierHeartbeater.Type())). + Msg("Failed to create a new notifier heartbeater") + } else { + heartbeaters = append(heartbeaters, notifierHeartbeater) + } + } + + return heartbeaters +} + +func (m *monitor) Start() { + m.tomb.Go(func() error { + w.NewWorker( + m.cfg.Name, + m.logger, + m.database.NewLock(m.cfg.LockName, m.cfg.LockTTL), + m.selfstateCheck, + ).Run(nil) + return nil + }) +} + +func (m *monitor) selfstateCheck(stop <-chan struct{}) error { + m.logger.Info().Msg(fmt.Sprintf("%s started", m.cfg.Name)) + + checkTicker := time.NewTicker(m.cfg.CheckInterval) + defer checkTicker.Stop() + + for { + select { + case <-stop: + m.logger.Info().Msg(fmt.Sprintf("%s stopped", m.cfg.Name)) + return nil + case <-checkTicker.C: + m.logger.Debug().Msg(fmt.Sprintf("%s selfstate check", m.cfg.Name)) + + m.check() + } + } +} + +func (m *monitor) check() { + pkgs := m.checkHeartbeats() + if len(pkgs) > 0 { + if err := m.sendNotifications(pkgs); err != nil { + m.logger.Error(). + Error(err). + String("type", m.cfg.Name). + Interface("notification_packages", pkgs). + Msg("Failed to send heartbeats notifications") + } + } +} + +func (m *monitor) checkHeartbeats() []notifier.NotificationPackage { + pkgs := make([]notifier.NotificationPackage, 0) + + for _, heartbeater := range m.heartbeaters { + heartbeatState, err := heartbeater.Check() + if err != nil { + m.logger.Error(). + Error(err). + String("name", m.cfg.Name). + String("heartbeater", string(heartbeater.Type())). + Msg("Heartbeat check failed") + } + + m.logger.Debug(). + String("name", m.cfg.Name). + String("heartbeater", string(heartbeater.Type())). + String("state", string(heartbeatState)). + Msg("Check heartbeat") + + pkg := m.generateHeartbeatNotificationPackage(heartbeater, heartbeatState) + if pkg != nil { + pkgs = append(pkgs, *pkg) + } + + m.heartbeatsInfo[heartbeater.Type()].lastCheckState = heartbeatState + } + + return pkgs +} + +func (m *monitor) generateHeartbeatNotificationPackage(heartbeater heartbeat.Heartbeater, heartbeatState heartbeat.State) *notifier.NotificationPackage { + heartbeatInfo := m.heartbeatsInfo[heartbeater.Type()] + + now := m.clock.NowUTC() + + isDegraded := heartbeatInfo.lastCheckState.IsDegraded(heartbeatState) + isRecovered := heartbeatInfo.lastCheckState.IsRecovered(heartbeatState) + allowNotify := now.Sub(heartbeatInfo.lastAlertTime) > m.cfg.NoticeInterval + + if isDegraded && allowNotify { + return m.createErrorNotificationPackage(heartbeater, m.clock) + } else if isRecovered { + return m.createOkNotificationPackage(heartbeater, m.clock) + } + + return nil +} + +func (m *monitor) createErrorNotificationPackage(heartbeater heartbeat.Heartbeater, clock moira.Clock) *notifier.NotificationPackage { + now := clock.NowUTC() + + m.heartbeatsInfo[heartbeater.Type()].lastAlertTime = now + + event := moira.NotificationEvent{ + Timestamp: now.Unix(), + OldState: moira.StateNODATA, + State: moira.StateERROR, + Metric: string(heartbeater.Type()), + Value: &errorValue, + } + + trigger := moira.TriggerData{ + Name: heartbeater.AlertSettings().Name, + Desc: heartbeater.AlertSettings().Desc, + ErrorValue: triggerErrorValue, + } + + return ¬ifier.NotificationPackage{ + Events: []moira.NotificationEvent{event}, + Trigger: trigger, + } +} + +func (m *monitor) createOkNotificationPackage(heartbeater heartbeat.Heartbeater, clock moira.Clock) *notifier.NotificationPackage { + now := clock.NowUTC() + + m.heartbeatsInfo[heartbeater.Type()].lastAlertTime = now + + event := moira.NotificationEvent{ + Timestamp: now.Unix(), + OldState: moira.StateERROR, + State: moira.StateOK, + Metric: string(heartbeater.Type()), + Value: &okValue, + } + + trigger := moira.TriggerData{ + Name: heartbeater.AlertSettings().Name, + Desc: heartbeater.AlertSettings().Desc, + ErrorValue: triggerErrorValue, + } + + return ¬ifier.NotificationPackage{ + Events: []moira.NotificationEvent{event}, + Trigger: trigger, + } +} + +func (m *monitor) Stop() error { + m.tomb.Kill(nil) + return m.tomb.Wait() +} diff --git a/notifier/selfstate/monitor/monitor_test.go b/notifier/selfstate/monitor/monitor_test.go new file mode 100644 index 000000000..cbd3c1392 --- /dev/null +++ b/notifier/selfstate/monitor/monitor_test.go @@ -0,0 +1,616 @@ +package monitor + +import ( + "sync" + "testing" + "time" + + "github.com/moira-alert/moira" + "github.com/moira-alert/moira/datatypes" + logging "github.com/moira-alert/moira/logging/zerolog_adapter" + mock_clock "github.com/moira-alert/moira/mock/clock" + mock_heartbeat "github.com/moira-alert/moira/mock/heartbeat" + mock_moira_alert "github.com/moira-alert/moira/mock/moira-alert" + mock_notifier "github.com/moira-alert/moira/mock/notifier" + "github.com/moira-alert/moira/notifier" + "github.com/moira-alert/moira/notifier/selfstate/heartbeat" + . "github.com/smartystreets/goconvey/convey" + "go.uber.org/mock/gomock" +) + +func TestCreateHeartbeaters(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + mockDatabase := mock_moira_alert.NewMockDatabase(mockCtrl) + mockClock := mock_clock.NewMockClock(mockCtrl) + mockLogger := mock_moira_alert.NewMockLogger(mockCtrl) + testTime := time.Date(2022, time.June, 6, 10, 0, 0, 0, time.UTC) + + Convey("Test createHeartbeaters", t, func() { + Convey("Without any heartbeater", func() { + hbCfg := heartbeat.HeartbeatersConfig{} + mockClock.EXPECT().NowUTC().Return(testTime) + heartbeaters := createHearbeaters(hbCfg, mockLogger, mockDatabase, mockClock) + So(heartbeaters, ShouldBeEmpty) + }) + + Convey("With database heartbeater", func() { + hbCfg := heartbeat.HeartbeatersConfig{ + DatabaseCfg: heartbeat.DatabaseHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + Enabled: true, + }, + RedisDisconnectDelay: time.Minute, + }, + } + mockClock.EXPECT().NowUTC().Return(testTime) + heartbeaters := createHearbeaters(hbCfg, mockLogger, mockDatabase, mockClock) + So(heartbeaters, ShouldHaveLength, 1) + }) + + Convey("With database and filter heartbeaters", func() { + hbCfg := heartbeat.HeartbeatersConfig{ + DatabaseCfg: heartbeat.DatabaseHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + Enabled: true, + }, + RedisDisconnectDelay: time.Minute, + }, + FilterCfg: heartbeat.FilterHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + Enabled: true, + }, + MetricReceivedDelay: time.Minute, + }, + } + mockClock.EXPECT().NowUTC().Return(testTime) + heartbeaters := createHearbeaters(hbCfg, mockLogger, mockDatabase, mockClock) + So(heartbeaters, ShouldHaveLength, 2) + }) + + Convey("With database, filter and local checker heartbeaters", func() { + hbCfg := heartbeat.HeartbeatersConfig{ + DatabaseCfg: heartbeat.DatabaseHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + Enabled: true, + }, + RedisDisconnectDelay: time.Minute, + }, + FilterCfg: heartbeat.FilterHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + Enabled: true, + }, + MetricReceivedDelay: time.Minute, + }, + LocalCheckerCfg: heartbeat.LocalCheckerHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + Enabled: true, + }, + LocalCheckDelay: time.Minute, + }, + } + mockClock.EXPECT().NowUTC().Return(testTime) + heartbeaters := createHearbeaters(hbCfg, mockLogger, mockDatabase, mockClock) + So(heartbeaters, ShouldHaveLength, 3) + }) + + Convey("With database, filter, local checker and remote checker heartbeaters", func() { + hbCfg := heartbeat.HeartbeatersConfig{ + DatabaseCfg: heartbeat.DatabaseHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + Enabled: true, + }, + RedisDisconnectDelay: time.Minute, + }, + FilterCfg: heartbeat.FilterHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + Enabled: true, + }, + MetricReceivedDelay: time.Minute, + }, + LocalCheckerCfg: heartbeat.LocalCheckerHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + Enabled: true, + }, + LocalCheckDelay: time.Minute, + }, + RemoteCheckerCfg: heartbeat.RemoteCheckerHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + Enabled: true, + }, + RemoteCheckDelay: time.Minute, + }, + } + mockClock.EXPECT().NowUTC().Return(testTime) + heartbeaters := createHearbeaters(hbCfg, mockLogger, mockDatabase, mockClock) + So(heartbeaters, ShouldHaveLength, 4) + }) + + Convey("With database, filter, local checker, remote checker and notifier heartbeaters", func() { + hbCfg := heartbeat.HeartbeatersConfig{ + DatabaseCfg: heartbeat.DatabaseHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + Enabled: true, + }, + RedisDisconnectDelay: time.Minute, + }, + FilterCfg: heartbeat.FilterHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + Enabled: true, + }, + MetricReceivedDelay: time.Minute, + }, + LocalCheckerCfg: heartbeat.LocalCheckerHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + Enabled: true, + }, + LocalCheckDelay: time.Minute, + }, + RemoteCheckerCfg: heartbeat.RemoteCheckerHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + Enabled: true, + }, + RemoteCheckDelay: time.Minute, + }, + NotifierCfg: heartbeat.NotifierHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + Enabled: true, + }, + }, + } + mockClock.EXPECT().NowUTC().Return(testTime) + heartbeaters := createHearbeaters(hbCfg, mockLogger, mockDatabase, mockClock) + So(heartbeaters, ShouldHaveLength, 5) + }) + }) +} + +func TestCreateErrorNotificationPackage(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + mockDatabase := mock_moira_alert.NewMockDatabase(mockCtrl) + mockClock := mock_clock.NewMockClock(mockCtrl) + mockLogger := mock_moira_alert.NewMockLogger(mockCtrl) + testTime := time.Date(2022, time.June, 6, 10, 0, 0, 0, time.UTC) + + Convey("Test createErrorNotificationPackage", t, func() { + mockClock.EXPECT().NowUTC().Return(testTime).Times(2) + + m := monitor{ + heartbeatsInfo: map[datatypes.HeartbeatType]*hearbeatInfo{ + datatypes.HeartbeatDatabase: {}, + }, + } + + heartbeaterBase := heartbeat.NewHeartbeaterBase(mockLogger, mockDatabase, mockClock) + databaseHeartbeater, err := heartbeat.NewDatabaseHeartbeater(heartbeat.DatabaseHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + AlertCfg: heartbeat.AlertConfig{ + Name: "Database Heartbeater", + Desc: "Some Database problems", + }, + }, + }, heartbeaterBase) + So(err, ShouldBeNil) + + expectedPkg := ¬ifier.NotificationPackage{ + Events: []moira.NotificationEvent{ + { + Timestamp: testTime.Unix(), + OldState: moira.StateNODATA, + State: moira.StateERROR, + Metric: string(databaseHeartbeater.Type()), + Value: &errorValue, + }, + }, + Trigger: moira.TriggerData{ + Name: "Database Heartbeater", + Desc: "Some Database problems", + ErrorValue: triggerErrorValue, + }, + } + + pkg := m.createErrorNotificationPackage(databaseHeartbeater, mockClock) + So(pkg, ShouldResemble, expectedPkg) + So(m.heartbeatsInfo[databaseHeartbeater.Type()].lastAlertTime, ShouldEqual, testTime) + }) +} + +func TestCheck(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + mockClock := mock_clock.NewMockClock(mockCtrl) + mockDatabase := mock_moira_alert.NewMockDatabase(mockCtrl) + logger, _ := logging.GetLogger("Test") + testTime := time.Date(2022, time.June, 6, 10, 0, 0, 0, time.UTC) + mockNotifier := mock_notifier.NewMockNotifier(mockCtrl) + + Convey("Test check", t, func() { + mockClock.EXPECT().NowUTC().Return(testTime).AnyTimes() + + databaseHeartbeater := mock_heartbeat.NewMockHeartbeater(mockCtrl) + localCheckerHeartbeater := mock_heartbeat.NewMockHeartbeater(mockCtrl) + notifierHeartbeater := mock_heartbeat.NewMockHeartbeater(mockCtrl) + + databaseHeartbeater.EXPECT().Check().Return(heartbeat.StateError, nil).AnyTimes() + databaseHeartbeater.EXPECT().Type().Return(datatypes.HeartbeatDatabase).AnyTimes() + databaseHeartbeater.EXPECT().AlertSettings().Return(heartbeat.AlertConfig{ + Name: "Database Heartbeater", + Desc: "Database Problems", + }).AnyTimes() + + localCheckerHeartbeater.EXPECT().Check().Return(heartbeat.StateError, nil).AnyTimes() + localCheckerHeartbeater.EXPECT().Type().Return(datatypes.HeartbeatLocalChecker).AnyTimes() + localCheckerHeartbeater.EXPECT().AlertSettings().Return(heartbeat.AlertConfig{ + Name: "Local Checker Heartbeater", + Desc: "Local Checker Problems", + }).AnyTimes() + + notifierHeartbeater.EXPECT().Check().Return(heartbeat.StateError, nil).AnyTimes() + notifierHeartbeater.EXPECT().Type().Return(datatypes.HeartbeatNotifier).AnyTimes() + notifierHeartbeater.EXPECT().AlertSettings().Return(heartbeat.AlertConfig{ + Name: "Notifier Heartbeater", + Desc: "Notifier Problems", + }).AnyTimes() + + sendingWG := &sync.WaitGroup{} + + Convey("With admin monitor", func() { + am := adminMonitor{ + adminCfg: AdminMonitorConfig{ + AdminContacts: []map[string]string{ + { + "type": "telegram", + "value": "@webcamsmodel", + }, + }, + }, + notifier: mockNotifier, + } + + m := monitor{ + cfg: monitorConfig{ + NoticeInterval: time.Minute, + }, + heartbeaters: []heartbeat.Heartbeater{databaseHeartbeater, localCheckerHeartbeater, notifierHeartbeater}, + heartbeatsInfo: map[datatypes.HeartbeatType]*hearbeatInfo{ + datatypes.HeartbeatDatabase: {}, + datatypes.HeartbeatLocalChecker: {}, + datatypes.HeartbeatNotifier: {}, + }, + notifier: mockNotifier, + logger: logger, + clock: mockClock, + sendNotifications: am.sendNotifications, + } + + contact := moira.ContactData{ + Type: "telegram", + Value: "@webcamsmodel", + } + + databaseErrorPkg := m.createErrorNotificationPackage(databaseHeartbeater, mockClock) + databaseErrorPkg.Contact = contact + localCheckerErrorPkg := m.createErrorNotificationPackage(localCheckerHeartbeater, mockClock) + localCheckerErrorPkg.Contact = contact + notifierErrorPkg := m.createErrorNotificationPackage(notifierHeartbeater, mockClock) + notifierErrorPkg.Contact = contact + + m.heartbeatsInfo = map[datatypes.HeartbeatType]*hearbeatInfo{ + datatypes.HeartbeatDatabase: { + lastAlertTime: testTime.Add(-2 * time.Minute), + }, + datatypes.HeartbeatLocalChecker: { + lastAlertTime: testTime.Add(-2 * time.Minute), + }, + datatypes.HeartbeatNotifier: { + lastAlertTime: testTime.Add(-2 * time.Minute), + }, + } + + mockNotifier.EXPECT().Send(databaseErrorPkg, sendingWG) + mockNotifier.EXPECT().Send(localCheckerErrorPkg, sendingWG) + mockNotifier.EXPECT().Send(notifierErrorPkg, sendingWG) + + m.check() + }) + + Convey("With user monitor", func() { + um := userMonitor{ + notifier: mockNotifier, + database: mockDatabase, + } + + m := monitor{ + cfg: monitorConfig{ + NoticeInterval: time.Minute, + }, + heartbeaters: []heartbeat.Heartbeater{databaseHeartbeater, localCheckerHeartbeater, notifierHeartbeater}, + heartbeatsInfo: map[datatypes.HeartbeatType]*hearbeatInfo{ + datatypes.HeartbeatDatabase: {}, + datatypes.HeartbeatLocalChecker: {}, + datatypes.HeartbeatNotifier: {}, + }, + notifier: mockNotifier, + logger: logger, + clock: mockClock, + sendNotifications: um.sendNotifications, + } + + contactIDs := []string{"test-contact-id"} + contact := moira.ContactData{ + Type: "telegram", + Value: "@webcamsmodel", + } + contacts := []*moira.ContactData{&contact} + + databaseErrorPkg := m.createErrorNotificationPackage(databaseHeartbeater, mockClock) + databaseErrorPkg.Contact = contact + localCheckerErrorPkg := m.createErrorNotificationPackage(localCheckerHeartbeater, mockClock) + localCheckerErrorPkg.Contact = contact + notifierErrorPkg := m.createErrorNotificationPackage(notifierHeartbeater, mockClock) + notifierErrorPkg.Contact = contact + + m.heartbeatsInfo = map[datatypes.HeartbeatType]*hearbeatInfo{ + datatypes.HeartbeatDatabase: { + lastAlertTime: testTime.Add(-2 * time.Minute), + }, + datatypes.HeartbeatLocalChecker: { + lastAlertTime: testTime.Add(-2 * time.Minute), + }, + datatypes.HeartbeatNotifier: { + lastAlertTime: testTime.Add(-2 * time.Minute), + }, + } + + mockDatabase.EXPECT().GetHeartbeatTypeContactIDs(datatypes.HeartbeatDatabase).Return(contactIDs, nil) + mockDatabase.EXPECT().GetHeartbeatTypeContactIDs(datatypes.HeartbeatLocalChecker).Return(contactIDs, nil) + mockDatabase.EXPECT().GetHeartbeatTypeContactIDs(datatypes.HeartbeatNotifier).Return(contactIDs, nil) + + mockDatabase.EXPECT().GetContacts(contactIDs).Return(contacts, nil).AnyTimes() + + mockNotifier.EXPECT().Send(databaseErrorPkg, sendingWG) + mockNotifier.EXPECT().Send(localCheckerErrorPkg, sendingWG) + mockNotifier.EXPECT().Send(notifierErrorPkg, sendingWG) + + m.check() + }) + }) +} + +func TestCheckHeartbeats(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + mockClock := mock_clock.NewMockClock(mockCtrl) + logger, _ := logging.GetLogger("Test") + testTime := time.Date(2022, time.June, 6, 10, 0, 0, 0, time.UTC) + + Convey("Test checkHeartbeats", t, func() { + mockClock.EXPECT().NowUTC().Return(testTime).AnyTimes() + + databaseHeartbeater := mock_heartbeat.NewMockHeartbeater(mockCtrl) + localCheckerHeartbeater := mock_heartbeat.NewMockHeartbeater(mockCtrl) + notifierHeartbeater := mock_heartbeat.NewMockHeartbeater(mockCtrl) + + m := monitor{ + cfg: monitorConfig{ + NoticeInterval: time.Minute, + }, + heartbeaters: []heartbeat.Heartbeater{databaseHeartbeater, localCheckerHeartbeater, notifierHeartbeater}, + heartbeatsInfo: map[datatypes.HeartbeatType]*hearbeatInfo{ + datatypes.HeartbeatDatabase: {}, + datatypes.HeartbeatLocalChecker: {}, + datatypes.HeartbeatNotifier: {}, + }, + logger: logger, + clock: mockClock, + } + + databaseHeartbeater.EXPECT().Check().Return(heartbeat.StateError, nil) + databaseHeartbeater.EXPECT().Type().Return(datatypes.HeartbeatDatabase).AnyTimes() + databaseHeartbeater.EXPECT().AlertSettings().Return(heartbeat.AlertConfig{ + Name: "Database Heartbeater", + Desc: "Database Problems", + }).AnyTimes() + + localCheckerHeartbeater.EXPECT().Check().Return(heartbeat.StateError, nil) + localCheckerHeartbeater.EXPECT().Type().Return(datatypes.HeartbeatLocalChecker).AnyTimes() + localCheckerHeartbeater.EXPECT().AlertSettings().Return(heartbeat.AlertConfig{ + Name: "Local Checker Heartbeater", + Desc: "Local Checker Problems", + }).AnyTimes() + + notifierHeartbeater.EXPECT().Check().Return(heartbeat.StateError, nil) + notifierHeartbeater.EXPECT().Type().Return(datatypes.HeartbeatNotifier).AnyTimes() + notifierHeartbeater.EXPECT().AlertSettings().Return(heartbeat.AlertConfig{ + Name: "Notifier Heartbeater", + Desc: "Notifier Problems", + }).AnyTimes() + + databaseErrorPkg := m.createErrorNotificationPackage(databaseHeartbeater, mockClock) + localCheckerErrorPkg := m.createErrorNotificationPackage(localCheckerHeartbeater, mockClock) + notifierErrorPkg := m.createErrorNotificationPackage(notifierHeartbeater, mockClock) + expectedPkgs := []notifier.NotificationPackage{*databaseErrorPkg, *localCheckerErrorPkg, *notifierErrorPkg} + + m.heartbeatsInfo = map[datatypes.HeartbeatType]*hearbeatInfo{ + datatypes.HeartbeatDatabase: { + lastAlertTime: testTime.Add(-2 * time.Minute), + }, + datatypes.HeartbeatLocalChecker: { + lastAlertTime: testTime.Add(-2 * time.Minute), + }, + datatypes.HeartbeatNotifier: { + lastAlertTime: testTime.Add(-2 * time.Minute), + }, + } + + pkgs := m.checkHeartbeats() + So(pkgs, ShouldResemble, expectedPkgs) + So(m.heartbeatsInfo, ShouldResemble, map[datatypes.HeartbeatType]*hearbeatInfo{ + datatypes.HeartbeatDatabase: { + lastAlertTime: testTime, + lastCheckState: heartbeat.StateError, + }, + datatypes.HeartbeatLocalChecker: { + lastAlertTime: testTime, + lastCheckState: heartbeat.StateError, + }, + datatypes.HeartbeatNotifier: { + lastAlertTime: testTime, + lastCheckState: heartbeat.StateError, + }, + }) + }) +} + +func TestGenerateHeartbeatNotificationPackage(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + mockDatabase := mock_moira_alert.NewMockDatabase(mockCtrl) + mockClock := mock_clock.NewMockClock(mockCtrl) + mockLogger := mock_moira_alert.NewMockLogger(mockCtrl) + testTime := time.Date(2022, time.June, 6, 10, 0, 0, 0, time.UTC) + + Convey("Test generateHeartbeatNotificationPackage", t, func() { + mockClock.EXPECT().NowUTC().Return(testTime).AnyTimes() + + m := monitor{ + cfg: monitorConfig{ + NoticeInterval: time.Minute, + }, + heartbeatsInfo: map[datatypes.HeartbeatType]*hearbeatInfo{ + datatypes.HeartbeatDatabase: {}, + }, + clock: mockClock, + } + + heartbeaterBase := heartbeat.NewHeartbeaterBase(mockLogger, mockDatabase, mockClock) + databaseHeartbeater, err := heartbeat.NewDatabaseHeartbeater(heartbeat.DatabaseHeartbeaterConfig{}, heartbeaterBase) + So(err, ShouldBeNil) + + Convey("isDegraded state and allowNotify", func() { + m.heartbeatsInfo[databaseHeartbeater.Type()] = &hearbeatInfo{ + lastCheckState: heartbeat.StateOK, + lastAlertTime: testTime.Add(-m.cfg.NoticeInterval - time.Minute), + } + defer func() { + m.heartbeatsInfo[databaseHeartbeater.Type()] = &hearbeatInfo{} + }() + + pkg := m.generateHeartbeatNotificationPackage(databaseHeartbeater, heartbeat.StateError) + So(pkg, ShouldNotBeNil) + So(pkg.Events[0].State, ShouldEqual, moira.StateERROR) + }) + + Convey("isDegraded state and not allowNotify", func() { + m.heartbeatsInfo[databaseHeartbeater.Type()] = &hearbeatInfo{ + lastCheckState: heartbeat.StateOK, + lastAlertTime: testTime.Add(-m.cfg.NoticeInterval + time.Minute), + } + defer func() { + m.heartbeatsInfo[databaseHeartbeater.Type()] = &hearbeatInfo{} + }() + + pkg := m.generateHeartbeatNotificationPackage(databaseHeartbeater, heartbeat.StateError) + So(pkg, ShouldBeNil) + }) + + Convey("isRecovered state and allowNotify", func() { + m.heartbeatsInfo[databaseHeartbeater.Type()] = &hearbeatInfo{ + lastCheckState: heartbeat.StateError, + lastAlertTime: testTime.Add(-m.cfg.NoticeInterval - time.Minute), + } + defer func() { + m.heartbeatsInfo[databaseHeartbeater.Type()] = &hearbeatInfo{} + }() + + pkg := m.generateHeartbeatNotificationPackage(databaseHeartbeater, heartbeat.StateOK) + So(pkg, ShouldNotBeNil) + So(pkg.Events[0].State, ShouldEqual, moira.StateOK) + }) + + Convey("isRecovered state and not allowNotify", func() { + m.heartbeatsInfo[databaseHeartbeater.Type()] = &hearbeatInfo{ + lastCheckState: heartbeat.StateError, + lastAlertTime: testTime.Add(-m.cfg.NoticeInterval + time.Minute), + } + defer func() { + m.heartbeatsInfo[databaseHeartbeater.Type()] = &hearbeatInfo{} + }() + + pkg := m.generateHeartbeatNotificationPackage(databaseHeartbeater, heartbeat.StateOK) + So(pkg, ShouldNotBeNil) + So(pkg.Events[0].State, ShouldEqual, moira.StateOK) + }) + + Convey("not degraded and not recovered", func() { + m.heartbeatsInfo[databaseHeartbeater.Type()] = &hearbeatInfo{ + lastCheckState: heartbeat.StateOK, + lastAlertTime: testTime.Add(-m.cfg.NoticeInterval - time.Minute), + } + defer func() { + m.heartbeatsInfo[databaseHeartbeater.Type()] = &hearbeatInfo{} + }() + + pkg := m.generateHeartbeatNotificationPackage(databaseHeartbeater, heartbeat.StateOK) + So(pkg, ShouldBeNil) + }) + }) +} + +func TestCreateOkNotificationPackage(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + mockDatabase := mock_moira_alert.NewMockDatabase(mockCtrl) + mockClock := mock_clock.NewMockClock(mockCtrl) + mockLogger := mock_moira_alert.NewMockLogger(mockCtrl) + testTime := time.Date(2022, time.June, 6, 10, 0, 0, 0, time.UTC) + + Convey("Test createOkNotificationPackage", t, func() { + mockClock.EXPECT().NowUTC().Return(testTime).Times(2) + + m := monitor{ + heartbeatsInfo: map[datatypes.HeartbeatType]*hearbeatInfo{ + datatypes.HeartbeatDatabase: {}, + }, + } + + heartbeaterBase := heartbeat.NewHeartbeaterBase(mockLogger, mockDatabase, mockClock) + databaseHeartbeater, err := heartbeat.NewDatabaseHeartbeater(heartbeat.DatabaseHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + AlertCfg: heartbeat.AlertConfig{ + Name: "Database Heartbeater", + Desc: "Some Database problems", + }, + }, + }, heartbeaterBase) + So(err, ShouldBeNil) + + expectedPkg := ¬ifier.NotificationPackage{ + Events: []moira.NotificationEvent{ + { + Timestamp: testTime.Unix(), + OldState: moira.StateERROR, + State: moira.StateOK, + Metric: string(databaseHeartbeater.Type()), + Value: &okValue, + }, + }, + Trigger: moira.TriggerData{ + Name: "Database Heartbeater", + Desc: "Some Database problems", + ErrorValue: triggerErrorValue, + }, + } + + pkg := m.createOkNotificationPackage(databaseHeartbeater, mockClock) + So(pkg, ShouldResemble, expectedPkg) + So(m.heartbeatsInfo[databaseHeartbeater.Type()].lastAlertTime, ShouldEqual, testTime) + }) +} diff --git a/notifier/selfstate/monitor/user.go b/notifier/selfstate/monitor/user.go new file mode 100644 index 000000000..2abef6abb --- /dev/null +++ b/notifier/selfstate/monitor/user.go @@ -0,0 +1,93 @@ +package monitor + +import ( + "fmt" + "sync" + "time" + + "github.com/moira-alert/moira" + "github.com/moira-alert/moira/datatypes" + "github.com/moira-alert/moira/notifier" +) + +const ( + userMonitorName = "Moira User Selfstate Monitoring" + userMonitorLockName = "moira-user-selfstate-monitor" + userMonitorLockTTL = 15 * time.Second +) + +type UserMonitorConfig struct { + MonitorBaseConfig +} + +type userMonitor struct { + userCfg UserMonitorConfig + database moira.Database + notifier notifier.Notifier +} + +func NewForUser( + userCfg UserMonitorConfig, + logger moira.Logger, + database moira.Database, + clock moira.Clock, + notifier notifier.Notifier, +) (*monitor, error) { + if err := moira.ValidateStruct(userCfg); err != nil { + return nil, fmt.Errorf("user config validation error: %w", err) + } + + userMonitor := userMonitor{ + userCfg: userCfg, + database: database, + notifier: notifier, + } + + cfg := monitorConfig{ + Name: userMonitorName, + LockName: userMonitorLockName, + LockTTL: userMonitorLockTTL, + NoticeInterval: userCfg.NoticeInterval, + CheckInterval: userCfg.CheckInterval, + } + + heartbeaters := createHearbeaters(userCfg.HeartbeatersCfg, logger, database, clock) + + return newMonitor( + cfg, + logger, + database, + clock, + notifier, + heartbeaters, + userMonitor.sendNotifications, + ) +} + +func (um *userMonitor) sendNotifications(pkgs []notifier.NotificationPackage) error { + sendingWG := &sync.WaitGroup{} + + for _, pkg := range pkgs { + event := pkg.Events[0] + heartbeatType := datatypes.HeartbeatType(event.Metric) + contactIDs, err := um.database.GetHeartbeatTypeContactIDs(heartbeatType) + if err != nil { + return fmt.Errorf("failed to get heartbeat type contact ids: %w", err) + } + + contacts, err := um.database.GetContacts(contactIDs) + if err != nil { + return fmt.Errorf("failed to get contacts by ids: %w", err) + } + + for _, contact := range contacts { + if contact != nil { + pkg.Contact = *contact + um.notifier.Send(&pkg, sendingWG) + sendingWG.Wait() + } + } + } + + return nil +} diff --git a/notifier/selfstate/monitor/user_test.go b/notifier/selfstate/monitor/user_test.go new file mode 100644 index 000000000..8f497460c --- /dev/null +++ b/notifier/selfstate/monitor/user_test.go @@ -0,0 +1,162 @@ +package monitor + +import ( + "errors" + "fmt" + "sync" + "testing" + "time" + + "github.com/moira-alert/moira" + "github.com/moira-alert/moira/datatypes" + mock_clock "github.com/moira-alert/moira/mock/clock" + mock_moira_alert "github.com/moira-alert/moira/mock/moira-alert" + mock_notifier "github.com/moira-alert/moira/mock/notifier" + "github.com/moira-alert/moira/notifier" + "github.com/moira-alert/moira/notifier/selfstate/heartbeat" + . "github.com/smartystreets/goconvey/convey" + "go.uber.org/mock/gomock" +) + +func TestNewForUser(t *testing.T) { + t.Skip() + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + mockLogger := mock_moira_alert.NewMockLogger(mockCtrl) + mockNotifier := mock_notifier.NewMockNotifier(mockCtrl) + mockDatabase := mock_moira_alert.NewMockDatabase(mockCtrl) + mockClock := mock_clock.NewMockClock(mockCtrl) + testTime := time.Date(2022, time.June, 6, 10, 0, 0, 0, time.UTC) + + Convey("Test NewForUser", t, func() { + userCfg := UserMonitorConfig{ + MonitorBaseConfig: MonitorBaseConfig{ + Enabled: true, + HeartbeatersCfg: defaultHeartbeatersConfig, + NoticeInterval: time.Minute, + CheckInterval: time.Minute, + }, + } + userMonitorCfg := monitorConfig{ + Name: userMonitorName, + LockName: userMonitorLockName, + LockTTL: userMonitorLockTTL, + NoticeInterval: time.Minute, + CheckInterval: time.Minute, + } + um := userMonitor{ + userCfg: userCfg, + database: mockDatabase, + notifier: mockNotifier, + } + mockClock.EXPECT().NowUTC().Return(testTime).Times(2) + heartbeaters := createHearbeaters(userCfg.HeartbeatersCfg, mockLogger, mockDatabase, mockClock) + hearbeatersInfo := make(map[datatypes.HeartbeatType]*hearbeatInfo, len(heartbeaters)) + for _, heartbeater := range heartbeaters { + hearbeatersInfo[heartbeater.Type()] = &hearbeatInfo{ + lastCheckState: heartbeat.StateOK, + } + } + + userMonitor, err := NewForUser(userCfg, mockLogger, mockDatabase, mockClock, mockNotifier) + So(err, ShouldNotBeNil) + So(userMonitor, ShouldResemble, &monitor{ + cfg: userMonitorCfg, + logger: mockLogger, + database: mockDatabase, + notifier: mockNotifier, + heartbeaters: heartbeaters, + clock: mockClock, + heartbeatsInfo: hearbeatersInfo, + sendNotifications: um.sendNotifications, + }) + }) +} + +func TestUserSendNotifications(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + mockNotifier := mock_notifier.NewMockNotifier(mockCtrl) + mockDatabase := mock_moira_alert.NewMockDatabase(mockCtrl) + + sendingWG := &sync.WaitGroup{} + + userMonitor := userMonitor{ + notifier: mockNotifier, + database: mockDatabase, + } + + contactIDs := []string{"test-contact-id"} + contact := &moira.ContactData{ + ID: "test-contact-id", + } + contacts := []*moira.ContactData{contact} + + testErr := errors.New("test error") + + Convey("Test sendNotifications", t, func() { + Convey("With empty notification packages", func() { + pkgs := []notifier.NotificationPackage{} + err := userMonitor.sendNotifications(pkgs) + So(err, ShouldBeNil) + }) + + Convey("With GetHeartbeatTypeContactIDs error", func() { + pkgs := []notifier.NotificationPackage{ + { + Events: []moira.NotificationEvent{ + { + Metric: string(datatypes.HeartbeatNotifier), + }, + }, + }, + } + mockDatabase.EXPECT().GetHeartbeatTypeContactIDs(datatypes.HeartbeatNotifier).Return([]string{}, testErr).Times(1) + err := userMonitor.sendNotifications(pkgs) + So(err, ShouldResemble, fmt.Errorf("failed to get heartbeat type contact ids: %w", testErr)) + }) + + Convey("With GetContacts error", func() { + pkgs := []notifier.NotificationPackage{ + { + Events: []moira.NotificationEvent{ + { + Metric: string(datatypes.HeartbeatNotifier), + }, + }, + }, + } + mockDatabase.EXPECT().GetHeartbeatTypeContactIDs(datatypes.HeartbeatNotifier).Return(contactIDs, nil).Times(1) + mockDatabase.EXPECT().GetContacts(contactIDs).Return(nil, testErr).Times(1) + err := userMonitor.sendNotifications(pkgs) + So(err, ShouldResemble, fmt.Errorf("failed to get contacts by ids: %w", testErr)) + }) + + Convey("With correct sending notification packages", func() { + pkgs := []notifier.NotificationPackage{ + { + Events: []moira.NotificationEvent{ + { + Metric: string(datatypes.HeartbeatNotifier), + }, + }, + }, + } + pkgWithContact := ¬ifier.NotificationPackage{ + Events: []moira.NotificationEvent{ + { + Metric: string(datatypes.HeartbeatNotifier), + }, + }, + Contact: *contacts[0], + } + mockDatabase.EXPECT().GetHeartbeatTypeContactIDs(datatypes.HeartbeatNotifier).Return(contactIDs, nil).Times(1) + mockDatabase.EXPECT().GetContacts(contactIDs).Return(contacts, nil).Times(1) + mockNotifier.EXPECT().Send(pkgWithContact, sendingWG).Times(1) + err := userMonitor.sendNotifications(pkgs) + So(err, ShouldBeNil) + }) + }) +} diff --git a/notifier/selfstate/selfstate.go b/notifier/selfstate/selfstate.go index 99bba0ed0..486fec29d 100644 --- a/notifier/selfstate/selfstate.go +++ b/notifier/selfstate/selfstate.go @@ -1,88 +1,101 @@ package selfstate import ( - "time" - - "github.com/moira-alert/moira/metrics" - - "github.com/moira-alert/moira/notifier/selfstate/heartbeat" - - "gopkg.in/tomb.v2" + "errors" "github.com/moira-alert/moira" "github.com/moira-alert/moira/notifier" - w "github.com/moira-alert/moira/worker" + "github.com/moira-alert/moira/notifier/selfstate/monitor" ) -const ( - selfStateLockName = "moira-self-state-monitor" - selfStateLockTTL = time.Second * 15 -) +var _ SelfstateWorker = (*selfstateWorker)(nil) -// SelfCheckWorker checks what all notifier services works correctly and send message when moira don't work. -type SelfCheckWorker struct { - Logger moira.Logger - Database moira.Database - Notifier notifier.Notifier - Config Config - tomb tomb.Tomb - heartbeats []heartbeat.Heartbeater +type SelfstateWorker interface { + Start() + Stop() error } -// NewSelfCheckWorker creates SelfCheckWorker. -func NewSelfCheckWorker(logger moira.Logger, database moira.Database, notifier notifier.Notifier, config Config, metrics *metrics.HeartBeatMetrics) *SelfCheckWorker { - heartbeats := createStandardHeartbeats(logger, database, config, metrics) - return &SelfCheckWorker{Logger: logger, Database: database, Notifier: notifier, Config: config, heartbeats: heartbeats} +type selfstateWorker struct { + monitors []monitor.Monitor } -// Start self check worker. -func (selfCheck *SelfCheckWorker) Start() error { - senders := selfCheck.Notifier.GetSenders() - if err := selfCheck.Config.checkConfig(senders); err != nil { - return err - } - - selfCheck.tomb.Go(func() error { - w.NewWorker( - "Moira Self State Monitoring", - selfCheck.Logger, - selfCheck.Database.NewLock(selfStateLockName, selfStateLockTTL), - selfCheck.selfStateChecker, - ).Run(selfCheck.tomb.Dying()) - return nil - }) - - return nil +func NewSelfstateWorker( + cfg Config, + logger moira.Logger, + database moira.Database, + notifier notifier.Notifier, + clock moira.Clock, +) (*selfstateWorker, error) { + monitors := createMonitors(cfg.MonitorCfg, logger, database, clock, notifier) + + return &selfstateWorker{ + monitors: monitors, + }, nil } -// Stop self check worker and wait for finish. -func (selfCheck *SelfCheckWorker) Stop() error { - selfCheck.tomb.Kill(nil) - return selfCheck.tomb.Wait() -} - -func createStandardHeartbeats(logger moira.Logger, database moira.Database, conf Config, metrics *metrics.HeartBeatMetrics) []heartbeat.Heartbeater { - heartbeats := make([]heartbeat.Heartbeater, 0) - - if hb := heartbeat.GetDatabase(conf.RedisDisconnectDelaySeconds, logger, database); hb != nil { - heartbeats = append(heartbeats, hb) +func createMonitors( + monitorCfg MonitorConfig, + logger moira.Logger, + database moira.Database, + clock moira.Clock, + notifier notifier.Notifier, +) []monitor.Monitor { + adminMonitorEnabled := monitorCfg.AdminCfg.Enabled + userMonitorEnabled := monitorCfg.UserCfg.Enabled + + monitors := make([]monitor.Monitor, 0) + + if adminMonitorEnabled { + adminMonitor, err := monitor.NewForAdmin( + monitorCfg.AdminCfg, + logger, + database, + clock, + notifier, + ) + if err != nil { + logger.Error(). + Error(err). + Msg("Failed to create a new admin monitor") + } else { + monitors = append(monitors, adminMonitor) + } } - if hb := heartbeat.GetFilter(conf.LastMetricReceivedDelaySeconds, logger, database); hb != nil { - heartbeats = append(heartbeats, hb) + if userMonitorEnabled { + userMonitor, err := monitor.NewForUser( + monitorCfg.UserCfg, + logger, + database, + clock, + notifier, + ) + if err != nil { + logger.Error(). + Error(err). + Msg("Failed to create a new user monitor") + } else { + monitors = append(monitors, userMonitor) + } } - if hb := heartbeat.GetLocalChecker(conf.LastCheckDelaySeconds, logger, database); hb != nil && hb.NeedToCheckOthers() { - heartbeats = append(heartbeats, hb) - } + return monitors +} - if hb := heartbeat.GetRemoteChecker(conf.LastRemoteCheckDelaySeconds, logger, database); hb != nil && hb.NeedToCheckOthers() { - heartbeats = append(heartbeats, hb) +func (selfstateWorker *selfstateWorker) Start() { + for _, monitor := range selfstateWorker.monitors { + monitor.Start() } +} + +func (selfstateWorker *selfstateWorker) Stop() error { + stopErrors := make([]error, 0) - if hb := heartbeat.GetNotifier(logger, database, metrics); hb != nil { - heartbeats = append(heartbeats, hb) + for _, monitor := range selfstateWorker.monitors { + if err := monitor.Stop(); err != nil { + stopErrors = append(stopErrors, err) + } } - return heartbeats + return errors.Join(stopErrors...) } diff --git a/notifier/selfstate/selfstate_test.go b/notifier/selfstate/selfstate_test.go deleted file mode 100644 index 71859dca9..000000000 --- a/notifier/selfstate/selfstate_test.go +++ /dev/null @@ -1,189 +0,0 @@ -package selfstate - -import ( - "errors" - "testing" - "time" - - "github.com/moira-alert/moira/metrics" - - mock_heartbeat "github.com/moira-alert/moira/mock/heartbeat" - "github.com/moira-alert/moira/notifier/selfstate/heartbeat" - - "github.com/moira-alert/moira" - - logging "github.com/moira-alert/moira/logging/zerolog_adapter" - mock_moira_alert "github.com/moira-alert/moira/mock/moira-alert" - mock_notifier "github.com/moira-alert/moira/mock/notifier" - . "github.com/smartystreets/goconvey/convey" - "go.uber.org/mock/gomock" -) - -type selfCheckWorkerMock struct { - selfCheckWorker *SelfCheckWorker - database *mock_moira_alert.MockDatabase - notif *mock_notifier.MockNotifier - conf Config - mockCtrl *gomock.Controller -} - -func TestSelfCheckWorker_selfStateChecker(t *testing.T) { - defaultLocalCluster := moira.MakeClusterKey(moira.GraphiteLocal, moira.DefaultCluster) - defaultRemoteCluster := moira.DefaultGraphiteRemoteCluster - - mock := configureWorker(t, true) - Convey("SelfCheckWorker should call all heartbeats checks", t, func() { - mock.database.EXPECT().GetChecksUpdatesCount().Return(int64(1), nil).Times(2) - mock.database.EXPECT().GetMetricsUpdatesCount().Return(int64(1), nil) - mock.database.EXPECT().GetRemoteChecksUpdatesCount().Return(int64(1), nil) - mock.database.EXPECT().GetNotifierState().Return(moira.SelfStateOK, nil) - mock.database.EXPECT().GetTriggersToCheckCount(defaultLocalCluster).Return(int64(1), nil).Times(2) - mock.database.EXPECT().GetTriggersToCheckCount(defaultRemoteCluster).Return(int64(1), nil) - - // Start worker after configuring Mock to avoid race conditions - err := mock.selfCheckWorker.Start() - So(err, ShouldBeNil) - - So(len(mock.selfCheckWorker.heartbeats), ShouldEqual, 5) - - const oneTickDelay = time.Millisecond * 1500 - time.Sleep(oneTickDelay) // wait for one tick of worker - - err = mock.selfCheckWorker.Stop() - So(err, ShouldBeNil) - }) - - mock.mockCtrl.Finish() -} - -func TestSelfCheckWorker_sendErrorMessages(t *testing.T) { - mock := configureWorker(t, true) - - Convey("Should call notifier send", t, func() { - err := mock.selfCheckWorker.Start() - So(err, ShouldBeNil) - - mock.notif.EXPECT().Send(gomock.Any(), gomock.Any()) - - var events []moira.NotificationEvent - mock.selfCheckWorker.sendErrorMessages(events) - - err = mock.selfCheckWorker.Stop() - So(err, ShouldBeNil) - }) - - mock.mockCtrl.Finish() -} - -func TestSelfCheckWorker_Start(t *testing.T) { - mock := configureWorker(t, false) - Convey("When Contact not corresponds to any Sender", t, func() { - mock.notif.EXPECT().GetSenders().Return(nil) - - Convey("Start should return error", func() { - err := mock.selfCheckWorker.Start() - So(err, ShouldNotBeNil) - }) - }) -} - -func TestSelfCheckWorker(t *testing.T) { - Convey("Test checked heartbeat", t, func() { - err := errors.New("test error") - now := time.Now().Unix() - - mock := configureWorker(t, false) - - Convey("Test handle error and no needed send events", func() { - check := mock_heartbeat.NewMockHeartbeater(mock.mockCtrl) - mock.selfCheckWorker.heartbeats = []heartbeat.Heartbeater{check} - - check.EXPECT().Check(now).Return(int64(0), false, err) - - events := mock.selfCheckWorker.handleCheckServices(now) - So(events, ShouldBeNil) - }) - - Convey("Test turn off notification", func() { - first := mock_heartbeat.NewMockHeartbeater(mock.mockCtrl) - second := mock_heartbeat.NewMockHeartbeater(mock.mockCtrl) - - mock.selfCheckWorker.heartbeats = []heartbeat.Heartbeater{first, second} - - first.EXPECT().NeedTurnOffNotifier().Return(true) - first.EXPECT().NeedToCheckOthers().Return(false) - first.EXPECT().GetErrorMessage().Return(moira.SelfStateERROR) - first.EXPECT().Check(now).Return(int64(0), true, nil) - mock.database.EXPECT().SetNotifierState(moira.SelfStateERROR) - - events := mock.selfCheckWorker.handleCheckServices(now) - So(len(events), ShouldEqual, 1) - }) - - Convey("Test of sending notifications from a check", func() { - now = time.Now().Unix() - first := mock_heartbeat.NewMockHeartbeater(mock.mockCtrl) - second := mock_heartbeat.NewMockHeartbeater(mock.mockCtrl) - - mock.selfCheckWorker.heartbeats = []heartbeat.Heartbeater{first, second} - nextSendErrorMessage := time.Now().Unix() - time.Hour.Milliseconds() - - first.EXPECT().Check(now).Return(int64(0), true, nil) - first.EXPECT().GetErrorMessage().Return(moira.SelfStateERROR) - first.EXPECT().NeedTurnOffNotifier().Return(true) - first.EXPECT().NeedToCheckOthers().Return(false) - mock.database.EXPECT().SetNotifierState(moira.SelfStateERROR).Return(err) - mock.notif.EXPECT().Send(gomock.Any(), gomock.Any()) - - nextSendErrorMessage = mock.selfCheckWorker.check(now, nextSendErrorMessage) - So(nextSendErrorMessage, ShouldEqual, now+60) - }) - - mock.mockCtrl.Finish() - }) -} - -func configureWorker(t *testing.T, isStart bool) *selfCheckWorkerMock { - adminContact := map[string]string{ - "type": "admin-mail", - "value": "admin@company.com", - } - conf := Config{ - Enabled: true, - Contacts: []map[string]string{ - adminContact, - }, - RedisDisconnectDelaySeconds: 10, - LastMetricReceivedDelaySeconds: 60, - LastCheckDelaySeconds: 120, - NoticeIntervalSeconds: 60, - LastRemoteCheckDelaySeconds: 120, - CheckInterval: 1 * time.Second, - } - - mockCtrl := gomock.NewController(t) - database := mock_moira_alert.NewMockDatabase(mockCtrl) - logger, _ := logging.GetLogger("SelfState") - notif := mock_notifier.NewMockNotifier(mockCtrl) - if isStart { - senders := map[string]bool{ - "admin-mail": true, - } - notif.EXPECT().GetSenders().Return(senders).MinTimes(1) - - lock := mock_moira_alert.NewMockLock(mockCtrl) - lock.EXPECT().Acquire(gomock.Any()).Return(nil, nil) - lock.EXPECT().Release() - database.EXPECT().NewLock(gomock.Any(), gomock.Any()).Return(lock) - } - - metric := &metrics.HeartBeatMetrics{} - - return &selfCheckWorkerMock{ - selfCheckWorker: NewSelfCheckWorker(logger, database, notif, conf, metric), - database: database, - notif: notif, - conf: conf, - mockCtrl: mockCtrl, - } -} From db7d251988b49d222faa83d739e2b40ce3903edd Mon Sep 17 00:00:00 2001 From: almostinf Date: Tue, 5 Nov 2024 18:29:42 +0300 Subject: [PATCH 2/6] fix linter --- mock/heartbeat/heartbeat.go | 14 -------------- notifier/selfstate/heartbeat/database_test.go | 6 +++--- notifier/selfstate/heartbeat/filter_test.go | 6 +++--- notifier/selfstate/heartbeat/local_checker_test.go | 6 +++--- notifier/selfstate/heartbeat/notifier_test.go | 6 +++--- .../selfstate/heartbeat/remote_checker_test.go | 6 +++--- notifier/selfstate/monitor/admin.go | 4 ++-- notifier/selfstate/monitor/admin_test.go | 6 +++--- notifier/selfstate/monitor/user.go | 4 ++-- notifier/selfstate/monitor/user_test.go | 10 +++++----- 10 files changed, 27 insertions(+), 41 deletions(-) diff --git a/mock/heartbeat/heartbeat.go b/mock/heartbeat/heartbeat.go index 359298067..694bee3c1 100644 --- a/mock/heartbeat/heartbeat.go +++ b/mock/heartbeat/heartbeat.go @@ -82,17 +82,3 @@ func (mr *MockHeartbeaterMockRecorder) Type() *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Type", reflect.TypeOf((*MockHeartbeater)(nil).Type)) } - -// Type mocks base method. -func (m *MockHeartbeater) Type() datatypes.HeartbeatType { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Type") - ret0, _ := ret[0].(datatypes.HeartbeatType) - return ret0 -} - -// Type indicates an expected call of Type. -func (mr *MockHeartbeaterMockRecorder) Type() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Type", reflect.TypeOf((*MockHeartbeater)(nil).Type)) -} diff --git a/notifier/selfstate/heartbeat/database_test.go b/notifier/selfstate/heartbeat/database_test.go index 66d4277f6..d379f96cd 100644 --- a/notifier/selfstate/heartbeat/database_test.go +++ b/notifier/selfstate/heartbeat/database_test.go @@ -15,7 +15,7 @@ const ( ) func TestNewDatabaseHeartbeater(t *testing.T) { - _, _, _, heartbeaterBase := heartbeaterHelper(t) + _, _, _, heartbeaterBase := heartbeaterHelper(t) //nolint:dogsled validationErr := validator.ValidationErrors{} @@ -112,7 +112,7 @@ func TestDatabaseHeartbeaterCheck(t *testing.T) { } func TestDatabaseHeartbeaterType(t *testing.T) { - _, _, _, heartbeaterBase := heartbeaterHelper(t) + _, _, _, heartbeaterBase := heartbeaterHelper(t) //nolint:dogsled Convey("Test databaseHeartbeater.Type", t, func() { cfg := DatabaseHeartbeaterConfig{ @@ -128,7 +128,7 @@ func TestDatabaseHeartbeaterType(t *testing.T) { } func TestDatabaseHeartbeaterAlertSettings(t *testing.T) { - _, _, _, heartbeaterBase := heartbeaterHelper(t) + _, _, _, heartbeaterBase := heartbeaterHelper(t) //nolint:dogsled Convey("Test databaseHeartbeater.AlertSettings", t, func() { alertCfg := AlertConfig{ diff --git a/notifier/selfstate/heartbeat/filter_test.go b/notifier/selfstate/heartbeat/filter_test.go index 52f29ebde..0ce4ed896 100644 --- a/notifier/selfstate/heartbeat/filter_test.go +++ b/notifier/selfstate/heartbeat/filter_test.go @@ -16,7 +16,7 @@ const ( ) func TestNewFilterHeartbeater(t *testing.T) { - _, _, _, heartbeaterBase := heartbeaterHelper(t) + _, _, _, heartbeaterBase := heartbeaterHelper(t) //nolint:dogsled validationErr := validator.ValidationErrors{} @@ -154,7 +154,7 @@ func TestFilterHeartbeaterCheck(t *testing.T) { } func TestFilterHeartbeaterType(t *testing.T) { - _, _, _, heartbeaterBase := heartbeaterHelper(t) + _, _, _, heartbeaterBase := heartbeaterHelper(t) //nolint:dogsled Convey("Test filterHeartbeater.Type", t, func() { cfg := FilterHeartbeaterConfig{ @@ -170,7 +170,7 @@ func TestFilterHeartbeaterType(t *testing.T) { } func TestFilterHeartbeaterAlertSettings(t *testing.T) { - _, _, _, heartbeaterBase := heartbeaterHelper(t) + _, _, _, heartbeaterBase := heartbeaterHelper(t) //nolint:dogsled Convey("Test filterHeartbeater.AlertSettings", t, func() { alertCfg := AlertConfig{ diff --git a/notifier/selfstate/heartbeat/local_checker_test.go b/notifier/selfstate/heartbeat/local_checker_test.go index cfe720674..37e9a53b6 100644 --- a/notifier/selfstate/heartbeat/local_checker_test.go +++ b/notifier/selfstate/heartbeat/local_checker_test.go @@ -16,7 +16,7 @@ const ( ) func TestNewLocalCheckerHeartbeater(t *testing.T) { - _, _, _, heartbeaterBase := heartbeaterHelper(t) + _, _, _, heartbeaterBase := heartbeaterHelper(t) //nolint:dogsled validationErr := validator.ValidationErrors{} @@ -154,7 +154,7 @@ func TestLocalCheckerHeartbeaterCheck(t *testing.T) { } func TestLocalCheckerHeartbeaterType(t *testing.T) { - _, _, _, heartbeaterBase := heartbeaterHelper(t) + _, _, _, heartbeaterBase := heartbeaterHelper(t) //nolint:dogsled Convey("Test localCheckerHeartbeater.Type", t, func() { cfg := LocalCheckerHeartbeaterConfig{ @@ -170,7 +170,7 @@ func TestLocalCheckerHeartbeaterType(t *testing.T) { } func TestLocalCheckerHeartbeaterAlertSettings(t *testing.T) { - _, _, _, heartbeaterBase := heartbeaterHelper(t) + _, _, _, heartbeaterBase := heartbeaterHelper(t) //nolint:dogsled Convey("Test localCheckerHeartbeater.AlertSettings", t, func() { alertCfg := AlertConfig{ diff --git a/notifier/selfstate/heartbeat/notifier_test.go b/notifier/selfstate/heartbeat/notifier_test.go index a0a76bbc2..3e99c73f8 100644 --- a/notifier/selfstate/heartbeat/notifier_test.go +++ b/notifier/selfstate/heartbeat/notifier_test.go @@ -11,7 +11,7 @@ import ( ) func TestNewNotifierHeartbeater(t *testing.T) { - _, _, _, heartbeaterBase := heartbeaterHelper(t) + _, _, _, heartbeaterBase := heartbeaterHelper(t) //nolint:dogsled Convey("Test NewNotifierHeartbeater", t, func() { Convey("With correct local checker heartbeater config", func() { @@ -66,7 +66,7 @@ func TestNotifierHeartbeaterCheck(t *testing.T) { } func TestNotifierHeartbeaterType(t *testing.T) { - _, _, _, heartbeaterBase := heartbeaterHelper(t) + _, _, _, heartbeaterBase := heartbeaterHelper(t) //nolint:dogsled Convey("Test notifierHeartbeater.Type", t, func() { cfg := NotifierHeartbeaterConfig{} @@ -80,7 +80,7 @@ func TestNotifierHeartbeaterType(t *testing.T) { } func TestNotifierHeartbeaterAlertSettings(t *testing.T) { - _, _, _, heartbeaterBase := heartbeaterHelper(t) + _, _, _, heartbeaterBase := heartbeaterHelper(t) //nolint:dogsled Convey("Test notifierHeartbeater.AlertSettings", t, func() { alertCfg := AlertConfig{ diff --git a/notifier/selfstate/heartbeat/remote_checker_test.go b/notifier/selfstate/heartbeat/remote_checker_test.go index 1c15e16bd..6de332577 100644 --- a/notifier/selfstate/heartbeat/remote_checker_test.go +++ b/notifier/selfstate/heartbeat/remote_checker_test.go @@ -16,7 +16,7 @@ const ( ) func TestNewRemoteCheckerHeartbeater(t *testing.T) { - _, _, _, heartbeaterBase := heartbeaterHelper(t) + _, _, _, heartbeaterBase := heartbeaterHelper(t) //nolint:dogsled validationErr := validator.ValidationErrors{} @@ -157,7 +157,7 @@ func TestRemoteCheckerHeartbeaterCheck(t *testing.T) { } func TestRemoteCheckerHeartbeaterType(t *testing.T) { - _, _, _, heartbeaterBase := heartbeaterHelper(t) + _, _, _, heartbeaterBase := heartbeaterHelper(t) //nolint:dogsled Convey("Test remoteCheckerHeartbeater.Type", t, func() { cfg := RemoteCheckerHeartbeaterConfig{ @@ -173,7 +173,7 @@ func TestRemoteCheckerHeartbeaterType(t *testing.T) { } func TestRemoteCheckerHeartbeaterAlertSettings(t *testing.T) { - _, _, _, heartbeaterBase := heartbeaterHelper(t) + _, _, _, heartbeaterBase := heartbeaterHelper(t) //nolint:dogsled Convey("Test remoteCheckerHeartbeater.AlertSettings", t, func() { alertCfg := AlertConfig{ diff --git a/notifier/selfstate/monitor/admin.go b/notifier/selfstate/monitor/admin.go index 65246cad4..45f72596f 100644 --- a/notifier/selfstate/monitor/admin.go +++ b/notifier/selfstate/monitor/admin.go @@ -59,7 +59,7 @@ func NewForAdmin( return nil, fmt.Errorf("admin config validation error: %w", err) } - adminMonitor := adminMonitor{ + am := adminMonitor{ adminCfg: adminCfg, database: database, notifier: notifier, @@ -82,7 +82,7 @@ func NewForAdmin( clock, notifier, heartbeaters, - adminMonitor.sendNotifications, + am.sendNotifications, ) } diff --git a/notifier/selfstate/monitor/admin_test.go b/notifier/selfstate/monitor/admin_test.go index caa2c0688..d8a9a3e21 100644 --- a/notifier/selfstate/monitor/admin_test.go +++ b/notifier/selfstate/monitor/admin_test.go @@ -190,7 +190,7 @@ func TestAdminSendNotifications(t *testing.T) { sendingWG := &sync.WaitGroup{} - adminMonitor := adminMonitor{ + am := adminMonitor{ notifier: mockNotifier, adminCfg: adminCfg, } @@ -198,7 +198,7 @@ func TestAdminSendNotifications(t *testing.T) { Convey("Test sendNotifications", t, func() { Convey("With empty notification packages", func() { pkgs := []notifier.NotificationPackage{} - err := adminMonitor.sendNotifications(pkgs) + err := am.sendNotifications(pkgs) So(err, ShouldBeNil) }) @@ -213,7 +213,7 @@ func TestAdminSendNotifications(t *testing.T) { }, } mockNotifier.EXPECT().Send(pkgWithContact, sendingWG).Times(1) - err := adminMonitor.sendNotifications(pkgs) + err := am.sendNotifications(pkgs) So(err, ShouldBeNil) }) }) diff --git a/notifier/selfstate/monitor/user.go b/notifier/selfstate/monitor/user.go index 2abef6abb..fa7664f7e 100644 --- a/notifier/selfstate/monitor/user.go +++ b/notifier/selfstate/monitor/user.go @@ -37,7 +37,7 @@ func NewForUser( return nil, fmt.Errorf("user config validation error: %w", err) } - userMonitor := userMonitor{ + um := userMonitor{ userCfg: userCfg, database: database, notifier: notifier, @@ -60,7 +60,7 @@ func NewForUser( clock, notifier, heartbeaters, - userMonitor.sendNotifications, + um.sendNotifications, ) } diff --git a/notifier/selfstate/monitor/user_test.go b/notifier/selfstate/monitor/user_test.go index 8f497460c..55349c16c 100644 --- a/notifier/selfstate/monitor/user_test.go +++ b/notifier/selfstate/monitor/user_test.go @@ -83,7 +83,7 @@ func TestUserSendNotifications(t *testing.T) { sendingWG := &sync.WaitGroup{} - userMonitor := userMonitor{ + um := userMonitor{ notifier: mockNotifier, database: mockDatabase, } @@ -99,7 +99,7 @@ func TestUserSendNotifications(t *testing.T) { Convey("Test sendNotifications", t, func() { Convey("With empty notification packages", func() { pkgs := []notifier.NotificationPackage{} - err := userMonitor.sendNotifications(pkgs) + err := um.sendNotifications(pkgs) So(err, ShouldBeNil) }) @@ -114,7 +114,7 @@ func TestUserSendNotifications(t *testing.T) { }, } mockDatabase.EXPECT().GetHeartbeatTypeContactIDs(datatypes.HeartbeatNotifier).Return([]string{}, testErr).Times(1) - err := userMonitor.sendNotifications(pkgs) + err := um.sendNotifications(pkgs) So(err, ShouldResemble, fmt.Errorf("failed to get heartbeat type contact ids: %w", testErr)) }) @@ -130,7 +130,7 @@ func TestUserSendNotifications(t *testing.T) { } mockDatabase.EXPECT().GetHeartbeatTypeContactIDs(datatypes.HeartbeatNotifier).Return(contactIDs, nil).Times(1) mockDatabase.EXPECT().GetContacts(contactIDs).Return(nil, testErr).Times(1) - err := userMonitor.sendNotifications(pkgs) + err := um.sendNotifications(pkgs) So(err, ShouldResemble, fmt.Errorf("failed to get contacts by ids: %w", testErr)) }) @@ -155,7 +155,7 @@ func TestUserSendNotifications(t *testing.T) { mockDatabase.EXPECT().GetHeartbeatTypeContactIDs(datatypes.HeartbeatNotifier).Return(contactIDs, nil).Times(1) mockDatabase.EXPECT().GetContacts(contactIDs).Return(contacts, nil).Times(1) mockNotifier.EXPECT().Send(pkgWithContact, sendingWG).Times(1) - err := userMonitor.sendNotifications(pkgs) + err := um.sendNotifications(pkgs) So(err, ShouldBeNil) }) }) From 758b5125104ae6127693297906113255ef8f31d8 Mon Sep 17 00:00:00 2001 From: almostinf Date: Tue, 5 Nov 2024 18:48:16 +0300 Subject: [PATCH 3/6] add documentation for code --- cmd/notifier/config.go | 13 +++++++++++++ notifier/selfstate/config.go | 2 ++ notifier/selfstate/monitor/admin.go | 2 ++ notifier/selfstate/monitor/monitor.go | 5 +++++ notifier/selfstate/monitor/user.go | 2 ++ notifier/selfstate/selfstate.go | 5 +++++ 6 files changed, 29 insertions(+) diff --git a/cmd/notifier/config.go b/cmd/notifier/config.go index 6e0571270..ed2f3b902 100644 --- a/cmd/notifier/config.go +++ b/cmd/notifier/config.go @@ -60,11 +60,13 @@ type notifierConfig struct { SetLogLevel setLogLevelConfig `yaml:"set_log_level"` } +// heartbeaterAlertConfig sets the configuration for the alert of a particular heartbeater. type heartbeaterAlertConfig struct { Name string `yaml:"name"` Desc string `yaml:"desc"` } +// heartbeaterBaseConfig sets the base configuration of heartbeater. type heartbeaterBaseConfig struct { Enabled bool `yaml:"enabled"` NeedTurnOffNotifier bool `yaml:"need_turn_off_notifier"` @@ -84,34 +86,40 @@ func (cfg heartbeaterBaseConfig) getSettings() heartbeat.HeartbeaterBaseConfig { } } +// databaseHeartbeaterConfig defines the database heartbeater configuration. type databaseHeartbeaterConfig struct { heartbeaterBaseConfig `yaml:",inline"` RedisDisconnectDelay string `yaml:"redis_disconnect_delay"` } +// filterHeartbeaterConfig defines the filter heartbeater configuration. type filterHeartbeaterConfig struct { heartbeaterBaseConfig `yaml:",inline"` MetricReceivedDelay string `yaml:"last_metric_received_delay"` } +// localCheckerHeartbeaterConfig defines the local checker heartbeater configuration. type localCheckerHeartbeaterConfig struct { heartbeaterBaseConfig `yaml:",inline"` LocalCheckDelay string `yaml:"last_check_delay"` } +// remoteCheckerHeartbeaterConfig defines the remote checker heartbeater configuration. type remoteCheckerHeartbeaterConfig struct { heartbeaterBaseConfig `yaml:",inline"` RemoteCheckDelay string `yaml:"last_remote_check_delay"` } +// notifierHeartbeaterConfig defines the notifier heartbeater configuration. type notifierHeartbeaterConfig struct { heartbeaterBaseConfig `yaml:",inline"` } +// heartbeatsConfig defines the configuration of heartbeaters. type heartbeatsConfig struct { DatabaseCfg databaseHeartbeaterConfig `yaml:"database"` FilterCfg filterHeartbeaterConfig `yaml:"filter"` @@ -144,6 +152,7 @@ func (cfg heartbeatsConfig) getSettings() heartbeat.HeartbeatersConfig { } } +// monitorBaseConfig defines the basic configuration of the monitor. type monitorBaseConfig struct { Enabled bool `yaml:"enabled"` HearbeatersCfg heartbeatsConfig `yaml:"heartbeaters"` @@ -151,21 +160,25 @@ type monitorBaseConfig struct { CheckInterval string `yaml:"check_interval"` } +// adminMonitorConfig defines the configuration for the admin monitor. type adminMonitorConfig struct { monitorBaseConfig `yaml:",inline"` AdminContacts []map[string]string `yaml:"contacts"` } +// userMonitorConfig defines the configuration for the user monitor. type userMonitorConfig struct { monitorBaseConfig `yaml:",inline"` } +// monitorConfig defines the configuration for all monitors. type monitorConfig struct { AdminCfg adminMonitorConfig `yaml:"admin"` UserCfg userMonitorConfig `yaml:"user"` } +// selfstateConfig defines the configuration of the selfstate worker. type selfstateConfig struct { Enabled bool `yaml:"enabled"` MonitorCfg monitorConfig `yaml:"monitor"` diff --git a/notifier/selfstate/config.go b/notifier/selfstate/config.go index 03a83960d..095bb62f3 100644 --- a/notifier/selfstate/config.go +++ b/notifier/selfstate/config.go @@ -4,11 +4,13 @@ import ( "github.com/moira-alert/moira/notifier/selfstate/monitor" ) +// MonitorConfig sets the configurations of all monitors. type MonitorConfig struct { UserCfg monitor.UserMonitorConfig AdminCfg monitor.AdminMonitorConfig } +// Config sets the configuration of the selfstate worker. type Config struct { Enabled bool MonitorCfg MonitorConfig diff --git a/notifier/selfstate/monitor/admin.go b/notifier/selfstate/monitor/admin.go index 45f72596f..754cdf060 100644 --- a/notifier/selfstate/monitor/admin.go +++ b/notifier/selfstate/monitor/admin.go @@ -15,6 +15,7 @@ const ( adminMonitorLockTTL = 15 * time.Second ) +// AdminMonitorConfig sets the configuration for admin monitor. type AdminMonitorConfig struct { MonitorBaseConfig @@ -48,6 +49,7 @@ type adminMonitor struct { notifier notifier.Notifier } +// NewForAdmin function that creates a new admin monitor. func NewForAdmin( adminCfg AdminMonitorConfig, logger moira.Logger, diff --git a/notifier/selfstate/monitor/monitor.go b/notifier/selfstate/monitor/monitor.go index 265c5a925..4d5c000bc 100644 --- a/notifier/selfstate/monitor/monitor.go +++ b/notifier/selfstate/monitor/monitor.go @@ -17,9 +17,11 @@ var ( errorValue = 1.0 triggerErrorValue = 1.0 + // Verify that monitor matches the Monitor interface. _ Monitor = (*monitor)(nil) ) +// MonitorBaseConfig sets the basic configuration for the monitor. type MonitorBaseConfig struct { Enabled bool HeartbeatersCfg heartbeat.HeartbeatersConfig `validate:"required_if=Enabled true"` @@ -40,6 +42,7 @@ type monitorConfig struct { CheckInterval time.Duration `validate:"required,gt=0"` } +// Monitor interface, which defines methods for starting and stopping the monitor. type Monitor interface { Start() Stop() error @@ -162,6 +165,7 @@ func createHearbeaters( return heartbeaters } +// Start is the method to start the monitor. func (m *monitor) Start() { m.tomb.Go(func() error { w.NewWorker( @@ -304,6 +308,7 @@ func (m *monitor) createOkNotificationPackage(heartbeater heartbeat.Heartbeater, } } +// Stop is a method for stopping the monitor. func (m *monitor) Stop() error { m.tomb.Kill(nil) return m.tomb.Wait() diff --git a/notifier/selfstate/monitor/user.go b/notifier/selfstate/monitor/user.go index fa7664f7e..f5f611564 100644 --- a/notifier/selfstate/monitor/user.go +++ b/notifier/selfstate/monitor/user.go @@ -16,6 +16,7 @@ const ( userMonitorLockTTL = 15 * time.Second ) +// UserMonitorConfig defines the configuration of the user monitor. type UserMonitorConfig struct { MonitorBaseConfig } @@ -26,6 +27,7 @@ type userMonitor struct { notifier notifier.Notifier } +// NewForUser is a method to create a user monitor. func NewForUser( userCfg UserMonitorConfig, logger moira.Logger, diff --git a/notifier/selfstate/selfstate.go b/notifier/selfstate/selfstate.go index 486fec29d..7a7d4f985 100644 --- a/notifier/selfstate/selfstate.go +++ b/notifier/selfstate/selfstate.go @@ -8,8 +8,10 @@ import ( "github.com/moira-alert/moira/notifier/selfstate/monitor" ) +// Verify that selfstateWorker matches the SelfstateWorker interface. var _ SelfstateWorker = (*selfstateWorker)(nil) +// SelfstateWorker interface, which defines methods for starting and stopping the selfstate worker. type SelfstateWorker interface { Start() Stop() error @@ -19,6 +21,7 @@ type selfstateWorker struct { monitors []monitor.Monitor } +// NewSelfstateWorker is a method to create a new selfstate worker. func NewSelfstateWorker( cfg Config, logger moira.Logger, @@ -82,12 +85,14 @@ func createMonitors( return monitors } +// Start is a method to start a selfstate worker. func (selfstateWorker *selfstateWorker) Start() { for _, monitor := range selfstateWorker.monitors { monitor.Start() } } +// Stop is a method for stopping a selfstate worker. func (selfstateWorker *selfstateWorker) Stop() error { stopErrors := make([]error, 0) From 1ff1dbad9ab90ecd8663f30f6946dee731d3bbc3 Mon Sep 17 00:00:00 2001 From: almostinf Date: Wed, 6 Nov 2024 12:36:17 +0300 Subject: [PATCH 4/6] add selfstate tests and new mocks --- generate_mocks.sh | 1 + mock/monitor/monitor.go | 65 +++++++++ notifier/selfstate/selfstate_test.go | 191 +++++++++++++++++++++++++++ 3 files changed, 257 insertions(+) create mode 100644 mock/monitor/monitor.go create mode 100644 notifier/selfstate/selfstate_test.go diff --git a/generate_mocks.sh b/generate_mocks.sh index 266519cdc..7074c02cb 100755 --- a/generate_mocks.sh +++ b/generate_mocks.sh @@ -19,6 +19,7 @@ mockgen -destination=mock/moira-alert/searcher.go -package=mock_moira_alert gith mockgen -destination=mock/metric_source/source.go -package=mock_metric_source github.com/moira-alert/moira/metric_source MetricSource mockgen -destination=mock/metric_source/fetch_result.go -package=mock_metric_source github.com/moira-alert/moira/metric_source FetchResult mockgen -destination=mock/heartbeat/heartbeat.go -package=mock_heartbeat github.com/moira-alert/moira/notifier/selfstate/heartbeat Heartbeater +mockgen -destination=mock/monitor/monitor.go -package=mock_monitor github.com/moira-alert/moira/notifier/selfstate/monitor Monitor mockgen -destination=mock/clock/clock.go -package=mock_clock github.com/moira-alert/moira Clock mockgen -destination=mock/notifier/mattermost/client.go -package=mock_mattermost github.com/moira-alert/moira/senders/mattermost Client diff --git a/mock/monitor/monitor.go b/mock/monitor/monitor.go new file mode 100644 index 000000000..2ccf57eec --- /dev/null +++ b/mock/monitor/monitor.go @@ -0,0 +1,65 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/moira-alert/moira/notifier/selfstate/monitor (interfaces: Monitor) +// +// Generated by this command: +// +// mockgen -destination=mock/monitor/monitor.go -package=mock_monitor github.com/moira-alert/moira/notifier/selfstate/monitor Monitor +// + +// Package mock_monitor is a generated GoMock package. +package mock_monitor + +import ( + reflect "reflect" + + gomock "go.uber.org/mock/gomock" +) + +// MockMonitor is a mock of Monitor interface. +type MockMonitor struct { + ctrl *gomock.Controller + recorder *MockMonitorMockRecorder +} + +// MockMonitorMockRecorder is the mock recorder for MockMonitor. +type MockMonitorMockRecorder struct { + mock *MockMonitor +} + +// NewMockMonitor creates a new mock instance. +func NewMockMonitor(ctrl *gomock.Controller) *MockMonitor { + mock := &MockMonitor{ctrl: ctrl} + mock.recorder = &MockMonitorMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockMonitor) EXPECT() *MockMonitorMockRecorder { + return m.recorder +} + +// Start mocks base method. +func (m *MockMonitor) Start() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Start") +} + +// Start indicates an expected call of Start. +func (mr *MockMonitorMockRecorder) Start() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*MockMonitor)(nil).Start)) +} + +// Stop mocks base method. +func (m *MockMonitor) Stop() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Stop") + ret0, _ := ret[0].(error) + return ret0 +} + +// Stop indicates an expected call of Stop. +func (mr *MockMonitorMockRecorder) Stop() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stop", reflect.TypeOf((*MockMonitor)(nil).Stop)) +} diff --git a/notifier/selfstate/selfstate_test.go b/notifier/selfstate/selfstate_test.go new file mode 100644 index 000000000..9058b4168 --- /dev/null +++ b/notifier/selfstate/selfstate_test.go @@ -0,0 +1,191 @@ +package selfstate + +import ( + "errors" + "testing" + "time" + + "go.uber.org/mock/gomock" + + logging "github.com/moira-alert/moira/logging/zerolog_adapter" + mock_clock "github.com/moira-alert/moira/mock/clock" + mock_moira_alert "github.com/moira-alert/moira/mock/moira-alert" + mock_monitor "github.com/moira-alert/moira/mock/monitor" + mock_notifier "github.com/moira-alert/moira/mock/notifier" + "github.com/moira-alert/moira/notifier/selfstate/heartbeat" + "github.com/moira-alert/moira/notifier/selfstate/monitor" + . "github.com/smartystreets/goconvey/convey" +) + +func TestNewSelfstateWorker(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + mockClock := mock_clock.NewMockClock(mockCtrl) + mockDatabase := mock_moira_alert.NewMockDatabase(mockCtrl) + logger, _ := logging.GetLogger("Test") + mockNotifier := mock_notifier.NewMockNotifier(mockCtrl) + + cfg := Config{ + Enabled: true, + MonitorCfg: MonitorConfig{}, + } + + Convey("Test NewSelfstateWorker", t, func() { + worker, err := NewSelfstateWorker(cfg, logger, mockDatabase, mockNotifier, mockClock) + So(err, ShouldBeNil) + So(worker.monitors, ShouldHaveLength, 0) + }) +} + +func TestCreateMonitors(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + mockClock := mock_clock.NewMockClock(mockCtrl) + mockDatabase := mock_moira_alert.NewMockDatabase(mockCtrl) + logger, _ := logging.GetLogger("Test") + mockNotifier := mock_notifier.NewMockNotifier(mockCtrl) + testTime := time.Date(2022, time.June, 6, 10, 0, 0, 0, time.UTC) + + Convey("Test createMonitors", t, func() { + mockClock.EXPECT().NowUTC().Return(testTime).AnyTimes() + + Convey("With disabled monitors", func() { + monitors := createMonitors(MonitorConfig{}, logger, mockDatabase, mockClock, mockNotifier) + So(monitors, ShouldHaveLength, 0) + }) + + Convey("With enabled user monitor", func() { + cfg := MonitorConfig{ + UserCfg: monitor.UserMonitorConfig{ + MonitorBaseConfig: monitor.MonitorBaseConfig{ + Enabled: true, + HeartbeatersCfg: heartbeat.HeartbeatersConfig{ + DatabaseCfg: heartbeat.DatabaseHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + Enabled: true, + }, + RedisDisconnectDelay: time.Minute, + }, + }, + NoticeInterval: time.Minute, + CheckInterval: time.Minute, + }, + }, + } + + monitors := createMonitors(cfg, logger, mockDatabase, mockClock, mockNotifier) + So(monitors, ShouldHaveLength, 1) + }) + + Convey("With enabled user and admin monitors", func() { + mockNotifier.EXPECT().GetSenders().Return(map[string]bool{}) + + cfg := MonitorConfig{ + UserCfg: monitor.UserMonitorConfig{ + MonitorBaseConfig: monitor.MonitorBaseConfig{ + Enabled: true, + HeartbeatersCfg: heartbeat.HeartbeatersConfig{ + DatabaseCfg: heartbeat.DatabaseHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + Enabled: true, + }, + RedisDisconnectDelay: time.Minute, + }, + }, + NoticeInterval: time.Minute, + CheckInterval: time.Minute, + }, + }, + AdminCfg: monitor.AdminMonitorConfig{ + MonitorBaseConfig: monitor.MonitorBaseConfig{ + Enabled: true, + HeartbeatersCfg: heartbeat.HeartbeatersConfig{ + DatabaseCfg: heartbeat.DatabaseHeartbeaterConfig{ + HeartbeaterBaseConfig: heartbeat.HeartbeaterBaseConfig{ + Enabled: true, + }, + RedisDisconnectDelay: time.Minute, + }, + }, + NoticeInterval: time.Minute, + CheckInterval: time.Minute, + }, + AdminContacts: []map[string]string{}, + }, + } + + monitors := createMonitors(cfg, logger, mockDatabase, mockClock, mockNotifier) + So(monitors, ShouldHaveLength, 2) + }) + }) +} + +func TestStart(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + mockUserMonitor := mock_monitor.NewMockMonitor(mockCtrl) + mockAdminMonitor := mock_monitor.NewMockMonitor(mockCtrl) + + worker := &selfstateWorker{ + monitors: []monitor.Monitor{mockUserMonitor, mockAdminMonitor}, + } + + Convey("Test Start", t, func() { + mockUserMonitor.EXPECT().Start() + mockAdminMonitor.EXPECT().Start() + + worker.Start() + }) +} + +func TestStop(t *testing.T) { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + mockUserMonitor := mock_monitor.NewMockMonitor(mockCtrl) + mockAdminMonitor := mock_monitor.NewMockMonitor(mockCtrl) + + userMonitorErr := errors.New("test user monitor error") + adminMonitorErr := errors.New("test admin monitor error") + + worker := &selfstateWorker{ + monitors: []monitor.Monitor{mockUserMonitor, mockAdminMonitor}, + } + + Convey("Test Stop", t, func() { + Convey("Without any errors", func() { + mockUserMonitor.EXPECT().Stop().Return(nil) + mockAdminMonitor.EXPECT().Stop().Return(nil) + + err := worker.Stop() + So(err, ShouldBeNil) + }) + + Convey("With user monitor error", func() { + mockUserMonitor.EXPECT().Stop().Return(userMonitorErr) + mockAdminMonitor.EXPECT().Stop().Return(nil) + + err := worker.Stop() + So(err, ShouldResemble, errors.Join(userMonitorErr)) + }) + + Convey("With admin monitor error", func() { + mockUserMonitor.EXPECT().Stop().Return(nil) + mockAdminMonitor.EXPECT().Stop().Return(adminMonitorErr) + + err := worker.Stop() + So(err, ShouldResemble, errors.Join(adminMonitorErr)) + }) + + Convey("With admin and user monitor errors", func() { + mockUserMonitor.EXPECT().Stop().Return(userMonitorErr) + mockAdminMonitor.EXPECT().Stop().Return(adminMonitorErr) + + err := worker.Stop() + So(err, ShouldResemble, errors.Join(userMonitorErr, adminMonitorErr)) + }) + }) +} From b259a65f94571872a976717e514781b41823e6d8 Mon Sep 17 00:00:00 2001 From: almostinf Date: Wed, 6 Nov 2024 12:37:31 +0300 Subject: [PATCH 5/6] remove unnecessary print --- cmd/notifier/main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/notifier/main.go b/cmd/notifier/main.go index 7eda85e05..9a3ab5e59 100644 --- a/cmd/notifier/main.go +++ b/cmd/notifier/main.go @@ -121,7 +121,6 @@ func main() { // Start moira selfstate checker if selfstateCfg.Enabled { - fmt.Println(selfstateCfg) logger.Info().Msg("Selfstate enabled") selfstateWorker, err := selfstate.NewSelfstateWorker(selfstateCfg, logger, database, sender, systemClock) if err != nil { From be63bb9294fae243ecbb1f79d72b3ff8028f494b Mon Sep 17 00:00:00 2001 From: almostinf Date: Fri, 8 Nov 2024 16:32:34 +0300 Subject: [PATCH 6/6] fix bug with reusing heartbeater base in all heartbeaters --- notifier/selfstate/monitor/monitor.go | 7 +++++-- notifier/selfstate/monitor/monitor_test.go | 12 ++++++------ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/notifier/selfstate/monitor/monitor.go b/notifier/selfstate/monitor/monitor.go index 4d5c000bc..c35e2c68d 100644 --- a/notifier/selfstate/monitor/monitor.go +++ b/notifier/selfstate/monitor/monitor.go @@ -98,11 +98,10 @@ func createHearbeaters( database moira.Database, clock moira.Clock, ) []heartbeat.Heartbeater { - hearbeaterBase := heartbeat.NewHeartbeaterBase(logger, database, clock) - heartbeaters := make([]heartbeat.Heartbeater, 0) if heartbeatersCfg.DatabaseCfg.Enabled { + hearbeaterBase := heartbeat.NewHeartbeaterBase(logger, database, clock) databaseHeartbeater, err := heartbeat.NewDatabaseHeartbeater(heartbeatersCfg.DatabaseCfg, hearbeaterBase) if err != nil { logger.Error(). @@ -115,6 +114,7 @@ func createHearbeaters( } if heartbeatersCfg.FilterCfg.Enabled { + hearbeaterBase := heartbeat.NewHeartbeaterBase(logger, database, clock) filterHeartbeater, err := heartbeat.NewFilterHeartbeater(heartbeatersCfg.FilterCfg, hearbeaterBase) if err != nil { logger.Error(). @@ -127,6 +127,7 @@ func createHearbeaters( } if heartbeatersCfg.LocalCheckerCfg.Enabled { + hearbeaterBase := heartbeat.NewHeartbeaterBase(logger, database, clock) localCheckerHeartbeater, err := heartbeat.NewLocalCheckerHeartbeater(heartbeatersCfg.LocalCheckerCfg, hearbeaterBase) if err != nil { logger.Error(). @@ -139,6 +140,7 @@ func createHearbeaters( } if heartbeatersCfg.RemoteCheckerCfg.Enabled { + hearbeaterBase := heartbeat.NewHeartbeaterBase(logger, database, clock) remoteCheckerHeartbeater, err := heartbeat.NewRemoteCheckerHeartbeater(heartbeatersCfg.RemoteCheckerCfg, hearbeaterBase) if err != nil { logger.Error(). @@ -151,6 +153,7 @@ func createHearbeaters( } if heartbeatersCfg.NotifierCfg.Enabled { + hearbeaterBase := heartbeat.NewHeartbeaterBase(logger, database, clock) notifierHeartbeater, err := heartbeat.NewNotifierHeartbeater(heartbeatersCfg.NotifierCfg, hearbeaterBase) if err != nil { logger.Error(). diff --git a/notifier/selfstate/monitor/monitor_test.go b/notifier/selfstate/monitor/monitor_test.go index cbd3c1392..0833d359e 100644 --- a/notifier/selfstate/monitor/monitor_test.go +++ b/notifier/selfstate/monitor/monitor_test.go @@ -30,7 +30,7 @@ func TestCreateHeartbeaters(t *testing.T) { Convey("Test createHeartbeaters", t, func() { Convey("Without any heartbeater", func() { hbCfg := heartbeat.HeartbeatersConfig{} - mockClock.EXPECT().NowUTC().Return(testTime) + mockClock.EXPECT().NowUTC().Return(testTime).AnyTimes() heartbeaters := createHearbeaters(hbCfg, mockLogger, mockDatabase, mockClock) So(heartbeaters, ShouldBeEmpty) }) @@ -44,7 +44,7 @@ func TestCreateHeartbeaters(t *testing.T) { RedisDisconnectDelay: time.Minute, }, } - mockClock.EXPECT().NowUTC().Return(testTime) + mockClock.EXPECT().NowUTC().Return(testTime).AnyTimes() heartbeaters := createHearbeaters(hbCfg, mockLogger, mockDatabase, mockClock) So(heartbeaters, ShouldHaveLength, 1) }) @@ -64,7 +64,7 @@ func TestCreateHeartbeaters(t *testing.T) { MetricReceivedDelay: time.Minute, }, } - mockClock.EXPECT().NowUTC().Return(testTime) + mockClock.EXPECT().NowUTC().Return(testTime).AnyTimes() heartbeaters := createHearbeaters(hbCfg, mockLogger, mockDatabase, mockClock) So(heartbeaters, ShouldHaveLength, 2) }) @@ -90,7 +90,7 @@ func TestCreateHeartbeaters(t *testing.T) { LocalCheckDelay: time.Minute, }, } - mockClock.EXPECT().NowUTC().Return(testTime) + mockClock.EXPECT().NowUTC().Return(testTime).AnyTimes() heartbeaters := createHearbeaters(hbCfg, mockLogger, mockDatabase, mockClock) So(heartbeaters, ShouldHaveLength, 3) }) @@ -122,7 +122,7 @@ func TestCreateHeartbeaters(t *testing.T) { RemoteCheckDelay: time.Minute, }, } - mockClock.EXPECT().NowUTC().Return(testTime) + mockClock.EXPECT().NowUTC().Return(testTime).AnyTimes() heartbeaters := createHearbeaters(hbCfg, mockLogger, mockDatabase, mockClock) So(heartbeaters, ShouldHaveLength, 4) }) @@ -159,7 +159,7 @@ func TestCreateHeartbeaters(t *testing.T) { }, }, } - mockClock.EXPECT().NowUTC().Return(testTime) + mockClock.EXPECT().NowUTC().Return(testTime).AnyTimes() heartbeaters := createHearbeaters(hbCfg, mockLogger, mockDatabase, mockClock) So(heartbeaters, ShouldHaveLength, 5) })