feat(cli): migrations for team validation (#1124)

* feat: migrations for team validation

* feat: write log than team will be renamed

* refactor: logging then renaming teams, add timeout between modifying teams and committing changes

cmd/cli/from_2.13_to_2.14.go

@@ -0,0 +1,27 @@
+package main
+
+import "github.com/moira-alert/moira"
+
+func updateFrom213(logger moira.Logger, database moira.Database) error {
+	logger.Info().Msg("Update 2.13 -> 2.14 started")
+
+	err := fillTeamNamesHash(logger, database)
+	if err != nil {
+		return err
+	}
+
+	logger.Info().Msg("Update 2.13 -> 2.14 was finished")
+	return nil
+}
+
+func downgradeTo213(logger moira.Logger, database moira.Database) error {
+	logger.Info().Msg("Downgrade 2.14 -> 2.13 started")
+
+	err := removeTeamNamesHash(logger, database)
+	if err != nil {
+		return err
+	}
+
+	logger.Info().Msg("Downgrade 2.14 -> 2.13 was finished")
+	return nil
+}

cmd/cli/main.go

@@ -25,7 +25,15 @@ var (
 	GoVersion    = "unknown"
 )
 
-var moiraValidVersions = []string{"2.3", "2.6", "2.7", "2.9", "2.11", "2.12"}
+var moiraValidVersions = []string{
+	"2.3",
+	"2.6",
+	"2.7",
+	"2.9",
+	"2.11",
+	"2.12",
+	"2.13",
+}
 
 var (
 	configFileName         = flag.String("config", "/etc/moira/cli.yml", "Path to configuration file")
@@ -125,6 +133,13 @@ func main() { //nolint
 					Error(err).
 					Msg("Fail to update from version 2.12")
 			}
+		case "2.13":
+			err := updateFrom213(logger, database)
+			if err != nil {
+				logger.Fatal().
+					Error(err).
+					Msg("Fail to update from version 2.13")
+			}
 		}
 	}
 
@@ -173,6 +188,13 @@ func main() { //nolint
 					Error(err).
 					Msg("Fail to update to version 2.12")
 			}
+		case "2.13":
+			err := downgradeTo213(logger, database)
+			if err != nil {
+				logger.Fatal().
+					Error(err).
+					Msg("Fail to update to version 2.13")
+			}
 		}
 	}
 

cmd/cli/teams_names.go

@@ -0,0 +1,264 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	goredis "github.com/go-redis/redis/v8"
+	"github.com/moira-alert/moira"
+	"github.com/moira-alert/moira/database/redis"
+)
+
+const (
+	teamsKey                 = "moira-teams"
+	teamsByNamesKey          = "moira-teams-by-names"
+	delayBeforeRenamingTeams = time.Second * 10
+)
+
+var errTeamsCountAndUniqueNamesCountMismatch = errors.New(
+	"count of teams does not match count of unique names after transformation")
+
+type renameInfo struct {
+	oldTeamName string
+	newTeamName string
+	teamID      string
+}
+
+func (info *renameInfo) String() string {
+	return fmt.Sprintf("%s: '%s' -> '%s'", info.teamID, info.oldTeamName, info.newTeamName)
+}
+
+// fillTeamNamesHash does the following
+//  1. Get all teams from DB.
+//  2. Group teams with same names.
+//  3. For teams with same names change name (example: ["name", "name", "name"] -> ["name", "name1", "name2"]).
+//  4. Update teams with name changed.
+//  5. Save pairs teamName:team.ID to "moira-teams-by-names" redis hash.
+func fillTeamNamesHash(logger moira.Logger, database moira.Database) error {
+	logger.Info().Msg("Start filling \"moira-teams-by-names\" hash")
+
+	switch db := database.(type) {
+	case *redis.DbConnector:
+		logger.Info().Msg("collecting teams from redis node...")
+
+		teamsMap, err := db.Client().HGetAll(db.Context(), teamsKey).Result()
+		if err != nil {
+			return fmt.Errorf("failed to fetch teams from redis node: %w", err)
+		}
+
+		logger.Info().
+			Int("total_teams_count", len(teamsMap)).
+			Msg("fetched teams")
+
+		teamsByNameMap, err := groupTeamsByNames(logger, teamsMap)
+		if err != nil {
+			return fmt.Errorf("failed to group teams by names: %w", err)
+		}
+
+		teamByUniqueName, renamingInfo := transformTeamsByNameMap(teamsByNameMap)
+
+		if len(teamByUniqueName) != len(teamsMap) {
+			return errTeamsCountAndUniqueNamesCountMismatch
+		}
+
+		builder := strings.Builder{}
+		for i, rename := range renamingInfo {
+			builder.WriteString(rename.String())
+			if i != len(renamingInfo)-1 {
+				builder.WriteString(", ")
+			}
+		}
+
+		logger.Info().
+			String("rename_info", builder.String()).
+			Msg("Would rename teams")
+
+		logger.Info().
+			Int("teams_renamed_count", len(renamingInfo)).
+			String("delay", delayBeforeRenamingTeams.String()).
+			Msg("Teams will be renamed after delay")
+
+		logger.Info().Msg("Press Ctrl+C to stop")
+		time.Sleep(delayBeforeRenamingTeams)
+
+		logger.Info().Msg("Start renaming teams and filling \"moira-teams-by-names\" hash")
+
+		client := db.Client()
+		ctx := db.Context()
+
+		_, pipeErr := client.TxPipelined(
+			ctx,
+			func(pipe goredis.Pipeliner) error {
+				return updateTeamsInPipe(ctx, logger, pipe, teamByUniqueName)
+			})
+		if pipeErr != nil {
+			return pipeErr
+		}
+
+		logger.Info().Msg("\"moira-teams-by-names\" hash successfully filled")
+
+	default:
+		return makeUnknownDBError(database)
+	}
+	return nil
+}
+
+func groupTeamsByNames(logger moira.Logger, teamsMap map[string]string) (map[string][]teamWithID, error) {
+	teamsByNameMap := make(map[string][]teamWithID, len(teamsMap))
+
+	for teamID, marshaledTeam := range teamsMap {
+		team, err := unmarshalTeam(teamID, []byte(marshaledTeam))
+		if err != nil {
+			return nil, err
+		}
+
+		lowercaseTeamName := strings.ToLower(team.Name)
+
+		teamWithNameList, exists := teamsByNameMap[lowercaseTeamName]
+		if exists {
+			teamWithNameList = append(teamWithNameList, team)
+			teamsByNameMap[lowercaseTeamName] = teamWithNameList
+		} else {
+			teamsByNameMap[lowercaseTeamName] = []teamWithID{team}
+		}
+	}
+
+	logger.Info().
+		Int("unique_team_names_count", len(teamsByNameMap)).
+		Msg("grouped teams with same names")
+
+	return teamsByNameMap, nil
+}
+
+func transformTeamsByNameMap(teamsByNameMap map[string][]teamWithID) (map[string]teamWithID, []renameInfo) {
+	teamByUniqueName := make(map[string]teamWithID, len(teamsByNameMap))
+	renamingTeams := make([]renameInfo, 0)
+
+	for _, teams := range teamsByNameMap {
+		for i, team := range teams {
+			iStr := strconv.FormatInt(int64(i), 10)
+
+			oldTeamName := team.Name
+			if i > 0 {
+				team.Name += iStr
+			}
+
+			for {
+				// sometimes we have the following situation in db (IDs and team names):
+				// moira-teams: {
+				//    team1: "team name",
+				//    team2: "team Name",
+				//    team3: "Team name1"
+				// }
+				// so we can't just add 1 to one of [team1, team2]
+				lowercasedTeamName := strings.ToLower(team.Name)
+
+				_, exists := teamByUniqueName[lowercasedTeamName]
+				if exists {
+					team.Name += "_" + iStr
+				} else {
+					teamByUniqueName[lowercasedTeamName] = team
+					if team.Name != oldTeamName {
+						renamingTeams = append(renamingTeams, renameInfo{
+							teamID:      team.ID,
+							oldTeamName: oldTeamName,
+							newTeamName: team.Name,
+						})
+					}
+					break
+				}
+			}
+		}
+	}
+
+	return teamByUniqueName, renamingTeams
+}
+
+func updateTeamsInPipe(ctx context.Context, logger moira.Logger, pipe goredis.Pipeliner, teamsByUniqueName map[string]teamWithID) error {
+	for _, team := range teamsByUniqueName {
+		teamBytes, err := getTeamBytes(team)
+		if err != nil {
+			return err
+		}
+
+		err = pipe.HSet(ctx, teamsKey, team.ID, teamBytes).Err()
+		if err != nil {
+			logger.Error().
+				Error(err).
+				String("team_id", team.ID).
+				String("new_team_name", team.Name).
+				Msg("failed to update team name")
+
+			return fmt.Errorf("failed to update team name: %w", err)
+		}
+
+		err = pipe.HSet(ctx, teamsByNamesKey, strings.ToLower(team.Name), team.ID).Err()
+		if err != nil {
+			logger.Error().
+				Error(err).
+				String("team_id", team.ID).
+				String("new_team_name", team.Name).
+				Msg("failed to add team name to redis hash")
+
+			return fmt.Errorf("failed to add team name to redis hash: %w", err)
+		}
+	}
+
+	return nil
+}
+
+// removeTeamNamesHash remove "moira-teams-by-names" redis hash.
+// Note that if fillTeamNamesHash have been called, then team names would not be changed back.
+func removeTeamNamesHash(logger moira.Logger, database moira.Database) error {
+	logger.Info().Msg("Start removing \"moira-teams-by-names\" redis hash")
+
+	switch db := database.(type) {
+	case *redis.DbConnector:
+		_, err := db.Client().Del(db.Context(), teamsByNamesKey).Result()
+		if err != nil {
+			return fmt.Errorf("failed to delete teamsByNameKey: %w", err)
+		}
+
+	default:
+		return makeUnknownDBError(database)
+	}
+
+	return nil
+}
+
+type teamStorageElement struct {
+	Name        string `json:"name"`
+	Description string `json:"description"`
+}
+
+type teamWithID struct {
+	teamStorageElement
+	ID string
+}
+
+func unmarshalTeam(teamID string, teamBytes []byte) (teamWithID, error) {
+	var storedTeam teamStorageElement
+	err := json.Unmarshal(teamBytes, &storedTeam)
+	if err != nil {
+		return teamWithID{}, fmt.Errorf("failed to deserialize team: %w", err)
+	}
+
+	return teamWithID{
+		teamStorageElement: storedTeam,
+		ID:                 teamID,
+	}, nil
+}
+
+func getTeamBytes(team teamWithID) ([]byte, error) {
+	bytes, err := json.Marshal(team.teamStorageElement)
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal team: %w", err)
+	}
+
+	return bytes, nil
+}