telemetry: fix tememetry data and update reported fields (#1327)

components/cluster/command/check.go

@@ -40,6 +40,10 @@ conflict checks with other clusters`,
 			if len(args) != 1 {
 				return cmd.Help()
 			}
+
+			if opt.ExistCluster {
+				clusterReport.ID = scrubClusterName(args[0])
+			}
 			// natvie ssh has it's own logic to find the default identity_file
 			if gOpt.SSHType == executor.SSHTypeSystem && !utils.IsFlagSetByUser(cmd.Flags(), "identity_file") {
 				opt.IdentityFile = ""

components/cluster/command/clean.go

@@ -42,6 +42,7 @@ You can retain some nodes and roles data when cleanup the cluster, eg:
 			}
 
 			clusterName := args[0]
+			clusterReport.ID = scrubClusterName(clusterName)
 			teleCommand = append(teleCommand, scrubClusterName(clusterName))
 
 			if cleanALl {

components/cluster/command/deploy.go

@@ -72,6 +72,7 @@ func newDeploy() *cobra.Command {
 			if err != nil {
 				return err
 			}
+			clusterReport.ID = scrubClusterName(clusterName)
 			teleCommand = append(teleCommand, scrubClusterName(clusterName))
 			teleCommand = append(teleCommand, version)
 
@@ -103,7 +104,7 @@ func postDeployHook(builder *task.Builder, topo spec.Topology) {
 		return nil
 	}).BuildAsStep("Check status").SetHidden(true)
 
-	if report.Enable() {
+	if report.Enabled() {
 		builder.ParallelStep("+ Check status", false, nodeInfoTask)
 	}
 

components/cluster/command/destroy.go

@@ -38,6 +38,7 @@ You can retain some nodes and roles data when destroy cluster, eg:
 			}
 
 			clusterName := args[0]
+			clusterReport.ID = scrubClusterName(clusterName)
 			teleCommand = append(teleCommand, scrubClusterName(clusterName))
 
 			// Validate the retained roles to prevent unexpected deleting data

components/cluster/command/disable.go

@@ -31,6 +31,7 @@ func newDisableCmd() *cobra.Command {
 			}
 
 			clusterName := args[0]
+			clusterReport.ID = scrubClusterName(clusterName)
 			teleCommand = append(teleCommand, scrubClusterName(clusterName))
 
 			return cm.EnableCluster(clusterName, gOpt, false)

components/cluster/command/display.go

@@ -44,6 +44,7 @@ func newDisplayCmd() *cobra.Command {
 			}
 
 			clusterName = args[0]
+			clusterReport.ID = scrubClusterName(clusterName)
 			teleCommand = append(teleCommand, scrubClusterName(clusterName))
 
 			exist, err := tidbSpec.Exist(clusterName)

components/cluster/command/edit_config.go

@@ -27,6 +27,7 @@ func newEditConfigCmd() *cobra.Command {
 			}
 
 			clusterName := args[0]
+			clusterReport.ID = scrubClusterName(clusterName)
 			teleCommand = append(teleCommand, scrubClusterName(clusterName))
 
 			return cm.EditConfig(clusterName, skipConfirm)

components/cluster/command/enable.go

@@ -31,6 +31,7 @@ func newEnableCmd() *cobra.Command {
 			}
 
 			clusterName := args[0]
+			clusterReport.ID = scrubClusterName(clusterName)
 			teleCommand = append(teleCommand, scrubClusterName(clusterName))
 
 			return cm.EnableCluster(clusterName, gOpt, true)

components/cluster/command/exec.go

@@ -30,6 +30,7 @@ func newExecCmd() *cobra.Command {
 			}
 
 			clusterName := args[0]
+			clusterReport.ID = scrubClusterName(clusterName)
 			teleCommand = append(teleCommand, scrubClusterName(clusterName))
 
 			return cm.Exec(clusterName, opt, gOpt)

components/cluster/command/patch.go

@@ -39,6 +39,7 @@ func newPatchCmd() *cobra.Command {
 				return perrs.New("the flag -R or -N must be specified at least one")
 			}
 			clusterName := args[0]
+			clusterReport.ID = scrubClusterName(clusterName)
 			teleCommand = append(teleCommand, scrubClusterName(clusterName))
 
 			return cm.Patch(clusterName, args[1], gOpt, overwrite, offlineMode)

components/cluster/command/prune.go

@@ -27,6 +27,8 @@ func newPruneCmd() *cobra.Command {
 			}
 
 			clusterName := args[0]
+			clusterReport.ID = scrubClusterName(clusterName)
+			teleCommand = append(teleCommand, scrubClusterName(clusterName))
 
 			return cm.DestroyTombstone(clusterName, gOpt, skipConfirm)
 		},

components/cluster/command/reload.go

@@ -34,6 +34,7 @@ func newReloadCmd() *cobra.Command {
 			}
 
 			clusterName := args[0]
+			clusterReport.ID = scrubClusterName(clusterName)
 			teleCommand = append(teleCommand, scrubClusterName(clusterName))
 
 			return cm.Reload(clusterName, gOpt, skipRestart)

components/cluster/command/rename.go

@@ -32,6 +32,7 @@ func newRenameCmd() *cobra.Command {
 
 			oldClusterName := args[0]
 			newClusterName := args[1]
+			clusterReport.ID = scrubClusterName(oldClusterName)
 			teleCommand = append(teleCommand, scrubClusterName(oldClusterName))
 
 			return cm.Rename(oldClusterName, gOpt, newClusterName)

components/cluster/command/restart.go

@@ -31,6 +31,7 @@ func newRestartCmd() *cobra.Command {
 			}
 
 			clusterName := args[0]
+			clusterReport.ID = scrubClusterName(clusterName)
 			teleCommand = append(teleCommand, scrubClusterName(clusterName))
 
 			return cm.RestartCluster(clusterName, gOpt)

components/cluster/command/root.go

@@ -45,17 +45,21 @@ import (
 )
 
 var (
-	errNS       = errorx.NewNamespace("cmd")
-	rootCmd     *cobra.Command
-	gOpt        operator.Options
-	skipConfirm bool
+	errNS         = errorx.NewNamespace("cmd")
+	rootCmd       *cobra.Command
+	gOpt          operator.Options
+	skipConfirm   bool
+	reportEnabled bool // is telemetry report enabled
 )
 
 var tidbSpec *spec.SpecManager
 var cm *manager.Manager
 
 func scrubClusterName(n string) string {
-	return "cluster_" + telemetry.HashReport(n)
+	// prepend the telemetry secret to cluster name, so that two installations
+	// of tiup with the same cluster name produce different hashes
+	cls := report.TelemetrySecret() + ":" + n
+	return "cluster_" + telemetry.HashReport(cls)
 }
 
 func getParentNames(cmd *cobra.Command) []string {
@@ -240,10 +244,12 @@ func Execute() {
 	teleReport = new(telemetry.Report)
 	clusterReport = new(telemetry.ClusterReport)
 	teleReport.EventDetail = &telemetry.Report_Cluster{Cluster: clusterReport}
-	if report.Enable() {
+	reportEnabled = report.Enabled()
+	if reportEnabled {
+		teleReport.InstallationUUID = report.TelemetryUUID()
 		teleReport.EventUUID = uuid.New().String()
 		teleReport.EventUnixTimestamp = time.Now().Unix()
-		clusterReport.UUID = report.UUID()
+		teleReport.Tiup = report.TiUPMeta()
 	}
 
 	start := time.Now()
@@ -255,20 +261,36 @@ func Execute() {
 
 	zap.L().Info("Execute command finished", zap.Int("code", code), zap.Error(err))
 
-	if report.Enable() {
+	if reportEnabled {
 		f := func() {
 			defer func() {
 				if r := recover(); r != nil {
 					if flags.DebugMode {
-						fmt.Println("Recovered in telemetry report", r)
+						log.Debugf("Recovered in telemetry report: %v", r)
 					}
 				}
 			}()
 
 			clusterReport.ExitCode = int32(code)
 			clusterReport.Nodes = teleNodeInfos
 			if teleTopology != "" {
-				if data, err := telemetry.ScrubYaml([]byte(teleTopology), map[string]struct{}{"host": {}}); err == nil {
+				if data, err := telemetry.ScrubYaml(
+					[]byte(teleTopology),
+					map[string]struct{}{
+						"host":       {},
+						"name":       {},
+						"user":       {},
+						"group":      {},
+						"deploy_dir": {},
+						"data_dir":   {},
+						"log_dir":    {},
+					}, // fields to hash
+					map[string]struct{}{
+						"config":         {},
+						"server_configs": {},
+					}, // fields to omit
+					report.TelemetrySecret(),
+				); err == nil {
 					clusterReport.Topology = (string(data))
 				}
 			}
@@ -283,7 +305,7 @@ func Execute() {
 				}
 				fmt.Printf("report: %s\n", teleReport.String())
 				if data, err := json.Marshal(teleReport); err == nil {
-					fmt.Printf("report: %s\n", string(data))
+					log.Debugf("report: %s\n", string(data))
 				}
 			}
 			cancel()

components/cluster/command/scale_in.go

@@ -32,6 +32,7 @@ func newScaleInCmd() *cobra.Command {
 			}
 
 			clusterName := args[0]
+			clusterReport.ID = scrubClusterName(clusterName)
 			teleCommand = append(teleCommand, scrubClusterName(clusterName))
 
 			scale := func(b *task.Builder, imetadata spec.Metadata, tlsCfg *tls.Config) {

components/cluster/command/scale_out.go

@@ -44,6 +44,7 @@ func newScaleOutCmd() *cobra.Command {
 			}
 
 			clusterName := args[0]
+			clusterReport.ID = scrubClusterName(clusterName)
 			teleCommand = append(teleCommand, scrubClusterName(clusterName))
 
 			topoFile := args[1]

components/cluster/command/start.go

@@ -33,6 +33,7 @@ func newStartCmd() *cobra.Command {
 			}
 
 			clusterName := args[0]
+			clusterReport.ID = scrubClusterName(clusterName)
 			teleCommand = append(teleCommand, scrubClusterName(clusterName))
 
 			return cm.StartCluster(clusterName, gOpt, func(b *task.Builder, metadata spec.Metadata) {

components/cluster/command/stop.go

@@ -31,6 +31,7 @@ func newStopCmd() *cobra.Command {
 			}
 
 			clusterName := args[0]
+			clusterReport.ID = scrubClusterName(clusterName)
 			teleCommand = append(teleCommand, scrubClusterName(clusterName))
 
 			return cm.StopCluster(clusterName, gOpt)

components/cluster/command/transfer.go

@@ -39,6 +39,7 @@ func newPullCmd() *cobra.Command {
 			clusterName := args[0]
 			opt.Remote = args[1]
 			opt.Local = args[2]
+			clusterReport.ID = scrubClusterName(clusterName)
 			teleCommand = append(teleCommand, scrubClusterName(clusterName))
 
 			return cm.Transfer(clusterName, opt, gOpt)
@@ -65,6 +66,7 @@ func newPushCmd() *cobra.Command {
 			clusterName := args[0]
 			opt.Local = args[1]
 			opt.Remote = args[2]
+			clusterReport.ID = scrubClusterName(clusterName)
 			teleCommand = append(teleCommand, scrubClusterName(clusterName))
 
 			return cm.Transfer(clusterName, opt, gOpt)

components/cluster/command/upgrade.go

@@ -34,6 +34,7 @@ func newUpgradeCmd() *cobra.Command {
 			if err != nil {
 				return err
 			}
+			clusterReport.ID = scrubClusterName(clusterName)
 			teleCommand = append(teleCommand, scrubClusterName(clusterName))
 			teleCommand = append(teleCommand, version)
 

pkg/cluster/flags/debug.go

@@ -26,5 +26,5 @@ var (
 
 func init() {
 	val := os.Getenv(localdata.EnvNameDebug)
-	DebugMode = val == "enable" || val == "enabled"
+	DebugMode = (val == "enable" || val == "enabled" || val == "true")
 }

pkg/cluster/report/report.go

@@ -16,24 +16,42 @@ package report
 import (
 	"bytes"
 	"os"
+	"runtime"
 
 	"github.com/gogo/protobuf/proto"
 	"github.com/pingcap/tiup/pkg/localdata"
 	tiuptele "github.com/pingcap/tiup/pkg/telemetry"
+	"github.com/pingcap/tiup/pkg/version"
 )
 
-// Enable return true if we enable telemetry.
-func Enable() bool {
+// Enabled return true if we enable telemetry.
+func Enabled() bool {
 	s := os.Getenv(localdata.EnvNameTelemetryStatus)
 	status := tiuptele.Status(s)
 	return status == tiuptele.EnableStatus
 }
 
-// UUID return telemetry uuid.
-func UUID() string {
+// TelemetryUUID return telemetry uuid.
+func TelemetryUUID() string {
 	return os.Getenv(localdata.EnvNameTelemetryUUID)
 }
 
+// TelemetrySecret return telemetry uuid.
+func TelemetrySecret() string {
+	return os.Getenv(localdata.EnvNameTelemetrySecret)
+}
+
+// TiUPMeta returns metadata of TiUP Cluster itself
+func TiUPMeta() *tiuptele.TiUPInfo {
+	return &tiuptele.TiUPInfo{
+		TiUPVersion:      os.Getenv(localdata.EnvNameTiUPVersion),
+		ComponentVersion: version.NewTiUPVersion().SemVer(),
+		GitCommit:        version.NewTiUPBuildInfo().GitHash,
+		Os:               runtime.GOOS,
+		Arch:             runtime.GOARCH,
+	}
+}
+
 // NodeInfoFromText get telemetry.NodeInfo from the text.
 func NodeInfoFromText(text string) (info *tiuptele.NodeInfo, err error) {
 	info = new(tiuptele.NodeInfo)

pkg/cluster/spec/validate.go

@@ -944,40 +944,22 @@ func (s *Specification) validateTiFlashConfigs() error {
 // Validate validates the topology specification and produce error if the
 // specification invalid (e.g: port conflicts or directory conflicts)
 func (s *Specification) Validate() error {
-	if err := s.validateTLSEnabled(); err != nil {
-		return err
-	}
-
-	if err := s.platformConflictsDetect(); err != nil {
-		return err
-	}
-
-	if err := s.portInvalidDetect(); err != nil {
-		return err
-	}
-
-	if err := s.portConflictsDetect(); err != nil {
-		return err
-	}
-
-	if err := s.dirConflictsDetect(); err != nil {
-		return err
-	}
-
-	if err := s.validateTiSparkSpec(); err != nil {
-		return err
-	}
-
-	if err := s.validateUserGroup(); err != nil {
-		return err
-	}
-
-	if err := s.validatePDNames(); err != nil {
-		return err
-	}
-
-	if err := s.validateTiFlashConfigs(); err != nil {
-		return err
+	validators := []func() error{
+		s.validateTLSEnabled,
+		s.platformConflictsDetect,
+		s.portInvalidDetect,
+		s.portConflictsDetect,
+		s.dirConflictsDetect,
+		s.validateUserGroup,
+		s.validatePDNames,
+		s.validateTiSparkSpec,
+		s.validateTiFlashConfigs,
+	}
+
+	for _, v := range validators {
+		if err := v(); err != nil {
+			return err
+		}
 	}
 
 	return RelativePathDetect(s, isSkipField)