Merge branch '1.5.1-airbnb-dev' of github.com:smineyev81/tidb-operato…

…r into 1.5.1-airbnb-dev
pingcap · Nov 15, 2023 · fbf1333 · fbf1333
2 parents 46a5ee5 + 249874c
commit fbf1333
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 4 deletions.
diff --git a/cmd/backup-manager/app/util/backup_size.go b/cmd/backup-manager/app/util/backup_size.go
@@ -292,9 +292,9 @@ func calculateChangedBlocksSize(volumeId, preSnapshotId, snapshotId string) (uin
 		// Each retry interval is around 1 second, and no more than 60 times retry (~1 minute)
 		backoff := wait.Backoff{
 			Duration: time.Second,
-			Steps:    60,
+			Steps:    600,
 			Factor:   1.0,
-			Jitter:   0.1,
+			Jitter:   0.5,
 		}
 
 		isAllChangeListed := false

diff --git a/pkg/fedvolumebackup/restore/restore_manager.go b/pkg/fedvolumebackup/restore/restore_manager.go
@@ -120,11 +120,17 @@ func (rm *restoreManager) syncRestore(volumeRestore *v1alpha1.VolumeRestore) err
 	if memberCreated {
 		return nil
 	}
-	rm.syncWarmUpStatus(volumeRestore, restoreMembers)
+
 	if err := rm.waitRestoreVolumeComplete(volumeRestore, restoreMembers); err != nil {
 		return err
 	}
 
+	rm.syncWarmUpStatus(volumeRestore, restoreMembers)
+
+	if err := rm.waitRestoreTiKVComplete(volumeRestore, restoreMembers); err != nil {
+		return err
+	}
+
 	memberUpdated, err := rm.executeRestoreDataPhase(ctx, volumeRestore, restoreMembers)
 	if err != nil {
 		return err
@@ -260,6 +266,30 @@ func (rm *restoreManager) waitRestoreVolumeComplete(volumeRestore *v1alpha1.Volu
 		rm.setVolumeRestoreVolumeComplete(volumeRestore)
 	}
 
+	return nil
+}
+
+func (rm *restoreManager) waitRestoreTiKVComplete(volumeRestore *v1alpha1.VolumeRestore, restoreMembers []*volumeRestoreMember) error {
+	// check if restore members failed in data plane
+	for _, restoreMember := range restoreMembers {
+		restoreMemberName := restoreMember.restore.Name
+		k8sClusterName := restoreMember.k8sClusterName
+		if pingcapv1alpha1.IsRestoreInvalid(restoreMember.restore) {
+			errMsg := fmt.Sprintf("restore member %s of cluster %s is invalid", restoreMemberName, k8sClusterName)
+			return &fedvolumebackup.BRDataPlaneFailedError{
+				Reason:  reasonVolumeRestoreMemberInvalid,
+				Message: errMsg,
+			}
+		}
+		if pingcapv1alpha1.IsRestoreFailed(restoreMember.restore) {
+			errMsg := fmt.Sprintf("restore member %s of cluster %s is failed", restoreMemberName, k8sClusterName)
+			return &fedvolumebackup.BRDataPlaneFailedError{
+				Reason:  reasonVolumeRestoreMemberFailed,
+				Message: errMsg,
+			}
+		}
+	}
+
 	for _, restoreMember := range restoreMembers {
 		restoreMemberName := restoreMember.restore.Name
 		k8sClusterName := restoreMember.k8sClusterName

diff --git a/pkg/fedvolumebackup/restore/restore_test.go b/pkg/fedvolumebackup/restore/restore_test.go
@@ -117,7 +117,7 @@ func (h *helper) assertRestoreVolumeComplete(volumeRestore *v1alpha1.VolumeResto
 func (h *helper) assertRestoreWarmUpStarted(volumeRestore *v1alpha1.VolumeRestore) {
 	h.g.Expect(volumeRestore.Status.Phase).To(gomega.Equal(v1alpha1.VolumeRestoreWarmUpStarted))
 	if volumeRestore.Spec.Template.Warmup == pingcapv1alpha1.RestoreWarmupModeASync {
-		// aysnc means start tikv and warm up simultaneously
+		// async means start tikv and warm up simultaneously
 		h.g.Expect(len(volumeRestore.Status.Steps)).To(gomega.Equal(3))
 	} else {
 		h.g.Expect(len(volumeRestore.Status.Steps)).To(gomega.Equal(2))