From 4064d7e0281eacc5ba762bc9b65365387a99cfcc Mon Sep 17 00:00:00 2001 From: BarkovBG Date: Wed, 27 Mar 2024 16:23:13 +0000 Subject: [PATCH 1/4] 831: [Disk Manager] validate crc32 for each block --- .../internal/pkg/clients/nbs/interface.go | 14 +++- .../pkg/clients/nbs/testing_client.go | 80 +++++++++++++++---- .../disk_service_test/disk_relocation_test.go | 44 +++++++--- .../disk_service_test/disk_service_test.go | 65 ++++++++++----- .../internal/pkg/facade/testcommon/common.go | 41 +++++++--- 5 files changed, 178 insertions(+), 66 deletions(-) diff --git a/cloud/disk_manager/internal/pkg/clients/nbs/interface.go b/cloud/disk_manager/internal/pkg/clients/nbs/interface.go index 3c3daedae3a..118cf3ef3f7 100644 --- a/cloud/disk_manager/internal/pkg/clients/nbs/interface.go +++ b/cloud/disk_manager/internal/pkg/clients/nbs/interface.go @@ -294,25 +294,33 @@ type Client interface { ) error // Used in tests. - ValidateCrc32(diskID string, contentSize uint64, expectedCrc32 uint32) error + ValidateCrc32( + ctx context.Context, + diskID string, + contentSize uint64, + expectedCrc32 uint32, + expectedBlocksCrc32 []uint32, + ) error // Used in tests. ValidateCrc32WithEncryption( + ctx context.Context, diskID string, contentSize uint64, encryption *types.EncryptionDesc, expectedCrc32 uint32, + expectedBlocksCrc32 []uint32, ) error // Used in tests. - CalculateCrc32(diskID string, contentSize uint64) (uint32, error) + CalculateCrc32(diskID string, contentSize uint64) (uint32, []uint32, error) // Used in tests. CalculateCrc32WithEncryption( diskID string, contentSize uint64, encryption *types.EncryptionDesc, - ) (uint32, error) + ) (uint32, []uint32, error) // Used in tests. MountForReadWrite(diskID string) (func(), error) diff --git a/cloud/disk_manager/internal/pkg/clients/nbs/testing_client.go b/cloud/disk_manager/internal/pkg/clients/nbs/testing_client.go index 65c07234834..21c573595c9 100644 --- a/cloud/disk_manager/internal/pkg/clients/nbs/testing_client.go +++ b/cloud/disk_manager/internal/pkg/clients/nbs/testing_client.go @@ -8,6 +8,7 @@ import ( "github.com/ydb-platform/nbs/cloud/blockstore/public/api/protos" nbs_client "github.com/ydb-platform/nbs/cloud/blockstore/public/sdk/go/client" "github.com/ydb-platform/nbs/cloud/disk_manager/internal/pkg/types" + "github.com/ydb-platform/nbs/cloud/tasks/logging" ) //////////////////////////////////////////////////////////////////////////////// @@ -15,7 +16,7 @@ import ( func (c *client) CalculateCrc32( diskID string, contentSize uint64, -) (uint32, error) { +) (uint32, []uint32, error) { return c.CalculateCrc32WithEncryption(diskID, contentSize, nil) } @@ -24,13 +25,13 @@ func (c *client) CalculateCrc32WithEncryption( diskID string, contentSize uint64, encryption *types.EncryptionDesc, -) (uint32, error) { +) (uint32, []uint32, error) { ctx := setupStderrLogger(context.Background()) nbsClient, _, err := c.nbs.DiscoverInstance(ctx) if err != nil { - return 0, err + return 0, []uint32{}, err } defer nbsClient.Close() @@ -42,7 +43,7 @@ func (c *client) CalculateCrc32WithEncryption( encryptionSpec, err := getEncryptionSpec(encryption) if err != nil { - return 0, err + return 0, []uint32{}, err } opts := nbs_client.MountVolumeOpts{ @@ -54,7 +55,7 @@ func (c *client) CalculateCrc32WithEncryption( } err = session.MountVolume(ctx, diskID, &opts) if err != nil { - return 0, err + return 0, []uint32{}, err } defer session.UnmountVolume(ctx) @@ -62,14 +63,14 @@ func (c *client) CalculateCrc32WithEncryption( volumeBlockSize := uint64(volume.BlockSize) if volumeBlockSize == 0 { - return 0, fmt.Errorf( + return 0, []uint32{}, fmt.Errorf( "%v volume block size should not be zero", diskID, ) } if contentSize%volumeBlockSize != 0 { - return 0, fmt.Errorf( + return 0, []uint32{}, fmt.Errorf( "%v contentSize %v should be multiple of volumeBlockSize %v", diskID, contentSize, @@ -81,7 +82,7 @@ func (c *client) CalculateCrc32WithEncryption( volumeSize := volume.BlocksCount * volumeBlockSize if contentSize > volumeSize { - return 0, fmt.Errorf( + return 0, []uint32{}, fmt.Errorf( "%v contentSize %v should not be greater than volumeSize %v", diskID, contentSize, @@ -92,12 +93,13 @@ func (c *client) CalculateCrc32WithEncryption( chunkSize := uint64(4 * 1024 * 1024) blocksInChunk := chunkSize / volumeBlockSize acc := crc32.NewIEEE() + blocksCrc32 := []uint32{} for offset := uint64(0); offset < contentBlocksCount; offset += blocksInChunk { blocksToRead := min(contentBlocksCount-offset, blocksInChunk) buffers, err := session.ReadBlocks(ctx, offset, uint32(blocksToRead), "") if err != nil { - return 0, fmt.Errorf( + return 0, []uint32{}, fmt.Errorf( "%v read blocks at (%v, %v) failed: %w", diskID, offset, @@ -107,14 +109,22 @@ func (c *client) CalculateCrc32WithEncryption( } for _, buffer := range buffers { + blockAcc := crc32.NewIEEE() if len(buffer) == 0 { buffer = make([]byte, volumeBlockSize) } _, err := acc.Write(buffer) if err != nil { - return 0, err + return 0, []uint32{}, err } + + _, err = blockAcc.Write(buffer) + if err != nil { + return 0, []uint32{}, err + } + + blocksCrc32 = append(blocksCrc32, blockAcc.Sum32()) } } @@ -123,7 +133,7 @@ func (c *client) CalculateCrc32WithEncryption( blocksToRead := min(volume.BlocksCount-offset, blocksInChunk) buffers, err := session.ReadBlocks(ctx, offset, uint32(blocksToRead), "") if err != nil { - return 0, fmt.Errorf( + return 0, []uint32{}, fmt.Errorf( "%v read blocks at (%v, %v) failed: %w", diskID, offset, @@ -136,7 +146,7 @@ func (c *client) CalculateCrc32WithEncryption( if len(buffer) != 0 { for j, b := range buffer { if b != 0 { - return 0, fmt.Errorf( + return 0, []uint32{}, fmt.Errorf( "%v non zero byte %v detected at (%v, %v)", diskID, b, @@ -149,33 +159,71 @@ func (c *client) CalculateCrc32WithEncryption( } } - return acc.Sum32(), nil + return acc.Sum32(), blocksCrc32, nil } func (c *client) ValidateCrc32( + ctx context.Context, diskID string, contentSize uint64, expectedCrc32 uint32, + expectedBlocksCrc32 []uint32, ) error { - return c.ValidateCrc32WithEncryption(diskID, contentSize, nil, expectedCrc32) + return c.ValidateCrc32WithEncryption( + ctx, + diskID, + contentSize, + nil, + expectedCrc32, + expectedBlocksCrc32, + ) } func (c *client) ValidateCrc32WithEncryption( + ctx context.Context, diskID string, contentSize uint64, encryption *types.EncryptionDesc, expectedCrc32 uint32, + expectedBlocksCrc32 []uint32, ) error { - actualCrc32, err := c.CalculateCrc32WithEncryption(diskID, contentSize, encryption) + actualCrc32, actualBlocksCrc32, err := c.CalculateCrc32WithEncryption( + diskID, + contentSize, + encryption, + ) if err != nil { return err } + if len(actualBlocksCrc32) != len(expectedBlocksCrc32) { + logging.Debug( + ctx, + "%v blocksCrc32 length doesn't match, expected %v, actual %v", + diskID, + len(expectedBlocksCrc32), + len(actualBlocksCrc32), + ) + } else { + for i := range expectedBlocksCrc32 { + if actualBlocksCrc32[i] != expectedBlocksCrc32[i] { + logging.Debug( + ctx, + "%v block with index %v crc32 doesn't match, expected %v, actual %v", + diskID, + i, + expectedBlocksCrc32[i], + actualBlocksCrc32[i], + ) + } + } + } + if expectedCrc32 != actualCrc32 { return fmt.Errorf( - "%v crc32 doesn't match, expected=%v, actual=%v", + "%v crc32 doesn't match, expected %v, actual %v", diskID, expectedCrc32, actualCrc32, diff --git a/cloud/disk_manager/internal/pkg/facade/disk_service_test/disk_relocation_test.go b/cloud/disk_manager/internal/pkg/facade/disk_service_test/disk_relocation_test.go index 33846e865c5..6469e11e5a4 100644 --- a/cloud/disk_manager/internal/pkg/facade/disk_service_test/disk_relocation_test.go +++ b/cloud/disk_manager/internal/pkg/facade/disk_service_test/disk_relocation_test.go @@ -59,7 +59,7 @@ func setupMigrationTest( if params.FillDisk { nbsClient := testcommon.NewNbsClient(t, ctx, params.SrcZoneID) - _, _, err = testcommon.FillDisk( + _, _, _, err = testcommon.FillDisk( nbsClient, params.DiskID, uint64(params.DiskSize), @@ -113,7 +113,7 @@ func successfullyMigrateDisk( diskSize := uint64(params.DiskSize) - srcCrc32, err := srcZoneNBSClient.CalculateCrc32(params.DiskID, diskSize) + srcCrc32, srcBlocksCrc32, err := srcZoneNBSClient.CalculateCrc32(params.DiskID, diskSize) require.NoError(t, err) err = client.SendMigrationSignal(ctx, &disk_manager.SendMigrationSignalRequest{ @@ -145,9 +145,15 @@ func successfullyMigrateDisk( require.ErrorContains(t, err, "Path not found") dstZoneNBSClient := testcommon.NewNbsClient(t, ctx, params.DstZoneID) - dstCrc32, err := dstZoneNBSClient.CalculateCrc32(params.DiskID, diskSize) + + err = dstZoneNBSClient.ValidateCrc32( + ctx, + params.DiskID, + diskSize, + srcCrc32, + srcBlocksCrc32, + ) require.NoError(t, err) - require.Equal(t, srcCrc32, dstCrc32) } func startAndCancelMigration( @@ -325,7 +331,7 @@ func migrateDiskInParallel( diskSize := uint64(params.DiskSize) - srcCrc32, err := srcZoneNBSClient.CalculateCrc32(params.DiskID, diskSize) + srcCrc32, srcBlocksCrc32, err := srcZoneNBSClient.CalculateCrc32(params.DiskID, diskSize) require.NoError(t, err) for _, operation := range operations { @@ -378,14 +384,22 @@ func migrateDiskInParallel( require.ErrorContains(t, err, "Path not found") dstZoneNBSClient := testcommon.NewNbsClient(t, ctx, dstZoneID) - dstCrc32, err := dstZoneNBSClient.CalculateCrc32(params.DiskID, diskSize) + err := dstZoneNBSClient.ValidateCrc32( + params.DiskID, + diskSize, + srcCrc32, + srcBlocksCrc32, + ) require.NoError(t, err) - require.Equal(t, srcCrc32, dstCrc32) } else { // All migrations are cancelled. Check that src disk is not affected. - crc32, err := srcZoneNBSClient.CalculateCrc32(params.DiskID, diskSize) + err := srcZoneNBSClient.ValidateCrc32( + params.DiskID, + diskSize, + srcCrc32, + srcBlocksCrc32, + ) require.NoError(t, err) - require.Equal(t, srcCrc32, crc32) } } @@ -421,7 +435,7 @@ func successfullyMigrateEmptyDisk( diskSize := uint64(migrationTestsDiskSize) - srcCrc32, err := srcZoneNBSClient.CalculateCrc32(params.DiskID, diskSize) + srcCrc32, blocksCrc32, err := srcZoneNBSClient.CalculateCrc32(params.DiskID, diskSize) require.NoError(t, err) metadata := &disk_manager.MigrateDiskMetadata{} @@ -443,9 +457,13 @@ func successfullyMigrateEmptyDisk( require.ErrorContains(t, err, "Path not found") dstZoneNBSClient := testcommon.NewNbsClient(t, ctx, params.DstZoneID) - dstCrc32, err := dstZoneNBSClient.CalculateCrc32(params.DiskID, diskSize) + err = dstZoneNBSClient.ValidateCrc32( + params.DiskID, + diskSize, + srcCrc32, + blocksCrc32, + ) require.NoError(t, err) - require.Equal(t, srcCrc32, dstCrc32) } func waitForMigrationStatus( @@ -703,7 +721,7 @@ func TestDiskServiceMigrateOverlayDisk(t *testing.T) { imageID := t.Name() diskSize := migrationTestsDiskSize imageSize := diskSize / 2 - _, _ = testcommon.CreateImage( + _, _, _ = testcommon.CreateImage( t, ctx, imageID, diff --git a/cloud/disk_manager/internal/pkg/facade/disk_service_test/disk_service_test.go b/cloud/disk_manager/internal/pkg/facade/disk_service_test/disk_service_test.go index 74ec9380863..4925bd8b5f9 100644 --- a/cloud/disk_manager/internal/pkg/facade/disk_service_test/disk_service_test.go +++ b/cloud/disk_manager/internal/pkg/facade/disk_service_test/disk_service_test.go @@ -134,6 +134,7 @@ func TestDiskServiceCreateDiskFromImageWithForceNotLayered(t *testing.T) { diskID, testcommon.GetRawImageSize(t), testcommon.GetRawImageCrc32(t), + []uint32{}, // We do not known expectedBlocksCrc32 for image. ) require.NoError(t, err) @@ -150,7 +151,7 @@ func TestDiskServiceCancelCreateDiskFromImage(t *testing.T) { imageID := t.Name() imageSize := uint64(64 * 1024 * 1024) - _, _ = testcommon.CreateImage( + _, _, _ = testcommon.CreateImage( t, ctx, imageID, @@ -191,7 +192,7 @@ func TestDiskServiceDeleteDiskWhenCreationIsInFlight(t *testing.T) { imageID := t.Name() imageSize := uint64(64 * 1024 * 1024) - _, _ = testcommon.CreateImage( + _, _, _ = testcommon.CreateImage( t, ctx, imageID, @@ -249,7 +250,7 @@ func TestDiskServiceCreateDisksFromImageWithConfiguredPool(t *testing.T) { imageID := t.Name() imageSize := uint64(64 * 1024 * 1024) - imageCrc32, _ := testcommon.CreateImage( + imageCrc32, blocksCrc32, _ := testcommon.CreateImage( t, ctx, imageID, @@ -305,7 +306,12 @@ func TestDiskServiceCreateDisksFromImageWithConfiguredPool(t *testing.T) { require.NoError(t, err) diskID := fmt.Sprintf("%v%v", t.Name(), i) - err = nbsClient.ValidateCrc32(diskID, imageSize, imageCrc32) + err = nbsClient.ValidateCrc32( + diskID, + imageSize, + imageCrc32, + blocksCrc32, + ) require.NoError(t, err) diskParams, err := nbsClient.Describe(ctx, diskID) @@ -376,7 +382,7 @@ func testCreateDiskFromIncrementalSnapshot( require.NoError(t, err) nbsClient := testcommon.NewNbsClient(t, ctx, "zone-a") - _, _, err = testcommon.FillDisk(nbsClient, diskID1, diskSize) + _, _, _, err = testcommon.FillDisk(nbsClient, diskID1, diskSize) require.NoError(t, err) snapshotID1 := t.Name() + "1" @@ -395,7 +401,7 @@ func testCreateDiskFromIncrementalSnapshot( err = internal_client.WaitOperation(ctx, client, operation.Id) require.NoError(t, err) - _, _, err = testcommon.FillDisk(nbsClient, diskID1, diskSize) + _, _, _, err = testcommon.FillDisk(nbsClient, diskID1, diskSize) require.NoError(t, err) snapshotID2 := t.Name() + "2" @@ -433,10 +439,10 @@ func testCreateDiskFromIncrementalSnapshot( err = internal_client.WaitOperation(ctx, client, operation.Id) require.NoError(t, err) - crc32, err := nbsClient.CalculateCrc32(diskID1, diskSize) + crc32, blocksCrc32, err := nbsClient.CalculateCrc32(diskID1, diskSize) require.NoError(t, err) - err = nbsClient.ValidateCrc32(diskID2, diskSize, crc32) + err = nbsClient.ValidateCrc32(diskID2, diskSize, crc32, blocksCrc32) require.NoError(t, err) testcommon.CheckConsistency(t, ctx) @@ -486,7 +492,7 @@ func TestDiskServiceCreateDiskFromSnapshot(t *testing.T) { require.NoError(t, err) nbsClient := testcommon.NewNbsClient(t, ctx, "zone-a") - crc32, _, err := testcommon.FillDisk(nbsClient, diskID1, diskSize) + crc32, blocksCrc32, _, err := testcommon.FillDisk(nbsClient, diskID1, diskSize) require.NoError(t, err) snapshotID1 := t.Name() + "1" @@ -534,10 +540,10 @@ func TestDiskServiceCreateDiskFromSnapshot(t *testing.T) { require.NoError(t, err) require.Equal(t, float64(1), diskMeta.Progress) - err = nbsClient.ValidateCrc32(diskID1, diskSize, crc32) + err = nbsClient.ValidateCrc32(diskID1, diskSize, crc32, blocksCrc32) require.NoError(t, err) - err = nbsClient.ValidateCrc32(diskID2, diskSize, crc32) + err = nbsClient.ValidateCrc32(diskID2, diskSize, crc32, blocksCrc32) require.NoError(t, err) testcommon.CheckConsistency(t, ctx) @@ -571,7 +577,7 @@ func TestDiskServiceCreateDiskFromImage(t *testing.T) { require.NoError(t, err) nbsClient := testcommon.NewNbsClient(t, ctx, "zone-a") - crc32, _, err := testcommon.FillDisk(nbsClient, diskID1, diskSize) + crc32, blocksCrc32, _, err := testcommon.FillDisk(nbsClient, diskID1, diskSize) require.NoError(t, err) imageID := t.Name() + "_image" @@ -616,10 +622,10 @@ func TestDiskServiceCreateDiskFromImage(t *testing.T) { err = internal_client.WaitOperation(ctx, client, operation.Id) require.NoError(t, err) - err = nbsClient.ValidateCrc32(diskID1, diskSize, crc32) + err = nbsClient.ValidateCrc32(diskID1, diskSize, crc32, blocksCrc32) require.NoError(t, err) - err = nbsClient.ValidateCrc32(diskID2, diskSize, crc32) + err = nbsClient.ValidateCrc32(diskID2, diskSize, crc32, blocksCrc32) require.NoError(t, err) testcommon.CheckConsistency(t, ctx) @@ -655,7 +661,7 @@ func TestDiskServiceCreateDiskFromSnapshotOfOverlayDisk(t *testing.T) { nbsClient := testcommon.NewNbsClient(t, ctx, "zone-a") - crc32, _, err := testcommon.FillDisk(nbsClient, diskID1, imageSize) + crc32, blocksCrc32, _, err := testcommon.FillDisk(nbsClient, diskID1, imageSize) require.NoError(t, err) imageID := t.Name() + "_image" @@ -748,13 +754,13 @@ func TestDiskServiceCreateDiskFromSnapshotOfOverlayDisk(t *testing.T) { err = internal_client.WaitOperation(ctx, client, operation.Id) require.NoError(t, err) - err = nbsClient.ValidateCrc32(diskID1, imageSize, crc32) + err = nbsClient.ValidateCrc32(diskID1, imageSize, crc32, blocksCrc32) require.NoError(t, err) - err = nbsClient.ValidateCrc32(diskID2, imageSize, crc32) + err = nbsClient.ValidateCrc32(diskID2, imageSize, crc32, blocksCrc32) require.NoError(t, err) - err = nbsClient.ValidateCrc32(diskID3, imageSize, crc32) + err = nbsClient.ValidateCrc32(diskID3, imageSize, crc32, blocksCrc32) require.NoError(t, err) testcommon.CheckConsistency(t, ctx) @@ -1048,7 +1054,12 @@ func TestDiskServiceCreateEncryptedDiskFromSnapshot(t *testing.T) { encryptionDesc, err := disks.PrepareEncryptionDesc(encryption) require.NoError(t, err) - crc32, _, err := testcommon.FillEncryptedDisk(nbsClient, diskID1, diskSize, encryptionDesc) + crc32, blocksCrc32, _, err := testcommon.FillEncryptedDisk( + nbsClient, + diskID1, + diskSize, + encryptionDesc, + ) require.NoError(t, err) diskParams1, err = nbsClient.Describe(ctx, diskID1) @@ -1135,9 +1146,21 @@ func TestDiskServiceCreateEncryptedDiskFromSnapshot(t *testing.T) { require.Error(t, err) require.ErrorContains(t, err, "encryption mode should be the same") - err = nbsClient.ValidateCrc32WithEncryption(diskID1, diskSize, encryptionDesc, crc32) + err = nbsClient.ValidateCrc32WithEncryption( + diskID1, + diskSize, + encryptionDesc, + crc32, + blocksCrc32, + ) require.NoError(t, err) - err = nbsClient.ValidateCrc32WithEncryption(diskID2, diskSize, encryptionDesc, crc32) + err = nbsClient.ValidateCrc32WithEncryption( + diskID2, + diskSize, + encryptionDesc, + crc32, + blocksCrc32, + ) require.NoError(t, err) testcommon.CheckConsistency(t, ctx) diff --git a/cloud/disk_manager/internal/pkg/facade/testcommon/common.go b/cloud/disk_manager/internal/pkg/facade/testcommon/common.go index 4248a0d39eb..2804f4050be 100644 --- a/cloud/disk_manager/internal/pkg/facade/testcommon/common.go +++ b/cloud/disk_manager/internal/pkg/facade/testcommon/common.go @@ -359,7 +359,7 @@ func FillDisk( nbsClient nbs.Client, diskID string, diskSize uint64, -) (uint32, uint64, error) { +) (uint32, []uint32, uint64, error) { return FillEncryptedDisk(nbsClient, diskID, diskSize, nil) } @@ -369,7 +369,7 @@ func FillEncryptedDisk( diskID string, diskSize uint64, encryption *types.EncryptionDesc, -) (uint32, uint64, error) { +) (uint32, []uint32, uint64, error) { ctx := NewContext() @@ -381,21 +381,22 @@ func FillEncryptedDisk( encryption, ) if err != nil { - return 0, 0, err + return 0, []uint32{}, 0, err } defer session.Close(ctx) chunkSize := uint64(1024 * 4096) // 4 MiB blockSize := uint64(session.BlockSize()) blocksInChunk := uint32(chunkSize / blockSize) - acc := crc32.NewIEEE() storageSize := uint64(0) zeroes := make([]byte, chunkSize) rand.Seed(time.Now().UnixNano()) + acc := crc32.NewIEEE() + blocksCrc32 := []uint32{} for offset := uint64(0); offset < diskSize; offset += chunkSize { - blockIndex := offset / blockSize + startIndex := offset / blockSize data := make([]byte, chunkSize) dice := rand.Intn(3) @@ -407,22 +408,36 @@ func FillEncryptedDisk( logging.Debug(ctx, "rand generated all zeroes") } - err = session.Write(ctx, blockIndex, data) + err = session.Write(ctx, startIndex, data) storageSize += chunkSize case 1: - err = session.Zero(ctx, blockIndex, blocksInChunk) + err = session.Zero(ctx, startIndex, blocksInChunk) } if err != nil { - return 0, 0, err + return 0, []uint32{}, 0, err } _, err = acc.Write(data) if err != nil { - return 0, 0, err + return 0, []uint32{}, 0, err + } + + for blockIndex := uint32(0); blockIndex < blocksInChunk; blockIndex++ { + blockAcc := crc32.NewIEEE() + + startOffset := uint64(blockIndex) * blockSize + endOffset := uint64(blockIndex+1) * blockSize + blockData := data[startOffset:endOffset] + + _, err = blockAcc.Write(blockData) + if err != nil { + return 0, []uint32{}, 0, err + } + blocksCrc32 = append(blocksCrc32, blockAcc.Sum32()) } } - return acc.Sum32(), storageSize, nil + return acc.Sum32(), blocksCrc32, storageSize, nil } func GoWriteRandomBlocksToNbsDisk( @@ -495,7 +510,7 @@ func CreateImage( imageSize uint64, folderID string, pooled bool, -) (crc32 uint32, storageSize uint64) { +) (uint32, []uint32, uint64) { client, err := NewClient(ctx) require.NoError(t, err) @@ -521,7 +536,7 @@ func CreateImage( require.NoError(t, err) nbsClient := NewNbsClient(t, ctx, "zone-a") - crc32, storageSize, err = FillDisk(nbsClient, diskID, imageSize) + crc32, blocksCrc32, storageSize, err := FillDisk(nbsClient, diskID, imageSize) require.NoError(t, err) reqCtx = GetRequestContext(t, ctx) @@ -545,7 +560,7 @@ func CreateImage( require.Equal(t, int64(imageSize), response.Size) require.Equal(t, int64(storageSize), response.StorageSize) - return crc32, storageSize + return crc32, blocksCrc32, storageSize } //////////////////////////////////////////////////////////////////////////////// From d3fe81ae479af1e52665306d5cdfe9882dca1650 Mon Sep 17 00:00:00 2001 From: BarkovBG Date: Thu, 28 Mar 2024 13:30:55 +0000 Subject: [PATCH 2/4] log new crc32 for each block after writing --- .../disk_service_test/disk_relocation_test.go | 7 ++++-- .../disk_service_test/disk_service_test.go | 20 +++++++++------- .../internal/pkg/facade/testcommon/common.go | 24 ++++++++++++++++++- 3 files changed, 40 insertions(+), 11 deletions(-) diff --git a/cloud/disk_manager/internal/pkg/facade/disk_service_test/disk_relocation_test.go b/cloud/disk_manager/internal/pkg/facade/disk_service_test/disk_relocation_test.go index 6469e11e5a4..a72dd79c9f1 100644 --- a/cloud/disk_manager/internal/pkg/facade/disk_service_test/disk_relocation_test.go +++ b/cloud/disk_manager/internal/pkg/facade/disk_service_test/disk_relocation_test.go @@ -385,6 +385,7 @@ func migrateDiskInParallel( dstZoneNBSClient := testcommon.NewNbsClient(t, ctx, dstZoneID) err := dstZoneNBSClient.ValidateCrc32( + ctx, params.DiskID, diskSize, srcCrc32, @@ -394,6 +395,7 @@ func migrateDiskInParallel( } else { // All migrations are cancelled. Check that src disk is not affected. err := srcZoneNBSClient.ValidateCrc32( + ctx, params.DiskID, diskSize, srcCrc32, @@ -435,7 +437,7 @@ func successfullyMigrateEmptyDisk( diskSize := uint64(migrationTestsDiskSize) - srcCrc32, blocksCrc32, err := srcZoneNBSClient.CalculateCrc32(params.DiskID, diskSize) + srcCrc32, srcBlocksCrc32, err := srcZoneNBSClient.CalculateCrc32(params.DiskID, diskSize) require.NoError(t, err) metadata := &disk_manager.MigrateDiskMetadata{} @@ -458,10 +460,11 @@ func successfullyMigrateEmptyDisk( dstZoneNBSClient := testcommon.NewNbsClient(t, ctx, params.DstZoneID) err = dstZoneNBSClient.ValidateCrc32( + ctx, params.DiskID, diskSize, srcCrc32, - blocksCrc32, + srcBlocksCrc32, ) require.NoError(t, err) } diff --git a/cloud/disk_manager/internal/pkg/facade/disk_service_test/disk_service_test.go b/cloud/disk_manager/internal/pkg/facade/disk_service_test/disk_service_test.go index 4925bd8b5f9..ce8ba956e4c 100644 --- a/cloud/disk_manager/internal/pkg/facade/disk_service_test/disk_service_test.go +++ b/cloud/disk_manager/internal/pkg/facade/disk_service_test/disk_service_test.go @@ -131,6 +131,7 @@ func TestDiskServiceCreateDiskFromImageWithForceNotLayered(t *testing.T) { nbsClient := testcommon.NewNbsClient(t, ctx, "zone-a") err = nbsClient.ValidateCrc32( + ctx, diskID, testcommon.GetRawImageSize(t), testcommon.GetRawImageCrc32(t), @@ -307,6 +308,7 @@ func TestDiskServiceCreateDisksFromImageWithConfiguredPool(t *testing.T) { diskID := fmt.Sprintf("%v%v", t.Name(), i) err = nbsClient.ValidateCrc32( + ctx, diskID, imageSize, imageCrc32, @@ -442,7 +444,7 @@ func testCreateDiskFromIncrementalSnapshot( crc32, blocksCrc32, err := nbsClient.CalculateCrc32(diskID1, diskSize) require.NoError(t, err) - err = nbsClient.ValidateCrc32(diskID2, diskSize, crc32, blocksCrc32) + err = nbsClient.ValidateCrc32(ctx, diskID2, diskSize, crc32, blocksCrc32) require.NoError(t, err) testcommon.CheckConsistency(t, ctx) @@ -540,10 +542,10 @@ func TestDiskServiceCreateDiskFromSnapshot(t *testing.T) { require.NoError(t, err) require.Equal(t, float64(1), diskMeta.Progress) - err = nbsClient.ValidateCrc32(diskID1, diskSize, crc32, blocksCrc32) + err = nbsClient.ValidateCrc32(ctx, diskID1, diskSize, crc32, blocksCrc32) require.NoError(t, err) - err = nbsClient.ValidateCrc32(diskID2, diskSize, crc32, blocksCrc32) + err = nbsClient.ValidateCrc32(ctx, diskID2, diskSize, crc32, blocksCrc32) require.NoError(t, err) testcommon.CheckConsistency(t, ctx) @@ -622,10 +624,10 @@ func TestDiskServiceCreateDiskFromImage(t *testing.T) { err = internal_client.WaitOperation(ctx, client, operation.Id) require.NoError(t, err) - err = nbsClient.ValidateCrc32(diskID1, diskSize, crc32, blocksCrc32) + err = nbsClient.ValidateCrc32(ctx, diskID1, diskSize, crc32, blocksCrc32) require.NoError(t, err) - err = nbsClient.ValidateCrc32(diskID2, diskSize, crc32, blocksCrc32) + err = nbsClient.ValidateCrc32(ctx, diskID2, diskSize, crc32, blocksCrc32) require.NoError(t, err) testcommon.CheckConsistency(t, ctx) @@ -754,13 +756,13 @@ func TestDiskServiceCreateDiskFromSnapshotOfOverlayDisk(t *testing.T) { err = internal_client.WaitOperation(ctx, client, operation.Id) require.NoError(t, err) - err = nbsClient.ValidateCrc32(diskID1, imageSize, crc32, blocksCrc32) + err = nbsClient.ValidateCrc32(ctx, diskID1, imageSize, crc32, blocksCrc32) require.NoError(t, err) - err = nbsClient.ValidateCrc32(diskID2, imageSize, crc32, blocksCrc32) + err = nbsClient.ValidateCrc32(ctx, diskID2, imageSize, crc32, blocksCrc32) require.NoError(t, err) - err = nbsClient.ValidateCrc32(diskID3, imageSize, crc32, blocksCrc32) + err = nbsClient.ValidateCrc32(ctx, diskID3, imageSize, crc32, blocksCrc32) require.NoError(t, err) testcommon.CheckConsistency(t, ctx) @@ -1147,6 +1149,7 @@ func TestDiskServiceCreateEncryptedDiskFromSnapshot(t *testing.T) { require.ErrorContains(t, err, "encryption mode should be the same") err = nbsClient.ValidateCrc32WithEncryption( + ctx, diskID1, diskSize, encryptionDesc, @@ -1155,6 +1158,7 @@ func TestDiskServiceCreateEncryptedDiskFromSnapshot(t *testing.T) { ) require.NoError(t, err) err = nbsClient.ValidateCrc32WithEncryption( + ctx, diskID2, diskSize, encryptionDesc, diff --git a/cloud/disk_manager/internal/pkg/facade/testcommon/common.go b/cloud/disk_manager/internal/pkg/facade/testcommon/common.go index 2804f4050be..708d9f08306 100644 --- a/cloud/disk_manager/internal/pkg/facade/testcommon/common.go +++ b/cloud/disk_manager/internal/pkg/facade/testcommon/common.go @@ -433,6 +433,14 @@ func FillEncryptedDisk( if err != nil { return 0, []uint32{}, 0, err } + + logging.Debug( + ctx, + "%v block with index %v crc32 now is %v", + diskID, + startIndex+uint64(blockIndex), + blockAcc.Sum32(), + ) blocksCrc32 = append(blocksCrc32, blockAcc.Sum32()) } } @@ -476,10 +484,11 @@ func GoWriteRandomBlocksToNbsDisk( dice := rand.Intn(2) var err error + blockAcc := crc32.NewIEEE() + data := make([]byte, blockSize) switch dice { case 0: - data := make([]byte, blockSize) rand.Read(data) if bytes.Equal(data, zeroes) { logging.Debug(ctx, "rand generated all zeroes") @@ -493,6 +502,19 @@ func GoWriteRandomBlocksToNbsDisk( if err != nil { return err } + + _, err = blockAcc.Write(data) + if err != nil { + return err + } + + logging.Debug( + ctx, + "%v block with index %v crc32 now is %v", + diskID, + blockIndex, + blockAcc.Sum32(), + ) } return nil From d1b09a39b3355855716b674d9682bf6e6e596124 Mon Sep 17 00:00:00 2001 From: BarkovBG Date: Thu, 28 Mar 2024 16:22:46 +0000 Subject: [PATCH 3/4] return structure --- .../internal/pkg/clients/nbs/interface.go | 20 +++-- .../pkg/clients/nbs/testing_client.go | 73 ++++++++++--------- .../disk_service_test/disk_relocation_test.go | 26 +++---- .../disk_service_test/disk_service_test.go | 57 +++++++-------- .../internal/pkg/facade/testcommon/common.go | 39 +++++----- 5 files changed, 108 insertions(+), 107 deletions(-) diff --git a/cloud/disk_manager/internal/pkg/clients/nbs/interface.go b/cloud/disk_manager/internal/pkg/clients/nbs/interface.go index 118cf3ef3f7..c757513aaed 100644 --- a/cloud/disk_manager/internal/pkg/clients/nbs/interface.go +++ b/cloud/disk_manager/internal/pkg/clients/nbs/interface.go @@ -113,6 +113,14 @@ type CheckpointParams struct { CheckpointType CheckpointType } +// Used in tests. +type DiskContentInfo struct { + ContentSize uint64 + StorageSize uint64 + Crc32 uint32 + BlockCrc32s []uint32 +} + //////////////////////////////////////////////////////////////////////////////// type Client interface { @@ -297,30 +305,26 @@ type Client interface { ValidateCrc32( ctx context.Context, diskID string, - contentSize uint64, - expectedCrc32 uint32, - expectedBlocksCrc32 []uint32, + expectedDiskContentInfo DiskContentInfo, ) error // Used in tests. ValidateCrc32WithEncryption( ctx context.Context, diskID string, - contentSize uint64, + expectedDiskContentInfo DiskContentInfo, encryption *types.EncryptionDesc, - expectedCrc32 uint32, - expectedBlocksCrc32 []uint32, ) error // Used in tests. - CalculateCrc32(diskID string, contentSize uint64) (uint32, []uint32, error) + CalculateCrc32(diskID string, contentSize uint64) (DiskContentInfo, error) // Used in tests. CalculateCrc32WithEncryption( diskID string, contentSize uint64, encryption *types.EncryptionDesc, - ) (uint32, []uint32, error) + ) (DiskContentInfo, error) // Used in tests. MountForReadWrite(diskID string) (func(), error) diff --git a/cloud/disk_manager/internal/pkg/clients/nbs/testing_client.go b/cloud/disk_manager/internal/pkg/clients/nbs/testing_client.go index 21c573595c9..ca03e1824a0 100644 --- a/cloud/disk_manager/internal/pkg/clients/nbs/testing_client.go +++ b/cloud/disk_manager/internal/pkg/clients/nbs/testing_client.go @@ -16,7 +16,7 @@ import ( func (c *client) CalculateCrc32( diskID string, contentSize uint64, -) (uint32, []uint32, error) { +) (DiskContentInfo, error) { return c.CalculateCrc32WithEncryption(diskID, contentSize, nil) } @@ -25,13 +25,13 @@ func (c *client) CalculateCrc32WithEncryption( diskID string, contentSize uint64, encryption *types.EncryptionDesc, -) (uint32, []uint32, error) { +) (DiskContentInfo, error) { ctx := setupStderrLogger(context.Background()) nbsClient, _, err := c.nbs.DiscoverInstance(ctx) if err != nil { - return 0, []uint32{}, err + return DiskContentInfo{}, err } defer nbsClient.Close() @@ -43,7 +43,7 @@ func (c *client) CalculateCrc32WithEncryption( encryptionSpec, err := getEncryptionSpec(encryption) if err != nil { - return 0, []uint32{}, err + return DiskContentInfo{}, err } opts := nbs_client.MountVolumeOpts{ @@ -55,7 +55,7 @@ func (c *client) CalculateCrc32WithEncryption( } err = session.MountVolume(ctx, diskID, &opts) if err != nil { - return 0, []uint32{}, err + return DiskContentInfo{}, err } defer session.UnmountVolume(ctx) @@ -63,14 +63,14 @@ func (c *client) CalculateCrc32WithEncryption( volumeBlockSize := uint64(volume.BlockSize) if volumeBlockSize == 0 { - return 0, []uint32{}, fmt.Errorf( + return DiskContentInfo{}, fmt.Errorf( "%v volume block size should not be zero", diskID, ) } if contentSize%volumeBlockSize != 0 { - return 0, []uint32{}, fmt.Errorf( + return DiskContentInfo{}, fmt.Errorf( "%v contentSize %v should be multiple of volumeBlockSize %v", diskID, contentSize, @@ -82,7 +82,7 @@ func (c *client) CalculateCrc32WithEncryption( volumeSize := volume.BlocksCount * volumeBlockSize if contentSize > volumeSize { - return 0, []uint32{}, fmt.Errorf( + return DiskContentInfo{}, fmt.Errorf( "%v contentSize %v should not be greater than volumeSize %v", diskID, contentSize, @@ -93,13 +93,13 @@ func (c *client) CalculateCrc32WithEncryption( chunkSize := uint64(4 * 1024 * 1024) blocksInChunk := chunkSize / volumeBlockSize acc := crc32.NewIEEE() - blocksCrc32 := []uint32{} + blockCrc32s := []uint32{} for offset := uint64(0); offset < contentBlocksCount; offset += blocksInChunk { blocksToRead := min(contentBlocksCount-offset, blocksInChunk) buffers, err := session.ReadBlocks(ctx, offset, uint32(blocksToRead), "") if err != nil { - return 0, []uint32{}, fmt.Errorf( + return DiskContentInfo{}, fmt.Errorf( "%v read blocks at (%v, %v) failed: %w", diskID, offset, @@ -116,15 +116,15 @@ func (c *client) CalculateCrc32WithEncryption( _, err := acc.Write(buffer) if err != nil { - return 0, []uint32{}, err + return DiskContentInfo{}, err } _, err = blockAcc.Write(buffer) if err != nil { - return 0, []uint32{}, err + return DiskContentInfo{}, err } - blocksCrc32 = append(blocksCrc32, blockAcc.Sum32()) + blockCrc32s = append(blockCrc32s, blockAcc.Sum32()) } } @@ -133,7 +133,7 @@ func (c *client) CalculateCrc32WithEncryption( blocksToRead := min(volume.BlocksCount-offset, blocksInChunk) buffers, err := session.ReadBlocks(ctx, offset, uint32(blocksToRead), "") if err != nil { - return 0, []uint32{}, fmt.Errorf( + return DiskContentInfo{}, fmt.Errorf( "%v read blocks at (%v, %v) failed: %w", diskID, offset, @@ -146,7 +146,7 @@ func (c *client) CalculateCrc32WithEncryption( if len(buffer) != 0 { for j, b := range buffer { if b != 0 { - return 0, []uint32{}, fmt.Errorf( + return DiskContentInfo{}, fmt.Errorf( "%v non zero byte %v detected at (%v, %v)", diskID, b, @@ -159,69 +159,72 @@ func (c *client) CalculateCrc32WithEncryption( } } - return acc.Sum32(), blocksCrc32, nil + return DiskContentInfo{ + ContentSize: contentSize, + Crc32: acc.Sum32(), + BlockCrc32s: blockCrc32s, + }, nil } func (c *client) ValidateCrc32( ctx context.Context, diskID string, - contentSize uint64, - expectedCrc32 uint32, - expectedBlocksCrc32 []uint32, + expectedDiskContentInfo DiskContentInfo, ) error { return c.ValidateCrc32WithEncryption( ctx, diskID, - contentSize, + expectedDiskContentInfo, nil, - expectedCrc32, - expectedBlocksCrc32, ) } func (c *client) ValidateCrc32WithEncryption( ctx context.Context, diskID string, - contentSize uint64, + expectedDiskContentInfo DiskContentInfo, encryption *types.EncryptionDesc, - expectedCrc32 uint32, - expectedBlocksCrc32 []uint32, ) error { - actualCrc32, actualBlocksCrc32, err := c.CalculateCrc32WithEncryption( + actualDiskContentInfo, err := c.CalculateCrc32WithEncryption( diskID, - contentSize, + expectedDiskContentInfo.ContentSize, encryption, ) if err != nil { return err } - if len(actualBlocksCrc32) != len(expectedBlocksCrc32) { + actualCrc32 := actualDiskContentInfo.Crc32 + expectedCrc32 := expectedDiskContentInfo.Crc32 + actualBlockCrc32s := actualDiskContentInfo.BlockCrc32s + expectedBlockCrc32s := expectedDiskContentInfo.BlockCrc32s + + if len(actualBlockCrc32s) != len(expectedBlockCrc32s) { logging.Debug( ctx, "%v blocksCrc32 length doesn't match, expected %v, actual %v", diskID, - len(expectedBlocksCrc32), - len(actualBlocksCrc32), + len(expectedBlockCrc32s), + len(actualBlockCrc32s), ) } else { - for i := range expectedBlocksCrc32 { - if actualBlocksCrc32[i] != expectedBlocksCrc32[i] { + for i := range expectedDiskContentInfo.BlockCrc32s { + if actualBlockCrc32s[i] != expectedBlockCrc32s[i] { logging.Debug( ctx, "%v block with index %v crc32 doesn't match, expected %v, actual %v", diskID, i, - expectedBlocksCrc32[i], - actualBlocksCrc32[i], + expectedBlockCrc32s[i], + actualBlockCrc32s[i], ) } } } - if expectedCrc32 != actualCrc32 { + if actualCrc32 != expectedCrc32 { return fmt.Errorf( "%v crc32 doesn't match, expected %v, actual %v", diskID, diff --git a/cloud/disk_manager/internal/pkg/facade/disk_service_test/disk_relocation_test.go b/cloud/disk_manager/internal/pkg/facade/disk_service_test/disk_relocation_test.go index a72dd79c9f1..372c10c89a0 100644 --- a/cloud/disk_manager/internal/pkg/facade/disk_service_test/disk_relocation_test.go +++ b/cloud/disk_manager/internal/pkg/facade/disk_service_test/disk_relocation_test.go @@ -59,7 +59,7 @@ func setupMigrationTest( if params.FillDisk { nbsClient := testcommon.NewNbsClient(t, ctx, params.SrcZoneID) - _, _, _, err = testcommon.FillDisk( + _, err = testcommon.FillDisk( nbsClient, params.DiskID, uint64(params.DiskSize), @@ -113,7 +113,7 @@ func successfullyMigrateDisk( diskSize := uint64(params.DiskSize) - srcCrc32, srcBlocksCrc32, err := srcZoneNBSClient.CalculateCrc32(params.DiskID, diskSize) + diskContentInfo, err := srcZoneNBSClient.CalculateCrc32(params.DiskID, diskSize) require.NoError(t, err) err = client.SendMigrationSignal(ctx, &disk_manager.SendMigrationSignalRequest{ @@ -149,9 +149,7 @@ func successfullyMigrateDisk( err = dstZoneNBSClient.ValidateCrc32( ctx, params.DiskID, - diskSize, - srcCrc32, - srcBlocksCrc32, + diskContentInfo, ) require.NoError(t, err) } @@ -331,7 +329,7 @@ func migrateDiskInParallel( diskSize := uint64(params.DiskSize) - srcCrc32, srcBlocksCrc32, err := srcZoneNBSClient.CalculateCrc32(params.DiskID, diskSize) + diskContentInfo, err := srcZoneNBSClient.CalculateCrc32(params.DiskID, diskSize) require.NoError(t, err) for _, operation := range operations { @@ -387,9 +385,7 @@ func migrateDiskInParallel( err := dstZoneNBSClient.ValidateCrc32( ctx, params.DiskID, - diskSize, - srcCrc32, - srcBlocksCrc32, + diskContentInfo, ) require.NoError(t, err) } else { @@ -397,9 +393,7 @@ func migrateDiskInParallel( err := srcZoneNBSClient.ValidateCrc32( ctx, params.DiskID, - diskSize, - srcCrc32, - srcBlocksCrc32, + diskContentInfo, ) require.NoError(t, err) } @@ -437,7 +431,7 @@ func successfullyMigrateEmptyDisk( diskSize := uint64(migrationTestsDiskSize) - srcCrc32, srcBlocksCrc32, err := srcZoneNBSClient.CalculateCrc32(params.DiskID, diskSize) + diskContentInfo, err := srcZoneNBSClient.CalculateCrc32(params.DiskID, diskSize) require.NoError(t, err) metadata := &disk_manager.MigrateDiskMetadata{} @@ -462,9 +456,7 @@ func successfullyMigrateEmptyDisk( err = dstZoneNBSClient.ValidateCrc32( ctx, params.DiskID, - diskSize, - srcCrc32, - srcBlocksCrc32, + diskContentInfo, ) require.NoError(t, err) } @@ -724,7 +716,7 @@ func TestDiskServiceMigrateOverlayDisk(t *testing.T) { imageID := t.Name() diskSize := migrationTestsDiskSize imageSize := diskSize / 2 - _, _, _ = testcommon.CreateImage( + _ = testcommon.CreateImage( t, ctx, imageID, diff --git a/cloud/disk_manager/internal/pkg/facade/disk_service_test/disk_service_test.go b/cloud/disk_manager/internal/pkg/facade/disk_service_test/disk_service_test.go index ce8ba956e4c..43d0dadb309 100644 --- a/cloud/disk_manager/internal/pkg/facade/disk_service_test/disk_service_test.go +++ b/cloud/disk_manager/internal/pkg/facade/disk_service_test/disk_service_test.go @@ -12,6 +12,7 @@ import ( "github.com/ydb-platform/nbs/cloud/disk_manager/api/yandex/cloud/priv/disk_manager/v1" "github.com/ydb-platform/nbs/cloud/disk_manager/internal/api" internal_client "github.com/ydb-platform/nbs/cloud/disk_manager/internal/pkg/client" + "github.com/ydb-platform/nbs/cloud/disk_manager/internal/pkg/clients/nbs" "github.com/ydb-platform/nbs/cloud/disk_manager/internal/pkg/facade/testcommon" "github.com/ydb-platform/nbs/cloud/disk_manager/internal/pkg/services/disks" "github.com/ydb-platform/nbs/cloud/disk_manager/internal/pkg/types" @@ -133,9 +134,11 @@ func TestDiskServiceCreateDiskFromImageWithForceNotLayered(t *testing.T) { err = nbsClient.ValidateCrc32( ctx, diskID, - testcommon.GetRawImageSize(t), - testcommon.GetRawImageCrc32(t), - []uint32{}, // We do not known expectedBlocksCrc32 for image. + nbs.DiskContentInfo{ + ContentSize: testcommon.GetRawImageSize(t), + Crc32: testcommon.GetRawImageCrc32(t), + BlockCrc32s: []uint32{}, // We do not know blockCrc32s for image. + }, ) require.NoError(t, err) @@ -152,7 +155,7 @@ func TestDiskServiceCancelCreateDiskFromImage(t *testing.T) { imageID := t.Name() imageSize := uint64(64 * 1024 * 1024) - _, _, _ = testcommon.CreateImage( + _ = testcommon.CreateImage( t, ctx, imageID, @@ -193,7 +196,7 @@ func TestDiskServiceDeleteDiskWhenCreationIsInFlight(t *testing.T) { imageID := t.Name() imageSize := uint64(64 * 1024 * 1024) - _, _, _ = testcommon.CreateImage( + _ = testcommon.CreateImage( t, ctx, imageID, @@ -251,7 +254,7 @@ func TestDiskServiceCreateDisksFromImageWithConfiguredPool(t *testing.T) { imageID := t.Name() imageSize := uint64(64 * 1024 * 1024) - imageCrc32, blocksCrc32, _ := testcommon.CreateImage( + diskContentInfo := testcommon.CreateImage( t, ctx, imageID, @@ -310,9 +313,7 @@ func TestDiskServiceCreateDisksFromImageWithConfiguredPool(t *testing.T) { err = nbsClient.ValidateCrc32( ctx, diskID, - imageSize, - imageCrc32, - blocksCrc32, + diskContentInfo, ) require.NoError(t, err) @@ -384,7 +385,7 @@ func testCreateDiskFromIncrementalSnapshot( require.NoError(t, err) nbsClient := testcommon.NewNbsClient(t, ctx, "zone-a") - _, _, _, err = testcommon.FillDisk(nbsClient, diskID1, diskSize) + _, err = testcommon.FillDisk(nbsClient, diskID1, diskSize) require.NoError(t, err) snapshotID1 := t.Name() + "1" @@ -403,7 +404,7 @@ func testCreateDiskFromIncrementalSnapshot( err = internal_client.WaitOperation(ctx, client, operation.Id) require.NoError(t, err) - _, _, _, err = testcommon.FillDisk(nbsClient, diskID1, diskSize) + _, err = testcommon.FillDisk(nbsClient, diskID1, diskSize) require.NoError(t, err) snapshotID2 := t.Name() + "2" @@ -441,10 +442,10 @@ func testCreateDiskFromIncrementalSnapshot( err = internal_client.WaitOperation(ctx, client, operation.Id) require.NoError(t, err) - crc32, blocksCrc32, err := nbsClient.CalculateCrc32(diskID1, diskSize) + diskContentInfo, err := nbsClient.CalculateCrc32(diskID1, diskSize) require.NoError(t, err) - err = nbsClient.ValidateCrc32(ctx, diskID2, diskSize, crc32, blocksCrc32) + err = nbsClient.ValidateCrc32(ctx, diskID2, diskContentInfo) require.NoError(t, err) testcommon.CheckConsistency(t, ctx) @@ -494,7 +495,7 @@ func TestDiskServiceCreateDiskFromSnapshot(t *testing.T) { require.NoError(t, err) nbsClient := testcommon.NewNbsClient(t, ctx, "zone-a") - crc32, blocksCrc32, _, err := testcommon.FillDisk(nbsClient, diskID1, diskSize) + diskContentInfo, err := testcommon.FillDisk(nbsClient, diskID1, diskSize) require.NoError(t, err) snapshotID1 := t.Name() + "1" @@ -542,10 +543,10 @@ func TestDiskServiceCreateDiskFromSnapshot(t *testing.T) { require.NoError(t, err) require.Equal(t, float64(1), diskMeta.Progress) - err = nbsClient.ValidateCrc32(ctx, diskID1, diskSize, crc32, blocksCrc32) + err = nbsClient.ValidateCrc32(ctx, diskID1, diskContentInfo) require.NoError(t, err) - err = nbsClient.ValidateCrc32(ctx, diskID2, diskSize, crc32, blocksCrc32) + err = nbsClient.ValidateCrc32(ctx, diskID2, diskContentInfo) require.NoError(t, err) testcommon.CheckConsistency(t, ctx) @@ -579,7 +580,7 @@ func TestDiskServiceCreateDiskFromImage(t *testing.T) { require.NoError(t, err) nbsClient := testcommon.NewNbsClient(t, ctx, "zone-a") - crc32, blocksCrc32, _, err := testcommon.FillDisk(nbsClient, diskID1, diskSize) + diskContentInfo, err := testcommon.FillDisk(nbsClient, diskID1, diskSize) require.NoError(t, err) imageID := t.Name() + "_image" @@ -624,10 +625,10 @@ func TestDiskServiceCreateDiskFromImage(t *testing.T) { err = internal_client.WaitOperation(ctx, client, operation.Id) require.NoError(t, err) - err = nbsClient.ValidateCrc32(ctx, diskID1, diskSize, crc32, blocksCrc32) + err = nbsClient.ValidateCrc32(ctx, diskID1, diskContentInfo) require.NoError(t, err) - err = nbsClient.ValidateCrc32(ctx, diskID2, diskSize, crc32, blocksCrc32) + err = nbsClient.ValidateCrc32(ctx, diskID2, diskContentInfo) require.NoError(t, err) testcommon.CheckConsistency(t, ctx) @@ -663,7 +664,7 @@ func TestDiskServiceCreateDiskFromSnapshotOfOverlayDisk(t *testing.T) { nbsClient := testcommon.NewNbsClient(t, ctx, "zone-a") - crc32, blocksCrc32, _, err := testcommon.FillDisk(nbsClient, diskID1, imageSize) + diskContentInfo, err := testcommon.FillDisk(nbsClient, diskID1, imageSize) require.NoError(t, err) imageID := t.Name() + "_image" @@ -756,13 +757,13 @@ func TestDiskServiceCreateDiskFromSnapshotOfOverlayDisk(t *testing.T) { err = internal_client.WaitOperation(ctx, client, operation.Id) require.NoError(t, err) - err = nbsClient.ValidateCrc32(ctx, diskID1, imageSize, crc32, blocksCrc32) + err = nbsClient.ValidateCrc32(ctx, diskID1, diskContentInfo) require.NoError(t, err) - err = nbsClient.ValidateCrc32(ctx, diskID2, imageSize, crc32, blocksCrc32) + err = nbsClient.ValidateCrc32(ctx, diskID2, diskContentInfo) require.NoError(t, err) - err = nbsClient.ValidateCrc32(ctx, diskID3, imageSize, crc32, blocksCrc32) + err = nbsClient.ValidateCrc32(ctx, diskID3, diskContentInfo) require.NoError(t, err) testcommon.CheckConsistency(t, ctx) @@ -1056,7 +1057,7 @@ func TestDiskServiceCreateEncryptedDiskFromSnapshot(t *testing.T) { encryptionDesc, err := disks.PrepareEncryptionDesc(encryption) require.NoError(t, err) - crc32, blocksCrc32, _, err := testcommon.FillEncryptedDisk( + diskContentInfo, err := testcommon.FillEncryptedDisk( nbsClient, diskID1, diskSize, @@ -1151,19 +1152,15 @@ func TestDiskServiceCreateEncryptedDiskFromSnapshot(t *testing.T) { err = nbsClient.ValidateCrc32WithEncryption( ctx, diskID1, - diskSize, + diskContentInfo, encryptionDesc, - crc32, - blocksCrc32, ) require.NoError(t, err) err = nbsClient.ValidateCrc32WithEncryption( ctx, diskID2, - diskSize, + diskContentInfo, encryptionDesc, - crc32, - blocksCrc32, ) require.NoError(t, err) diff --git a/cloud/disk_manager/internal/pkg/facade/testcommon/common.go b/cloud/disk_manager/internal/pkg/facade/testcommon/common.go index 708d9f08306..f0e4bc796fa 100644 --- a/cloud/disk_manager/internal/pkg/facade/testcommon/common.go +++ b/cloud/disk_manager/internal/pkg/facade/testcommon/common.go @@ -358,18 +358,18 @@ func WaitForCheckpointsAreEmpty( func FillDisk( nbsClient nbs.Client, diskID string, - diskSize uint64, -) (uint32, []uint32, uint64, error) { + contentSize uint64, +) (nbs.DiskContentInfo, error) { - return FillEncryptedDisk(nbsClient, diskID, diskSize, nil) + return FillEncryptedDisk(nbsClient, diskID, contentSize, nil) } func FillEncryptedDisk( nbsClient nbs.Client, diskID string, - diskSize uint64, + contentSize uint64, encryption *types.EncryptionDesc, -) (uint32, []uint32, uint64, error) { +) (nbs.DiskContentInfo, error) { ctx := NewContext() @@ -381,7 +381,7 @@ func FillEncryptedDisk( encryption, ) if err != nil { - return 0, []uint32{}, 0, err + return nbs.DiskContentInfo{}, err } defer session.Close(ctx) @@ -394,8 +394,8 @@ func FillEncryptedDisk( rand.Seed(time.Now().UnixNano()) acc := crc32.NewIEEE() - blocksCrc32 := []uint32{} - for offset := uint64(0); offset < diskSize; offset += chunkSize { + blockCrc32s := []uint32{} + for offset := uint64(0); offset < contentSize; offset += chunkSize { startIndex := offset / blockSize data := make([]byte, chunkSize) dice := rand.Intn(3) @@ -414,12 +414,12 @@ func FillEncryptedDisk( err = session.Zero(ctx, startIndex, blocksInChunk) } if err != nil { - return 0, []uint32{}, 0, err + return nbs.DiskContentInfo{}, err } _, err = acc.Write(data) if err != nil { - return 0, []uint32{}, 0, err + return nbs.DiskContentInfo{}, err } for blockIndex := uint32(0); blockIndex < blocksInChunk; blockIndex++ { @@ -431,7 +431,7 @@ func FillEncryptedDisk( _, err = blockAcc.Write(blockData) if err != nil { - return 0, []uint32{}, 0, err + return nbs.DiskContentInfo{}, err } logging.Debug( @@ -441,11 +441,16 @@ func FillEncryptedDisk( startIndex+uint64(blockIndex), blockAcc.Sum32(), ) - blocksCrc32 = append(blocksCrc32, blockAcc.Sum32()) + blockCrc32s = append(blockCrc32s, blockAcc.Sum32()) } } - return acc.Sum32(), blocksCrc32, storageSize, nil + return nbs.DiskContentInfo{ + ContentSize: contentSize, + StorageSize: storageSize, + Crc32: acc.Sum32(), + BlockCrc32s: blockCrc32s, + }, nil } func GoWriteRandomBlocksToNbsDisk( @@ -532,7 +537,7 @@ func CreateImage( imageSize uint64, folderID string, pooled bool, -) (uint32, []uint32, uint64) { +) nbs.DiskContentInfo { client, err := NewClient(ctx) require.NoError(t, err) @@ -558,7 +563,7 @@ func CreateImage( require.NoError(t, err) nbsClient := NewNbsClient(t, ctx, "zone-a") - crc32, blocksCrc32, storageSize, err := FillDisk(nbsClient, diskID, imageSize) + diskContentInfo, err := FillDisk(nbsClient, diskID, imageSize) require.NoError(t, err) reqCtx = GetRequestContext(t, ctx) @@ -580,9 +585,9 @@ func CreateImage( err = internal_client.WaitResponse(ctx, client, operation.Id, &response) require.NoError(t, err) require.Equal(t, int64(imageSize), response.Size) - require.Equal(t, int64(storageSize), response.StorageSize) + require.Equal(t, int64(diskContentInfo.StorageSize), response.StorageSize) - return crc32, blocksCrc32, storageSize + return diskContentInfo } //////////////////////////////////////////////////////////////////////////////// From 4bc6beb1014a58e627fe9ead3ad83bcf7b26d2ca Mon Sep 17 00:00:00 2001 From: BarkovBG Date: Fri, 29 Mar 2024 09:50:12 +0000 Subject: [PATCH 4/4] reolve issues --- .../internal/pkg/clients/nbs/interface.go | 2 +- .../pkg/clients/nbs/mocks/client_mock.go | 23 ++++--- .../image_service_test/image_service_test.go | 68 +++++++++++++++---- .../private_service_test.go | 16 ++--- .../snapshot_service_test.go | 18 +++-- cloud/disk_manager/test/remote/cmd/main.go | 32 +++++++-- 6 files changed, 117 insertions(+), 42 deletions(-) diff --git a/cloud/disk_manager/internal/pkg/clients/nbs/interface.go b/cloud/disk_manager/internal/pkg/clients/nbs/interface.go index c757513aaed..2d3f8dcbcb9 100644 --- a/cloud/disk_manager/internal/pkg/clients/nbs/interface.go +++ b/cloud/disk_manager/internal/pkg/clients/nbs/interface.go @@ -115,7 +115,7 @@ type CheckpointParams struct { // Used in tests. type DiskContentInfo struct { - ContentSize uint64 + ContentSize uint64 // The coordinate of the last non-zero byte. StorageSize uint64 Crc32 uint32 BlockCrc32s []uint32 diff --git a/cloud/disk_manager/internal/pkg/clients/nbs/mocks/client_mock.go b/cloud/disk_manager/internal/pkg/clients/nbs/mocks/client_mock.go index 7548822ba29..ad63fc87a59 100644 --- a/cloud/disk_manager/internal/pkg/clients/nbs/mocks/client_mock.go +++ b/cloud/disk_manager/internal/pkg/clients/nbs/mocks/client_mock.go @@ -323,29 +323,34 @@ func (c *ClientMock) Stat( //////////////////////////////////////////////////////////////////////////////// func (c *ClientMock) ValidateCrc32( + ctx context.Context, diskID string, - contentSize uint64, - expectedCrc32 uint32, + expectedDiskContentInfo nbs.DiskContentInfo, ) error { - return c.ValidateCrc32WithEncryption(diskID, contentSize, nil, expectedCrc32) + return c.ValidateCrc32WithEncryption( + ctx, + diskID, + expectedDiskContentInfo, + nil, + ) } func (c *ClientMock) ValidateCrc32WithEncryption( + ctx context.Context, diskID string, - contentSize uint64, + expectedDiskContentInfo nbs.DiskContentInfo, encryption *types.EncryptionDesc, - expectedCrc32 uint32, ) error { - args := c.Called(diskID, contentSize, encryption, expectedCrc32) + args := c.Called(ctx, diskID, expectedDiskContentInfo, encryption) return args.Error(0) } func (c *ClientMock) CalculateCrc32( diskID string, contentSize uint64, -) (uint32, error) { +) (nbs.DiskContentInfo, error) { return c.CalculateCrc32WithEncryption(diskID, contentSize, nil) } @@ -354,10 +359,10 @@ func (c *ClientMock) CalculateCrc32WithEncryption( diskID string, contentSize uint64, encryption *types.EncryptionDesc, -) (uint32, error) { +) (nbs.DiskContentInfo, error) { args := c.Called(diskID, contentSize, encryption) - return args.Get(0).(uint32), args.Error(1) + return args.Get(0).(nbs.DiskContentInfo), args.Error(1) } func (c *ClientMock) MountForReadWrite( diff --git a/cloud/disk_manager/internal/pkg/facade/image_service_test/image_service_test.go b/cloud/disk_manager/internal/pkg/facade/image_service_test/image_service_test.go index fb05c53b0e9..b7765a3d4ab 100644 --- a/cloud/disk_manager/internal/pkg/facade/image_service_test/image_service_test.go +++ b/cloud/disk_manager/internal/pkg/facade/image_service_test/image_service_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/require" "github.com/ydb-platform/nbs/cloud/disk_manager/api/yandex/cloud/priv/disk_manager/v1" internal_client "github.com/ydb-platform/nbs/cloud/disk_manager/internal/pkg/client" + "github.com/ydb-platform/nbs/cloud/disk_manager/internal/pkg/clients/nbs" "github.com/ydb-platform/nbs/cloud/disk_manager/internal/pkg/facade/testcommon" "github.com/ydb-platform/nbs/cloud/disk_manager/internal/pkg/services/disks" "github.com/ydb-platform/nbs/cloud/disk_manager/internal/pkg/types" @@ -74,7 +75,10 @@ func checkEncryptedSource( encryption, err := disks.PrepareEncryptionDesc(defaultEncryptionDescWithKey) require.NoError(t, err) - err = nbsClient.ValidateCrc32WithEncryption(diskID1, uint64(diskSize), encryption, crc32) + err = nbsClient.ValidateCrc32WithEncryption(ctx, diskID1, nbs.DiskContentInfo{ + ContentSize: uint64(diskSize), + Crc32: crc32, + }, encryption) require.NoError(t, err) diskID2 := t.Name() + "-bad-disk-from-encrypted-" + tag @@ -139,7 +143,10 @@ func checkUnencryptedImage( encryption, err := disks.PrepareEncryptionDesc(defaultEncryptionDescWithKey) require.NoError(t, err) - err = nbsClient.ValidateCrc32WithEncryption(diskID1, uint64(diskSize), encryption, crc32) + err = nbsClient.ValidateCrc32WithEncryption(ctx, diskID1, nbs.DiskContentInfo{ + ContentSize: uint64(diskSize), + Crc32: crc32, + }, encryption) require.NoError(t, err) diskID2 := t.Name() + "-bad-enrypted-disk-from-image" @@ -179,7 +186,7 @@ func TestImageServiceCreateImageFromImage(t *testing.T) { imageID1 := t.Name() + "_image1" imageSize := uint64(40 * 1024 * 1024) - imageCrc32, _ := testcommon.CreateImage( + diskContentInfo := testcommon.CreateImage( t, ctx, imageID1, @@ -233,7 +240,7 @@ func TestImageServiceCreateImageFromImage(t *testing.T) { nbsClient := testcommon.NewNbsClient(t, ctx, "zone-a") - err = nbsClient.ValidateCrc32(diskID2, imageSize, imageCrc32) + err = nbsClient.ValidateCrc32(ctx, diskID2, diskContentInfo) require.NoError(t, err) reqCtx = testcommon.GetRequestContext(t, ctx) @@ -300,7 +307,7 @@ func TestImageServiceCreateImageFromSnapshot(t *testing.T) { encryption, err := disks.PrepareEncryptionDesc(defaultEncryptionDescWithKey) require.NoError(t, err) - crc32, _, err := testcommon.FillEncryptedDisk(nbsClient, diskID, diskSize, encryption) + diskContentInfo, err := testcommon.FillEncryptedDisk(nbsClient, diskID, diskSize, encryption) require.NoError(t, err) diskParams, err = nbsClient.Describe(ctx, diskID) @@ -335,7 +342,15 @@ func TestImageServiceCreateImageFromSnapshot(t *testing.T) { SrcSnapshotId: snapshotID, }, } - checkEncryptedSource(t, client, ctx, snapshotSrc, int64(diskSize), crc32, "snapshot") + checkEncryptedSource( + t, + client, + ctx, + snapshotSrc, + int64(diskSize), + diskContentInfo.Crc32, + "snapshot", + ) imageID := t.Name() + "_image" @@ -366,7 +381,15 @@ func TestImageServiceCreateImageFromSnapshot(t *testing.T) { SrcImageId: imageID, }, } - checkEncryptedSource(t, client, ctx, imageSrc, int64(diskSize), crc32, "image") + checkEncryptedSource( + t, + client, + ctx, + imageSrc, + int64(diskSize), + diskContentInfo.Crc32, + "image", + ) reqCtx = testcommon.GetRequestContext(t, ctx) operation1, err := client.DeleteImage(reqCtx, &disk_manager.DeleteImageRequest{ @@ -453,9 +476,12 @@ func testCreateImageFromURL( nbsClient := testcommon.NewNbsClient(t, ctx, "zone-a") err = nbsClient.ValidateCrc32( + ctx, diskID, - imageSize, - diskCRC32, + nbs.DiskContentInfo{ + ContentSize: imageSize, + Crc32: diskCRC32, + }, ) require.NoError(t, err) @@ -601,7 +627,14 @@ func TestImageServiceCreateQCOW2ImageFromURLWhichIsOverwrittenInProcess( nbsClient := testcommon.NewNbsClient(t, ctx, "zone-a") - err = nbsClient.ValidateCrc32(diskID, imageSize, imageCrc32) + err = nbsClient.ValidateCrc32( + ctx, + diskID, + nbs.DiskContentInfo{ + ContentSize: imageSize, + Crc32: imageCrc32, + }, + ) require.NoError(t, err) testcommon.CheckConsistency(t, ctx) @@ -703,7 +736,7 @@ func TestImageServiceCancelCreateImageFromImage(t *testing.T) { imageID1 := t.Name() + "1" imageSize := uint64(64 * 1024 * 1024) - _, _ = testcommon.CreateImage( + _ = testcommon.CreateImage( t, ctx, imageID1, @@ -816,7 +849,7 @@ func TestImageServiceCreateImageFromDisk(t *testing.T) { require.NoError(t, err) nbsClient := testcommon.NewNbsClient(t, ctx, "zone-a") - crc32, _, err := testcommon.FillDisk(nbsClient, diskID, diskSize) + diskContentInfo, err := testcommon.FillDisk(nbsClient, diskID, diskSize) require.NoError(t, err) imageID := t.Name() @@ -848,7 +881,14 @@ func TestImageServiceCreateImageFromDisk(t *testing.T) { testcommon.RequireCheckpointsAreEmpty(t, ctx, diskID) - checkUnencryptedImage(t, client, ctx, imageID, int64(diskSize), crc32) + checkUnencryptedImage( + t, + client, + ctx, + imageID, + int64(diskSize), + diskContentInfo.Crc32, + ) reqCtx = testcommon.GetRequestContext(t, ctx) operation, err = client.DeleteImage(reqCtx, &disk_manager.DeleteImageRequest{ @@ -918,7 +958,7 @@ func TestImageServiceDeleteImage(t *testing.T) { imageID := t.Name() imageSize := uint64(40 * 1024 * 1024) - _, _ = testcommon.CreateImage( + _ = testcommon.CreateImage( t, ctx, imageID, diff --git a/cloud/disk_manager/internal/pkg/facade/private_service_test/private_service_test.go b/cloud/disk_manager/internal/pkg/facade/private_service_test/private_service_test.go index 2b17a6d60b5..a6d9355ddba 100644 --- a/cloud/disk_manager/internal/pkg/facade/private_service_test/private_service_test.go +++ b/cloud/disk_manager/internal/pkg/facade/private_service_test/private_service_test.go @@ -42,7 +42,7 @@ func TestPrivateServiceRetireBaseDisks(t *testing.T) { imageID := t.Name() imageSize := uint64(64 * 1024 * 1024) - imageCrc32, _ := testcommon.CreateImage( + diskContentInfo := testcommon.CreateImage( t, ctx, imageID, @@ -121,7 +121,7 @@ func TestPrivateServiceRetireBaseDisks(t *testing.T) { return } - err = nbsClient.ValidateCrc32(diskID, imageSize, imageCrc32) + err = nbsClient.ValidateCrc32(ctx, diskID, diskContentInfo) errs <- err }() } @@ -171,7 +171,7 @@ func TestPrivateServiceRetireBaseDisksUsingBaseDiskAsSrc(t *testing.T) { imageID := t.Name() imageSize := uint64(64 * 1024 * 1024) - imageCrc32, _ := testcommon.CreateImage( + diskContentInfo := testcommon.CreateImage( t, ctx, imageID, @@ -225,7 +225,7 @@ func TestPrivateServiceRetireBaseDisksUsingBaseDiskAsSrc(t *testing.T) { require.NoError(t, err) nbsClient := testcommon.NewNbsClient(t, ctx, "zone-a") - err = nbsClient.ValidateCrc32(diskID, imageSize, imageCrc32) + err = nbsClient.ValidateCrc32(ctx, diskID, diskContentInfo) require.NoError(t, err) reqCtx = testcommon.GetRequestContext(t, ctx) @@ -252,7 +252,7 @@ func TestPrivateServiceOptimizeBaseDisks(t *testing.T) { imageID := t.Name() imageSize := uint64(64 * 1024 * 1024) - imageCrc32, _ := testcommon.CreateImage( + diskContentInfo := testcommon.CreateImage( t, ctx, imageID, @@ -317,7 +317,7 @@ func TestPrivateServiceOptimizeBaseDisks(t *testing.T) { return } - err = nbsClient.ValidateCrc32(diskID, imageSize, imageCrc32) + err = nbsClient.ValidateCrc32(ctx, diskID, diskContentInfo) errs <- err }() } @@ -362,7 +362,7 @@ func TestPrivateServiceConfigurePool(t *testing.T) { imageSize := uint64(4 * 1024 * 1024) - _, _ = testcommon.CreateImage( + _ = testcommon.CreateImage( t, ctx, imageID, @@ -406,7 +406,7 @@ func TestPrivateServiceDeletePool(t *testing.T) { imageID := t.Name() imageSize := uint64(64 * 1024 * 1024) - _, _ = testcommon.CreateImage( + _ = testcommon.CreateImage( t, ctx, imageID, diff --git a/cloud/disk_manager/internal/pkg/facade/snapshot_service_test/snapshot_service_test.go b/cloud/disk_manager/internal/pkg/facade/snapshot_service_test/snapshot_service_test.go index e904e9ef627..f57cb7be089 100644 --- a/cloud/disk_manager/internal/pkg/facade/snapshot_service_test/snapshot_service_test.go +++ b/cloud/disk_manager/internal/pkg/facade/snapshot_service_test/snapshot_service_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/require" "github.com/ydb-platform/nbs/cloud/disk_manager/api/yandex/cloud/priv/disk_manager/v1" internal_client "github.com/ydb-platform/nbs/cloud/disk_manager/internal/pkg/client" + "github.com/ydb-platform/nbs/cloud/disk_manager/internal/pkg/clients/nbs" "github.com/ydb-platform/nbs/cloud/disk_manager/internal/pkg/facade/testcommon" ) @@ -48,7 +49,7 @@ func testCreateSnapshotFromDisk( require.NoError(t, err) nbsClient := testcommon.NewNbsClient(t, ctx, "zone-a") - _, _, err = testcommon.FillDisk(nbsClient, diskID, 64*4096) + _, err = testcommon.FillDisk(nbsClient, diskID, 64*4096) require.NoError(t, err) snapshotID := t.Name() @@ -280,7 +281,10 @@ func testCreateIncrementalSnapshotFromDisk( _, err = acc.Write(bytes) require.NoError(t, err) - err = nbsClient.ValidateCrc32(diskID2, uint64(contentSize), acc.Sum32()) + err = nbsClient.ValidateCrc32(ctx, diskID2, nbs.DiskContentInfo{ + ContentSize: uint64(contentSize), + Crc32: acc.Sum32(), + }) require.NoError(t, err) testcommon.CheckConsistency(t, ctx) @@ -407,7 +411,10 @@ func TestSnapshotServiceCreateIncrementalSnapshotAfterDeletionOfBaseSnapshot(t * _, err = acc.Write(bytes) require.NoError(t, err) - err = nbsClient.ValidateCrc32(diskID2, uint64(diskSize), acc.Sum32()) + err = nbsClient.ValidateCrc32(ctx, diskID2, nbs.DiskContentInfo{ + ContentSize: uint64(diskSize), + Crc32: acc.Sum32(), + }) require.NoError(t, err) testcommon.CheckConsistency(t, ctx) @@ -516,7 +523,10 @@ func TestSnapshotServiceCreateIncrementalSnapshotWhileDeletingBaseSnapshot(t *te _, err = acc.Write(bytes) require.NoError(t, err) - err = nbsClient.ValidateCrc32(diskID2, uint64(diskSize), acc.Sum32()) + err = nbsClient.ValidateCrc32(ctx, diskID2, nbs.DiskContentInfo{ + ContentSize: uint64(diskSize), + Crc32: acc.Sum32(), + }) require.NoError(t, err) testcommon.CheckConsistency(t, ctx) diff --git a/cloud/disk_manager/test/remote/cmd/main.go b/cloud/disk_manager/test/remote/cmd/main.go index ac7e9bf4da0..3bb275591e0 100644 --- a/cloud/disk_manager/test/remote/cmd/main.go +++ b/cloud/disk_manager/test/remote/cmd/main.go @@ -959,9 +959,12 @@ func testCreateDiskFromImageImpl( } err = nbsClient.ValidateCrc32( + ctx, diskID, - testConfig.GetImageSize(), - testConfig.GetImageCrc32(), + nbs.DiskContentInfo{ + ContentSize: testConfig.GetImageSize(), + Crc32: testConfig.GetImageCrc32(), + }, ) if err != nil { return resources{}, err @@ -1156,9 +1159,12 @@ func testRetireBaseDisks( } err = nbsClient.ValidateCrc32( + ctx, diskID, - testConfig.GetImageSize(), - testConfig.GetImageCrc32(), + nbs.DiskContentInfo{ + ContentSize: testConfig.GetImageSize(), + Crc32: testConfig.GetImageCrc32(), + }, ) errs <- err }() @@ -1268,7 +1274,14 @@ func testCreateDiskFromSnapshotImpl( return resources{}, err } - err = nbsClient.ValidateCrc32(diskID2, diskSize, expectedCrc32) + err = nbsClient.ValidateCrc32( + ctx, + diskID2, + nbs.DiskContentInfo{ + ContentSize: diskSize, + Crc32: expectedCrc32, + }, + ) if err != nil { return resources{}, err } @@ -1455,7 +1468,14 @@ func testCreateImageFromImageImpl( return resources{}, err } - err = nbsClient.ValidateCrc32(diskID2, uint64(diskSize), expectedCrc32) + err = nbsClient.ValidateCrc32( + ctx, + diskID2, + nbs.DiskContentInfo{ + ContentSize: diskSize, + Crc32: expectedCrc32, + }, + ) if err != nil { return resources{}, err }