GO SDK track2 migration (#1380)

CHANGELOG.md

@@ -1,17 +1,23 @@
-## 2.3.0 (Unreleased)
+## 2.3.0~preview.1 (Unreleased)
 **Bug Fixes**
+- [#1057](https://github.com/Azure/azure-storage-fuse/issues/1057) Fixed the issue where user-assigned identity is not used to authenticate when system-assigned identity is enabled.
+- Listing blobs is now supported for blob names that contain characters that aren't valid in XML (U+FFFE or U+FFFF).
 - [#1359](https://github.com/Azure/azure-storage-fuse/issues/1359), [#1368](https://github.com/Azure/azure-storage-fuse/issues/1368) Fixed RHEL 8.6 mount failure
 
 **Features**
-- lazy-write support for async flush and close file call. Actual upload will be scheduled in background when this feature is enable.
+- Migrated to the latest [azblob SDK](https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/storage/azblob).
+- Migrated to the latest [azdatalake SDK](https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake).
+- Migrated from deprecated ADAL to MSAL through the latest [azidentity SDK](https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/azidentity).
+- Added support for uploading blobs in cold and premium tier.
+- Support CPK for adls storage accounts.
+- Lazy-write support for async flush and close file call. Actual upload will be scheduled in background when this feature is enabled.
 
 ## 2.2.1 (2024-02-28)
 **Bug Fixes**
 - Fixed panic while truncating a file to a very large size.
 - Fixed block-cache panic on flush of a file which has no active changeset
 - Fixed block-cache panic on renaming a file and then flushing older handle
 - Fixed block-cache flush resulting in invalid-block-list error
-
 
 ## 2.2.0 (2024-01-24)
 **Bug Fixes**

TSG.md

@@ -3,8 +3,6 @@
 Please ensure logging is turned on DEBUG mode when trying to reproduce an issue.
 This can help in many instances to understand what the underlying issue is.
 
-A useful setting in your configuration file to utilize when debugging is `sdk-trace: true` under the azstorage component. This will log all outgoing REST calls.
-
 # BlobFuse2 Health Monitor
 
 One of the biggest BlobFuse2 features is our brand new health monitor. It allows customers gain more insight into how their BlobFuse2 instance is behaving with the rest of their machine. Visit [here](https://github.com/Azure/azure-storage-fuse/blob/main/tools/health-monitor/README.md) to set it up.

common/config/keys_tree_test.go

@@ -73,7 +73,7 @@ func (suite *keysTreeTestSuite) TestParseValue() {
 		{val: "65535", toType: reflect.Uint16, result: 65535},
 		{val: "4294967295", toType: reflect.Uint32, result: 4294967295},
 		{val: "18446744073709551615", toType: reflect.Uint64, result: uint64(18446744073709551615)},
-		{val: "6.24321908234", toType: reflect.Float32, result: 6.24321908234},
+		{val: "6.24321908234", toType: reflect.Float32, result: (float32)(6.24321908234)},
 		{val: "31247921747687123.123871293791263", toType: reflect.Float64, result: 31247921747687123.123871293791263},
 		{val: "6-8i", toType: reflect.Complex64, result: 6 - 8i},
 		{val: "2341241-910284i", toType: reflect.Complex128, result: 2341241 - 910284i},

common/types.go

@@ -47,7 +47,7 @@ import (
 
 // Standard config default values
 const (
-	blobfuse2Version_ = "2.2.1"
+	blobfuse2Version_ = "2.3.0~preview.1"
 
 	DefaultMaxLogFileSize = 512
 	DefaultLogFileCount   = 10
@@ -68,6 +68,8 @@ const (
 	BfuseStats = "blobfuse_stats"
 
 	FuseAllowedFlags = "invalid FUSE options. Allowed FUSE configurations are: `-o attr_timeout=TIMEOUT`, `-o negative_timeout=TIMEOUT`, `-o entry_timeout=TIMEOUT` `-o allow_other`, `-o allow_root`, `-o umask=PERMISSIONS -o default_permissions`, `-o ro`"
+
+	UserAgentHeader = "User-Agent"
 )
 
 func FuseIgnoredFlags() []string {

common/version.go

@@ -49,26 +49,26 @@ type Version struct {
 }
 
 // To keep the code simple, we assume we only use a simple subset of semantic versions.
-// Namely, the version is either a normal stable version, or a pre-release version with '-preview' attached.
-// Examples: 10.1.0, 11.2.0-preview.1
+// Namely, the version is either a normal stable version, or a pre-release version with '~preview' or '-preview' attached.
+// Examples: 10.1.0, 11.2.0-preview.1, 11.2.0~preview.1
 func ParseVersion(raw string) (*Version, error) {
 	const standardError = "invalid version string"
 
 	rawSegments := strings.Split(raw, ".")
-	if !(len(rawSegments) == 3 || (len(rawSegments) == 4 && strings.Contains(rawSegments[2], "-"))) {
+	if !(len(rawSegments) == 3 || (len(rawSegments) == 4 && (strings.Contains(rawSegments[2], "-") || strings.Contains(rawSegments[2], "~")))) {
 		return nil, errors.New(standardError)
 	}
 
 	v := &Version{segments: make([]int64, 4), original: raw}
 	for i, str := range rawSegments {
 		//For any case such as SemVer-preview.1, SemVer-beta.1, SemVer-alpha.1 this would be true, and we assume the version to be a preview version.
-		if strings.Contains(str, "-") {
+		if strings.Contains(str, "-") || strings.Contains(str, "~") {
 			if i != 2 {
 				return nil, errors.New(standardError)
 			}
 			v.preview = true
 			//Splitting the string into two pieces and extracting SemVer which is always at 0th index
-			str = strings.Split(str, "-")[0]
+			str = strings.Split(strings.Split(str, "-")[0], "~")[0]
 		}
 
 		val, err := strconv.ParseInt(str, 10, 64)

common/version_test.go

@@ -58,6 +58,14 @@ func (vSuite *versionTestSuite) TestVersionEquality() {
 	v1, _ = ParseVersion("10.0.0-beta.5")
 	v2, _ = ParseVersion("10.0.0-beta.5")
 	assert.Equal(v1.compare(*v2), 0)
+
+	v1, _ = ParseVersion("10.0.0~preview.1")
+	v2, _ = ParseVersion("10.0.0~preview.1")
+	assert.Equal(v1.compare(*v2), 0)
+
+	v1, _ = ParseVersion("10.0.0~beta.5")
+	v2, _ = ParseVersion("10.0.0~beta.5")
+	assert.Equal(v1.compare(*v2), 0)
 }
 
 func (vSuite *versionTestSuite) TestVersionSuperiority() {
@@ -82,6 +90,18 @@ func (vSuite *versionTestSuite) TestVersionSuperiority() {
 	v1, _ = ParseVersion("15.5.5-preview.6")
 	v2, _ = ParseVersion("15.5.5-preview.3")
 	assert.Equal(v1.compare(*v2), 1)
+
+	v1, _ = ParseVersion("15.5.6")
+	v2, _ = ParseVersion("15.5.6~preview.3")
+	assert.Equal(v1.compare(*v2), 1)
+
+	v1, _ = ParseVersion("15.5.6~preview.6")
+	v2, _ = ParseVersion("15.5.6~preview.3")
+	assert.Equal(v1.compare(*v2), 1)
+
+	v1, _ = ParseVersion("15.5.7~preview.6")
+	v2, _ = ParseVersion("15.5.7-preview.3")
+	assert.Equal(v1.compare(*v2), 1)
 }
 
 func (vSuite *versionTestSuite) TestVersionInferiority() {
@@ -106,6 +126,18 @@ func (vSuite *versionTestSuite) TestVersionInferiority() {
 	v1, _ = ParseVersion("15.5.5-preview.3")
 	v2, _ = ParseVersion("15.5.5-preview.6")
 	assert.Equal(v1.compare(*v2), -1)
+
+	v1, _ = ParseVersion("15.5.6~preview.6")
+	v2, _ = ParseVersion("15.5.6")
+	assert.Equal(v1.compare(*v2), -1)
+
+	v1, _ = ParseVersion("15.5.6~preview.3")
+	v2, _ = ParseVersion("15.5.6~preview.6")
+	assert.Equal(v1.compare(*v2), -1)
+
+	v1, _ = ParseVersion("15.5.7-preview.3")
+	v2, _ = ParseVersion("15.5.7~preview.6")
+	assert.Equal(v1.compare(*v2), -1)
 }
 
 func TestVersionTestSuite(t *testing.T) {

component/azstorage/azauth.go

@@ -34,6 +34,8 @@
 package azstorage
 
 import (
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud"
 	"github.com/Azure/azure-storage-fuse/v2/common/log"
 )
 
@@ -71,13 +73,13 @@ type azAuthConfig struct {
 type azAuth interface {
 	getEndpoint() string
 	setOption(key, value string)
-	getCredential() interface{}
+	getServiceClient(stConfig *AzStorageConfig) (interface{}, error)
 }
 
 // getAzAuth returns a new AzAuth
 // config: Defines the AzAuthConfig
 func getAzAuth(config azAuthConfig) azAuth {
-	log.Debug("azAuth::getAzAuth : account %s, account-type %s, protocol %s, endpoint %s",
+	log.Debug("azAuth::getAzAuth : Account: %s, AccountType: %s, Protocol: %s, Endpoint: %s",
 		config.AccountName,
 		config.AccountType,
 		func(useHttp bool) string {
@@ -89,14 +91,14 @@ func getAzAuth(config azAuthConfig) azAuth {
 		config.Endpoint)
 
 	if EAccountType.BLOCK() == config.AccountType {
-		return getAzAuthBlob(config)
+		return getAzBlobAuth(config)
 	} else if EAccountType.ADLS() == config.AccountType {
-		return getAzAuthBfs(config)
+		return getAzDatalakeAuth(config)
 	}
 	return nil
 }
 
-func getAzAuthBlob(config azAuthConfig) azAuth {
+func getAzBlobAuth(config azAuthConfig) azAuth {
 	base := azAuthBase{config: config}
 	if config.AuthMode == EAuthType.KEY() {
 		return &azAuthBlobKey{
@@ -123,39 +125,39 @@ func getAzAuthBlob(config azAuthConfig) azAuth {
 			},
 		}
 	} else {
-		log.Crit("azAuth::getAzAuthBlob : Auth type %s not supported. Failed to create Auth object", config.AuthMode)
+		log.Crit("azAuth::getAzBlobAuth : Auth type %s not supported. Failed to create Auth object", config.AuthMode)
 	}
 	return nil
 }
 
-func getAzAuthBfs(config azAuthConfig) azAuth {
+func getAzDatalakeAuth(config azAuthConfig) azAuth {
 	base := azAuthBase{config: config}
 	if config.AuthMode == EAuthType.KEY() {
-		return &azAuthBfsKey{
+		return &azAuthDatalakeKey{
 			azAuthKey{
 				azAuthBase: base,
 			},
 		}
 	} else if config.AuthMode == EAuthType.SAS() {
-		return &azAuthBfsSAS{
+		return &azAuthDatalakeSAS{
 			azAuthSAS{
 				azAuthBase: base,
 			},
 		}
 	} else if config.AuthMode == EAuthType.MSI() {
-		return &azAuthBfsMSI{
+		return &azAuthDatalakeMSI{
 			azAuthMSI{
 				azAuthBase: base,
 			},
 		}
 	} else if config.AuthMode == EAuthType.SPN() {
-		return &azAuthBfsSPN{
+		return &azAuthDatalakeSPN{
 			azAuthSPN{
 				azAuthBase: base,
 			},
 		}
 	} else {
-		log.Crit("azAuth::getAzAuthBfs : Auth type %s not supported. Failed to create Auth object", config.AuthMode)
+		log.Crit("azAuth::getAzDatalakeAuth : Auth type %s not supported. Failed to create Auth object", config.AuthMode)
 	}
 	return nil
 }
@@ -171,3 +173,32 @@ func (base *azAuthBase) setOption(_, _ string) {}
 func (base *azAuthBase) getEndpoint() string {
 	return base.config.Endpoint
 }
+
+// this type is included in OAuth modes - spn and msi
+type azOAuthBase struct{}
+
+// TODO:: track2 : check ActiveDirectoryEndpoint and AuthResource part
+func (base *azOAuthBase) getAzIdentityClientOptions(config *azAuthConfig) azcore.ClientOptions {
+	if config == nil {
+		log.Err("azAuth::getAzIdentityClientOptions : azAuthConfig is nil")
+		return azcore.ClientOptions{}
+	}
+	opts := azcore.ClientOptions{
+		Cloud:   getCloudConfiguration(config.Endpoint),
+		Logging: getSDKLogOptions(),
+	}
+
+	if config.ActiveDirectoryEndpoint != "" {
+		log.Debug("azAuthBase::getAzIdentityClientOptions : ActiveDirectoryAuthorityHost = %s", config.ActiveDirectoryEndpoint)
+		opts.Cloud.ActiveDirectoryAuthorityHost = config.ActiveDirectoryEndpoint
+	}
+	if config.AuthResource != "" {
+		if val, ok := opts.Cloud.Services[cloud.ResourceManager]; ok {
+			log.Debug("azAuthBase::getAzIdentityClientOptions : AuthResource = %s", config.AuthResource)
+			val.Endpoint = config.AuthResource
+			opts.Cloud.Services[cloud.ResourceManager] = val
+		}
+	}
+
+	return opts
+}

component/azstorage/azauth_test.go

@@ -458,7 +458,7 @@ func (suite *authTestSuite) TestBlockSasKeySetOption() {
 		assert.Fail("TestBlockSasKeySetOption : Failed to create Storage object")
 	}
 	stg.SetupPipeline()
-	stg.NewCredentialKey("saskey", storageTestConfigurationParameters.BlockSas)
+	stg.UpdateServiceClient("saskey", storageTestConfigurationParameters.BlockSas)
 	if err := stg.SetupPipeline(); err != nil {
 		assert.Fail("TestBlockSasKeySetOption : Failed to setup pipeline")
 	}
@@ -591,7 +591,7 @@ func (suite *authTestSuite) TestAdlsSasKeySetOption() {
 		assert.Fail("TestBlockSasKeySetOption : Failed to create Storage object")
 	}
 	stg.SetupPipeline()
-	stg.NewCredentialKey("saskey", storageTestConfigurationParameters.AdlsSas)
+	stg.UpdateServiceClient("saskey", storageTestConfigurationParameters.AdlsSas)
 	if err := stg.SetupPipeline(); err != nil {
 		assert.Fail("TestBlockSasKeySetOption : Failed to setup pipeline")
 	}
@@ -634,22 +634,23 @@ func (suite *authTestSuite) TestBlockMsiResId() {
 	}
 }
 
-func (suite *authTestSuite) TestBlockMsiObjId() {
-	defer suite.cleanupTest()
-	if !storageTestConfigurationParameters.SkipMsi {
-		stgConfig := AzStorageConfig{
-			container: storageTestConfigurationParameters.BlockContainer,
-			authConfig: azAuthConfig{
-				AuthMode:    EAuthType.MSI(),
-				AccountType: EAccountType.BLOCK(),
-				AccountName: storageTestConfigurationParameters.BlockAccount,
-				ObjectID:    storageTestConfigurationParameters.MsiObjId,
-				Endpoint:    generateEndpoint(false, storageTestConfigurationParameters.BlockAccount, EAccountType.BLOCK()),
-			},
-		}
-		suite.validateStorageTest("TestBlockMsiObjId", stgConfig)
-	}
-}
+// ObjectID is not supported in msal. So, commenting this.
+// func (suite *authTestSuite) TestBlockMsiObjId() {
+// 	defer suite.cleanupTest()
+// 	if !storageTestConfigurationParameters.SkipMsi {
+// 		stgConfig := AzStorageConfig{
+// 			container: storageTestConfigurationParameters.BlockContainer,
+// 			authConfig: azAuthConfig{
+// 				AuthMode:    EAuthType.MSI(),
+// 				AccountType: EAccountType.BLOCK(),
+// 				AccountName: storageTestConfigurationParameters.BlockAccount,
+// 				ObjectID:    storageTestConfigurationParameters.MsiObjId,
+// 				Endpoint:    generateEndpoint(false, storageTestConfigurationParameters.BlockAccount, EAccountType.BLOCK()),
+// 			},
+// 		}
+// 		suite.validateStorageTest("TestBlockMsiObjId", stgConfig)
+// 	}
+// }
 
 // Can't use HTTP requests with MSI/SPN credentials
 func (suite *authTestSuite) TestAdlsMsiAppId() {