Telemetry header addition (#1214)

* Adding telemetry string

CHANGELOG.md

@@ -11,6 +11,7 @@
 - If `du` command is not found skip checking for disk usage in LRU cache-eviction policy.
 - V1 flag of `file-cache-timeout-in-seconds` not interpreted correctly by V2 and causing eviction policy to assume its 0. 
 - If `du` is not found on standard path try paths where it can potentially be found.
+- Fix uid/gid marshalling for `mountv1` command, which was resulting in panic.
 
 ## 2.0.4 (2023-07-03)
 **Features**

component/azstorage/azstorage.go

@@ -636,6 +636,10 @@ func init() {
 	disableCompression := config.AddBoolFlag("disable-compression", false, "Disable transport layer compression.")
 	config.BindPFlag(compName+".disable-compression", disableCompression)
 
+	telemetry := config.AddStringFlag("telemetry", "", "Additional telemetry information.")
+	config.BindPFlag(compName+".telemetry", telemetry)
+	telemetry.Hidden = true
+
 	honourACL := config.AddBoolFlag("honour-acl", false, "Match ObjectID in ACL against the one used for authentication.")
 	config.BindPFlag(compName+".honour-acl", honourACL)
 	honourACL.Hidden = true

component/azstorage/config.go

@@ -176,6 +176,7 @@ type AzStorageOptions struct {
 	VirtualDirectory        bool   `config:"virtual-directory" yaml:"virtual-directory"`
 	MaxResultsForList       int32  `config:"max-results-for-list" yaml:"max-results-for-list"`
 	DisableCompression      bool   `config:"disable-compression" yaml:"disable-compression"`
+	Telemetry               string `config:"telemetry" yaml:"telemetry"`
 	HonourACL               bool   `config:"honour-acl" yaml:"honour-acl"`
 
 	// v1 support
@@ -361,6 +362,8 @@ func ParseAndValidateConfig(az *AzStorage, opt AzStorageOptions) error {
 	// Block list call on mount for given amount of time
 	az.stConfig.cancelListForSeconds = opt.CancelListForSeconds
 
+	az.stConfig.telemetry = opt.Telemetry
+
 	httpProxyProvided := opt.HttpProxyAddress != ""
 	httpsProxyProvided := opt.HttpsProxyAddress != ""
 
@@ -486,6 +489,8 @@ func ParseAndValidateConfig(az *AzStorage, opt AzStorageOptions) error {
 	log.Info("ParseAndValidateConfig : Retry Config: Retry count %d, Max Timeout %d, BackOff Time %d, Max Delay %d",
 		az.stConfig.maxRetries, az.stConfig.maxTimeout, az.stConfig.backoffTime, az.stConfig.maxRetryDelay)
 
+	log.Info("ParseAndValidateConfig : Telemetry : %s", az.stConfig.telemetry)
+
 	return nil
 }
 

component/azstorage/connection.go

@@ -76,7 +76,9 @@ type AzStorageConfig struct {
 	virtualDirectory   bool
 	maxResultsForList  int32
 	disableCompression bool
-	HonourACL          bool
+
+	telemetry string
+	HonourACL bool
 }
 
 type AzStorageConnection struct {

component/azstorage/utils.go

@@ -89,8 +89,16 @@ func getAzBlobPipelineOptions(conf AzStorageConfig) (azblob.PipelineOptions, ste
 		RetryDelay:    time.Second * time.Duration(conf.backoffTime),   // Backoff amount for each retry (exponential or linear)
 		MaxRetryDelay: time.Second * time.Duration(conf.maxRetryDelay), // Max delay between retries
 	}
+
+	telemetryValue := conf.telemetry
+	if telemetryValue != "" {
+		telemetryValue += " "
+	}
+
+	telemetryValue += UserAgent() + " (" + common.GetCurrentDistro() + ")"
+
 	telemetryOptions := azblob.TelemetryOptions{
-		Value: UserAgent() + " (" + common.GetCurrentDistro() + ")",
+		Value: telemetryValue,
 	}
 
 	sysLogDisabled := log.GetType() == "silent" // If logging is enabled, allow the SDK to log retries to syslog.
@@ -120,9 +128,17 @@ func getAzBfsPipelineOptions(conf AzStorageConfig) (azbfs.PipelineOptions, ste.X
 		RetryDelay:    time.Second * time.Duration(conf.backoffTime),   // Backoff amount for each retry (exponential or linear)
 		MaxRetryDelay: time.Second * time.Duration(conf.maxRetryDelay), // Max delay between retries
 	}
+
+	telemetryValue := conf.telemetry
+	if telemetryValue != "" {
+		telemetryValue += " "
+	}
+
+	telemetryValue += UserAgent() + " (" + common.GetCurrentDistro() + ")"
 	telemetryOptions := azbfs.TelemetryOptions{
-		Value: UserAgent() + " (" + common.GetCurrentDistro() + ")",
+		Value: telemetryValue,
 	}
+
 	sysLogDisabled := log.GetType() == "silent" // If logging is enabled, allow the SDK to log retries to syslog.
 	requestLogOptions := azbfs.RequestLogOptions{
 		// TODO: We can potentially consider making LogWarningIfTryOverThreshold a user settable option. For now lets use the default

component/libfuse/libfuse.go

@@ -100,7 +100,7 @@ type LibfuseOptions struct {
 	IgnoreOpenFlags         bool   `config:"ignore-open-flags" yaml:"ignore-open-flags,omitempty"`
 	nonEmptyMount           bool   `config:"nonempty" yaml:"nonempty,omitempty"`
 	Uid                     uint32 `config:"uid" yaml:"uid,omitempty"`
-	Gid                     uint32 `config:"gid" yaml:"uid,omitempty"`
+	Gid                     uint32 `config:"gid" yaml:"gid,omitempty"`
 	MaxFuseThreads          uint32 `config:"max-fuse-threads" yaml:"max-fuse-threads,omitempty"`
 	DirectIO                bool   `config:"direct-io" yaml:"direct-io,omitempty"`
 }

setup/baseConfig.yaml

@@ -73,7 +73,7 @@ libfuse:
   # Streaming configuration
 stream:
   # If block-size-mb, blocks-per-file or cache-size-mb are 0, the stream component will not cache blocks. 
-  block-size-mb: <for read only mode:: size of each block to be cached in memory while streaming (in MB). For read/write:: size of newly created blocks. Default - 0 MB>
+  block-size-mb: <for read only mode size of each block to be cached in memory while streaming (in MB). For read/write size of newly created blocks. Default - 0 MB>
   max-buffers: <total number of buffers to store blocks in. Default - 0 MB>
   buffer-size-mb: <size for each buffer. Default - 0>
   file-caching: <read/write mode file level caching or handle level caching. Default - false (handle level caching ON)>
@@ -151,6 +151,7 @@ azstorage:
   virtual-directory: true|false <support virtual directories without existence of a special marker blob>
   disable-compression: true|false <disable transport layer content encoding like gzip, set this flag to true if blobs have content-encoding set in container>
   max-results-for-list: <maximum number of results returned in a single list API call while getting file attributes. Default - 2>
+  telemetry : <additional information that customer want to push in user-agent>
   honour-acl: true|false <honour ACLs on files and directories when mounted using MSI Auth and object-ID is provided in config>
 
 # Mount all configuration

test/longhaul/longhaul.sh

@@ -2,6 +2,9 @@
 SERVICE="blobfuse2"
 SCRIPT="longhaul.sh"
 
+# To create ramdisk
+# sudo mount -t tmpfs -o rw,size=4G tmpfs /mnt/ramdisk
+
 cd /home/vibhansa/go/src/azure-storage-fuse/
 
 if pgrep -x "$SERVICE" > /dev/null
@@ -23,18 +26,23 @@ then
 
 		rm -rf /home/vibhansa/blob_mnt2/stress	
 		rm -rf /home/vibhansa/blob_mnt2/myfile*
+
 		#go test -timeout 120m -v ./test/stress_test/stress_test.go -args -mnt-path=/home/vibhansa/blob_mnt2 -quick=false 2&> ./stress.log
 		./test/longhaul/stresstest.sh
 		echo "`whoami` : `date` :: Ending stress test " >> ./longhaul2.log
 		cp  ./longhaul2.log  /home/vibhansa/blob_mnt2/
 		cp ./stress.log /home/vibhansa/blob_mnt2/
+
 		sleep 30
+
+		rm -rf /mnt/ramdisk/*
 		rm -rf /home/vibhansa/blob_mnt2/stress	
 		sudo rm -rf /var/log/blob*.gz
 	fi
 else
 	echo "`date` :: Re-Starting blobfuse2 *******************" >> ./longhaul2.log
 	rm -rf /home/vibhansa/blob_mnt2/*
+	rm -rf /mnt/ramdisk/*
 	sudo fusermount -u ~/blob_mnt2
 	rm -rf /mnt/ramdisk2/*
 	./blobfuse2 mount ~/blob_mnt2 --config-file=./config.yaml
@@ -47,7 +55,7 @@ else
 	echo "`date`: Restart : `./blobfuse2 --version`" >> ./restart2.log
 
 	# Send email that blobfuse2 has crashed
-	echo "Blobfuse2 Failure" | mail -s "Blobfuse2 Restart" -A ./restart2.log -a "From: [email protected]" <mail id here>
+	echo "Blobfuse2 Failure" | mail -s "Blobfuse2 Restart" -A ./restart2.log -a "From: [email protected]" [email protected]
 
 	cp /var/log/blobfuse2.log /home/vibhansa/blob_mnt2/
 	cp ./longhaul2.log  /home/vibhansa/blob_mnt2/

test/longhaul/stresstest.sh

@@ -2,15 +2,14 @@
 
 # Small file test
 echo "Creating small files"
-for i in {1..2000}; do echo $i; done | parallel --will-cite -j 20 'head -c 1M < /dev/zero > /home/vibhansa/blob_mnt2/myfile_small_{}'
+for i in {1..500}; do echo $i; done | parallel --will-cite -j 20 'head -c 50M < /dev/random > /home/vibhansa/blob_mnt2/myfile_small_{}'
 sleep 5
 
 echo "Reading small files"
-for i in {1..2000}; do echo $i; done | parallel --will-cite -j 20 'hexdump /home/vibhansa/blob_mnt2/myfile_small_{} > /dev/null'
+for i in {1..500}; do echo $i; done | parallel --will-cite -j 20 'hexdump /home/vibhansa/blob_mnt2/myfile_small_{} > /dev/null'
 sleep 5
 
-rm -rf /home/vibhansa/blob_mnt2/myfile_small*
-
+rm -rf /home/vibhansa/blob_mnt2/myfile_small_*
 
 
 # Medium file test

test/longhaul/telemetrytest.sh

@@ -0,0 +1,31 @@
+
+#!/bin/bash
+
+while true
+do
+        while read -r line
+        do
+                echo "Running with : " $line
+
+                rm -rf ~/blob_mnt2/stress*
+                rm -rf ~/blob_mnt2/myfile*
+
+                echo "Un-Mounting Blobfuse2"
+                ./blobfuse2 unmount all
+                rm -rf ~/blob_mnt2/*
+                rm -rf ~/blob_mnt2/.*
+                rm -rf /mnt/ramdisk/*
+
+                echo "Mounting Blobfuse2"
+                ./blobfuse2 mount ~/blob_mnt2 --config-file=./config.yaml --file-cache-timeout=0 --telemetry="$line"
+                sleep 2
+
+                echo "Blobfuse2 pid : " `pidof blobfuse2`
+                echo "`whoami` : `date` :: Starting stress test " >> ./longhaul2.log
+                ./test/longhaul/stresstest.sh
+                echo "`whoami` : `date` :: Ending stress test " >> ./longhaul2.log
+
+                sleep 300
+        done < $1
+done
+