Custom component feature

CHANGELOG.md

@@ -1,6 +1,9 @@
 ## 2.3.3 (Unreleased)
 **Bug Fixes**
 
+**Features**
+- Added support for custom component via go plugin.
+
 ## 2.3.2 (2024-09-03)
 **Bug Fixes**
 - Fixed the case where file creation using SAS on HNS accounts was returning back wrong error code.

README.md

@@ -203,6 +203,8 @@ To learn about a specific command, just include the name of the command (For exa
 - CPK options: 
     * `AZURE_STORAGE_CPK_ENCRYPTION_KEY`: Customer provided base64-encoded AES-256 encryption key value.
     * `AZURE_STORAGE_CPK_ENCRYPTION_KEY_SHA256`: Base64-encoded SHA256 of the cpk encryption key.
+- Custom component options:
+    * `BLOBFUSE_PLUGIN_PATH`: Specifies plugin file path as a colon-separated list of `.so` files. Example BLOBFUSE_PLUGIN_PATH="/path/to/plugin1.so:/path/to/plugin2.so".
 
 
 ## Config Guide

SampleCustomConfig.yaml

@@ -0,0 +1,30 @@
+# sample config to test the custom component
+allow-other: true
+
+logging:
+  type: base
+  level: log_debug
+
+components:
+  - libfuse
+  - block_cache
+  - sample_custom_component1
+  - sample_custom_component2
+
+sample_custom_component1:
+  block-size-mb: 16
+
+sample_custom_component2:
+  block-size-mb: 16
+libfuse:
+  attribute-expiration-sec: 120
+  entry-expiration-sec: 120
+  negative-entry-expiration-sec: 240
+
+block_cache:
+  block-size-mb: 16
+  path: /tmp/cache
+  parallelism: 128
+
+attr_cache:
+  timeout-sec: 7200

cmd/imports.go

@@ -37,6 +37,7 @@ import (
 	_ "github.com/Azure/azure-storage-fuse/v2/component/attr_cache"
 	_ "github.com/Azure/azure-storage-fuse/v2/component/azstorage"
 	_ "github.com/Azure/azure-storage-fuse/v2/component/block_cache"
+	_ "github.com/Azure/azure-storage-fuse/v2/component/custom"
 	_ "github.com/Azure/azure-storage-fuse/v2/component/file_cache"
 	_ "github.com/Azure/azure-storage-fuse/v2/component/libfuse"
 	_ "github.com/Azure/azure-storage-fuse/v2/component/loopback"

component/block_cache/block_cache.go

@@ -1350,9 +1350,10 @@ func (bc *BlockCache) upload(item *workItem) {
 
 	// This block is updated so we need to stage it now
 	err := bc.NextComponent().StageData(internal.StageDataOptions{
-		Name: item.handle.Path,
-		Data: item.block.data[0 : item.block.endIndex-item.block.offset],
-		Id:   item.blockId})
+		Name:   item.handle.Path,
+		Data:   item.block.data[0 : item.block.endIndex-item.block.offset],
+		Offset: uint64(item.block.offset),
+		Id:     item.blockId})
 	if err != nil {
 		// Fail to write the data so just reschedule this request
 		log.Err("BlockCache::upload : Failed to write %v=>%s from offset %v [%s]", item.handle.ID, item.handle.Path, item.block.id, err.Error())

component/custom/custom.go

@@ -0,0 +1,96 @@
+/*
+    _____           _____   _____   ____          ______  _____  ------
+   |     |  |      |     | |     | |     |     | |       |            |
+   |     |  |      |     | |     | |     |     | |       |            |
+   | --- |  |      |     | |-----| |---- |     | |-----| |-----  ------
+   |     |  |      |     | |     | |     |     |       | |       |
+   | ____|  |_____ | ____| | ____| |     |_____|  _____| |_____  |_____
+
+
+   Licensed under the MIT License <http://opensource.org/licenses/MIT>.
+
+   Copyright © 2020-2024 Microsoft Corporation. All rights reserved.
+   Author : <[email protected]>
+
+   Permission is hereby granted, free of charge, to any person obtaining a copy
+   of this software and associated documentation files (the "Software"), to deal
+   in the Software without restriction, including without limitation the rights
+   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+   copies of the Software, and to permit persons to whom the Software is
+   furnished to do so, subject to the following conditions:
+
+   The above copyright notice and this permission notice shall be included in all
+   copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE
+*/
+
+package custom
+
+import (
+	"fmt"
+	"os"
+	"plugin"
+	"strings"
+	"time"
+
+	"github.com/Azure/azure-storage-fuse/v2/common/log"
+	"github.com/Azure/azure-storage-fuse/v2/exported"
+	"github.com/Azure/azure-storage-fuse/v2/internal"
+)
+
+func initializePlugins() error {
+
+	// Environment variable which expects file path as a colon-separated list of `.so` files.
+	// Example BLOBFUSE_PLUGIN_PATH="/path/to/plugin1.so:/path/to/plugin2.so"
+	pluginFilesPath := os.Getenv("BLOBFUSE_PLUGIN_PATH")
+	if pluginFilesPath == "" {
+		return nil
+	}
+
+	pluginFiles := strings.Split(pluginFilesPath, ":")
+
+	for _, file := range pluginFiles {
+		if strings.HasSuffix(file, ".so") {
+			log.Info("loading plugin %s", file)
+			startTime := time.Now()
+			p, err := plugin.Open(file)
+			if err != nil {
+				return fmt.Errorf("error opening plugin %s: %s", file, err.Error())
+			}
+
+			getExternalComponentFunc, err := p.Lookup("GetExternalComponent")
+			if err != nil {
+				return fmt.Errorf("error looking up GetExternalComponent function in %s: %s", file, err.Error())
+			}
+
+			getExternalComponent, ok := getExternalComponentFunc.(func() (string, func() exported.Component))
+			if !ok {
+				return fmt.Errorf("error casting GetExternalComponent function in %s", file)
+			}
+
+			compName, initExternalComponent := getExternalComponent()
+			internal.AddComponent(compName, initExternalComponent)
+			duration := time.Since(startTime)
+			log.Info("plugin %s loaded successfully in %v", file, duration)
+		} else {
+			return fmt.Errorf("invalid plugin file extension: %s", file)
+		}
+	}
+	return nil
+}
+
+func init() {
+	err := initializePlugins()
+	if err != nil {
+		log.Err("custom::Error initializing plugins: %s", err.Error())
+		fmt.Printf("failed to initialize plugin: %s\n", err.Error())
+		os.Exit(1)
+	}
+}

component/custom/custom_test.go

@@ -0,0 +1,111 @@
+/*
+    _____           _____   _____   ____          ______  _____  ------
+   |     |  |      |     | |     | |     |     | |       |            |
+   |     |  |      |     | |     | |     |     | |       |            |
+   | --- |  |      |     | |-----| |---- |     | |-----| |-----  ------
+   |     |  |      |     | |     | |     |     |       | |       |
+   | ____|  |_____ | ____| | ____| |     |_____|  _____| |_____  |_____
+
+
+   Licensed under the MIT License <http://opensource.org/licenses/MIT>.
+
+   Copyright © 2020-2024 Microsoft Corporation. All rights reserved.
+   Author : <[email protected]>
+
+   Permission is hereby granted, free of charge, to any person obtaining a copy
+   of this software and associated documentation files (the "Software"), to deal
+   in the Software without restriction, including without limitation the rights
+   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+   copies of the Software, and to permit persons to whom the Software is
+   furnished to do so, subject to the following conditions:
+
+   The above copyright notice and this permission notice shall be included in all
+   copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE
+*/
+
+package custom
+
+import (
+	"os"
+	"os/exec"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/suite"
+)
+
+type customTestSuite struct {
+	suite.Suite
+	assert *assert.Assertions
+}
+
+func (suite *customTestSuite) SetupTest() {
+	suite.assert = assert.New(suite.T())
+}
+
+// This test builds a custom component and then tries to load the .so file
+// Though both .so file and tests are being built using same go packages, it still gives an error in loading .so file
+//
+//	"plugin was built with a different version of package"
+//
+// Hence, this test is disabled for now.
+// Same .so file loads fine when blobfuse is run from CLI.
+// If you wish to debug blobfuse2 with a custom component then always build .so file with "-gcflags=all=-N -l" option
+//
+//	e.g. go build -buildmode=plugin -gcflags=all=-N -l -o x.so <path to source>
+//
+// This flag disables all optimizations and inline replacements and then .so will load in debug mode as well.
+// However same .so will not work with cli mount and there you need to build .so without these flags.
+func (suite *customTestSuite) _TestInitializePluginsValidPath() {
+	// Direct paths to the Go plugin source files
+	source1 := "../../test/sample_custom_component1/main.go"
+	source2 := "../../test/sample_custom_component2/main.go"
+
+	// Paths to the compiled .so files in the current directory
+	plugin1 := "./sample_custom_component1.so"
+	plugin2 := "./sample_custom_component2.so"
+
+	// Compile the Go plugin source files into .so files
+	cmd := exec.Command("go", "build", "-buildmode=plugin", "-gcflags=all=-N -l", "-o", plugin1, source1)
+	err := cmd.Run()
+	suite.assert.Nil(err)
+	cmd = exec.Command("go", "build", "-buildmode=plugin", "-gcflags=all=-N -l", "-o", plugin2, source2)
+	err = cmd.Run()
+	suite.assert.Nil(err)
+
+	os.Setenv("BLOBFUSE_PLUGIN_PATH", plugin1+":"+plugin2)
+
+	err = initializePlugins()
+	suite.assert.Nil(err)
+
+	// Clean up the generated .so files
+	os.Remove(plugin1)
+	os.Remove(plugin2)
+}
+
+func (suite *customTestSuite) TestInitializePluginsInvalidPath() {
+	dummyPath := "/invalid/path/plugin1.so"
+	os.Setenv("BLOBFUSE_PLUGIN_PATH", dummyPath)
+
+	err := initializePlugins()
+	suite.assert.NotNil(err)
+}
+
+func (suite *customTestSuite) TestInitializePluginsEmptyPath() {
+	os.Setenv("BLOBFUSE_PLUGIN_PATH", "")
+
+	err := initializePlugins()
+	suite.assert.Nil(err)
+}
+
+func TestCustomSuite(t *testing.T) {
+	suite.Run(t, new(customTestSuite))
+}