[cinder-csi-plugin] Add http endpoint of CSI container

cmd/cinder-csi-plugin/main.go

@@ -19,23 +19,29 @@ package main
 import (
 	"flag"
 	"fmt"
+	"net/http"
 	"os"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"k8s.io/cloud-provider-openstack/pkg/csi/cinder"
 	"k8s.io/cloud-provider-openstack/pkg/csi/cinder/openstack"
+	"k8s.io/cloud-provider-openstack/pkg/metrics"
 	"k8s.io/cloud-provider-openstack/pkg/util/metadata"
 	"k8s.io/cloud-provider-openstack/pkg/util/mount"
 	"k8s.io/component-base/logs"
 	"k8s.io/klog/v2"
+
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 var (
-	endpoint    string
-	nodeID      string
-	cloudconfig string
-	cluster     string
+	endpoint     string
+	nodeID       string
+	cloudconfig  string
+	cluster      string
+	httpEndpoint string
+	metricsPath  string
 )
 
 func init() {
@@ -85,6 +91,10 @@ func main() {
 
 	cmd.PersistentFlags().StringVar(&cluster, "cluster", "", "The identifier of the cluster that the plugin is running in.")
 
+	cmd.PersistentFlags().StringVar(&httpEndpoint, "http-endpoint", "", "The TCP network address where the HTTP server for diagnostics, including metrics and leader election health check, will listen (example: `:8080`). The default is empty string, which means the server is disabled.")
+
+	cmd.PersistentFlags().StringVar(&metricsPath, "metrics-path", "/metrics", "The HTTP path where prometheus metrics will be exposed. Default is `/metrics`.")
+
 	openstack.AddExtraFlags(pflag.CommandLine)
 
 	logs.InitLogs()
@@ -100,6 +110,18 @@ func main() {
 
 func handle() {
 
+	if httpEndpoint != "" {
+		metrics.RegisterAPIPrometheusMetrics()
+		http.Handle("/metrics", promhttp.Handler())
+		go func() {
+			klog.Infof("ServeMux listening at %v", httpEndpoint)
+			err := http.ListenAndServe(httpEndpoint, nil)
+			if err != nil {
+				klog.Fatalf("Failed to start HTTP serv")
+			}
+		}()
+	}
+
 	d := cinder.NewDriver(nodeID, endpoint, cluster)
 	// Initiliaze cloud
 	openstack.InitOpenStackProvider(cloudconfig)

go.mod

@@ -25,6 +25,7 @@ require (
 	github.com/onsi/gomega v1.9.0
 	github.com/pborman/uuid v1.2.0
 	github.com/pelletier/go-toml v1.4.0 // indirect
+	github.com/prometheus/client_golang v1.7.1
 	github.com/sirupsen/logrus v1.6.0
 	github.com/spf13/cobra v1.1.1
 	github.com/spf13/pflag v1.0.5

go.sum

@@ -103,7 +103,6 @@ github.com/clusterhq/flocker-go v0.0.0-20160920122132-2b8b7259d313/go.mod h1:P1w
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/codegangsta/negroni v1.0.0/go.mod h1:v0y3T5G7Y1UlFfyxFn/QLRU4a2EuNau2iZY63YTKWo0=
-github.com/container-storage-interface/spec v1.2.0 h1:bD9KIVgaVKKkQ/UbVUY9kCaH/CJbhNxe0eeB4JeJV2s=
 github.com/container-storage-interface/spec v1.2.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4=
 github.com/container-storage-interface/spec v1.3.0 h1:wMH4UIoWnK/TXYw8mbcIHgZmB6kHOeIsYsiaTJwa6bc=
 github.com/container-storage-interface/spec v1.3.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4=
@@ -309,8 +308,6 @@ github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyyc
 github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
 github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
 github.com/gophercloud/gophercloud v0.6.1-0.20191122030953-d8ac278c1c9d/go.mod h1:ozGNgr9KYOVATV5jsgHl/ceCDXGuguqOZAzoQ/2vcNM=
-github.com/gophercloud/gophercloud v0.12.1-0.20200922152735-900f5ba8c05b h1:dcAizxhbIBU4A7Iu9uARqZJz63uesA5MtXjHD1XSg3M=
-github.com/gophercloud/gophercloud v0.12.1-0.20200922152735-900f5ba8c05b/go.mod h1:VX0Ibx85B60B5XOrZr6kaNwrmPUzcmMpwxvQ1WQIIWM=
 github.com/gophercloud/gophercloud v0.15.1-0.20210105012856-e34a44dc6580 h1:EWFhxn19tw6t3LjFe95Kli30sBI02+fPYBqMvXmCvtU=
 github.com/gophercloud/gophercloud v0.15.1-0.20210105012856-e34a44dc6580/go.mod h1:VX0Ibx85B60B5XOrZr6kaNwrmPUzcmMpwxvQ1WQIIWM=
 github.com/gophercloud/utils v0.0.0-20200423144003-7c72efc7435d h1:fduaPzWwIfvOMLuHk2Al3GZH0XbUqG8MbElPop+Igzs=

manifests/cinder-csi-plugin/cinder-csi-controllerplugin.yaml

@@ -104,6 +104,7 @@ spec:
             - "--endpoint=$(CSI_ENDPOINT)"
             - "--cloud-config=$(CLOUD_CONFIG)"
             - "--cluster=$(CLUSTER_NAME)"
+            - "--http-endpoint=$(HTTP_ENDPOINT)"
           env:
             - name: NODE_ID
               valueFrom:
@@ -115,6 +116,8 @@ spec:
               value: /etc/config/cloud.conf
             - name: CLUSTER_NAME
               value: kubernetes
+            - name: HTTP_ENDPOINT
+              value: :8080
           imagePullPolicy: "IfNotPresent"
           ports:
             - containerPort: 9808

pkg/csi/cinder/openstack/openstack_snapshots.go

@@ -27,6 +27,7 @@ import (
 	"github.com/gophercloud/gophercloud/openstack/blockstorage/v3/snapshots"
 	"github.com/gophercloud/gophercloud/pagination"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/cloud-provider-openstack/pkg/metrics"
 	"k8s.io/klog/v2"
 )
 
@@ -67,8 +68,12 @@ func (os *OpenStack) CreateSnapshot(name, volID string, tags *map[string]string)
 		opts.Metadata = *tags
 	}
 	// TODO: Do some check before really call openstack API on the input
+	mc := metrics.NewMetricPrometheusContext("snapshot", "create")
 
 	snap, err := snapshots.Create(os.blockstorage, opts).Extract()
+	if mc.ObserveRequest(err) != nil {
+		return nil, err
+	}
 	if err != nil {
 		return &snapshots.Snapshot{}, err
 	}
@@ -85,6 +90,8 @@ func (os *OpenStack) ListSnapshots(filters map[string]string) ([]snapshots.Snaps
 	var nextPageToken string
 	var snaps []snapshots.Snapshot
 
+	mc := metrics.NewMetricPrometheusContext("snapshot", "list")
+
 	// Build the Opts
 	opts := snapshots.ListOpts{}
 	for key, val := range filters {
@@ -126,6 +133,9 @@ func (os *OpenStack) ListSnapshots(filters map[string]string) ([]snapshots.Snaps
 
 		return false, nil
 	})
+	if mc.ObserveRequest(err) != nil {
+		return nil, nextPageToken, err
+	}
 	if err != nil {
 		return nil, nextPageToken, err
 	}
@@ -135,7 +145,12 @@ func (os *OpenStack) ListSnapshots(filters map[string]string) ([]snapshots.Snaps
 
 // DeleteSnapshot issues a request to delete the Snapshot with the specified ID from the Cinder backend
 func (os *OpenStack) DeleteSnapshot(snapID string) error {
+	mc := metrics.NewMetricPrometheusContext("snapshot", "delete")
 	err := snapshots.Delete(os.blockstorage, snapID).ExtractErr()
+	if mc.ObserveRequest(err) != nil {
+		return err
+	}
+
 	if err != nil {
 		klog.Errorf("Failed to delete snapshot: %v", err)
 	}
@@ -144,7 +159,12 @@ func (os *OpenStack) DeleteSnapshot(snapID string) error {
 
 //GetSnapshotByID returns snapshot details by id
 func (os *OpenStack) GetSnapshotByID(snapshotID string) (*snapshots.Snapshot, error) {
+	mc := metrics.NewMetricPrometheusContext("snapshot", "get")
 	s, err := snapshots.Get(os.blockstorage, snapshotID).Extract()
+	if mc.ObserveRequest(err) != nil {
+		return nil, err
+	}
+
 	if err != nil {
 		klog.Errorf("Failed to get snapshot: %v", err)
 		return nil, err

pkg/csi/cinder/openstack/openstack_volumes.go

@@ -28,6 +28,7 @@ import (
 	"github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/volumeattach"
 	"github.com/gophercloud/gophercloud/pagination"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/cloud-provider-openstack/pkg/metrics"
 	cpoerrors "k8s.io/cloud-provider-openstack/pkg/util/errors"
 
 	"k8s.io/klog/v2"
@@ -63,6 +64,7 @@ func (os *OpenStack) CheckBlockStorageAPI() error {
 
 // CreateVolume creates a volume of given size
 func (os *OpenStack) CreateVolume(name string, size int, vtype, availability string, snapshotID string, sourcevolID string, tags *map[string]string) (*volumes.Volume, error) {
+	mc := metrics.NewMetricPrometheusContext("volume", "create")
 
 	opts := &volumes.CreateOpts{
 		Name:             name,
@@ -78,6 +80,9 @@ func (os *OpenStack) CreateVolume(name string, size int, vtype, availability str
 	}
 
 	vol, err := volumes.Create(os.blockstorage, opts).Extract()
+	if mc.ObserveRequest(err) != nil {
+		return nil, err
+	}
 	if err != nil {
 		return nil, err
 	}
@@ -90,6 +95,8 @@ func (os *OpenStack) ListVolumes(limit int, startingToken string) ([]volumes.Vol
 	var nextPageToken string
 	var vols []volumes.Volume
 
+	mc := metrics.NewMetricPrometheusContext("volume", "list")
+
 	opts := volumes.ListOpts{Limit: limit, Marker: startingToken}
 	err := volumes.List(os.blockstorage, opts).EachPage(func(page pagination.Page) (bool, error) {
 		var err error
@@ -114,6 +121,9 @@ func (os *OpenStack) ListVolumes(limit int, startingToken string) ([]volumes.Vol
 
 		return false, nil
 	})
+	if mc.ObserveRequest(err) != nil {
+		return nil, nextPageToken, err
+	}
 	if err != nil {
 		return nil, nextPageToken, err
 	}
@@ -124,11 +134,16 @@ func (os *OpenStack) ListVolumes(limit int, startingToken string) ([]volumes.Vol
 // GetVolumesByName is a wrapper around ListVolumes that creates a Name filter to act as a GetByName
 // Returns a list of Volume references with the specified name
 func (os *OpenStack) GetVolumesByName(n string) ([]volumes.Volume, error) {
+	mc := metrics.NewMetricPrometheusContext("volume", "get")
+
 	opts := volumes.ListOpts{Name: n}
 	pages, err := volumes.List(os.blockstorage, opts).AllPages()
 	if err != nil {
 		return nil, err
 	}
+	if mc.ObserveRequest(err) != nil {
+		return nil, err
+	}
 
 	vols, err := volumes.ExtractVolumes(pages)
 	if err != nil {
@@ -140,6 +155,8 @@ func (os *OpenStack) GetVolumesByName(n string) ([]volumes.Volume, error) {
 
 // DeleteVolume delete a volume
 func (os *OpenStack) DeleteVolume(volumeID string) error {
+	mc := metrics.NewMetricPrometheusContext("volume", "delete")
+
 	used, err := os.diskIsUsed(volumeID)
 	if err != nil {
 		return err
@@ -149,16 +166,23 @@ func (os *OpenStack) DeleteVolume(volumeID string) error {
 	}
 
 	err = volumes.Delete(os.blockstorage, volumeID, nil).ExtractErr()
+	if mc.ObserveRequest(err) != nil {
+		return err
+	}
 	return err
 }
 
 // GetVolume retrieves Volume by its ID.
 func (os *OpenStack) GetVolume(volumeID string) (*volumes.Volume, error) {
+	mc := metrics.NewMetricPrometheusContext("volume", "get")
 
 	vol, err := volumes.Get(os.blockstorage, volumeID).Extract()
 	if err != nil {
 		return nil, err
 	}
+	if mc.ObserveRequest(err) != nil {
+		return nil, err
+	}
 
 	return vol, nil
 }
@@ -189,9 +213,14 @@ func (os *OpenStack) AttachVolume(instanceID, volumeID string) (string, error) {
 		computeServiceClient.Microversion = "2.60"
 	}
 
+	mc := metrics.NewMetricPrometheusContext("volumeattachment", "create")
+
 	_, err = volumeattach.Create(computeServiceClient, instanceID, &volumeattach.CreateOpts{
 		VolumeID: volume.ID,
 	}).Extract()
+	if mc.ObserveRequest(err) != nil {
+		return "", err
+	}
 
 	if err != nil {
 		return "", fmt.Errorf("failed to attach %s volume to %s compute: %v", volumeID, instanceID, err)
@@ -276,7 +305,12 @@ func (os *OpenStack) DetachVolume(instanceID, volumeID string) error {
 	// Incase volume is of type multiattach, it could be attached to more than one instance
 	for _, att := range volume.Attachments {
 		if att.ServerID == instanceID {
+			mc := metrics.NewMetricPrometheusContext("volumeattachment", "delete")
+
 			err = volumeattach.Delete(os.compute, instanceID, volume.ID).ExtractErr()
+			if mc.ObserveRequest(err) != nil {
+				return err
+			}
 			if err != nil {
 				return fmt.Errorf("failed to detach volume %s from compute %s : %v", volume.ID, instanceID, err)
 			}
@@ -349,10 +383,21 @@ func (os *OpenStack) ExpandVolume(volumeID string, status string, newSize int) e
 		// cinder online resize is available since 3.42 microversion
 		// https://docs.openstack.org/cinder/latest/contributor/api_microversion_history.html#id40
 		blockstorageClient.Microversion = "3.42"
+		mc := metrics.NewMetricPrometheusContext("volume", "update")
+
+		err = volumeexpand.ExtendSize(blockstorageClient, volumeID, extendOpts).ExtractErr()
+		if mc.ObserveRequest(err) != nil {
+			return err
+		}
+		return err
 
-		return volumeexpand.ExtendSize(blockstorageClient, volumeID, extendOpts).ExtractErr()
 	case VolumeAvailableStatus:
-		return volumeexpand.ExtendSize(os.blockstorage, volumeID, extendOpts).ExtractErr()
+		mc := metrics.NewMetricPrometheusContext("volume", "update")
+		err := volumeexpand.ExtendSize(os.blockstorage, volumeID, extendOpts).ExtractErr()
+		if mc.ObserveRequest(err) != nil {
+			return err
+		}
+		return err
 	}
 
 	// cinder volume can not be expanded when volume status is not volumeInUseStatus or not volumeAvailableStatus