add prometheus metrics (#22)

add prometheus metrics

cmd/injector/injector.go

@@ -6,8 +6,8 @@ import (
 	"k8s.io/klog/v2"
 	"log"
 	"net/http"
+	"rusi/internal/kube"
 	"rusi/pkg/injector"
-	"rusi/pkg/kube"
 )
 
 func main() {

cmd/operator/operator.go

@@ -3,7 +3,7 @@ package main
 import (
 	"flag"
 	"k8s.io/klog/v2"
-	"rusi/pkg/kube"
+	"rusi/internal/kube"
 	"rusi/pkg/operator"
 )
 

cmd/rusid/sidecar.go

@@ -7,6 +7,8 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
+	"rusi/internal/diagnostics"
+	"rusi/internal/metrics"
 	"rusi/internal/tracing"
 	grpc_api "rusi/pkg/api/runtime/grpc"
 	components_loader "rusi/pkg/custom-resource/components/loader"
@@ -43,7 +45,7 @@ func main() {
 	}
 
 	//setup tracing
-	go tracing.WatchConfig(mainCtx, configLoader, tracing.SetJaegerTracing, "dev", cfg.AppID)
+	go diagnostics.WatchConfig(mainCtx, configLoader, tracing.SetJaegerTracing("dev", cfg.AppID))
 
 	compManager, err := runtime.NewComponentsManager(mainCtx, cfg.AppID, compLoader,
 		RegisterComponentFactories()...)
@@ -63,8 +65,8 @@ func main() {
 		"app id", cfg.AppID, "mode", cfg.Mode)
 	klog.InfoS("Rusid is using", "config", cfg)
 
-	//Start healthz server
-	go startHealthzServer(mainCtx, wg, cfg.HealthzPort,
+	//Start diagnostics server
+	go startDiagnosticsServer(mainCtx, wg, cfg.DiagnosticsPort, cfg.EnableMetrics,
 		// WithTimeout allows you to set a max overall timeout.
 		healthcheck.WithTimeout(5*time.Second),
 		healthcheck.WithChecker("component manager", compManager))
@@ -77,6 +79,7 @@ func main() {
 	}
 
 	wg.Wait() // wait for app to close gracefully
+	klog.Info("Rusid closed gracefully")
 }
 
 func shutdownOnInterrupt(cancel func()) {
@@ -88,16 +91,25 @@ func shutdownOnInterrupt(cancel func()) {
 		klog.InfoS("Shutdown requested")
 		cancel()
 	}()
-
 }
 
-func startHealthzServer(ctx context.Context, wg *sync.WaitGroup, healthzPort int, options ...healthcheck.Option) {
+func startDiagnosticsServer(ctx context.Context, wg *sync.WaitGroup, port int,
+	enableMetrics bool, options ...healthcheck.Option) {
 	wg.Add(1)
 	defer wg.Done()
 
-	if err := healthcheck.Run(ctx, healthzPort, options...); err != nil {
+	router := http.NewServeMux()
+	router.Handle("/healthz", healthcheck.HandlerFunc(options...))
+
+	if enableMetrics {
+		router.Handle("/metrics", metrics.GetPrometheusMetricHandler())
+	} else {
+		metrics.SetNoopMeterProvider()
+	}
+
+	if err := diagnostics.Run(ctx, port, router); err != nil {
 		if err != http.ErrServerClosed {
-			klog.ErrorS(err, "failed to start healthz server")
+			klog.ErrorS(err, "failed to start diagnostics server")
 		}
 	}
 }

go.mod

@@ -4,64 +4,71 @@ go 1.17
 
 require (
 	github.com/google/uuid v1.3.0
-	github.com/json-iterator/go v1.1.11
+	github.com/json-iterator/go v1.1.12
 	github.com/kelseyhightower/envconfig v1.4.0
-	github.com/nats-io/nats-server/v2 v2.3.4 // indirect
-	github.com/nats-io/nats-streaming-server v0.22.1 // indirect
-	github.com/nats-io/nats.go v1.12.0
+	github.com/nats-io/nats.go v1.13.0
 	github.com/nats-io/stan.go v0.10.0
 	github.com/pkg/errors v0.9.1
 	github.com/stretchr/testify v1.7.0
-	go.opentelemetry.io/contrib/propagators/jaeger v0.0.0-20210904165346-f650585d08df
-	go.opentelemetry.io/otel v1.0.0-RC3
-	go.opentelemetry.io/otel/exporters/jaeger v1.0.0-RC3
-	go.opentelemetry.io/otel/sdk v1.0.0-RC3
-	go.opentelemetry.io/otel/trace v1.0.0-RC3
+	go.opentelemetry.io/contrib/propagators/jaeger v1.0.0
+	go.opentelemetry.io/otel v1.0.1
+	go.opentelemetry.io/otel/exporters/jaeger v1.0.1
+	go.opentelemetry.io/otel/exporters/prometheus v0.24.0
+	go.opentelemetry.io/otel/metric v0.24.0
+	go.opentelemetry.io/otel/sdk v1.0.1
+	go.opentelemetry.io/otel/sdk/export/metric v0.24.0
+	go.opentelemetry.io/otel/sdk/metric v0.24.0
+	go.opentelemetry.io/otel/trace v1.0.1
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	google.golang.org/grpc v1.40.0
+	google.golang.org/grpc v1.41.0
 	google.golang.org/protobuf v1.27.1
 	gopkg.in/yaml.v2 v2.4.0
-	k8s.io/api v0.22.1
-	k8s.io/apiextensions-apiserver v0.22.1
-	k8s.io/apimachinery v0.22.1
-	k8s.io/client-go v0.22.1
-	k8s.io/klog/v2 v2.10.0
+	k8s.io/api v0.22.2
+	k8s.io/apiextensions-apiserver v0.22.2
+	k8s.io/apimachinery v0.22.2
+	k8s.io/client-go v0.22.2
+	k8s.io/klog/v2 v2.20.0
 )
 
 require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/evanphx/json-patch v4.11.0+incompatible // indirect
-	github.com/go-logr/logr v0.4.0 // indirect
+	github.com/go-logr/logr v1.1.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/go-cmp v0.5.6 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/googleapis/gnostic v0.5.5 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/nats-io/nats-server/v2 v2.6.2 // indirect
+	github.com/nats-io/nats-streaming-server v0.23.0 // indirect
 	github.com/nats-io/nkeys v0.3.0 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_golang v1.11.0 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.31.1 // indirect
+	github.com/prometheus/procfs v0.7.3 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect
-	golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 // indirect
-	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
-	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
-	golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect
-	golang.org/x/text v0.3.6 // indirect
+	go.opentelemetry.io/otel/internal/metric v0.24.0 // indirect
+	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
+	golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f // indirect
+	golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1 // indirect
+	golang.org/x/sys v0.0.0-20211015200801-69063c4bb744 // indirect
+	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
+	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71 // indirect
+	google.golang.org/genproto v0.0.0-20211018162055-cf77aa76bad2 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
-	k8s.io/kube-openapi v0.0.0-20210817084001-7fbd8d59e5b8 // indirect
-	k8s.io/utils v0.0.0-20210820185131-d34e5cb4466e // indirect
+	k8s.io/kube-openapi v0.0.0-20211014175136-b3fe75cc9b2f // indirect
+	k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect
-	sigs.k8s.io/yaml v1.2.0 // indirect
-)
-
-exclude (
-	github.com/go-logr/logr v1.0.0
-	github.com/go-logr/logr v1.1.0
+	sigs.k8s.io/yaml v1.3.0 // indirect
 )