managerd: Exposing metrics as a prometheus endpoint

doc/sshproxy-managerd.yaml.txt

@@ -32,6 +32,10 @@ The following keys can be defined:
 	a string specifying the listening address. The format is '[host]:port'
 	and the default is '127.0.0.1:55555'.
 
+*promlisten*::
+	a string specifying the listening address of the prometheus exporter. The format is '[host]:port'
+	and the default is ':55556'.
+
 *log*::
 	a string which can be:
 	- empty ('""') to display the logs on the standard output. It is the
@@ -92,6 +96,8 @@ debug: false
 
 listen: 127.0.0.1:55555
 
+listen: :55556
+
 log: syslog
 
 check_interval: 10m

glide.yaml

@@ -1,9 +1,13 @@
-package:  github.com/cea-hpc/sshproxy
+package: github.com/cea-hpc/sshproxy
 homepage: https://github.com/cea-hpc/sshproxy
-license:  CECILL-B
+license: CECILL-B
 owners:
-- name:  Arnaud Guignard
+- name: Arnaud Guignard
   email: [email protected]
+ignore:
+- github.com/Azure/go-ansiterm
+- github.com/docker/docker/pkg/term/windows
+- github.com/sirupsen/logrus
 import:
 - package: github.com/docker/docker
   version: v17.05.0-ce
@@ -15,8 +19,9 @@ import:
   version: v1
 - package: gopkg.in/yaml.v2
   version: v2
-  repo:    https://github.com/go-yaml/yaml.git
-ignore:
-- github.com/Azure/go-ansiterm
-- github.com/docker/docker/pkg/term/windows
-- github.com/sirupsen/logrus
+  repo: https://github.com/go-yaml/yaml.git
+- package: github.com/prometheus/client_golang
+  version: ~0.9.0-pre1
+  subpackages:
+  - prometheus
+  - prometheus/promhttp

sshproxy-managerd/sshproxy-managerd.go

@@ -17,6 +17,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net"
+	"net/http"
 	"os"
 	"os/signal"
 	"regexp"
@@ -28,14 +29,18 @@ import (
 	"github.com/cea-hpc/sshproxy/utils"
 
 	"github.com/op/go-logging"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+
 	"gopkg.in/yaml.v2"
 )
 
 var (
 	// SshproxyVersion is set in the Makefile.
-	SshproxyVersion   = "0.0.0+notproperlybuilt"
-	defaultConfig     = "/etc/sshproxy/sshproxy-managerd.yaml"
-	defaultListenAddr = "127.0.0.1:55555"
+	SshproxyVersion       = "0.0.0+notproperlybuilt"
+	defaultConfig         = "/etc/sshproxy/sshproxy-managerd.yaml"
+	defaultListenAddr     = "127.0.0.1:55555"
+	defaultPromListenAddr = ":55556"
 )
 
 var (
@@ -56,11 +61,54 @@ var (
 	// map of proxied connections (keys are user@host)
 	proxiedConnections = make(map[string]*proxiedConn)
 )
+var (
+	promUserStats = prometheus.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name:       "sshproxy_user_connections_summary",
+			Help:       "SSH Connection distribution",
+			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+		},
+		[]string{"user", "server"},
+	)
+
+	promServerStats = prometheus.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name:       "sshproxy_server_connections_summary",
+			Help:       "SSH Connection distribution",
+			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+		},
+		[]string{"server"},
+	)
+
+	promInstUserConnection = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "sshproxy_user_connections",
+			Help: "Current number of Proxied connections",
+		},
+		[]string{"user", "server"},
+	)
+
+	promServers = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "sshproxy_server_up",
+			Help: "Is this server Up ?",
+		},
+		[]string{"server"},
+	)
+
+	promManagerdLatency = prometheus.NewSummary(
+		prometheus.SummaryOpts{
+			Name: "sshproxy_managerd_latency",
+			Help: "sshproxy-managerd request handling statistics in microseconds",
+		},
+	)
+)
 
 // Configuration
 type managerdConfig struct {
 	Debug         bool                 // Debug mode
 	Listen        string               // Listen address [host]:port
+	PromListen    string               // Prometheus Metrics Listen address [host]:port
 	Log           string               // Where to log: empty is for stdout, "syslog" or a file
 	CheckInterval utils.Duration       `yaml:"check_interval"` // Minimum interval between host checks
 	RouteSelect   string               `yaml:"route_select"`   // Algorithm used to select a destination
@@ -115,6 +163,10 @@ func loadConfig(filename string) error {
 		config.RouteSelect = route.DefaultAlgorithm
 	}
 
+	if config.PromListen == "" {
+		config.PromListen = defaultPromListenAddr
+	}
+
 	return nil
 }
 
@@ -185,6 +237,11 @@ func (hc *hostChecker) DoCheck(hostport string) State {
 		state = Up
 	}
 	hc.Update(hostport, state, time.Now())
+	if state == Up {
+		promServers.WithLabelValues(hostport).Set(1)
+	} else {
+		promServers.WithLabelValues(hostport).Set(0)
+	}
 	return state
 }
 
@@ -227,7 +284,7 @@ func genKey(user, host string) string {
 func getUserFromKey(key string) (string, error) {
 	match := regexp.MustCompile(`^(\w*)@`).FindStringSubmatch(key)
 	if len(match) < 2 {
-		return "", fmt.Errorf("Unable to extract user from given key (%s)",key)
+		return "", fmt.Errorf("Unable to extract user from given key (%s)", key)
 	}
 	return match[1], nil
 }
@@ -368,6 +425,7 @@ func connectHandler(args []string) (string, error) {
 	}
 
 	log.Info("new connection for %s: %s", key, dst)
+	promInstUserConnection.WithLabelValues(user, dst).Inc()
 	return fmt.Sprintf("+%s", dst), nil
 }
 
@@ -386,6 +444,7 @@ func disableHandler(args []string) (string, error) {
 
 	managerHostChecker.Update(hostport, Disabled, time.Now())
 
+	promServers.WithLabelValues(hostport).Set(0)
 	return "+OK", nil
 }
 
@@ -415,6 +474,7 @@ func disconnectHandler(args []string) (string, error) {
 		delete(proxiedConnections, key)
 	}
 
+	promInstUserConnection.WithLabelValues(user, pc.Dest).Set(float64(pc.N))
 	return "+OK", nil
 }
 
@@ -509,6 +569,7 @@ type request struct {
 // It either writes a response in the request.response channel or an error in
 // the request.errc channel.
 func handle(r *request) {
+	start := time.Now()
 	fields := strings.Fields(r.request)
 	if len(fields) == 0 {
 		r.errc <- errors.New("empty request")
@@ -531,6 +592,8 @@ func handle(r *request) {
 
 	r.response <- response
 	close(r.response)
+	elapsed := time.Since(start)
+	promManagerdLatency.Observe(float64(elapsed / time.Microsecond))
 }
 
 // serve processes requests written in the queue channel and quits when the
@@ -676,6 +739,46 @@ func main() {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
+	prometheus.MustRegister(promUserStats)
+	prometheus.MustRegister(promServerStats)
+	prometheus.MustRegister(promInstUserConnection)
+	prometheus.MustRegister(promServers)
+	prometheus.MustRegister(promManagerdLatency)
+
+	go func() {
+		var UserStats map[string]map[string]uint64
+		var ServerStats map[string]uint64
+		var user string
+		for {
+			UserStats = make(map[string]map[string]uint64)
+			ServerStats = make(map[string]uint64)
+			for k, v := range proxiedConnections {
+				user, err = getUserFromKey(k)
+				if err == nil {
+					if _, ok := UserStats[user]; !ok {
+						UserStats[user] = make(map[string]uint64)
+					}
+					UserStats[user][v.Dest] = uint64(v.N)
+					ServerStats[v.Dest] += uint64(v.N)
+				} else {
+					continue
+				}
+			}
+			for observed_user, observed_servers := range UserStats {
+				for observed_server, nb_connections := range observed_servers {
+					promUserStats.WithLabelValues(observed_user, observed_server).Observe(float64(nb_connections))
+				}
+			}
+			for observed_server, nb_connections := range ServerStats {
+				promServerStats.WithLabelValues(observed_server).Observe(float64(nb_connections))
+			}
+			time.Sleep(time.Second)
+		}
+	}()
+
+	http.Handle("/metrics", promhttp.Handler())
+	go http.ListenAndServe(config.PromListen, nil)
+
 	go serve(ctx, queue)
 
 	for {