Support OpenTelemetry tracing (#14)

* Update to Go 1.18 module format

And run `go mod tidy`.

And remove `GO111MODULE`, as modules are default-on since 1.16 <https://go.dev/blog/go116-module-changes>.

* Support OpenTelemetry tracing

With the `otlp` and `stdout` exporters. Environment variable interface is based on <https://opentelemetry.io/docs/reference/specification/sdk-environment-variables/>. However, we default to the `none` exporter as previous versions of prommsd did not support tracing.

Based on the library estalished internally for Go programs. Internal ticket is SESINST-98.

Dockerfile

@@ -19,7 +19,7 @@ WORKDIR /src
 # version to be used, to avoid pulling from GitHub again (and make this build
 # work for forks, etc).
 ENV GOPRIVATE="github.com/G-Research/prommsd"
-RUN GO111MODULE=on go install github.com/G-Research/prommsd/cmd/prommsd@$(cat /git-commit-id)
+RUN go install github.com/G-Research/prommsd/cmd/prommsd@$(cat /git-commit-id)
 
 # final stage
 FROM alpine

cmd/prommsd/main.go

@@ -6,15 +6,20 @@
 package main
 
 import (
+	"context"
 	"flag"
 	"fmt"
+	"log"
 	"os"
 	"runtime/debug"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"go.opentelemetry.io/otel/sdk/resource"
+	semconv "go.opentelemetry.io/otel/semconv/v1.12.0"
 
 	"github.com/G-Research/prommsd/pkg/alertchecker"
 	"github.com/G-Research/prommsd/pkg/alerthook"
+	"github.com/G-Research/prommsd/pkg/tracing"
 )
 
 var (
@@ -31,6 +36,20 @@ func main() {
 		os.Exit(0)
 	}
 
+	ctx := context.Background()
+
+	shutdownTracing, err := tracing.SetProviderFromEnv(
+		ctx,
+		resource.WithAttributes(
+			semconv.ServiceNameKey.String("prommsd"),
+			semconv.ServiceNamespaceKey.String("github.com/G-Research"),
+		),
+	)
+	if err != nil {
+		log.Fatalf("Cannot initialise tracing: %v", err)
+	}
+	defer shutdownTracing(ctx)
+
 	reg := prometheus.DefaultRegisterer
 	reg.MustRegister(prometheus.NewBuildInfoCollector())
 

go.mod

@@ -1,10 +1,40 @@
 module github.com/G-Research/prommsd
 
-go 1.12
+go 1.18
 
 require (
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.1 // indirect
 	github.com/prometheus/client_golang v1.2.1
-	golang.org/x/net v0.0.0-20190613194153-d28f0bde5980
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.37.0
+	go.opentelemetry.io/otel v1.11.2
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.2
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.11.2
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.11.2
+	go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.11.2
+	go.opentelemetry.io/otel/sdk v1.11.2
+	go.opentelemetry.io/otel/trace v1.11.2
+	golang.org/x/exp v0.0.0-20230105202349-8879d0199aa3
+	golang.org/x/net v0.0.0-20220722155237-a158d28d115b
+)
+
+require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cenkalti/backoff/v4 v4.2.0 // indirect
+	github.com/cespare/xxhash/v2 v2.1.1 // indirect
+	github.com/felixge/httpsnoop v1.0.3 // indirect
+	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 // indirect
+	github.com/prometheus/common v0.7.0 // indirect
+	github.com/prometheus/procfs v0.0.5 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.2 // indirect
+	go.opentelemetry.io/otel/metric v0.34.0 // indirect
+	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
+	golang.org/x/sys v0.3.0 // indirect
+	golang.org/x/text v0.4.0 // indirect
+	google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1 // indirect
+	google.golang.org/grpc v1.51.0 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
 )

pkg/alerthook/serve.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
 	"golang.org/x/net/trace"
 )
 
@@ -22,7 +23,7 @@ func Serve(listenAddr string, alertHandler AlertHandler, registerer prometheus.R
 }
 
 func registerHandlers(serveMux *http.ServeMux, handler *AlertHook) {
-	serveMux.Handle("/alert", handler)
+	serveMux.Handle("/alert", otelhttp.NewHandler(handler, "/alert"))
 	serveMux.Handle("/metrics", promhttp.Handler())
 
 	serveMux.HandleFunc("/-/healthy", func(w http.ResponseWriter, req *http.Request) {

pkg/tracing/processor.go

@@ -0,0 +1,75 @@
+package tracing
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
+	"go.opentelemetry.io/otel/exporters/stdout/stdouttrace"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	"golang.org/x/exp/slices"
+)
+
+var newExporters = map[string]func(context.Context) (sdktrace.SpanExporter, error){
+	"stdout": func(_ context.Context) (sdktrace.SpanExporter, error) { return stdouttrace.New() },
+	"otlp":   NewOTLPExporterFromEnv,
+}
+
+func NewOTLPExporterFromEnv(ctx context.Context) (sdktrace.SpanExporter, error) {
+	protocol := "http/protobuf"
+	if p, ok := os.LookupEnv("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL"); ok {
+		protocol = strings.TrimSpace(p)
+	} else if p, ok = os.LookupEnv("OTEL_EXPORTER_OTLP_PROTOCOL"); ok {
+		protocol = strings.TrimSpace(p)
+	}
+
+	var otlpExporter *otlptrace.Exporter
+	var err error
+	switch protocol {
+	case "grpc":
+		otlpExporter, err = otlptracegrpc.New(ctx)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create OTLP gRPC trace exporter: %w", err)
+		}
+	case "http/protobuf":
+		otlpExporter, err = otlptracehttp.New(ctx)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create OTLP HTTP trace exporter: %w", err)
+		}
+	default:
+		return nil, fmt.Errorf("unimplemented OTLP protocol %s", protocol)
+	}
+	return otlpExporter, nil
+}
+
+func NewProcessorsFromEnv(ctx context.Context) ([]sdktrace.SpanProcessor, error) {
+	enabled := strings.Split(strings.TrimSpace(os.Getenv("OTEL_TRACES_EXPORTER")), ",")
+
+	// https://opentelemetry.io/docs/reference/specification/sdk-environment-variables/#exporter-selection
+	// Default exporter should be "otlp"; however to preserve compatibiltiy
+	// we will default to "none".
+	if slices.Contains(enabled, "none") {
+		// Short-circuit: If "none" is present, ignore everything else.
+		enabled = nil
+	}
+
+	var spanProcessors []sdktrace.SpanProcessor
+
+	for _, name := range enabled {
+		if new, ok := newExporters[name]; ok {
+			if exporter, err := new(ctx); err == nil {
+				spanProcessors = append(spanProcessors, sdktrace.NewBatchSpanProcessor(exporter))
+			} else {
+				return nil, fmt.Errorf("failed to create exporter \"%s\": %w", name, err)
+			}
+		} else {
+			return nil, fmt.Errorf("unknown trace exporter: \"%s\"", name)
+		}
+	}
+
+	return spanProcessors, nil
+}

pkg/tracing/propagator.go

@@ -0,0 +1,40 @@
+package tracing
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"go.opentelemetry.io/otel/propagation"
+)
+
+var newPropagators = map[string]func() propagation.TextMapPropagator{
+	"baggage":      func() propagation.TextMapPropagator { return propagation.Baggage{} },
+	"tracecontext": func() propagation.TextMapPropagator { return propagation.TraceContext{} },
+}
+
+func NewPropagatorsFromEnv() (propagation.TextMapPropagator, error) {
+	enabled := []string{"tracecontext", "baggage"}
+
+	if v, ok := os.LookupEnv("OTEL_PROPAGATORS"); ok {
+		enabled = strings.Split(strings.TrimSpace(v), ",")
+	}
+
+	var propagators []propagation.TextMapPropagator
+
+	for _, name := range enabled {
+		if new, ok := newPropagators[name]; ok {
+			propagators = append(propagators, new())
+		} else {
+			return nil, fmt.Errorf("unknown propagator: \"%s\"", name)
+		}
+	}
+
+	if len(propagators) > 1 {
+		return propagation.NewCompositeTextMapPropagator(propagators...), nil
+	}
+	if len(propagators) == 1 {
+		return propagators[0], nil
+	}
+	return nil, nil
+}

pkg/tracing/provider.go

@@ -0,0 +1,70 @@
+package tracing
+
+import (
+	"context"
+	"fmt"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/sdk/resource"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/trace"
+)
+
+func NewProviderFromEnv(ctx context.Context, resourceOptions ...resource.Option) (*sdktrace.TracerProvider, error) {
+	spanProcessors, err := NewProcessorsFromEnv(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get tracing processors: %w", err)
+	}
+	if spanProcessors == nil {
+		// Nothing to export.
+		return nil, nil
+	}
+
+	res, err := resource.New(
+		ctx,
+		append(
+			resourceOptions,
+			// Allow environment variables to override constant attributes provided by the caller.
+			resource.WithFromEnv(),
+			resource.WithProcess(),
+		)...,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create resource: %w", err)
+	}
+
+	tracerProvider := sdktrace.NewTracerProvider(
+		sdktrace.WithResource(res),
+	)
+
+	for _, sp := range spanProcessors {
+		tracerProvider.RegisterSpanProcessor(sp)
+	}
+
+	return tracerProvider, nil
+}
+
+func SetProviderFromEnv(ctx context.Context, resourceOptions ...resource.Option) (func(context.Context) error, error) {
+	tracerProvider, err := NewProviderFromEnv(ctx, resourceOptions...)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create tracing provider: %w", err)
+	}
+
+	if tracerProvider == nil {
+		// Make sure there's always something to use. (And that we
+		// clear out any previous provider.)
+		otel.SetTracerProvider(trace.NewNoopTracerProvider())
+		// It has nothing to shutdown; return a do-nothing func.
+		return func(_ context.Context) error { return nil }, nil
+	}
+
+	if p, err := NewPropagatorsFromEnv(); err != nil {
+		return nil, fmt.Errorf("failed to create propagators: %w", err)
+	} else if p != nil {
+		otel.SetTextMapPropagator(p)
+	}
+
+	otel.SetTracerProvider(tracerProvider)
+
+	return tracerProvider.Shutdown, nil
+}