diff --git a/go.mod b/go.mod index da77bae4..4a94b96c 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,8 @@ go 1.21 require ( github.com/Masterminds/squirrel v1.5.2 + github.com/aquilax/go-perlin v1.1.0 + github.com/docker/docker v24.0.5+incompatible github.com/fission/fission v1.19.0 github.com/fxamacker/cbor/v2 v2.4.0 github.com/go-chi/chi/v5 v5.0.7 @@ -21,6 +23,7 @@ require ( github.com/stretchr/testify v1.8.4 github.com/testcontainers/testcontainers-go v0.21.0 github.com/valyala/quicktemplate v1.7.0 + golang.org/x/crypto v0.13.0 golang.org/x/oauth2 v0.8.0 k8s.io/api v0.25.4 k8s.io/apimachinery v0.26.2 @@ -39,7 +42,6 @@ require ( github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/andybalholm/brotli v1.0.4 // indirect - github.com/aquilax/go-perlin v1.1.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect @@ -47,7 +49,6 @@ require ( github.com/cpuguy83/dockercfg v0.3.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/docker/distribution v2.8.2+incompatible // indirect - github.com/docker/docker v24.0.5+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect @@ -113,7 +114,6 @@ require ( go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.24.0 // indirect - golang.org/x/crypto v0.13.0 // indirect golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea // indirect golang.org/x/mod v0.10.0 // indirect golang.org/x/net v0.15.0 // indirect diff --git a/go.sum b/go.sum index 1ebf57eb..c5aedaeb 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,7 @@ cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdi cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 h1:EKPd1INOIyr5hWOWhvpmQpY6tKjeG0hT1s3AMC/9fic= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1/go.mod h1:VzwV+t+dZ9j/H867F1M2ziD+yLHtB46oM35FxxMJ4d0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= @@ -28,6 +29,7 @@ github.com/Masterminds/squirrel v1.5.2/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA4 github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/Microsoft/hcsshim v0.10.0-rc.5 h1:JfkknPHBtfdC2Ezd+jpl8Kicw7UyhvUSzoy6xsqirwY= +github.com/Microsoft/hcsshim v0.10.0-rc.5/go.mod h1:NNb9uh/cgA52AVhc9+Y7U+YyHQS9nHHA4STcAjdg2xk= github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.0.3/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= @@ -57,12 +59,14 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/cyphar/filepath-securejoin v0.2.3 h1:YX6ebbZCZP7VkM3scTTokDgBL2TY741X51MTk3ycuNI= github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dhui/dktest v0.3.16 h1:i6gq2YQEtcrjKbeJpBkWjE8MmLZPYllcjOFbTZuPDnw= +github.com/dhui/dktest v0.3.16/go.mod h1:gYaA3LRmM8Z4vJl2MA0THIigJoZrwOansEOsp+kqxp0= github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v24.0.5+incompatible h1:WmgcE4fxyI6EEXxBRxsHnZXrO1pQ3smi0k/jho4HLeY= @@ -84,6 +88,7 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/fission/fission v1.19.0 h1:fzSjZpWL4IJG/8FjIfcLuKwDT0gG0+aRJUJnduqjYy4= @@ -196,6 +201,7 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -213,6 +219,7 @@ github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= +github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mholt/archiver/v3 v3.5.1 h1:rDjOBX9JSF5BvoJGvjqK479aL70qh9DIpZCl+k7Clwo= @@ -237,9 +244,13 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ= github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs= +github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys= +github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0-rc4 h1:oOxKUJWnFC4YGHCCMNql1x4YaDfYBTS5Y4x/Cgeo1E0= @@ -270,6 +281,7 @@ github.com/rabbitmq/amqp091-go v1.3.4/go.mod h1:ogQDLSOACsLPsIq0NpbtiifNZi2YOz0V github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rs/cors v1.8.2 h1:KCooALfAYGs415Cwu5ABvv9n9509fSiG5SQJn/AQo4U= github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -327,6 +339,7 @@ go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= +go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= @@ -377,6 +390,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -461,6 +475,7 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= @@ -472,6 +487,7 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/api v0.25.4 h1:3YO8J4RtmG7elEgaWMb4HgmpS2CfY1QlaOz9nwB+ZSs= diff --git a/pkg/health/health.go b/pkg/health/health.go new file mode 100644 index 00000000..667951b8 --- /dev/null +++ b/pkg/health/health.go @@ -0,0 +1,122 @@ +package health + +import ( + "context" + "errors" + "fmt" + "log" + "net/http" + "time" + + "github.com/go-chi/chi/v5" + + "sensorbucket.nl/sensorbucket/internal/web" +) + +type Check func() bool + +type checks map[string]Check + +func (checks checks) Perform() checksResult { + if len(checks) == 0 { + return checksResult{Success: false} + } + + success := []string{} + failed := []string{} + for name, check := range checks { + if check() { + success = append(success, name) + continue + } + failed = append(failed, name) + } + return checksResult{ + Success: len(failed) == 0, + ChecksSucess: success, + ChecksFailed: failed, + } +} + +type HealthChecker struct { + livelinessChecks checks + readinessChecks checks + router chi.Router +} + +func NewHealthEndpoint() *HealthChecker { + hc := HealthChecker{ + router: chi.NewRouter(), + livelinessChecks: checks{}, + readinessChecks: checks{}, + } + hc.setupRoutes(hc.router) + return &hc +} + +func (hc *HealthChecker) WithLiveChecks(checks checks) *HealthChecker { + for name, c := range checks { + hc.livelinessChecks[name] = c + } + return hc +} + +func (hc *HealthChecker) WithReadyChecks(checks checks) *HealthChecker { + for name, c := range checks { + hc.readinessChecks[name] = c + } + return hc +} + +func (hc *HealthChecker) setupRoutes(r chi.Router) { + r.Get("/livez", hc.httpLivelinessCheck) + r.Get("/readyz", hc.httpReadinessCheck) +} + +func (hc HealthChecker) ServeHTTP(w http.ResponseWriter, r *http.Request) { + hc.router.ServeHTTP(w, r) +} + +func (hc *HealthChecker) httpReadinessCheck(w http.ResponseWriter, r *http.Request) { + checksResponse(w, hc.readinessChecks) +} + +func (hc *HealthChecker) httpLivelinessCheck(w http.ResponseWriter, r *http.Request) { + checksResponse(w, hc.livelinessChecks) +} + +func (hc *HealthChecker) RunAsServer(address string) func(context.Context) error { + srv := &http.Server{ + Addr: address, + WriteTimeout: 5 * time.Second, + ReadTimeout: 5 * time.Second, + Handler: hc, + } + go func() { + log.Printf("HealthChecker endpoint available at: %s\n", srv.Addr) + if err := srv.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) && err != nil { + log.Printf("HealthChecker server closed unexpectedly: %s\n", err.Error()) + } + }() + return func(ctx context.Context) error { + return srv.Shutdown(ctx) + } +} + +func checksResponse(w http.ResponseWriter, checks checks) { + results := checks.Perform() + statusCode := http.StatusOK + if !results.Success { + statusCode = http.StatusServiceUnavailable + } + web.HTTPResponse(w, statusCode, web.APIResponse[checksResult]{ + Message: fmt.Sprintf("%d/%d checks passed", len(results.ChecksSucess), len(checks)), + Data: results, + }) +} + +type checksResult struct { + Success bool `json:"success"` + ChecksSucess []string `json:"checks_success"` + ChecksFailed []string `json:"checks_failed"` +} diff --git a/pkg/health/health_test.go b/pkg/health/health_test.go new file mode 100644 index 00000000..56b71295 --- /dev/null +++ b/pkg/health/health_test.go @@ -0,0 +1,263 @@ +package health + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/go-chi/chi/v5" + "github.com/stretchr/testify/assert" +) + +func TestReadyEndpoint(t *testing.T) { + type scenario struct { + checks map[string]Check + expected result + expectedStatus int + } + + // Arrange + scenarios := map[string]scenario{ + "only 1 out of 3 checks succeed": { + checks: map[string]Check{"failing-1": func() bool { return false }, "failing-2": func() bool { return false }, "success-1": func() bool { return true }}, + expected: result{ + Message: "1/3 checks passed", + Data: resultData{ + Success: false, + ChecksSucess: []string{"success-1"}, + ChecksFailed: []string{"failing-1", "failing-2"}, + }, + }, + expectedStatus: http.StatusServiceUnavailable, + }, + "all checks fail": { + checks: map[string]Check{"failing-1": func() bool { return false }, "failing-2": func() bool { return false }, "failing-3": func() bool { return false }}, + expected: result{ + Message: "0/3 checks passed", + Data: resultData{ + Success: false, + ChecksSucess: []string{}, + ChecksFailed: []string{"failing-1", "failing-2", "failing-3"}, + }, + }, + expectedStatus: http.StatusServiceUnavailable, + }, + "all checks succeed": { + checks: map[string]Check{"success-1": func() bool { return true }, "success-2": func() bool { return true }, "success-3": func() bool { return true }, "success-4": func() bool { return true }}, + expected: result{ + Message: "4/4 checks passed", + Data: resultData{ + Success: true, + ChecksSucess: []string{"success-1", "success-2", "success-3", "success-4"}, + ChecksFailed: []string{}, + }, + }, + expectedStatus: http.StatusOK, + }, + "only 1 check fails": { + checks: map[string]Check{"failing-1": func() bool { return false }, "success-1": func() bool { return true }, "success-2": func() bool { return true }, "success-3": func() bool { return true }, "success-4": func() bool { return true }}, + expected: result{ + Message: "4/5 checks passed", + Data: resultData{ + Success: false, + ChecksSucess: []string{"success-1", "success-2", "success-3", "success-4"}, + ChecksFailed: []string{"failing-1"}, + }, + }, + expectedStatus: http.StatusServiceUnavailable, + }, + "no checks configured": { + checks: map[string]Check{}, + expected: result{ + Message: "0/0 checks passed", + Data: resultData{}, + }, + expectedStatus: http.StatusServiceUnavailable, + }, + } + + for scene, cfg := range scenarios { + t.Run(scene, func(t *testing.T) { + transport := testHealthEndpoint(nil, cfg.checks) + req, _ := http.NewRequest("GET", "/readyz", nil) + + // Act + rr := httptest.NewRecorder() + transport.router.ServeHTTP(rr, req) + + // Assert + result := asResult(rr.Body.String()) + assert.Equal(t, cfg.expectedStatus, rr.Code) + assert.Equal(t, cfg.expected.Message, result.Message) + assert.Equal(t, cfg.expected.Data.Success, result.Data.Success) + assert.True(t, sliceEqual(cfg.expected.Data.ChecksFailed, result.Data.ChecksFailed)) + assert.True(t, sliceEqual(cfg.expected.Data.ChecksSucess, result.Data.ChecksSucess)) + }) + } +} + +func TestLivelinessEndpoint(t *testing.T) { + type scenario struct { + checks map[string]Check + expected result + expectedStatus int + } + + // Arrange + scenarios := map[string]scenario{ + "only 1 out of 3 checks succeed": { + checks: map[string]Check{"failing-1": func() bool { return false }, "failing-2": func() bool { return false }, "success-1": func() bool { return true }}, + expected: result{ + Message: "1/3 checks passed", + Data: resultData{ + Success: false, + ChecksSucess: []string{"success-1"}, + ChecksFailed: []string{"failing-1", "failing-2"}, + }, + }, + expectedStatus: http.StatusServiceUnavailable, + }, + "all checks fail": { + checks: map[string]Check{"failing-1": func() bool { return false }, "failing-2": func() bool { return false }, "failing-3": func() bool { return false }}, + expected: result{ + Message: "0/3 checks passed", + Data: resultData{ + Success: false, + ChecksSucess: []string{}, + ChecksFailed: []string{"failing-1", "failing-2", "failing-3"}, + }, + }, + expectedStatus: http.StatusServiceUnavailable, + }, + "all checks succeed": { + checks: map[string]Check{"success-1": func() bool { return true }, "success-2": func() bool { return true }, "success-3": func() bool { return true }, "success-4": func() bool { return true }}, + expected: result{ + Message: "4/4 checks passed", + Data: resultData{ + Success: true, + ChecksSucess: []string{"success-1", "success-2", "success-3", "success-4"}, + ChecksFailed: []string{}, + }, + }, + expectedStatus: http.StatusOK, + }, + "only 1 check fails": { + checks: map[string]Check{"failing-1": func() bool { return false }, "success-1": func() bool { return true }, "success-2": func() bool { return true }, "success-3": func() bool { return true }, "success-4": func() bool { return true }}, + expected: result{ + Message: "4/5 checks passed", + Data: resultData{ + Success: false, + ChecksSucess: []string{"success-1", "success-2", "success-3", "success-4"}, + ChecksFailed: []string{"failing-1"}, + }, + }, + expectedStatus: http.StatusServiceUnavailable, + }, + "no checks configured": { + checks: map[string]Check{}, + expected: result{ + Message: "0/0 checks passed", + Data: resultData{}, + }, + expectedStatus: http.StatusServiceUnavailable, + }, + } + + for scene, cfg := range scenarios { + t.Run(scene, func(t *testing.T) { + transport := testHealthEndpoint(cfg.checks, nil) + req, _ := http.NewRequest("GET", "/livez", nil) + + // Act + rr := httptest.NewRecorder() + transport.router.ServeHTTP(rr, req) + + // Assert + result := asResult(rr.Body.String()) + assert.Equal(t, cfg.expectedStatus, rr.Code) + assert.Equal(t, cfg.expected.Message, result.Message) + assert.Equal(t, cfg.expected.Data.Success, result.Data.Success) + assert.True(t, sliceEqual(cfg.expected.Data.ChecksFailed, result.Data.ChecksFailed)) + assert.True(t, sliceEqual(cfg.expected.Data.ChecksSucess, result.Data.ChecksSucess)) + }) + } +} + +func TestWithLiveChecks(t *testing.T) { + // Arrange + hc := testHealthEndpoint(map[string]Check{}, map[string]Check{}) + checks := map[string]Check{ + "check1": func() bool { return false }, + "check2": func() bool { return true }, + } + + // Act + hc.WithLiveChecks(checks) + + // Assert + assert.False(t, hc.livelinessChecks["check1"]()) + assert.True(t, hc.livelinessChecks["check2"]()) +} + +func TestWithReadyChecks(t *testing.T) { + // Arrange + hc := testHealthEndpoint(map[string]Check{}, map[string]Check{}) + checks := map[string]Check{ + "check1": func() bool { return false }, + "check2": func() bool { return true }, + } + + // Act + hc.WithReadyChecks(checks) + + // Assert + assert.False(t, hc.readinessChecks["check1"]()) + assert.True(t, hc.readinessChecks["check2"]()) +} + +type result struct { + Message string `json:"message"` + Data resultData `json:"data"` +} + +type resultData struct { + Success bool `json:"success"` + ChecksSucess []string `json:"checks_success"` + ChecksFailed []string `json:"checks_failed"` +} + +func asResult(jsonStr string) result { + r := result{} + _ = json.NewDecoder(strings.NewReader(jsonStr)).Decode(&r) + return r +} + +func sliceEqual(sl1 []string, sl2 []string) bool { + if len(sl1) != len(sl2) { + return false + } + for _, x := range sl1 { + found := false + for _, y := range sl2 { + if x == y { + found = true + } + } + if !found { + return false + } + } + return true +} + +func testHealthEndpoint(liveChecks, readyChecks map[string]Check) *HealthChecker { + hc := HealthChecker{ + livelinessChecks: liveChecks, + readinessChecks: readyChecks, + router: chi.NewRouter(), + } + hc.setupRoutes(hc.router) + return &hc +} diff --git a/pkg/mq/amqp_connection.go b/pkg/mq/amqp_connection.go index a5643dcd..56cbed23 100644 --- a/pkg/mq/amqp_connection.go +++ b/pkg/mq/amqp_connection.go @@ -16,6 +16,7 @@ const ( AMQP_DISCONNECTED AMQPState = iota AMQP_CONNECTED AMQP_RECONNECTING + AMQP_UNREACHABLE AMQP_QUEUE_LEN = 10 ) @@ -51,13 +52,13 @@ func NewConnection(host string) *AMQPConnection { func (c *AMQPConnection) Start() { defer func() { log.Println("AMQPConnection stopping") - c.state = AMQP_DISCONNECTED c.usersLock.Lock() for _, user := range c.users { close(user) } c.usersLock.Unlock() if c.connection != nil { + c.state = AMQP_DISCONNECTED c.connection.Close() } log.Println("AMQPConnection stopped") @@ -68,11 +69,13 @@ func (c *AMQPConnection) Start() { // Keep reconnecting until we get a 'done' signal for { log.Println("AMQPConnection (re)connecting...") + c.connection = nil c.state = AMQP_RECONNECTING connection, err := amqp.Dial(c.amqpHost) if err != nil { log.Printf("AMQPConnection connect failed: %v\n", err) if retries > c.maximumRetries { + c.state = AMQP_UNREACHABLE log.Printf("AMQPConnection maximum retries of %d reached, quitting...\n", retries) return } @@ -112,6 +115,14 @@ func (c *AMQPConnection) Start() { } } +func (c *AMQPConnection) Ready() bool { + return c.state == AMQP_CONNECTED +} + +func (c *AMQPConnection) Healthy() bool { + return c.state != AMQP_UNREACHABLE +} + func (c *AMQPConnection) Shutdown() { close(c.done) } diff --git a/services/httpimporter/main.go b/services/httpimporter/main.go index e978674e..1f7f9d65 100644 --- a/services/httpimporter/main.go +++ b/services/httpimporter/main.go @@ -1,21 +1,27 @@ package main import ( + "context" + "errors" "fmt" "log" "net/http" + "os" + "os/signal" "time" "github.com/rs/cors" "sensorbucket.nl/sensorbucket/internal/env" "sensorbucket.nl/sensorbucket/internal/web" + "sensorbucket.nl/sensorbucket/pkg/health" "sensorbucket.nl/sensorbucket/pkg/mq" "sensorbucket.nl/sensorbucket/services/httpimporter/service" ) var ( HTTP_ADDR = env.Could("HTTP_ADDR", ":3000") + HEALTH_ADDR = env.Could("HEALTH_ADDR", ":3030") AMQP_HOST = env.Could("AMQP_HOST", "amqp://guest:guest@localhost/") AMQP_XCHG = env.Could("AMQP_XCHG", "ingress") AMQP_XCHG_TOPIC = env.Could("AMQP_XCHG_TOPIC", "ingress.httpimporter") @@ -34,6 +40,9 @@ func main() { } func Run() error { + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + // Create AMQP Message Queue mqConn := mq.NewConnection(AMQP_HOST) go mqConn.Start() @@ -52,6 +61,38 @@ func Run() error { ReadTimeout: 5 * time.Second, } - log.Printf("HTTP Server listening on: %s\n", srv.Addr) - return srv.ListenAndServe() + shutdownHealthEndpoint := health.NewHealthEndpoint(). + WithReadyChecks( + map[string]health.Check{ + "mqconn-ready": mqConn.Ready, + }, + ). + WithLiveChecks( + map[string]health.Check{ + "mqconn-healthy": mqConn.Healthy, + }, + ). + RunAsServer(HEALTH_ADDR) + + errC := make(chan error) + go func() { + log.Printf("HTTP Server listening on: %s\n", srv.Addr) + if err := srv.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) && err != nil { + errC <- err + } + }() + + var err error + select { + case <-ctx.Done(): + case err = <-errC: + } + + ctxTO, cancelTO := context.WithTimeout(context.Background(), 5*time.Second) + defer cancelTO() + + srv.Shutdown(ctxTO) + shutdownHealthEndpoint(ctxTO) + + return err } diff --git a/tools/docker-compose/caddy/Caddyfile b/tools/docker-compose/caddy/Caddyfile index 4e3c0dba..1e8e7df3 100644 --- a/tools/docker-compose/caddy/Caddyfile +++ b/tools/docker-compose/caddy/Caddyfile @@ -15,6 +15,12 @@ reverse_proxy core:3000 } + handle_path /health/* { + handle_path /httpimporter* { + reverse_proxy httpimporter:3030 + } + } + redir /dev/mq /dev/mq/ handle_path /dev/mq/* { reverse_proxy mq:15672