From a122d11b260e429d7ec21432a8a8176de016f207 Mon Sep 17 00:00:00 2001 From: Paul Abel <128620221+pdabelf5@users.noreply.github.com> Date: Tue, 12 Dec 2023 11:50:13 +0000 Subject: [PATCH] update prometheus exporter to 1.0.0 (#4769) * update prometheus exporter to 1.0.0 --- cmd/nginx-ingress/main.go | 14 ++++--- go.mod | 6 ++- go.sum | 8 +++- internal/configs/configurator_test.go | 56 +++++++++++++++++++++++++- internal/metrics/listener.go | 7 +++- tests/Makefile | 9 +++-- tests/suite/test_prometheus_metrics.py | 42 +++++++++++++++++++ 7 files changed, 126 insertions(+), 16 deletions(-) diff --git a/cmd/nginx-ingress/main.go b/cmd/nginx-ingress/main.go index 1ff67bcd25..bece95587c 100644 --- a/cmd/nginx-ingress/main.go +++ b/cmd/nginx-ingress/main.go @@ -28,6 +28,7 @@ import ( "github.com/nginxinc/nginx-plus-go-client/client" nginxCollector "github.com/nginxinc/nginx-prometheus-exporter/collector" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/promlog" api_v1 "k8s.io/api/core/v1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" util_version "k8s.io/apimachinery/pkg/util/version" @@ -694,16 +695,17 @@ func createPlusAndLatencyCollectors( serverZoneVariableLabels := []string{"resource_type", "resource_name", "resource_namespace"} streamServerZoneVariableLabels := []string{"resource_type", "resource_name", "resource_namespace"} + cacheZoneLabels := []string{"resource_type", "resource_name", "resource_namespace"} + workerPIDVariableLabels := []string{"resource_type", "resource_name", "resource_namespace"} variableLabelNames := nginxCollector.NewVariableLabelNames(upstreamServerVariableLabels, serverZoneVariableLabels, upstreamServerPeerVariableLabelNames, - streamUpstreamServerVariableLabels, streamServerZoneVariableLabels, streamUpstreamServerPeerVariableLabelNames) - plusCollector = nginxCollector.NewNginxPlusCollector(plusClient, "nginx_ingress_nginxplus", variableLabelNames, constLabels) + streamUpstreamServerVariableLabels, streamServerZoneVariableLabels, streamUpstreamServerPeerVariableLabelNames, cacheZoneLabels, workerPIDVariableLabels) + promlogConfig := &promlog.Config{} + logger := promlog.New(promlogConfig) + plusCollector = nginxCollector.NewNginxPlusCollector(plusClient, "nginx_ingress_nginxplus", variableLabelNames, constLabels, logger) go metrics.RunPrometheusListenerForNginxPlus(*prometheusMetricsListenPort, plusCollector, registry, prometheusSecret) } else { httpClient := getSocketClient("/var/lib/nginx/nginx-status.sock") - client, err := metrics.NewNginxMetricsClient(httpClient) - if err != nil { - glog.Errorf("Error creating the Nginx client for Prometheus metrics: %v", err) - } + client := metrics.NewNginxMetricsClient(httpClient) go metrics.RunPrometheusListenerForNginx(*prometheusMetricsListenPort, client, registry, constLabels, prometheusSecret) } if *enableLatencyMetrics { diff --git a/go.mod b/go.mod index 1fc268ba43..25da2a773e 100644 --- a/go.mod +++ b/go.mod @@ -13,9 +13,10 @@ require ( github.com/google/go-cmp v0.6.0 github.com/kr/pretty v0.3.1 github.com/nginxinc/nginx-plus-go-client v1.2.0 - github.com/nginxinc/nginx-prometheus-exporter v0.11.0 + github.com/nginxinc/nginx-prometheus-exporter v1.0.0 github.com/nginxinc/nginx-service-mesh v1.7.0 github.com/prometheus/client_golang v1.17.0 + github.com/prometheus/common v0.45.0 github.com/spiffe/go-spiffe/v2 v2.1.6 github.com/stretchr/testify v1.8.4 golang.org/x/exp v0.0.0-20231006140011-7918f672742d @@ -55,7 +56,9 @@ require ( github.com/felixge/httpsnoop v1.0.3 // indirect github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect github.com/go-jose/go-jose/v3 v3.0.1 // indirect + github.com/go-kit/log v0.2.1 // indirect github.com/go-ldap/ldap/v3 v3.4.5 // indirect + github.com/go-logfmt/logfmt v0.5.1 // indirect github.com/go-logr/logr v1.3.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.2.4 // indirect @@ -86,7 +89,6 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect - github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.11.1 // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect github.com/spf13/cobra v1.7.0 // indirect diff --git a/go.sum b/go.sum index 36d98116e9..d784f101fd 100644 --- a/go.sum +++ b/go.sum @@ -84,8 +84,12 @@ github.com/go-chi/chi/v5 v5.0.10 h1:rLz5avzKpjqxrYwXNfmjkrYYXOyLJd37pz53UFHC6vk= github.com/go-chi/chi/v5 v5.0.10/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-jose/go-jose/v3 v3.0.1 h1:pWmKFVtt+Jl0vBZTIpz/eAKwsm6LkIxDVVbFHKkchhA= github.com/go-jose/go-jose/v3 v3.0.1/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8= +github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= +github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-ldap/ldap/v3 v3.4.5 h1:ekEKmaDrpvR2yf5Nc/DClsGG9lAmdDixe44mLzlW5r8= github.com/go-ldap/ldap/v3 v3.4.5/go.mod h1:bMGIq3AGbytbaMwf8wdv5Phdxz0FWHTIYMSzyrYgnQs= +github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -186,8 +190,8 @@ github.com/nginxinc/glog v1.1.2 h1:zyoZXhCoYvNMJq4qMsKislUCGyJ4eU2gNl3Nt7cjESg= github.com/nginxinc/glog v1.1.2/go.mod h1:Q2FpGp/qFhJEVnuC88BVfbLDPmio9aHYUj4al6w0138= github.com/nginxinc/nginx-plus-go-client v1.2.0 h1:NVfRsHbMJ7lOhkqMG52uvODiDBhQZNp20c0tV2lU3wg= github.com/nginxinc/nginx-plus-go-client v1.2.0/go.mod h1:n8OFLzrJulJ2fur28Cwa1Qp5DZNS2VicLV+Adt30LQ4= -github.com/nginxinc/nginx-prometheus-exporter v0.11.0 h1:21xjnqNgxtni2jDgAQ90bl15uDnrTreO9sIlu1YsX/U= -github.com/nginxinc/nginx-prometheus-exporter v0.11.0/go.mod h1:GdyHnWAb8q8OW1Pssrrqbcqra0SH0Vn6UXICMmyWkw8= +github.com/nginxinc/nginx-prometheus-exporter v1.0.0 h1:rw5q6j6FQe9EWzJy5HzRgRBJ2tSVyC9By6k9ZFQ7lD8= +github.com/nginxinc/nginx-prometheus-exporter v1.0.0/go.mod h1:SPohlKx0SiOuZYi04js53GWWb0HhD281AT8q4ApVMIE= github.com/nginxinc/nginx-service-mesh v1.7.0 h1:oxKr+Jdbxkos10VTy5xF2UHCcmfIhqWNlsOK/zPnZDM= github.com/nginxinc/nginx-service-mesh v1.7.0/go.mod h1:8tREM3kSEUGyk8JT8hdCf/9ol2kEo7hLR8b+m5Yd8Fs= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= diff --git a/internal/configs/configurator_test.go b/internal/configs/configurator_test.go index 565605e6ce..a20240f438 100644 --- a/internal/configs/configurator_test.go +++ b/internal/configs/configurator_test.go @@ -388,6 +388,8 @@ type mockLabelUpdater struct { streamUpstreamServerPeerLabels map[string][]string streamUpstreamServerLabels map[string][]string streamServerZoneLabels map[string][]string + cacheZoneLabels map[string][]string + workerPIDVariableLabels map[string][]string } func newFakeLabelUpdater() *mockLabelUpdater { @@ -398,6 +400,8 @@ func newFakeLabelUpdater() *mockLabelUpdater { streamUpstreamServerPeerLabels: make(map[string][]string), streamUpstreamServerLabels: make(map[string][]string), streamServerZoneLabels: make(map[string][]string), + cacheZoneLabels: make(map[string][]string), + workerPIDVariableLabels: make(map[string][]string), } } @@ -485,6 +489,34 @@ func (u *mockLabelUpdater) DeleteStreamServerZoneLabels(zoneNames []string) { } } +// UpdateCacheZoneLabels updates the Cache Zone Labels +func (u *mockLabelUpdater) UpdateCacheZoneLabels(cacheZoneLabelValues map[string][]string) { + for k, v := range cacheZoneLabelValues { + u.cacheZoneLabels[k] = v + } +} + +// DeleteCacheZoneLabels deletes the Cache Zone Labels +func (u *mockLabelUpdater) DeleteCacheZoneLabels(cacheZoneNames []string) { + for _, k := range cacheZoneNames { + delete(u.cacheZoneLabels, k) + } +} + +// UpdateWorkerLabels updates the Worker Labels +func (u *mockLabelUpdater) UpdateWorkerLabels(workerValues map[string][]string) { + for k, v := range workerValues { + u.workerPIDVariableLabels[k] = v + } +} + +// DeleteWorkerLabels deletes the Worker Labels +func (u *mockLabelUpdater) DeleteWorkerLabels(workerNames []string) { + for _, k := range workerNames { + delete(u.workerPIDVariableLabels, k) + } +} + type mockLatencyCollector struct { upstreamServerLabels map[string][]string upstreamServerPeerLabels map[string][]string @@ -613,6 +645,8 @@ func TestUpdateIngressMetricsLabels(t *testing.T) { streamUpstreamServerPeerLabels: make(map[string][]string), streamUpstreamServerLabels: make(map[string][]string), streamServerZoneLabels: make(map[string][]string), + cacheZoneLabels: make(map[string][]string), + workerPIDVariableLabels: make(map[string][]string), } expectedLatencyCollector := &mockLatencyCollector{ upstreamServerLabels: upstreamServerLabels, @@ -661,7 +695,9 @@ func TestUpdateIngressMetricsLabels(t *testing.T) { upstreamServerPeerLabels: upstreamServerPeerLabels, streamUpstreamServerPeerLabels: make(map[string][]string), streamUpstreamServerLabels: make(map[string][]string), - streamServerZoneLabels: map[string][]string{}, + streamServerZoneLabels: make(map[string][]string), + cacheZoneLabels: make(map[string][]string), + workerPIDVariableLabels: make(map[string][]string), } expectedLatencyCollector = &mockLatencyCollector{ upstreamServerLabels: upstreamServerLabels, @@ -688,6 +724,8 @@ func TestUpdateIngressMetricsLabels(t *testing.T) { streamUpstreamServerPeerLabels: map[string][]string{}, streamUpstreamServerLabels: map[string][]string{}, streamServerZoneLabels: map[string][]string{}, + cacheZoneLabels: map[string][]string{}, + workerPIDVariableLabels: map[string][]string{}, } expectedLatencyCollector = &mockLatencyCollector{ upstreamServerLabels: upstreamServerLabels, @@ -780,6 +818,8 @@ func TestUpdateVirtualServerMetricsLabels(t *testing.T) { streamUpstreamServerPeerLabels: map[string][]string{}, streamUpstreamServerLabels: map[string][]string{}, streamServerZoneLabels: map[string][]string{}, + cacheZoneLabels: map[string][]string{}, + workerPIDVariableLabels: map[string][]string{}, } expectedLatencyCollector := &mockLatencyCollector{ @@ -829,6 +869,8 @@ func TestUpdateVirtualServerMetricsLabels(t *testing.T) { streamUpstreamServerPeerLabels: map[string][]string{}, streamUpstreamServerLabels: map[string][]string{}, streamServerZoneLabels: map[string][]string{}, + cacheZoneLabels: map[string][]string{}, + workerPIDVariableLabels: map[string][]string{}, } expectedLatencyCollector = &mockLatencyCollector{ @@ -853,6 +895,8 @@ func TestUpdateVirtualServerMetricsLabels(t *testing.T) { streamUpstreamServerPeerLabels: map[string][]string{}, streamUpstreamServerLabels: map[string][]string{}, streamServerZoneLabels: map[string][]string{}, + cacheZoneLabels: map[string][]string{}, + workerPIDVariableLabels: map[string][]string{}, } expectedLatencyCollector = &mockLatencyCollector{ @@ -948,6 +992,8 @@ func TestUpdateTransportServerMetricsLabels(t *testing.T) { upstreamServerPeerLabels: make(map[string][]string), upstreamServerLabels: make(map[string][]string), serverZoneLabels: make(map[string][]string), + cacheZoneLabels: make(map[string][]string), + workerPIDVariableLabels: make(map[string][]string), } cnf.updateTransportServerMetricsLabels(tsEx, streamUpstreams) @@ -989,6 +1035,8 @@ func TestUpdateTransportServerMetricsLabels(t *testing.T) { upstreamServerPeerLabels: map[string][]string{}, upstreamServerLabels: map[string][]string{}, serverZoneLabels: map[string][]string{}, + cacheZoneLabels: map[string][]string{}, + workerPIDVariableLabels: map[string][]string{}, } cnf.updateTransportServerMetricsLabels(tsEx, updatedStreamUpstreams) @@ -1003,6 +1051,8 @@ func TestUpdateTransportServerMetricsLabels(t *testing.T) { streamUpstreamServerPeerLabels: map[string][]string{}, streamUpstreamServerLabels: map[string][]string{}, streamServerZoneLabels: map[string][]string{}, + cacheZoneLabels: map[string][]string{}, + workerPIDVariableLabels: map[string][]string{}, } cnf.deleteTransportServerMetricsLabels("default/test-transportserver") @@ -1063,6 +1113,8 @@ func TestUpdateTransportServerMetricsLabels(t *testing.T) { upstreamServerPeerLabels: make(map[string][]string), upstreamServerLabels: make(map[string][]string), serverZoneLabels: make(map[string][]string), + cacheZoneLabels: make(map[string][]string), + workerPIDVariableLabels: make(map[string][]string), } cnf.updateTransportServerMetricsLabels(tsExTLS, streamUpstreams) @@ -1077,6 +1129,8 @@ func TestUpdateTransportServerMetricsLabels(t *testing.T) { streamUpstreamServerPeerLabels: map[string][]string{}, streamUpstreamServerLabels: map[string][]string{}, streamServerZoneLabels: map[string][]string{}, + cacheZoneLabels: map[string][]string{}, + workerPIDVariableLabels: map[string][]string{}, } cnf.deleteTransportServerMetricsLabels("default/test-transportserver-tls") diff --git a/internal/metrics/listener.go b/internal/metrics/listener.go index 4d56b50a70..4f247eca1e 100644 --- a/internal/metrics/listener.go +++ b/internal/metrics/listener.go @@ -15,17 +15,20 @@ import ( nginxCollector "github.com/nginxinc/nginx-prometheus-exporter/collector" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/prometheus/common/promlog" v1 "k8s.io/api/core/v1" ) // NewNginxMetricsClient creates an NginxClient to fetch stats from NGINX over an unix socket -func NewNginxMetricsClient(httpClient *http.Client) (*prometheusClient.NginxClient, error) { +func NewNginxMetricsClient(httpClient *http.Client) *prometheusClient.NginxClient { return prometheusClient.NewNginxClient(httpClient, "http://config-status/stub_status") } // RunPrometheusListenerForNginx runs an http server to expose Prometheus metrics for NGINX func RunPrometheusListenerForNginx(port int, client *prometheusClient.NginxClient, registry *prometheus.Registry, constLabels map[string]string, prometheusSecret *v1.Secret) { - registry.MustRegister(nginxCollector.NewNginxCollector(client, "nginx_ingress_nginx", constLabels)) + promlogConfig := &promlog.Config{} + logger := promlog.New(promlogConfig) + registry.MustRegister(nginxCollector.NewNginxCollector(client, "nginx_ingress_nginx", constLabels, logger)) runServer(strconv.Itoa(port), registry, prometheusSecret) } diff --git a/tests/Makefile b/tests/Makefile index fc4c4110d7..a680077c70 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -3,15 +3,15 @@ ROOT_DIR = $(shell git rev-parse --show-toplevel) CONTEXT = PULL_POLICY = IfNotPresent DEPLOYMENT_TYPE = deployment -IC_TYPE = nginx-ingress SERVICE = nodeport NODE_IP = TEST_PREFIX = test-runner KUBE_CONFIG_FOLDER = ${HOME}/.kube KIND_KUBE_CONFIG_FOLDER = $(KUBE_CONFIG_FOLDER)/kind -SHOW_IC_LOGS = no DOCKERFILEPATH := ${ROOT_DIR}/tests/Dockerfile IP_FAMILY = dual +IC_TYPE ?= nginx-ingress ## The Ingress Controller type to use, "nginx-ingress" or "nginx-plus-ingress". Defaults to "nginx-ingress" +SHOW_IC_LOGS ?= no ## Should the tests show the Ingress Controller logs on failure, "yes" or "no". Defaults to "no" TEST_TAG ?= latest ## The Tag to use for the test image. e.g. commitsha PREFIX ?= nginx/nginx-ingress ## The name of the image. For example, nginx/nginx-ingress TAG ?= edge ## The tag of the image. For example, edge @@ -52,7 +52,10 @@ update-test-kind-config: .PHONY: run-tests-in-kind run-tests-in-kind: update-test-kind-config ## Run tests in Kind - docker run --network=kind --rm -v $(KIND_KUBE_CONFIG_FOLDER):/root/.kube $(TEST_PREFIX):$(TEST_TAG) \ + docker run --network=kind --rm \ + -v $(KIND_KUBE_CONFIG_FOLDER):/root/.kube \ + -v $(ROOT_DIR)/tests:/workspace/tests \ + $(TEST_PREFIX):$(TEST_TAG) \ --context=kind-$(strip $(K8S_CLUSTER_NAME)) \ --image=$(BUILD_IMAGE) --image-pull-policy=$(PULL_POLICY) \ --deployment-type=$(DEPLOYMENT_TYPE) \ diff --git a/tests/suite/test_prometheus_metrics.py b/tests/suite/test_prometheus_metrics.py index f3bbf79633..d1136ae0db 100644 --- a/tests/suite/test_prometheus_metrics.py +++ b/tests/suite/test_prometheus_metrics.py @@ -298,3 +298,45 @@ def test_total_metrics( wait_before_test() assert_ts_total_metric(ingress_controller_endpoint, ts_type, 0) + + +@pytest.mark.ingresses +@pytest.mark.smoke +@pytest.mark.skip_for_nginx_oss +class TestPrometheusExporterPlus: + @pytest.mark.parametrize( + "ingress_controller, expected_metrics", + [ + pytest.param( + {"extra_args": ["-enable-prometheus-metrics", "-enable-oidc"]}, + [ + 'nginx_ingress_nginxplus_cache_bypass_bytes{class="nginx",zone="jwk"}', + 'nginx_ingress_nginxplus_cache_expired_bytes{class="nginx",zone="jwk"}', + 'nginx_ingress_nginxplus_cache_hit_responses{class="nginx",zone="jwk"}', + 'nginx_ingress_nginxplus_cache_miss_responses{class="nginx",zone="jwk"}', + 'nginx_ingress_nginxplus_cache_size{class="nginx",zone="jwk"}', + 'nginxplus_worker_connection_accepted{class="nginx"', + 'nginxplus_worker_connection_active{class="nginx"', + 'nginxplus_worker_http_requests_total{class="nginx"', + ], + ) + ], + indirect=["ingress_controller"], + ) + def test_plus_metrics( + self, + ingress_controller_endpoint, + ingress_controller, + expected_metrics, + ingress_setup, + ): + ensure_connection(ingress_setup.req_url, 200, {"host": ingress_setup.ingress_host}) + resp = requests.get(ingress_setup.req_url, headers={"host": ingress_setup.ingress_host}, verify=False) + assert resp.status_code == 200 + req_url = f"http://{ingress_controller_endpoint.public_ip}:{ingress_controller_endpoint.metrics_port}/metrics" + ensure_connection(req_url, 200) + resp = requests.get(req_url) + assert resp.status_code == 200, f"Expected 200 code for /metrics but got {resp.status_code}" + resp_content = resp.content.decode("utf-8") + for item in expected_metrics: + assert item in resp_content