From 3c691d43f80ca24cd889e20c7aedec4d804695fb Mon Sep 17 00:00:00 2001 From: Gabor Boros Date: Mon, 27 Nov 2023 09:45:06 +0400 Subject: [PATCH 1/6] feat: add promstack to chart --- charts/harmony-chart/Chart.lock | 13 ++++++++++--- charts/harmony-chart/Chart.yaml | 20 +++++++++++++------- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/charts/harmony-chart/Chart.lock b/charts/harmony-chart/Chart.lock index 6be0d35..2db676b 100644 --- a/charts/harmony-chart/Chart.lock +++ b/charts/harmony-chart/Chart.lock @@ -5,20 +5,27 @@ dependencies: - name: cert-manager repository: https://charts.jetstack.io version: v1.11.0 -- name: elasticsearch - repository: https://helm.elastic.co - version: 7.17.3 - name: metrics-server repository: https://kubernetes-sigs.github.io/metrics-server/ version: 3.8.4 - name: vertical-pod-autoscaler repository: https://cowboysysop.github.io/charts/ version: 6.0.3 +- name: elasticsearch + repository: https://helm.elastic.co + version: 7.17.3 - name: opensearch repository: https://opensearch-project.github.io/helm-charts version: 2.13.3 + - name: karpenter repository: oci://public.ecr.aws/karpenter version: v0.29.2 digest: sha256:453b9f734e2d770948d3cbd36529d98da284b96de051581ea8d11a3c05e7a78e generated: "2023-10-03T10:52:43.453442762-05:00" + +- name: kube-prometheus-stack + repository: https://prometheus-community.github.io/helm-charts + version: 54.2.2 +digest: sha256:3296573c744d3a87a4daab2b5cbd63d85b283e72206f8ddddae94a76e7ccfcc3 +generated: "2023-11-24T13:29:51.649052+04:00" diff --git a/charts/harmony-chart/Chart.yaml b/charts/harmony-chart/Chart.yaml index e4fd749..5af59d0 100644 --- a/charts/harmony-chart/Chart.yaml +++ b/charts/harmony-chart/Chart.yaml @@ -26,23 +26,23 @@ dependencies: repository: https://charts.jetstack.io condition: cert-manager.enabled -- name: elasticsearch - version: "7.17.3" - condition: elasticsearch.enabled - repository: https://helm.elastic.co - - name: metrics-server + alias: metricsserver version: "~3.8.3" repository: https://kubernetes-sigs.github.io/metrics-server/ - alias: metricsserver condition: metricsserver.enabled - name: vertical-pod-autoscaler + alias: vpa version: "~6.0.3" repository: https://cowboysysop.github.io/charts/ - alias: vpa condition: vpa.enabled +- name: elasticsearch + version: "7.17.3" + condition: elasticsearch.enabled + repository: https://helm.elastic.co + - name: opensearch version: "2.13.3" condition: opensearch.enabled @@ -52,3 +52,9 @@ dependencies: version: "v0.29.2" repository: oci://public.ecr.aws/karpenter condition: karpenter.enabled + +- name: kube-prometheus-stack + alias: prometheusstack + version: "54.2.2" + condition: prometheusstack.enabled + repository: https://prometheus-community.github.io/helm-charts From 2a665a835c1dddc3c9060d1fba64ed975b0a1029 Mon Sep 17 00:00:00 2001 From: Gabor Boros Date: Mon, 27 Nov 2023 09:45:45 +0400 Subject: [PATCH 2/6] feat: add default values --- charts/harmony-chart/templates/NOTES.txt | 7 +- charts/harmony-chart/templates/echo.yaml | 3 +- charts/harmony-chart/values.yaml | 104 +++++++++++++++++++---- values-example.yaml | 26 ++++++ values-minikube.yaml | 5 ++ 5 files changed, 126 insertions(+), 19 deletions(-) diff --git a/charts/harmony-chart/templates/NOTES.txt b/charts/harmony-chart/templates/NOTES.txt index 37c2f34..dff52ce 100644 --- a/charts/harmony-chart/templates/NOTES.txt +++ b/charts/harmony-chart/templates/NOTES.txt @@ -7,9 +7,14 @@ that the load balancer is working. First, get its external IP using kubectl get svc -n {{ .Release.Namespace }} harmony-ingress-nginx-controller -Next, go to http://the.external.ip.shown/cluster-echo-test and make sure you get +Next, go to http://{{ .Values.clusterDomain }}/cluster-echo-test and make sure you get a JSON response. +{{ if and .Values.prometheusstack.grafana.enabled .Values.prometheusstack.grafana.ingress.enabled }} +Grafana shipped with the default admin user password as a bug prevents +changing it. Since is enabled on the cluster and exposed to the internet. +Please make sure you update the default admin user password! +{{- end }} diff --git a/charts/harmony-chart/templates/echo.yaml b/charts/harmony-chart/templates/echo.yaml index 87f5227..093dbd9 100644 --- a/charts/harmony-chart/templates/echo.yaml +++ b/charts/harmony-chart/templates/echo.yaml @@ -47,7 +47,8 @@ metadata: spec: ingressClassName: {{ (index .Values "ingress-nginx" "controller" "ingressClass") }} rules: - - http: + - host: {{ .Values.clusterDomain }} + http: paths: - path: /cluster-echo-test pathType: Prefix diff --git a/charts/harmony-chart/values.yaml b/charts/harmony-chart/values.yaml index d52ba8b..f6b45f2 100644 --- a/charts/harmony-chart/values.yaml +++ b/charts/harmony-chart/values.yaml @@ -2,6 +2,8 @@ # This is a YAML-formatted file. # Declare variables to be passed into your templates. +clusterDomain: "*" + ingress-nginx: # Use ingress-nginx as a default controller. enabled: true @@ -14,6 +16,23 @@ cert-manager: # certificates. email: "" +# Configuration for the metrics server chart +metricsserver: + # Control the chart inclusion + enabled: false + # See https://github.com/kubernetes-sigs/metrics-server/blob/master/charts/metrics-server/values.yaml + # for all available options + replicas: 1 + +# Configuration for the Vertical Pod Autoscaler chart +vpa: + # Control the chart inclusion + enabled: false + # See https://github.com/cowboysysop/charts/blob/master/charts/vertical-pod-autoscaler/values.yaml + # for all available options + admissionController: + replicaCount: 1 + # Multi-tenant ElasticSearch elasticsearch: enabled: false @@ -66,22 +85,6 @@ elasticsearch: xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certs/ca.crt xpack.security.transport.ssl.verification_mode: certificate -# Configuration for the metrics server chart -metricsserver: - # Control the chart inclusion - enabled: false - # See https://github.com/kubernetes-sigs/metrics-server/blob/master/charts/metrics-server/values.yaml - # for all available options - replicas: 1 -# Configuration for the Vertical Pod Autoscaler chart -vpa: - # Control the chart inclusion - enabled: false - # See https://github.com/cowboysysop/charts/blob/master/charts/vertical-pod-autoscaler/values.yaml - # for all available options - admissionController: - replicaCount: 1 - # Multi-tenant OpenSearch opensearch: enabled: false @@ -99,7 +102,6 @@ opensearch: # K8S_HARMONY_USE_SHARED_OPENSEARCH: true # HARMONY_SEARCH_HTTP_AUTH: "username:actual_password" - # # This secret will contain the ssl certificates. secretMounts: - name: opensearch-certificates @@ -235,3 +237,71 @@ karpenter: # Node template reference. More details in https://karpenter.sh/docs/concepts/node-templates/ nodeTemplate: name: "default" + +# Prometheus stack +prometheusstack: + enabled: false + + kubeStateMetrics: + enabled: true + + nodeExporter: + enabled: true + + prometheus: + enabled: true + + prometheusSpec: + resources: + requests: + cpu: "200m" + memory: "450Mi" + + alertmanager: + enabled: true + + alertmanagerSpec: + resources: + requests: + cpu: "100m" + memory: "50Mi" + + # Admin password is not pre-generated, because it is not picked up by the + # grafana pod yet -- this is a bug on their end. For more information, visit: + # https://github.com/prometheus-community/helm-charts/issues/3679 + grafana: + enabled: false + + ingress: + enabled: false + + dashboardProviders: + dashboardproviders.yaml: + apiVersion: 1 + providers: + - name: 'default' + orgId: 1 + folder: '' + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards/default + + dashboards: + default: + kubernetes-views-global: + # url: https://grafana.com/api/dashboards/15757/revisions/31/download + gnetId: 15757 + revision: 31 + datasource: Prometheus + + grafana.ini: + dashboards: + default_home_dashboard_path: /var/lib/grafana/dashboards/default/kubernetes-views-global.json + + + resources: + requests: + cpu: 200m + memory: 256Mi diff --git a/values-example.yaml b/values-example.yaml index 0df602e..be68230 100644 --- a/values-example.yaml +++ b/values-example.yaml @@ -1,7 +1,10 @@ +clusterDomain: "example.com" + ingress-nginx: controller: config: proxy-body-size: 100m + cert-manager: # Set your email address here so auto-generated HTTPS certs will work: email: "email@example.com" @@ -11,8 +14,31 @@ elasticsearch: metricsserver: enabled: false + vpa: enabled: false opensearch: enabled: false + +prometheusstack: + enabled: false + + grafana: + enabled: false + + ingress: + enabled: true + ingressClassName: nginx + annotations: + cert-manager.io/cluster-issuer: "harmony-letsencrypt-global" + nginx.ingress.kubernetes.io/ssl-redirect: "true" + hosts: + - grafana.example.com + tls: + - secretName: promstack-ingress-tls + hosts: + - grafana.example.com + + # alertmanager: + # config: {} # Set it using `--set-file prometheusstack.alertmanager.config=` diff --git a/values-minikube.yaml b/values-minikube.yaml index c739977..4528485 100644 --- a/values-minikube.yaml +++ b/values-minikube.yaml @@ -2,6 +2,8 @@ cert-manager: enabled: false +clusterDomain: "example.local" + elasticsearch: enabled: false @@ -22,3 +24,6 @@ opensearch: persistence: size: 8Gi + +prometheusstack: + enabled: false From 16d4aef7e33e46160bc0541cac709d4c26231ee5 Mon Sep 17 00:00:00 2001 From: Gabor Boros Date: Mon, 27 Nov 2023 10:48:59 +0400 Subject: [PATCH 3/6] chore: file formatting --- charts/harmony-chart/values.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/charts/harmony-chart/values.yaml b/charts/harmony-chart/values.yaml index f6b45f2..fbd6f75 100644 --- a/charts/harmony-chart/values.yaml +++ b/charts/harmony-chart/values.yaml @@ -300,7 +300,6 @@ prometheusstack: dashboards: default_home_dashboard_path: /var/lib/grafana/dashboards/default/kubernetes-views-global.json - resources: requests: cpu: 200m From 2acc5c6d0b8f7d483c6e6c2106f5da39bd3f3cb1 Mon Sep 17 00:00:00 2001 From: Gabor Boros Date: Mon, 5 Feb 2024 14:49:00 +0400 Subject: [PATCH 4/6] chore: bump chart version --- charts/harmony-chart/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/harmony-chart/Chart.yaml b/charts/harmony-chart/Chart.yaml index 5af59d0..b44bd57 100644 --- a/charts/harmony-chart/Chart.yaml +++ b/charts/harmony-chart/Chart.yaml @@ -5,7 +5,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes to the chart and its # templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.3.0 +version: 0.4.0 # This is the version number of the application being deployed. This version number should be incremented each time you # make changes to the application. Versions are not expected to follow Semantic Versioning. They should reflect the # version the application is using. It is recommended to use it with quotes. From 45481026d1657107d3e87017cf9bb0e118b5553d Mon Sep 17 00:00:00 2001 From: Gabor Boros Date: Mon, 5 Feb 2024 15:01:05 +0400 Subject: [PATCH 5/6] refactor: bump prometheus version --- charts/harmony-chart/Chart.lock | 10 +++------- charts/harmony-chart/Chart.yaml | 2 +- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/charts/harmony-chart/Chart.lock b/charts/harmony-chart/Chart.lock index 2db676b..dca39cd 100644 --- a/charts/harmony-chart/Chart.lock +++ b/charts/harmony-chart/Chart.lock @@ -17,15 +17,11 @@ dependencies: - name: opensearch repository: https://opensearch-project.github.io/helm-charts version: 2.13.3 - - name: karpenter repository: oci://public.ecr.aws/karpenter version: v0.29.2 -digest: sha256:453b9f734e2d770948d3cbd36529d98da284b96de051581ea8d11a3c05e7a78e -generated: "2023-10-03T10:52:43.453442762-05:00" - - name: kube-prometheus-stack repository: https://prometheus-community.github.io/helm-charts - version: 54.2.2 -digest: sha256:3296573c744d3a87a4daab2b5cbd63d85b283e72206f8ddddae94a76e7ccfcc3 -generated: "2023-11-24T13:29:51.649052+04:00" + version: 56.6.2 +digest: sha256:8d3ab6c27a75b2ab6106164e38ecfaeaecfb329abc3526c25565f4d6fa6d4ed5 +generated: "2024-02-05T14:59:50.438116+04:00" diff --git a/charts/harmony-chart/Chart.yaml b/charts/harmony-chart/Chart.yaml index b44bd57..565a897 100644 --- a/charts/harmony-chart/Chart.yaml +++ b/charts/harmony-chart/Chart.yaml @@ -55,6 +55,6 @@ dependencies: - name: kube-prometheus-stack alias: prometheusstack - version: "54.2.2" + version: "56.6.2" condition: prometheusstack.enabled repository: https://prometheus-community.github.io/helm-charts From 433e418e73c8ac825bb76c72aaaafc5b4d0c117c Mon Sep 17 00:00:00 2001 From: Gabor Boros Date: Mon, 5 Feb 2024 15:01:26 +0400 Subject: [PATCH 6/6] docs: add storage instructions --- charts/harmony-chart/values.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/charts/harmony-chart/values.yaml b/charts/harmony-chart/values.yaml index fbd6f75..13d8350 100644 --- a/charts/harmony-chart/values.yaml +++ b/charts/harmony-chart/values.yaml @@ -239,6 +239,13 @@ karpenter: name: "default" # Prometheus stack +# +# If no storage is defined, the Prometheus and Grafana data is stored on +# empheral storage. You can find more information about storages in the +# Prometheus operator user guides: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/storage.md +# +# - To make Grafana persistent, you may want to set Grafana storage as shown in https://github.com/prometheus-community/helm-charts/blob/main/charts/kube-prometheus-stack/values.yaml#L999-L1009 +# - To set Prometheus storage, you can see the example storage spec at https://github.com/prometheus-community/helm-charts/blob/main/charts/kube-prometheus-stack/values.yaml#L3557-L3570 prometheusstack: enabled: false