From d4621298cb1089e3f61c86580ca0a67c8e704f13 Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Tue, 5 Nov 2024 11:49:10 -0500 Subject: [PATCH 1/4] feat: add ClickHouse operator sub-chart --- charts/harmony-chart/Chart.lock | 7 +++++-- charts/harmony-chart/Chart.yaml | 5 +++++ charts/harmony-chart/values.yaml | 10 ++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/charts/harmony-chart/Chart.lock b/charts/harmony-chart/Chart.lock index 9e609b7..82f1b4d 100644 --- a/charts/harmony-chart/Chart.lock +++ b/charts/harmony-chart/Chart.lock @@ -35,5 +35,8 @@ dependencies: - name: vector repository: https://helm.vector.dev version: 0.37.0 -digest: sha256:6620a3f1bac1f49165da56dc845b6435d73fd3d6c83e2ff4876e2d5c7394ae7b -generated: "2024-12-08T13:24:03.513574144-05:00" +- name: altinity-clickhouse-operator + repository: https://docs.altinity.com/clickhouse-operator/ + version: 0.24.0 +digest: sha256:ebf7f57f1f5ac2067105096a2e2961cd428b801ca04349e1aba5390cf3cf83dd +generated: "2024-12-09T12:19:37.000035907-05:00" diff --git a/charts/harmony-chart/Chart.yaml b/charts/harmony-chart/Chart.yaml index d5b0c13..435139c 100644 --- a/charts/harmony-chart/Chart.yaml +++ b/charts/harmony-chart/Chart.yaml @@ -79,3 +79,8 @@ dependencies: version: 0.37.0 repository: https://helm.vector.dev condition: vector.enabled + +- name: altinity-clickhouse-operator + version: "0.24.0" + repository: https://docs.altinity.com/clickhouse-operator/ + condition: clickhouse-operator.enabled diff --git a/charts/harmony-chart/values.yaml b/charts/harmony-chart/values.yaml index ab26460..d0688ff 100644 --- a/charts/harmony-chart/values.yaml +++ b/charts/harmony-chart/values.yaml @@ -486,3 +486,13 @@ vector: # Make sure to check out values-example.yml to now how to sink logs to S3, CloudWatch and other services sinks: {} + +clickhouse-operator: + enabled: false + dashboards: # Change this if you have monitoring disabled + enabled: true + serviceMonitor: # Change this if you have monitoring disabled + enabled: true + secret: + username: "change_me" + password: "change_me" From afd8fa475cc4d1c15326429f6de5f69b4cf5378b Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Tue, 5 Nov 2024 12:37:35 -0500 Subject: [PATCH 2/4] docs: add ClickHouse cluster documentation --- README.md | 38 ++++++++ .../clickhouse/clickhouse-installation.yml | 76 ++++++++++++++++ .../examples/clickhouse/clickhouse-keeper.yml | 86 +++++++++++++++++++ charts/harmony-chart/values.yaml | 1 + 4 files changed, 201 insertions(+) create mode 100644 charts/examples/clickhouse/clickhouse-installation.yml create mode 100644 charts/examples/clickhouse/clickhouse-keeper.yml diff --git a/README.md b/README.md index f849db3..aff34f8 100644 --- a/README.md +++ b/README.md @@ -316,6 +316,44 @@ populated with random value to ensure uniqueness. In order for SSL to work without warnings the CA certificate needs to be mounted in the relevant pods. This is not yet implemented as due to an [outstanding issue in tutor](https://github.com/overhangio/tutor/issues/791) that had not yet been completed at the time of writing. +### ClickHouse Cluster + +ClickHouse is needed for running Aspects, however for medium/large instances one single ClickHouse node can be +a bottleneck for Aspects and the default ClickHouse deployment in Aspects can take down other services running on the +same node as the ClickHouse pod. In case you are interested on running a ClickHouse cluster, you can enable the +Altinity ClickHouse Operator and follow the templates available on `charts/examples/clickhouse` to setup a ClickHouseKeeper +quorum (needed for replication) and a ClickHouse cluster based on your needs. + +Once your cluster is created and working on Kubernetes, you need to update your installation settings: + +```yaml +# See the clickhouse-installation.yml template for more details +CLICKHOUSE_ADMIN_USER: default +CLICKHOUSE_ADMIN_PASSWORD: change_me +CLICKHOUSE_CLUSTER_NAME: openedx-demo +# Set the first ClickHouse node as the DDL node. +CLICKHOUSE_CLUSTER_DDL_NODE_HOST: chi-clickhouse-{{CLICKHOUSE_CLUSTER_NAME}}-0-0.{{namespace}} +CLICKHOUSE_HOST: clickhouse-clickhouse.{{namespace}} +CLICKHOUSE_SECURE_CONNECTION: false +RUN_CLICKHOUSE: false +``` + +For multitenancy you have two options, either have multiple ClickHouse clusters or use different databases and users: + +*Using different users and databases*: Make sure to update the users and databases on your config: + +```yaml +ASPECTS_CLICKHOUSE_CMS_USER: openedx_demo_ch_cms +ASPECTS_CLICKHOUSE_LRS_USER: openedx_demo_ch_lrs +ASPECTS_CLICKHOUSE_REPORT_USER: openedx_demo_ch_report +ASPECTS_CLICKHOUSE_VECTOR_USER: openedx_demo_ch_vector +ASPECTS_XAPI_DATABASE: openedx_demo_xapi +ASPECTS_EVENT_SINK_DATABASE: openedx_demo_event_sink +ASPECTS_VECTOR_DATABASE: openedx_demo_openedx +DBT_PROFILE_TARGET_DATABASE: openedx_demo_reporting +``` + + ## Extended Documentation ### How to uninstall this chart diff --git a/charts/examples/clickhouse/clickhouse-installation.yml b/charts/examples/clickhouse/clickhouse-installation.yml new file mode 100644 index 0000000..0b9e09e --- /dev/null +++ b/charts/examples/clickhouse/clickhouse-installation.yml @@ -0,0 +1,76 @@ +--- +apiVersion: "clickhouse.altinity.com/v1" +kind: "ClickHouseInstallation" +metadata: + name: "clickhouse" +spec: + configuration: + clusters: + - name: "openedx-demo" + layout: + shardsCount: 1 # Shards have not been tested with Aspects and we don't recommend it. + replicasCount: 2 # Scale as you need/can + templates: + podTemplate: server + volumeClaimTemplate: storage + users: + test/networks/ip: + - "::/0" + test/profile: default + test/password: change_me + test/quota: default + # Default permissions needed for user creation + test/access_management: 1 + test/named_collection_control: 1 + test/show_named_collections: 1 + test/show_named_collections_secrets: 1 + zookeeper: + nodes: + - host: clickhouse-keeper-0.clickhouse-keeper-headless + - host: clickhouse-keeper-1.clickhouse-keeper-headless + - host: clickhouse-keeper-2.clickhouse-keeper-headless + files: + # Enable user replication + users-replication.xml: | + + + + /etc/clickhouse-server/users.xml + + + /clickhouse/access/ + + + + # Enable function replication + functions-replication.xml: | + + /udf + + templates: + podTemplates: + - name: server + spec: + containers: + - name: clickhouse + image: clickhouse/clickhouse-server:24.8 + # If you are running a dedicated node group for ClickHouse (and you should) + # make sure to add it tolerations. + tolerations: + - key: "clickhouseInstance" + operator: "Exists" + effect: "NoSchedule" + # Optional: set the nodegroup name + nodeSelector: + eks.amazonaws.com/nodegroup: clickhouse_worker + volumeClaimTemplates: + - name: storage + # Do not delete PV if installation is deleted. If a new ClickHouseInstallation is created + # data will be re-used, allowing recovery of data + reclaimPolicy: Retain + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Gi diff --git a/charts/examples/clickhouse/clickhouse-keeper.yml b/charts/examples/clickhouse/clickhouse-keeper.yml new file mode 100644 index 0000000..7223ccd --- /dev/null +++ b/charts/examples/clickhouse/clickhouse-keeper.yml @@ -0,0 +1,86 @@ +apiVersion: "clickhouse-keeper.altinity.com/v1" +kind: "ClickHouseKeeperInstallation" +metadata: + name: clickhouse-keeper +spec: + configuration: + clusters: + - name: "openedx-demo" + layout: + # ClickHouseKeeper needs at least tree pods to form a Quorum for high + # availability. + replicasCount: 3 + settings: + logger/level: "trace" + logger/console: "true" + listen_host: "0.0.0.0" + keeper_server/storage_path: /var/lib/clickhouse-keeper + keeper_server/tcp_port: "2181" + keeper_server/four_letter_word_white_list: "*" + keeper_server/coordination_settings/raft_logs_level: "information" + keeper_server/raft_configuration/server/port: "9444" + prometheus/endpoint: "/metrics" + prometheus/port: "7000" + prometheus/metrics: "true" + prometheus/events: "true" + prometheus/asynchronous_metrics: "true" + prometheus/status_info: "false" + templates: + podTemplates: + - name: default + spec: + # affinity removed to allow use in single node test environment + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app" + operator: In + values: + - clickhouse-keeper + topologyKey: "kubernetes.io/hostname" + containers: + - name: clickhouse-keeper + imagePullPolicy: IfNotPresent + # Make sure to keep this up to date with the ClickHouse compatible version + image: "clickhouse/clickhouse-keeper:24.8-alpine" + resources: + requests: + memory: "256M" + cpu: "0.25" + limits: + memory: "1Gi" + cpu: "1" + priorityClassName: clickhouse-priority + volumeClaimTemplates: + - name: default + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + - name: snapshot-storage-path + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + - name: log-storage-path + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + +--- +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: clickhouse-priority +value: 1000000 +globalDefault: false +description: "This priority class should be used for ClickHouse service pods only." diff --git a/charts/harmony-chart/values.yaml b/charts/harmony-chart/values.yaml index d0688ff..dc80108 100644 --- a/charts/harmony-chart/values.yaml +++ b/charts/harmony-chart/values.yaml @@ -496,3 +496,4 @@ clickhouse-operator: secret: username: "change_me" password: "change_me" + fullnameOverride: "clickhouse-operator" From d9807d7298d67d29feca29689a36a51005d917e1 Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Tue, 5 Nov 2024 13:42:18 -0500 Subject: [PATCH 3/4] build: add clickhouse integration testing --- .github/workflows/integration-test.yml | 1 + integration-test/clickhouse/cluster.sh | 0 .../openedx-demo/clickhouse-installation.yml | 76 ++++++++++++++++ .../openedx-demo/clickhouse-keeper.yml | 86 +++++++++++++++++++ .../clickhouse/openedx-demo/config.yml | 22 +++++ .../openedx-demo/post-installation.sh | 2 + .../clickhouse/openedx-demo/pre-init.sh | 3 + .../clickhouse/openedx-demo/requirements.txt | 3 + integration-test/clickhouse/values.yaml | 53 ++++++++++++ 9 files changed, 246 insertions(+) create mode 100644 integration-test/clickhouse/cluster.sh create mode 100644 integration-test/clickhouse/openedx-demo/clickhouse-installation.yml create mode 100644 integration-test/clickhouse/openedx-demo/clickhouse-keeper.yml create mode 100644 integration-test/clickhouse/openedx-demo/config.yml create mode 100644 integration-test/clickhouse/openedx-demo/post-installation.sh create mode 100644 integration-test/clickhouse/openedx-demo/pre-init.sh create mode 100644 integration-test/clickhouse/openedx-demo/requirements.txt create mode 100644 integration-test/clickhouse/values.yaml diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 3633716..03fd8a5 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -11,6 +11,7 @@ jobs: matrix: cluster: - elasticsearch + - clickhouse steps: - name: Checkout diff --git a/integration-test/clickhouse/cluster.sh b/integration-test/clickhouse/cluster.sh new file mode 100644 index 0000000..e69de29 diff --git a/integration-test/clickhouse/openedx-demo/clickhouse-installation.yml b/integration-test/clickhouse/openedx-demo/clickhouse-installation.yml new file mode 100644 index 0000000..d11eb18 --- /dev/null +++ b/integration-test/clickhouse/openedx-demo/clickhouse-installation.yml @@ -0,0 +1,76 @@ +--- +apiVersion: "clickhouse.altinity.com/v1" +kind: "ClickHouseInstallation" +metadata: + name: "clickhouse" +spec: + configuration: + clusters: + - name: "openedx-demo" + layout: + shardsCount: 1 # Shards have not been tested with Aspects and we don't recommend it. + replicasCount: 1 # Scale as you need/can + templates: + podTemplate: server + volumeClaimTemplate: storage + users: + test/networks/ip: + - "::/0" + test/profile: default + test/password: change_me + test/quota: default + # Default permissions needed for user creation + test/access_management: 1 + test/named_collection_control: 1 + test/show_named_collections: 1 + test/show_named_collections_secrets: 1 + zookeeper: + nodes: + - host: chk-clickhouse-keeper-openedx-demo-0-0 + - host: chk-clickhouse-keeper-openedx-demo-0-1 + - host: chk-clickhouse-keeper-openedx-demo-0-2 + files: + # Enable user replication + users-replication.xml: | + + + + /etc/clickhouse-server/users.xml + + + /clickhouse/access/ + + + + # Enable function replication + functions-replication.xml: | + + /udf + + templates: + podTemplates: + - name: server + spec: + containers: + - name: clickhouse + image: clickhouse/clickhouse-server:24.8 + # If you are running a dedicated node group for ClickHouse (and you should) + # make sure to add it tolerations. + # tolerations: + # - key: "clickhouseInstance" + # operator: "Exists" + # effect: "NoSchedule" + # Optional: set the nodegroup name + # nodeSelector: + # eks.amazonaws.com/nodegroup: clickhouse_worker + volumeClaimTemplates: + - name: storage + # Do not delete PV if installation is deleted. If a new ClickHouseInstallation is created + # data will be re-used, allowing recovery of data + reclaimPolicy: Retain + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Gi diff --git a/integration-test/clickhouse/openedx-demo/clickhouse-keeper.yml b/integration-test/clickhouse/openedx-demo/clickhouse-keeper.yml new file mode 100644 index 0000000..32c6a11 --- /dev/null +++ b/integration-test/clickhouse/openedx-demo/clickhouse-keeper.yml @@ -0,0 +1,86 @@ +apiVersion: "clickhouse-keeper.altinity.com/v1" +kind: "ClickHouseKeeperInstallation" +metadata: + name: clickhouse-keeper +spec: + configuration: + clusters: + - name: "openedx-demo" + layout: + # ClickHouseKeeper needs at least tree pods to form a Quorum for high + # availability. + replicasCount: 3 + settings: + logger/level: "trace" + logger/console: "true" + listen_host: "0.0.0.0" + keeper_server/storage_path: /var/lib/clickhouse-keeper + keeper_server/tcp_port: "2181" + keeper_server/four_letter_word_white_list: "*" + keeper_server/coordination_settings/raft_logs_level: "information" + keeper_server/raft_configuration/server/port: "9444" + prometheus/endpoint: "/metrics" + prometheus/port: "7000" + prometheus/metrics: "true" + prometheus/events: "true" + prometheus/asynchronous_metrics: "true" + prometheus/status_info: "false" + templates: + podTemplates: + - name: default + spec: + # affinity removed to allow use in single node test environment + # affinity: + # podAntiAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # - labelSelector: + # matchExpressions: + # - key: "app" + # operator: In + # values: + # - clickhouse-keeper + # topologyKey: "kubernetes.io/hostname" + containers: + - name: clickhouse-keeper + imagePullPolicy: IfNotPresent + # Make sure to keep this up to date with the ClickHouse compatible version + image: "clickhouse/clickhouse-keeper:24.8-alpine" + resources: + requests: + memory: "256M" + cpu: "0.25" + limits: + memory: "1Gi" + cpu: "1" + priorityClassName: clickhouse-priority + volumeClaimTemplates: + - name: default + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + - name: snapshot-storage-path + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + - name: log-storage-path + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + +--- +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: clickhouse-priority +value: 1000000 +globalDefault: false +description: "This priority class should be used for ClickHouse service pods only." diff --git a/integration-test/clickhouse/openedx-demo/config.yml b/integration-test/clickhouse/openedx-demo/config.yml new file mode 100644 index 0000000..0b3ace0 --- /dev/null +++ b/integration-test/clickhouse/openedx-demo/config.yml @@ -0,0 +1,22 @@ +ENABLE_HTTPS: true +ENABLE_WEB_PROXY: false +K8S_HARMONY_NAMESPACE: openedx-harmony +K8S_NAMESPACE: openedx-demo +LMS_HOST: local.openedx.io +PLUGINS: +- k8s_harmony +- mfe +- aspects +DOCKER_IMAGE_OPENEDX: edunext/openedx-aspects:1.2.0 +PLUGIN_INDEXES: +- https://overhang.io/tutor/main + +# Aspects settings +CLICKHOUSE_ADMIN_PASSWORD: change_me +CLICKHOUSE_ADMIN_USER: test +CLICKHOUSE_CLUSTER_NAME: openedx-demo +# Set the first ClickHouse node as the DDL node. +CLICKHOUSE_CLUSTER_DDL_NODE_HOST: chi-clickhouse-{{CLICKHOUSE_CLUSTER_NAME}}-0-0.{{K8S_HARMONY_NAMESPACE}} +CLICKHOUSE_HOST: clickhouse-clickhouse.{{K8S_HARMONY_NAMESPACE}} +CLICKHOUSE_SECURE_CONNECTION: false +RUN_CLICKHOUSE: false diff --git a/integration-test/clickhouse/openedx-demo/post-installation.sh b/integration-test/clickhouse/openedx-demo/post-installation.sh new file mode 100644 index 0000000..5b23663 --- /dev/null +++ b/integration-test/clickhouse/openedx-demo/post-installation.sh @@ -0,0 +1,2 @@ +# Run any arbitrary commands necessary to verify the installation is working +echo "Make sure to change this script to verify your installation is tested correctly" diff --git a/integration-test/clickhouse/openedx-demo/pre-init.sh b/integration-test/clickhouse/openedx-demo/pre-init.sh new file mode 100644 index 0000000..b63bb3c --- /dev/null +++ b/integration-test/clickhouse/openedx-demo/pre-init.sh @@ -0,0 +1,3 @@ +export HARMONY_NAMESPACE=$(tutor config printvalue K8S_HARMONY_NAMESPACE) +kubectl apply -f clickhouse-keeper.yml -n "$K8S_HARMONY_NAMESPACE" --wait +kubectl apply -f clickhouse-installation.yml -n "$K8S_HARMONY_NAMESPACE" --wait diff --git a/integration-test/clickhouse/openedx-demo/requirements.txt b/integration-test/clickhouse/openedx-demo/requirements.txt new file mode 100644 index 0000000..f91bb40 --- /dev/null +++ b/integration-test/clickhouse/openedx-demo/requirements.txt @@ -0,0 +1,3 @@ +../../../tutor-contrib-harmony-plugin +tutor<19 +tutor-contrib-aspects==1.2.0 diff --git a/integration-test/clickhouse/values.yaml b/integration-test/clickhouse/values.yaml new file mode 100644 index 0000000..4b8ba7a --- /dev/null +++ b/integration-test/clickhouse/values.yaml @@ -0,0 +1,53 @@ +# Disable HTTPS cert provisioning for testing with minikube +cert-manager: + enabled: false + +ingress-nginx: + # Use ingress-nginx as a default controller. + enabled: true + controller: + # All these needed for local development + service: + type: NodePort + hostPort: + enabled: true + publishService: + enabled: false + extraArgs: + publish-status-address: localhost + +clusterDomain: harmony.test + +elasticsearch: + enabled: false + + # TODO: move this to a separate PR + # Permit co-located instances for solitary minikube virtual machines. + antiAffinity: "soft" + + volumeClaimTemplate: + resources: + requests: + storage: 8Gi + replicas: 1 + +opensearch: + enabled: false + + # Permit co-located instances for solitary minikube virtual machines. + antiAffinity: "soft" + + persistence: + size: 8Gi + +prometheusstack: + enabled: true + +k8sdashboard: + enabled: false + +openfaas: + enabled: false + +clickhouse-operator: + enabled: true From 90ccc2a32a90c6e1bce6687481d2014bdd67abce Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Mon, 9 Dec 2024 12:22:07 -0500 Subject: [PATCH 4/4] chore: bump version to 0.10.0 --- charts/harmony-chart/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/harmony-chart/Chart.yaml b/charts/harmony-chart/Chart.yaml index 435139c..e913851 100644 --- a/charts/harmony-chart/Chart.yaml +++ b/charts/harmony-chart/Chart.yaml @@ -5,7 +5,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes to the chart and its # templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.9.0 +version: 0.10.0 # This is the version number of the application being deployed. This version number should be incremented each time you # make changes to the application. Versions are not expected to follow Semantic Versioning. They should reflect the # version the application is using. It is recommended to use it with quotes.