build: add clickhouse integration testing

.github/workflows/integration-test.yml

@@ -11,6 +11,7 @@ jobs:
       matrix:
         cluster:
           - elasticsearch
+          - clickhouse
 
     steps:
       - name: Checkout

integration-test/clickhouse/openedx-demo/clickhouse-installation.yml

@@ -0,0 +1,76 @@
+---
+apiVersion: "clickhouse.altinity.com/v1"
+kind: "ClickHouseInstallation"
+metadata:
+  name: "clickhouse"
+spec:
+  configuration:
+    clusters:
+      - name: "openedx-demo"
+        layout:
+          shardsCount: 1 # Shards have not been tested with Aspects and we don't recommend it.
+          replicasCount: 1 # Scale as you need/can
+        templates:
+          podTemplate: server
+          volumeClaimTemplate: storage
+    users:
+      test/networks/ip:
+        - "::/0"
+      test/profile: default
+      test/password: change_me
+      test/quota: default
+      # Default permissions needed for user creation
+      test/access_management: 1
+      test/named_collection_control: 1
+      test/show_named_collections: 1
+      test/show_named_collections_secrets: 1
+    zookeeper:
+      nodes:
+        - host: chk-clickhouse-keeper-openedx-demo-0-0
+        - host: chk-clickhouse-keeper-openedx-demo-0-1
+        - host: chk-clickhouse-keeper-openedx-demo-0-2
+    files:
+      # Enable user replication
+      users-replication.xml: |
+        <clickhouse>
+          <user_directories replace="replace">
+            <users_xml>
+              <path>/etc/clickhouse-server/users.xml</path>
+            </users_xml>
+            <replicated>
+              <zookeeper_path>/clickhouse/access/</zookeeper_path>
+            </replicated>
+          </user_directories>
+        </clickhouse>
+      # Enable function replication
+      functions-replication.xml: |
+        <clickhouse>
+            <user_defined_zookeeper_path>/udf</user_defined_zookeeper_path>
+        </clickhouse>
+  templates:
+    podTemplates:
+      - name: server
+        spec:
+          containers:
+            - name: clickhouse
+              image: clickhouse/clickhouse-server:24.8
+          # If you are running a dedicated node group for ClickHouse (and you should)
+          # make sure to add it tolerations.
+          # tolerations:
+          # - key: "clickhouseInstance"
+          #   operator: "Exists"
+          #   effect: "NoSchedule"
+          # Optional: set the nodegroup name
+          # nodeSelector:
+          #   eks.amazonaws.com/nodegroup: clickhouse_worker
+    volumeClaimTemplates:
+      - name: storage
+        # Do not delete PV if installation is deleted. If a new ClickHouseInstallation is created
+        # data will be re-used, allowing recovery of data
+        reclaimPolicy: Retain
+        spec:
+          accessModes:
+            - ReadWriteOnce
+          resources:
+            requests:
+              storage: 50Gi

integration-test/clickhouse/openedx-demo/clickhouse-keeper.yml

@@ -0,0 +1,86 @@
+apiVersion: "clickhouse-keeper.altinity.com/v1"
+kind: "ClickHouseKeeperInstallation"
+metadata:
+  name: clickhouse-keeper
+spec:
+  configuration:
+    clusters:
+      - name: "openedx-demo"
+        layout:
+          # ClickHouseKeeper needs at least tree pods to form a Quorum for high
+          # availability.
+          replicasCount: 3
+    settings:
+      logger/level: "trace"
+      logger/console: "true"
+      listen_host: "0.0.0.0"
+      keeper_server/storage_path: /var/lib/clickhouse-keeper
+      keeper_server/tcp_port: "2181"
+      keeper_server/four_letter_word_white_list: "*"
+      keeper_server/coordination_settings/raft_logs_level: "information"
+      keeper_server/raft_configuration/server/port: "9444"
+      prometheus/endpoint: "/metrics"
+      prometheus/port: "7000"
+      prometheus/metrics: "true"
+      prometheus/events: "true"
+      prometheus/asynchronous_metrics: "true"
+      prometheus/status_info: "false"
+  templates:
+    podTemplates:
+      - name: default
+        spec:
+          # affinity removed to allow use in single node test environment
+          # affinity:
+          #   podAntiAffinity:
+          #     requiredDuringSchedulingIgnoredDuringExecution:
+          #       - labelSelector:
+          #           matchExpressions:
+          #             - key: "app"
+          #               operator: In
+          #               values:
+          #                 - clickhouse-keeper
+          #         topologyKey: "kubernetes.io/hostname"
+          containers:
+            - name: clickhouse-keeper
+              imagePullPolicy: IfNotPresent
+              # Make sure to keep this up to date with the ClickHouse compatible version
+              image: "clickhouse/clickhouse-keeper:24.8-alpine"
+              resources:
+                requests:
+                  memory: "256M"
+                  cpu: "0.25"
+                limits:
+                  memory: "1Gi"
+                  cpu: "1"
+          priorityClassName: clickhouse-priority
+    volumeClaimTemplates:
+      - name: default
+        spec:
+          accessModes:
+            - ReadWriteOnce
+          resources:
+            requests:
+              storage: 10Gi
+      - name: snapshot-storage-path
+        spec:
+          accessModes:
+            - ReadWriteOnce
+          resources:
+            requests:
+              storage: 10Gi
+      - name: log-storage-path
+        spec:
+          accessModes:
+            - ReadWriteOnce
+          resources:
+            requests:
+              storage: 10Gi
+
+---
+apiVersion: scheduling.k8s.io/v1
+kind: PriorityClass
+metadata:
+  name: clickhouse-priority
+value: 1000000
+globalDefault: false
+description: "This priority class should be used for ClickHouse service pods only."

integration-test/clickhouse/openedx-demo/config.yml

@@ -0,0 +1,22 @@
+ENABLE_HTTPS: true
+ENABLE_WEB_PROXY: false
+K8S_HARMONY_NAMESPACE: openedx-harmony
+K8S_NAMESPACE: openedx-demo
+LMS_HOST: local.openedx.io
+PLUGINS:
+- k8s_harmony
+- mfe
+- aspects
+DOCKER_IMAGE_OPENEDX: edunext/openedx-aspects:1.2.0
+PLUGIN_INDEXES:
+- https://overhang.io/tutor/main
+
+# Aspects settings
+CLICKHOUSE_ADMIN_PASSWORD: change_me
+CLICKHOUSE_ADMIN_USER: test
+CLICKHOUSE_CLUSTER_NAME: openedx-demo
+# Set the first ClickHouse node as the DDL node.
+CLICKHOUSE_CLUSTER_DDL_NODE_HOST: chi-clickhouse-{{CLICKHOUSE_CLUSTER_NAME}}-0-0.{{K8S_HARMONY_NAMESPACE}}
+CLICKHOUSE_HOST: clickhouse-clickhouse.{{K8S_HARMONY_NAMESPACE}}
+CLICKHOUSE_SECURE_CONNECTION: false
+RUN_CLICKHOUSE: false

integration-test/clickhouse/openedx-demo/post-installation.sh

@@ -0,0 +1,2 @@
+# Run any arbitrary commands necessary to verify the installation is working
+echo "Make sure to change this script to verify your installation is tested correctly"

integration-test/clickhouse/openedx-demo/pre-init.sh

@@ -0,0 +1,3 @@
+export HARMONY_NAMESPACE=$(tutor config printvalue K8S_HARMONY_NAMESPACE)
+kubectl apply -f clickhouse-keeper.yml -n "$K8S_HARMONY_NAMESPACE" --wait
+kubectl apply -f clickhouse-installation.yml -n "$K8S_HARMONY_NAMESPACE" --wait

integration-test/clickhouse/openedx-demo/requirements.txt

@@ -0,0 +1,3 @@
+../../../tutor-contrib-harmony-plugin
+tutor<19
+tutor-contrib-aspects==1.2.0

integration-test/clickhouse/values.yaml

@@ -0,0 +1,53 @@
+# Disable HTTPS cert provisioning for testing with minikube
+cert-manager:
+   enabled: false
+
+ingress-nginx:
+  # Use ingress-nginx as a default controller.
+  enabled: true
+  controller:
+    # All these needed for local development
+    service:
+      type: NodePort
+    hostPort:
+      enabled: true
+    publishService:
+      enabled: false
+    extraArgs:
+      publish-status-address: localhost
+
+clusterDomain: harmony.test
+
+elasticsearch:
+  enabled: false
+
+   # TODO: move this to a separate PR
+   # Permit co-located instances for solitary minikube virtual machines.
+  antiAffinity: "soft"
+
+  volumeClaimTemplate:
+    resources:
+      requests:
+        storage: 8Gi
+  replicas: 1
+
+opensearch:
+  enabled: false
+
+   # Permit co-located instances for solitary minikube virtual machines.
+  antiAffinity: "soft"
+
+  persistence:
+    size: 8Gi
+
+prometheusstack:
+  enabled: true
+
+k8sdashboard:
+  enabled: false
+
+openfaas:
+  enabled: false
+
+clickhouse-operator:
+  enabled: true