From d9807d7298d67d29feca29689a36a51005d917e1 Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Tue, 5 Nov 2024 13:42:18 -0500 Subject: [PATCH] build: add clickhouse integration testing --- .github/workflows/integration-test.yml | 1 + integration-test/clickhouse/cluster.sh | 0 .../openedx-demo/clickhouse-installation.yml | 76 ++++++++++++++++ .../openedx-demo/clickhouse-keeper.yml | 86 +++++++++++++++++++ .../clickhouse/openedx-demo/config.yml | 22 +++++ .../openedx-demo/post-installation.sh | 2 + .../clickhouse/openedx-demo/pre-init.sh | 3 + .../clickhouse/openedx-demo/requirements.txt | 3 + integration-test/clickhouse/values.yaml | 53 ++++++++++++ 9 files changed, 246 insertions(+) create mode 100644 integration-test/clickhouse/cluster.sh create mode 100644 integration-test/clickhouse/openedx-demo/clickhouse-installation.yml create mode 100644 integration-test/clickhouse/openedx-demo/clickhouse-keeper.yml create mode 100644 integration-test/clickhouse/openedx-demo/config.yml create mode 100644 integration-test/clickhouse/openedx-demo/post-installation.sh create mode 100644 integration-test/clickhouse/openedx-demo/pre-init.sh create mode 100644 integration-test/clickhouse/openedx-demo/requirements.txt create mode 100644 integration-test/clickhouse/values.yaml diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 3633716..03fd8a5 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -11,6 +11,7 @@ jobs: matrix: cluster: - elasticsearch + - clickhouse steps: - name: Checkout diff --git a/integration-test/clickhouse/cluster.sh b/integration-test/clickhouse/cluster.sh new file mode 100644 index 0000000..e69de29 diff --git a/integration-test/clickhouse/openedx-demo/clickhouse-installation.yml b/integration-test/clickhouse/openedx-demo/clickhouse-installation.yml new file mode 100644 index 0000000..d11eb18 --- /dev/null +++ b/integration-test/clickhouse/openedx-demo/clickhouse-installation.yml @@ -0,0 +1,76 @@ +--- +apiVersion: "clickhouse.altinity.com/v1" +kind: "ClickHouseInstallation" +metadata: + name: "clickhouse" +spec: + configuration: + clusters: + - name: "openedx-demo" + layout: + shardsCount: 1 # Shards have not been tested with Aspects and we don't recommend it. + replicasCount: 1 # Scale as you need/can + templates: + podTemplate: server + volumeClaimTemplate: storage + users: + test/networks/ip: + - "::/0" + test/profile: default + test/password: change_me + test/quota: default + # Default permissions needed for user creation + test/access_management: 1 + test/named_collection_control: 1 + test/show_named_collections: 1 + test/show_named_collections_secrets: 1 + zookeeper: + nodes: + - host: chk-clickhouse-keeper-openedx-demo-0-0 + - host: chk-clickhouse-keeper-openedx-demo-0-1 + - host: chk-clickhouse-keeper-openedx-demo-0-2 + files: + # Enable user replication + users-replication.xml: | + + + + /etc/clickhouse-server/users.xml + + + /clickhouse/access/ + + + + # Enable function replication + functions-replication.xml: | + + /udf + + templates: + podTemplates: + - name: server + spec: + containers: + - name: clickhouse + image: clickhouse/clickhouse-server:24.8 + # If you are running a dedicated node group for ClickHouse (and you should) + # make sure to add it tolerations. + # tolerations: + # - key: "clickhouseInstance" + # operator: "Exists" + # effect: "NoSchedule" + # Optional: set the nodegroup name + # nodeSelector: + # eks.amazonaws.com/nodegroup: clickhouse_worker + volumeClaimTemplates: + - name: storage + # Do not delete PV if installation is deleted. If a new ClickHouseInstallation is created + # data will be re-used, allowing recovery of data + reclaimPolicy: Retain + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Gi diff --git a/integration-test/clickhouse/openedx-demo/clickhouse-keeper.yml b/integration-test/clickhouse/openedx-demo/clickhouse-keeper.yml new file mode 100644 index 0000000..32c6a11 --- /dev/null +++ b/integration-test/clickhouse/openedx-demo/clickhouse-keeper.yml @@ -0,0 +1,86 @@ +apiVersion: "clickhouse-keeper.altinity.com/v1" +kind: "ClickHouseKeeperInstallation" +metadata: + name: clickhouse-keeper +spec: + configuration: + clusters: + - name: "openedx-demo" + layout: + # ClickHouseKeeper needs at least tree pods to form a Quorum for high + # availability. + replicasCount: 3 + settings: + logger/level: "trace" + logger/console: "true" + listen_host: "0.0.0.0" + keeper_server/storage_path: /var/lib/clickhouse-keeper + keeper_server/tcp_port: "2181" + keeper_server/four_letter_word_white_list: "*" + keeper_server/coordination_settings/raft_logs_level: "information" + keeper_server/raft_configuration/server/port: "9444" + prometheus/endpoint: "/metrics" + prometheus/port: "7000" + prometheus/metrics: "true" + prometheus/events: "true" + prometheus/asynchronous_metrics: "true" + prometheus/status_info: "false" + templates: + podTemplates: + - name: default + spec: + # affinity removed to allow use in single node test environment + # affinity: + # podAntiAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # - labelSelector: + # matchExpressions: + # - key: "app" + # operator: In + # values: + # - clickhouse-keeper + # topologyKey: "kubernetes.io/hostname" + containers: + - name: clickhouse-keeper + imagePullPolicy: IfNotPresent + # Make sure to keep this up to date with the ClickHouse compatible version + image: "clickhouse/clickhouse-keeper:24.8-alpine" + resources: + requests: + memory: "256M" + cpu: "0.25" + limits: + memory: "1Gi" + cpu: "1" + priorityClassName: clickhouse-priority + volumeClaimTemplates: + - name: default + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + - name: snapshot-storage-path + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + - name: log-storage-path + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + +--- +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: clickhouse-priority +value: 1000000 +globalDefault: false +description: "This priority class should be used for ClickHouse service pods only." diff --git a/integration-test/clickhouse/openedx-demo/config.yml b/integration-test/clickhouse/openedx-demo/config.yml new file mode 100644 index 0000000..0b3ace0 --- /dev/null +++ b/integration-test/clickhouse/openedx-demo/config.yml @@ -0,0 +1,22 @@ +ENABLE_HTTPS: true +ENABLE_WEB_PROXY: false +K8S_HARMONY_NAMESPACE: openedx-harmony +K8S_NAMESPACE: openedx-demo +LMS_HOST: local.openedx.io +PLUGINS: +- k8s_harmony +- mfe +- aspects +DOCKER_IMAGE_OPENEDX: edunext/openedx-aspects:1.2.0 +PLUGIN_INDEXES: +- https://overhang.io/tutor/main + +# Aspects settings +CLICKHOUSE_ADMIN_PASSWORD: change_me +CLICKHOUSE_ADMIN_USER: test +CLICKHOUSE_CLUSTER_NAME: openedx-demo +# Set the first ClickHouse node as the DDL node. +CLICKHOUSE_CLUSTER_DDL_NODE_HOST: chi-clickhouse-{{CLICKHOUSE_CLUSTER_NAME}}-0-0.{{K8S_HARMONY_NAMESPACE}} +CLICKHOUSE_HOST: clickhouse-clickhouse.{{K8S_HARMONY_NAMESPACE}} +CLICKHOUSE_SECURE_CONNECTION: false +RUN_CLICKHOUSE: false diff --git a/integration-test/clickhouse/openedx-demo/post-installation.sh b/integration-test/clickhouse/openedx-demo/post-installation.sh new file mode 100644 index 0000000..5b23663 --- /dev/null +++ b/integration-test/clickhouse/openedx-demo/post-installation.sh @@ -0,0 +1,2 @@ +# Run any arbitrary commands necessary to verify the installation is working +echo "Make sure to change this script to verify your installation is tested correctly" diff --git a/integration-test/clickhouse/openedx-demo/pre-init.sh b/integration-test/clickhouse/openedx-demo/pre-init.sh new file mode 100644 index 0000000..b63bb3c --- /dev/null +++ b/integration-test/clickhouse/openedx-demo/pre-init.sh @@ -0,0 +1,3 @@ +export HARMONY_NAMESPACE=$(tutor config printvalue K8S_HARMONY_NAMESPACE) +kubectl apply -f clickhouse-keeper.yml -n "$K8S_HARMONY_NAMESPACE" --wait +kubectl apply -f clickhouse-installation.yml -n "$K8S_HARMONY_NAMESPACE" --wait diff --git a/integration-test/clickhouse/openedx-demo/requirements.txt b/integration-test/clickhouse/openedx-demo/requirements.txt new file mode 100644 index 0000000..f91bb40 --- /dev/null +++ b/integration-test/clickhouse/openedx-demo/requirements.txt @@ -0,0 +1,3 @@ +../../../tutor-contrib-harmony-plugin +tutor<19 +tutor-contrib-aspects==1.2.0 diff --git a/integration-test/clickhouse/values.yaml b/integration-test/clickhouse/values.yaml new file mode 100644 index 0000000..4b8ba7a --- /dev/null +++ b/integration-test/clickhouse/values.yaml @@ -0,0 +1,53 @@ +# Disable HTTPS cert provisioning for testing with minikube +cert-manager: + enabled: false + +ingress-nginx: + # Use ingress-nginx as a default controller. + enabled: true + controller: + # All these needed for local development + service: + type: NodePort + hostPort: + enabled: true + publishService: + enabled: false + extraArgs: + publish-status-address: localhost + +clusterDomain: harmony.test + +elasticsearch: + enabled: false + + # TODO: move this to a separate PR + # Permit co-located instances for solitary minikube virtual machines. + antiAffinity: "soft" + + volumeClaimTemplate: + resources: + requests: + storage: 8Gi + replicas: 1 + +opensearch: + enabled: false + + # Permit co-located instances for solitary minikube virtual machines. + antiAffinity: "soft" + + persistence: + size: 8Gi + +prometheusstack: + enabled: true + +k8sdashboard: + enabled: false + +openfaas: + enabled: false + +clickhouse-operator: + enabled: true