OZ-671: Add Keycloak-Superset EIP integration

README.md

@@ -82,9 +82,11 @@ export EXPORT_DESTINATION_TABLES_PATH=$DISTRO_PATH/distro/configs/analytics/dsl/
 export EXPORT_SOURCE_QUERIES_PATH=$DISTRO_PATH/distro/configs/analytics/dsl/export/queries; \
 export EXPORT_OUTPUT_PATH=./data/parquet; \
 export EXPORT_OUTPUT_TAG=h1; \
-export SUPERSET_CONFIG_PATH=$DISTRO_PATH/distro/configs/superset/ ; \
-export SUPERSET_DASHBOARDS_PATH=$DISTRO_PATH/distro/configs/superset/assets/ ; \
-export SQL_SCRIPTS_PATH=$DISTRO_PATH/distro/data
+export SUPERSET_CONFIG_PATH=$DISTRO_PATH/distro/configs/superset/; \
+export SUPERSET_DASHBOARDS_PATH=$DISTRO_PATH/distro/configs/superset/assets/; \
+export EIP_KEYCLOAK_SUPERSET_ROUTES_PATH=$DISTRO_PATH/distro/binaries/eip-keycloak-superset; \
+export EIP_KEYCLOAK_SUPERSET_PROPERTIES_PATH=$DISTRO_PATH/distro/configs/eip-keycloak-superset/properties; \
+export SQL_SCRIPTS_PATH=$DISTRO_PATH/distro/data; \
 ```
 
 For Linux 

docker/.env

@@ -63,6 +63,7 @@ ANALYTICS_KAFKA_URL=kafka:9092
 
 # Kafka
 CREATE_TOPICS=openmrs.openmrs.drug_order,openmrs.openmrs.appointment_service,openmrs.openmrs.appointment_service_type,openmrs.openmrs.care_setting,openmrs.openmrs.concept,openmrs.openmrs.concept_set,openmrs.openmrs.concept_answer,openmrs.openmrs.concept_name,openmrs.openmrs.concept_reference_map,openmrs.openmrs.concept_reference_source,openmrs.openmrs.concept_reference_term,openmrs.openmrs.conditions,openmrs.openmrs.encounter,openmrs.openmrs.encounter_diagnosis,openmrs.openmrs.encounter_type,openmrs.openmrs.location,openmrs.openmrs.location_tag_map,openmrs.openmrs.location_tag,openmrs.openmrs.form,openmrs.openmrs.obs,openmrs.openmrs.order_type,openmrs.openmrs.orders,openmrs.openmrs.patient,openmrs.openmrs.patient_appointment,openmrs.openmrs.patient_appointment_provider,openmrs.openmrs.patient_identifier,openmrs.openmrs.patient_identifier_type,openmrs.openmrs.patient_program,openmrs.openmrs.program,openmrs.openmrs.person,openmrs.openmrs.person_attribute,openmrs.openmrs.person_attribute_type,openmrs.openmrs.person_name,openmrs.openmrs.person_address,openmrs.openmrs.visit_type,openmrs.openmrs.visit,openmrs.openmrs.visit_attribute,openmrs.openmrs.visit_attribute_type,odoo.public.sale_order,odoo.public.sale_order_line,odoo.public.res_partner,odoo.public.product_product,odoo.public.product_template,odoo.public.ir_model_data
+KAFKA_HEAP_OPTS=-Xmx2G -Xms2G
 
 # Postgres
 POSTGRES_USER=postgres
@@ -120,4 +121,8 @@ ZOOKEEPER_URL=zookeeper:2181
 
 #Keycloak
 KEYCLOAK_HOSTNAME=
-ISSUER_URL=
+ISSUER_URL=
+
+#EIP
+EIP_KEYCLOAK_SUPERSET_PROPERTIES_PATH=
+EIP_KEYCLOAK_SUPERSET_ROUTES_PATH=

docker/docker-compose-batch-etl.yaml

@@ -1,4 +1,3 @@
-version: '3.8'
 services:
   batch-etl:
     networks:

docker/docker-compose-data-pipelines-local.yaml

@@ -1,4 +1,3 @@
-version: '3.8'
 services:
   jobmanager:
     depends_on:

docker/docker-compose-db.yaml

@@ -1,4 +1,3 @@
-version: '3.8'
 services:
   mysql:
     networks:

docker/docker-compose-drill.yaml

@@ -1,4 +1,3 @@
-version: "3.8"
 services:
   drill:
     restart: unless-stopped

docker/docker-compose-eip-keycloak-superset.yaml

@@ -0,0 +1,41 @@
+services:
+  env-substitution:
+    image: mekomsolutions/env-substitution
+    networks:
+      - ozone-analytics
+    restart: on-failure
+    volumes:
+       - "${DISTRO_PATH}:/opt/env-substitution/files"
+    environment:
+      - KEYCLOAK_URL=https://${KEYCLOAK_HOSTNAME}
+      - HOST_URL=https://${O3_HOSTNAME}
+      - KEYCLOAK_AUTH_SERVER_URL=https://${KEYCLOAK_HOSTNAME}
+      - SUPERSET_CLIENT_UUID=${SUPERSET_CLIENT_UUID}
+      - SUPERSET_CLIENT_SECRET=${SUPERSET_CLIENT_SECRET}
+      - KEYCLOAK_ADMIN_SA_CLIENT_SECRET=${KEYCLOAK_ADMIN_SA_CLIENT_SECRET}
+  # Keycloak - Superset integration service
+  eip-keycloak-superset:
+    image: mekomsolutions/eip-pro-client
+    restart: unless-stopped
+    environment:
+      KEYCLOAK_SERVER_URL: https://${KEYCLOAK_HOSTNAME}
+      KEYCLOAK_SA_CLIENT_ID: keycloak-admin-sa
+      SUPERSET_USERNAME: ${SUPERSET_ADMIN_USERNAME}
+      SUPERSET_PASSWORD: ${SUPERSET_ADMIN_PASSWORD}
+      SUPERSET_SERVER_URL: http://superset:8088
+      SUPERSET_REFRESH: "true"
+      SUPERSET_CLIENT_UUID: ${SUPERSET_CLIENT_UUID}
+      SUPERSET_CLIENT_SECRET: ${SUPERSET_CLIENT_SECRET}
+      KEYCLOAK_ADMIN_SA_CLIENT_SECRET: ${KEYCLOAK_ADMIN_SA_CLIENT_SECRET}
+    volumes:
+      - "${EIP_KEYCLOAK_SUPERSET_ROUTES_PATH}/:/eip-pro-client/routes"
+      - "${EIP_KEYCLOAK_SUPERSET_PROPERTIES_PATH}/:/eip-pro-client/config"
+    networks:
+      - ozone-analytics
+    depends_on:
+      superset:
+        condition: service_healthy
+      env-substitution:
+        condition: service_completed_successfully
+    ports:
+      - "9083:8080"

docker/docker-compose-export.yaml

@@ -1,4 +1,3 @@
-version: '3.8'
 services:
   parquet-export:
     networks:

docker/docker-compose-kowl.yaml

@@ -1,4 +1,3 @@
-version: '3.8'
 services:
   kowl:
     image: rsmnarts/kowl

docker/docker-compose-migration.yaml

@@ -1,4 +1,3 @@
-version: '3.8'
 services:
   odoo-replica-identity-migration:
     image: liquibase/liquibase:4.24

docker/docker-compose-minio.yaml

@@ -1,4 +1,3 @@
-version: '3.8'
 services:
   minio:
     networks:

docker/docker-compose-streaming-common.yaml

@@ -1,4 +1,3 @@
-version: '3.8'
 services:
   zookeeper:
     networks:
@@ -29,6 +28,7 @@ services:
       - KAFKA_LISTENERS=PLAINTEXT://kafka:9092,CONTROLLER://kafka:9093,PLAINTEXT_HOST://0.0.0.0:29092
       - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
       - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,PLAINTEXT_HOST://kafka:29092
+      - HEAP_OPTS="${KAFKA_HEAP_OPTS}"
     volumes:
       - kafka-data:/kafka/data
     healthcheck:

docker/docker-compose-superset.yaml

@@ -43,7 +43,7 @@ services:
       - ${SUPERSET_CONFIG_PATH}/superset-init.sh:/app/superset-init.sh
 
   superset-worker:
-    command: "celery --app=superset.tasks.celery_app:app worker --pool=gevent -Ofair -n worker1@%h --loglevel=INFO"
+    command: "celery --app=superset.tasks.celery_app:app worker -Ofair -n worker1@%h --loglevel=INFO"
     depends_on:
       redis:
         condition: service_started

docker/secrets.env

@@ -0,0 +1,8 @@
+#
+# Sample SSO Client Secrets used in the demo script
+#
+
+export SUPERSET_CLIENT_UUID=891b980a-9edb-4c72-a63d-1f8e488d6ad4
+export SUPERSET_CLIENT_SECRET=znZK8dvk7hLOpwfU
+export KEYCLOAK_ADMIN_SA_CLIENT_UUID=1fba824a-9468-4bae-a0b8-ec8a26eda2d4
+export KEYCLOAK_ADMIN_SA_CLIENT_SECRET=k3Q94Q506zO6OSBz

scripts/destroy.sh

@@ -13,4 +13,6 @@ setTraefikIP
 
 setTraefikHostnames
 
-docker compose -p ozone-analytics -f ../docker/docker-compose-db.yaml -f ../docker/docker-compose-migration.yaml -f ../docker/docker-compose-streaming-common.yaml -f ../docker/docker-compose-kowl.yaml  -f ../docker/docker-compose-superset.yaml down -v 
+setDockerComposeCLIOptions
+
+docker compose -p ozone-analytics $dockerComposeCLIOptions down -v 

scripts/docker-compose-files.txt

@@ -0,0 +1,6 @@
+docker-compose-db.yaml
+docker-compose-migration.yaml
+docker-compose-streaming-common.yaml 
+docker-compose-kowl.yaml
+docker-compose-superset.yaml
+docker-compose-superset-ports.yaml

scripts/start.sh

@@ -15,6 +15,8 @@ setTraefikIP
 
 setTraefikHostnames
 
-echo $CONNECT_ODOO_DB_NAME
+setDockerComposeCLIOptions
 
-docker compose -p ozone-analytics -f ../docker/docker-compose-db.yaml -f ../docker/docker-compose-migration.yaml -f ../docker/docker-compose-streaming-common.yaml -f ../docker/docker-compose-kowl.yaml  -f ../docker/docker-compose-superset.yaml -f ../docker/docker-compose-superset-ports.yaml up -d
+echo $dockerComposeCLIOptions
+
+docker compose -p ozone-analytics $dockerComposeCLIOptions up -d --build --remove-orphans

scripts/utils.sh

@@ -51,6 +51,8 @@ function exportEnvs () {
     export SQL_SCRIPTS_PATH=$DISTRO_PATH/data
     export SUPERSET_CONFIG_PATH=$DISTRO_PATH/configs/superset/
     export SUPERSET_DASHBOARDS_PATH=$DISTRO_PATH/configs/superset/assets/
+    export EIP_KEYCLOAK_SUPERSET_ROUTES_PATH=$DISTRO_PATH/binaries/eip-keycloak-superset
+    export EIP_KEYCLOAK_SUPERSET_PROPERTIES_PATH=$DISTRO_PATH/configs/eip-keycloak-superset/properties
     export JAVA_OPTS='-Xms2048m -Xmx8192m';
 
     echo "→ ANALYTICS_CONFIG_FILE_PATH=$ANALYTICS_CONFIG_FILE_PATH"
@@ -80,6 +82,8 @@ function exportEnvs () {
     echo "→ SQL_SCRIPTS_PATH=$SQL_SCRIPTS_PATH"
     echo "→ SUPERSET_CONFIG_PATH=$SUPERSET_CONFIG_PATH"
     echo "→ SUPERSET_DASHBOARDS_PATH=$SUPERSET_DASHBOARDS_PATH"
+    echo "→ EIP_KEYCLOAK_SUPERSET_ROUTES_PATH=$EIP_KEYCLOAK_SUPERSET_ROUTES_PATH"
+    echo "→ EIP_KEYCLOAK_SUPERSET_PROPERTIES_PATH=$EIP_KEYCLOAK_SUPERSET_PROPERTIES_PATH"
     echo "→ JAVA_OPTS=$JAVA_OPTS"
 }
 
@@ -89,9 +93,9 @@ function setDockerHost {
         export DOCKER_GATEWAY_HOST="172.17.0.1" 
     elif [[ "$OSTYPE" == "darwin"* ]]; then
         # Mac OSX
-        export DOCKER_GATEWAY_HOST="gateway.docker.internal"
+        export DOCKER_GATEWAY_HOST="host.docker.internal"
     fi
-    export DOCKER_GATEWAY_HOST="gateway.docker.internal"
+    export DOCKER_GATEWAY_HOST="host.docker.internal"
     export CONNECT_MYSQL_HOSTNAME=$DOCKER_GATEWAY_HOST
     export CONNECT_ODOO_DB_HOSTNAME=$DOCKER_GATEWAY_HOST
     export ODOO_DB_HOST=$DOCKER_GATEWAY_HOST
@@ -123,4 +127,16 @@ function setTraefikHostnames {
     export KEYCLOAK_HOSTNAME=auth-"${IP_WITH_DASHES}.traefik.me"
     echo "→ SUPERSET_HOSTNAME=$SUPERSET_HOSTNAME"
     echo "→ KEYCLOAK_HOSTNAME=$KEYCLOAK_HOSTNAME"
+}
+
+function setDockerComposeCLIOptions () {
+    # Parse 'docker-compose-files.txt' to get the list of Docker Compose files to run
+    dockerComposeFiles=$(cat docker-compose-files.txt)
+    for file in ${dockerComposeFiles}
+    do
+        export dockerComposeCLIOptions="$dockerComposeCLIOptions -f ../docker/$file"
+    done
+    if [ "$withOzoneSSO" = "true" ]; then
+        export dockerComposeCLIOptions="$dockerComposeCLIOptions --env-file ../docker/.env --env-file ../docker/secrets.env  -f ../docker-compose-eip-keycloak-superset.yaml"
+    fi
 }