docker-compose.yml

services:
  zookeeper:
    image: debezium/zookeeper:2.7.3.Final
    container_name: zookeeper
    ports:
      - "2181:2181"
      - "2888:2888"
      - "3888:3888"
    networks:
      - de_vault

  kafka: # Kafka broker
    image: debezium/kafka:2.7.3.Final
    container_name: kafka
    ports:
      - "9093:9092"
    links:
      - zookeeper
    environment:
      - ZOOKEEPER_CONNECT=zookeeper:2181
    command: "bash -c 'kafka-server-start.sh /kafka/config/server.properties & ./kafka_create_topics.sh'"
    volumes:
      - ./src/kafka_create_topics.sh:/kafka_create_topics.sh
    networks:
      - de_vault

  mysql:
    image: debezium/example-mysql:2.7.3.Final
    container_name: mysql
    volumes:
      - ./mysql/data:/var/lib/mysql
    ports:
      - "3306:3306"
    env_file:
      - .env
    networks:
      - de_vault

  kafka-connect: # Debezium connector for capturing CDC events
    image: debezium/connect:2.7.3.Final
    container_name: kafka-connect
    ports:
      - "8083:8083"
    links:
      - kafka
      - mysql
    environment:
      - BOOTSTRAP_SERVERS=kafka:9092
      - GROUP_ID=1 # group id of kafka connect cluster
      - CONFIG_STORAGE_TOPIC=my_connect_configs
      - OFFSET_STORAGE_TOPIC=my_connect_offsets
      - STATUS_STORAGE_TOPIC=my_connect_statuses
    networks:
      - de_vault

  psql:
    image: postgres:latest
    container_name: psql
    volumes:
      - ./postgresql:/var/lib/postgresql/data
    ports:
      - "5432:5432"
    env_file:
      - .env
    networks:
      - de_vault

  minio:
    hostname: minio
    image: minio/minio
    container_name: minio
    ports:
      - "9000:9001"
    command: ["server", "/data", "--console-address", ":9001"]
    volumes:
      - ./minio:/data
    env_file:
      - .env
    networks:
      - de_vault

  # Dagster
  de_dagster:
    container_name: dagster
    image: de_dagster

  de_dagster_dagit:
    image: de_dagster:latest
    entrypoint:
      - dagit
      - -h
      - "0.0.0.0"
      - -p
      - "3001"
      - -w
      - workspace.yaml
    container_name: dagster_dagit
    expose:
      - "3001"
    ports:
      - "3001:3001"
    volumes: 
      - /var/run/docker.sock:/var/run/docker.sock
    #  - ./dagster_home:/opt/dagster/dagster_home
    env_file:
      - .env
    networks:
      - de_vault

  de_dagster_daemon:
    image: de_dagster:latest
    entrypoint:
      - dagster-daemon
      - run
    container_name: dagster_daemon
    volumes: 
      - /var/run/docker.sock:/var/run/docker.sock
    #  - ./dagster_home:/opt/dagster/dagster_home
    env_file:
      - .env
    networks:
      - de_vault

  # Minio Client
  mc:
    image: minio/mc
    container_name: mc
    hostname: mc
    env_file:
      - .env
    entrypoint: >
      /bin/sh -c " until (/usr/bin/mc config host add minio
      http://localhost:9001 minio minio123) do echo '...waiting...' && sleep 1;
      done; /usr/bin/mc mb minio/warehouse; /usr/bin/mc policy set public
      minio/warehouse; exit 0; "
    depends_on:
      - minio
    networks:
      - de_vault

  # Pipelines
#  etl_pipeline:
#    build:
#      context: etl_pipeline/
#      dockerfile: ./Dockerfile
#    container_name: etl_pipeline
#    image: etl_pipeline:latest
#    volumes:
#      - ./etl_pipeline:/opt/dagster/app
#    env_file:
#      - .env
#    networks:
#      - de_vault

networks:
  de_vault:
    driver: bridge
    name: de_vault