docker-compose.yaml

services:
  de_zookeeper: # Manage Kafka brokers
    image: debezium/zookeeper:2.0
    container_name: debezium_zookeeper
    ports:
      - 2181:2181
      - 2888:2888
      - 3888:3888
    networks:
      - de_vault

  de_kafka: # Message broker for streaming data
    image: debezium/kafka:2.0
    container_name: debezium_kafka
    ports:
      - "9093:9092"
    links:
      - de_zookeeper
    environment:
      - ZOOKEEPER_CONNECT=zookeeper:2181
    networks:
      - de_vault

  de_mysql:
    image: debezium/example-mysql:2.0
    container_name: debezium_mysql
    volumes:
      - ./mysql/data:/var/lib/mysql
    ports:
      - "3306:3306"
    env_file:
      - .env
    networks:
      - de_vault

  connect: # Debezium connector for capturing CDC events
    image: debezium/connect:2.0
    container_name: debezium_connect
    ports:
      - 8083:8083
    links:
      - de_kafka
      - de_mysql
    environment:
      BOOTSTRAP_SERVERS: kafka:9092
      GROUP_ID: 1 # group id of kafka connect cluster
      CONFIG_STORAGE_TOPIC: my_connect_configs
      OFFSET_STORAGE_TOPIC: my_connect_offsets
      STATUS_STORAGE_TOPIC: my_connect_statuses
    networks:
      - de_vault

  de_psql:
    image: postgres:latest
    container_name: de_psql
    volumes:
      - ./postgresql:/var/lib/postgresql/data
    ports:
      - "5432:5432"
    env_file:
      - .env
    networks:
      - de_vault

  de_minio:
    hostname: minio
    image: minio/minio
    container_name: de_minio
    ports:
      # "9001:9001" #maps port 9001 on the host machine to port 9000 on the container
      - "9000:9001"
    command: ["server", "/data", "--console-address", ":9001"]
    volumes:
      - ./minio:/data
    env_file:
      - .env
    networks:
      - de_vault

  # Dagster
  de_dagster:
    container_name: de_dagster
    image: de_dagster

  de_dagster_dagit:
    image: de_dagster:latest
    entrypoint:
      - dagit
      - -h
      - "0.0.0.0"
      - -p
      - "3001"
      - -w
      - workspace.yaml
    container_name: de_dagster_dagit
    expose:
      - "3001"
    ports:
      - "3001:3001"
    volumes: # Make docker client accessible so we can terminate containers from dagit
      - /var/run/docker.sock:/var/run/docker.sock
      - ./dagster_home:/opt/dagster/dagster_home
    env_file:
      - .env
    networks:
      - de_vault

  de_dagster_daemon:
    image: de_dagster:latest
    entrypoint:
      - dagster-daemon
      - run
    container_name: de_dagster_daemon
    volumes: # Make docker client accessible so we can launch containers using host docker
      - /var/run/docker.sock:/var/run/docker.sock
      - ./dagster_home:/opt/dagster/dagster_home
    env_file:
      - .env
    networks:
      - de_vault

  # Minio Client
  de_mc:
    image: minio/mc
    container_name: de_mc
    hostname: mc
    env_file:
      - .env
    entrypoint: >
      /bin/sh -c " until (/usr/bin/mc config host add minio
      http://localhost:9001 minio minio123) do echo '...waiting...' && sleep 1;
      done; /usr/bin/mc mb minio/warehouse; /usr/bin/mc policy set public
      minio/warehouse; exit 0; "
    depends_on:
      - de_minio
    networks:
      - de_vault

  # Pipelines
#  etl_pipeline:
#    build:
#      context: etl_pipeline/
#      dockerfile: ./Dockerfile
#    container_name: etl_pipeline
#    image: etl_pipeline:latest
#    volumes:
#      - ./etl_pipeline:/opt/dagster/app
#    env_file:
#      - .env
#    networks:
#      - de_vault
#    command: ["tail", "-f", "/dev/null"]

networks:
  de_vault:
    driver: bridge
    name: de_vault