docker-compose.yml

services:

  # Prefect Agent
  agent:
    image: prefecthq/prefect:2.12-python3.11
    command: prefect agent start --pool default-agent-pool --work-queue default
    environment:
      EXTRA_PIP_PACKAGES: adlfs databricks-sdk
      PREFECT_API_KEY: ${PREFECT_API_KEY}
      PREFECT_API_URL: ${PREFECT_API_URL}
      DATABRICKS_HOST: ${DATABRICKS_HOST}
      DATABRICKS_TOKEN: ${DATABRICKS_TOKEN}

  # Prefect CLI
  cli:
    image: prefecthq/prefect:2.12-python3.11
    tty: true  # keep running
    working_dir: "/root/databricks_pipelines"
    volumes:
      - "./:/root/databricks_pipelines"
    environment:
      PYTHONPATH: "/root/databricks_pipelines"
      EXTRA_PIP_PACKAGES: adlfs databricks-sdk
      PREFECT_API_KEY: ${PREFECT_API_KEY}
      PREFECT_API_URL: ${PREFECT_API_URL}
      AZURE_STORAGE_ACCOUNT_NAME: ${AZURE_STORAGE_ACCOUNT_NAME}
      AZURE_STORAGE_ACCOUNT_KEY: ${AZURE_STORAGE_ACCOUNT_KEY}

networks:
  default:
    name: prefect-network