diff --git a/README.md b/README.md index 4e935ca..fc45327 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ This repository contains the code we wrote during [Rock the JVM's Spark Essenti - either clone the repo or download as zip - open with IntelliJ as an SBT project - Windows users, you need to set up some Hadoop-related configs - use [this guide](/HadoopWindowsUserSetup.md) -- in a terminal window, navigate to the folder where you downloaded this repo and run `docker-compose up` to build and start the PostgreSQL container - we will interact with it from Spark +- in a terminal window, navigate to the folder where you downloaded this repo and run `docker compose up` to build and start the PostgreSQL container - we will interact with it from Spark - in another terminal window, navigate to `spark-cluster/` - Linux/Mac users: build the Docker-based Spark cluster with ``` @@ -19,7 +19,7 @@ chmod +x build-images.sh ``` build-images.bat ``` -- when prompted to start the Spark cluster, go to the `spark-cluster` directory and run `docker-compose up --scale spark-worker=3` to spin up the Spark containers with 3 worker nodes +- when prompted to start the Spark cluster, go to the `spark-cluster` directory and run `docker compose up --scale spark-worker=3` to spin up the Spark containers with 3 worker nodes ### Spark Cluster Troubleshooting diff --git a/spark-cluster/README.md b/spark-cluster/README.md index df696c0..4348bbc 100644 --- a/spark-cluster/README.md +++ b/spark-cluster/README.md @@ -51,7 +51,7 @@ This will create the following docker images: The final step to create your test cluster will be to run the compose file: ```sh -docker-compose up --scale spark-worker=3 +docker compose up --scale spark-worker=3 ``` ## Validate your cluster diff --git a/spark-cluster/docker/base/Dockerfile b/spark-cluster/docker/base/Dockerfile index 61193d3..5739bb1 100644 --- a/spark-cluster/docker/base/Dockerfile +++ b/spark-cluster/docker/base/Dockerfile @@ -37,4 +37,4 @@ RUN wget --no-verbose https://archive.apache.org/dist/spark/spark-${SPARK_VERSIO # Fix the value of PYTHONHASHSEED # Note: this is needed when you use Python 3.3 or greater -ENV PYTHONHASHSEED 1 +ENV PYTHONHASHSEED=1 diff --git a/spark-cluster/docker/spark-master/Dockerfile b/spark-cluster/docker/spark-master/Dockerfile index 274b6fb..1efd35a 100644 --- a/spark-cluster/docker/spark-master/Dockerfile +++ b/spark-cluster/docker/spark-master/Dockerfile @@ -4,9 +4,9 @@ RUN apt-get update && apt-get install -y dos2unix COPY start-master.sh / RUN dos2unix /start-master.sh && apt-get --purge remove -y dos2unix && rm -rf /var/lib/apt/lists/* -ENV SPARK_MASTER_PORT 7077 -ENV SPARK_MASTER_WEBUI_PORT 8080 -ENV SPARK_MASTER_LOG /spark/logs +ENV SPARK_MASTER_PORT=7077 +ENV SPARK_MASTER_WEBUI_PORT=8080 +ENV SPARK_MASTER_LOG=/spark/logs EXPOSE 8080 7077 6066 diff --git a/spark-cluster/docker/spark-submit/Dockerfile b/spark-cluster/docker/spark-submit/Dockerfile index 41fb492..bdb3afe 100644 --- a/spark-cluster/docker/spark-submit/Dockerfile +++ b/spark-cluster/docker/spark-submit/Dockerfile @@ -6,7 +6,7 @@ RUN dos2unix /spark-submit.sh && apt-get --purge remove -y dos2unix && rm -rf /v ENV SPARK_MASTER_URL="spark://spark-master:7077" ENV SPARK_SUBMIT_ARGS="" -ENV SPARK_APPLICATION_ARGS "" +ENV SPARK_APPLICATION_ARGS="" #ENV SPARK_APPLICATION_JAR_LOCATION /opt/spark-apps/myjar.jar #ENV SPARK_APPLICATION_MAIN_CLASS my.main.Application diff --git a/spark-cluster/docker/spark-worker/Dockerfile b/spark-cluster/docker/spark-worker/Dockerfile index 463f3c6..2bf1d2b 100644 --- a/spark-cluster/docker/spark-worker/Dockerfile +++ b/spark-cluster/docker/spark-worker/Dockerfile @@ -4,9 +4,9 @@ RUN apt-get update && apt-get install -y dos2unix COPY start-worker.sh / RUN dos2unix /start-worker.sh && apt-get --purge remove -y dos2unix && rm -rf /var/lib/apt/lists/* -ENV SPARK_WORKER_WEBUI_PORT 8081 -ENV SPARK_WORKER_LOG /spark/logs -ENV SPARK_MASTER "spark://spark-master:7077" +ENV SPARK_WORKER_WEBUI_PORT=8081 +ENV SPARK_WORKER_LOG=/spark/logs +ENV SPARK_MASTER="spark://spark-master:7077" EXPOSE 8081