docker-compose.yml

version: "2"
services:
  spark-master:
    image: spark-base:latest
    entrypoint:
      - '/spark/sbin/start-master.sh'
    ports:
      - "8080:8080"
      - "7077:7077"
      - "6066:6066"
    environment:
      - SPARK_LOCAL_IP=spark-master
      - SPARK_MASTER_IP=spark-master
      - SPARK_MASTER_PORT=7077
      - SPARK_MASTER_WEBUI_PORT=8080
      - SPARK_MASTER_LOG=/spark/logs
      - SPARK_NO_DAEMONIZE=true
      - SPARK_HOME=/spark
    volumes:
      - app-volume:/target/scala-2.12
      - ./data:/data
      - ./logs:/spark/logs

  spark-worker-1:
    image: spark-base:latest
    entrypoint:
      - '/spark/sbin/start-slave.sh'
      - 'spark://spark-master:7077'
    depends_on:
      - spark-master
    ports:
      - "8081:8081"
    environment:
      - SPARK_MASTER=spark://spark-master:7077
      - SPARK_WORKER_CORES=1
      - SPARK_WORKER_MEMORY=3G
      - SPARK_DRIVER_MEMORY=2G
      - SPARK_WORKER_WEBUI_PORT=8081
      - SPARK_WORKER_LOG=/spark/logs
      - SPARK_LOCAL_IP=spark-worker-1
      - SPARK_MASTER_IP=spark-master
      - SPARK_NO_DAEMONIZE=true
      - SPARK_HOME=/spark
    volumes:
      - app-volume:/target/scala-2.12
      - ./data:/data
      - ./logs:/spark/logs

  spark-worker-2:
    image: spark-base:latest
    depends_on:
      - spark-master
    entrypoint:
      - '/spark/sbin/start-slave.sh'
      - 'spark://spark-master:7077'
    ports:
      - "8082:8082"
    environment:
      - SPARK_MASTER=spark://spark-master:7077
      - SPARK_WORKER_CORES=1
      - SPARK_WORKER_MEMORY=3G
      - SPARK_DRIVER_MEMORY=2G
      - SPARK_WORKER_WEBUI_PORT=8082
      - SPARK_WORKER_LOG=/spark/logs
      - SPARK_LOCAL_IP=spark-worker-2
      - SPARK_MASTER_IP=spark-master
      - SPARK_NO_DAEMONIZE=true
      - SPARK_HOME=/spark
    volumes:
      - app-volume:/target/scala-2.12
      - ./data:/data
      - ./logs:/spark/logs

  spark-submit:
    image: spark-base:latest
    depends_on:
      - spark-master
      - spark-worker-1
      - spark-worker-2
    environment:
      - SPARK_APPLICATION_MAIN_CLASS=DataChallenge
      - SPARK_MASTER=spark://spark-master:7077
      - SPARK_MASTER_IP=spark-master
      - SPARK_SUBMIT_ARGS=
      - SPARK_APPLICATION_JAR_LOCATION=/target/scala-2.12/data-challenge_2.12-1.0.jar
      - SPARK_APPLICATION_ARGS=
      - SPARK_NO_DAEMONIZE=true
      - SPARK_HOME=/spark
    volumes:
      - app-volume:/target/scala-2.12
      - ./data:/data
      - ./logs:/spark/logs

volumes:
  app-volume:
  # data: