revert back

snowplow · Aug 8, 2024 · 641eba5 · 641eba5
1 parent fbe1789
commit 641eba5
Show file tree

Hide file tree

Showing 3 changed files with 65 additions and 88 deletions.
diff --git a/.github/workflows/pr_tests_spark.yml b/.github/workflows/pr_tests_spark.yml
@@ -3,14 +3,15 @@ name: pr_tests_spark
 on:
   pull_request:
 
+concurrency: dbt_integration_tests
+
 env:
   DBT_PROFILES_DIR: ./ci
   SPARK_MASTER_HOST: localhost
   SPARK_USER: spark
   SPARK_SCHEMA: default
   AWS_REGION: eu-west-1
   AWS_DEFAULT_REGION: eu-west-1
-  DOCKER_PLATFORM: linux/amd64
 
 jobs:
   pr_tests_spark:
@@ -29,12 +30,6 @@ jobs:
       - name: Check out
         uses: actions/checkout@v3
 
-      - name: Log in to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
       - name: Set SCHEMA_SUFFIX env
         run: >-
           echo "SCHEMA_SUFFIX=$(echo ${DBT_VERSION%.*} | tr . _)" >> $GITHUB_ENV
@@ -64,53 +59,65 @@ jobs:
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: eu-west-1
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
       - name: Install Docker Compose
         run: |
           sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
           sudo chmod +x /usr/local/bin/docker-compose
-      - name: Pull Docker image
-        run: docker pull snowplow/spark-s3-iceberg:v2
-      - name: Check image architecture
-        run: docker inspect --format='{{.Architecture}}' snowplow/spark-s3-iceberg:v2
-      - name: Diagnostic Information
-        run: |
-          echo "Docker version:"
-          docker --version
-          echo "Docker info:"
-          docker info
-          echo "Available images:"
-          docker images
-          echo "System information:"
-          uname -a
-
-      - name: Start Spark cluster
+
+      - name: Build and start Spark cluster
         working-directory: .github/workflows/spark_deployment
         run: |
-          docker-compose config
-          docker-compose pull
+          docker-compose build
           docker-compose up -d
           echo "Waiting for Spark services to start..."
-          sleep 30
-          docker-compose ps
-          docker-compose logs
+          sleep 90
 
       - name: Check running containers
+        working-directory: .github/workflows/spark_deployment
+        run: docker ps
+
+      - name: Check Docker network
         working-directory: .github/workflows/spark_deployment
         run: |
-          docker ps -a
-          echo "Container details:"
-          docker inspect $(docker ps -aq)
+          docker network ls
+          # docker network inspect spark-network
 
       - name: Print Docker logs
-        if: failure()
         working-directory: .github/workflows/spark_deployment
         run: |
           echo "Docker logs for spark-master:"
-          docker-compose logs spark-master
+          docker-compose logs --tail=1000 spark-master
           echo "Docker logs for spark-worker:"
-          docker-compose logs spark-worker
+          docker-compose logs --tail=1000 spark-worker
           echo "Docker logs for thrift-server:"
-          docker-compose logs thrift-server
+          docker-compose logs --tail=1000 thrift-server
+
+      - name: Verify Spark configuration
+        working-directory: .github/workflows/spark_deployment
+        run: |
+          echo "Verifying Spark configuration..."
+          docker-compose exec -T spark-master bash -c "cat /spark/conf/spark-defaults.conf"
+
+      - name: Wait for Thrift Server
+        run: |
+          echo "Waiting for Thrift Server to be fully operational..."
+          sleep 60
+
+      - name: Check ThriftServer Process
+        working-directory: .github/workflows/spark_deployment
+        run: docker-compose exec -T thrift-server bash -c "ps aux | grep ThriftServer"
+
+      - name: Check Latest ThriftServer Log
+        working-directory: .github/workflows/spark_deployment
+        run: docker-compose exec -T thrift-server bash -c "tail -n 50 /spark/logs/\$(ls -t /spark/logs/ | grep thriftserver | head -n1)"
+
+      - name: Test ThriftServer connection with Beeline
+        working-directory: .github/workflows/spark_deployment
+        run: |
+          docker-compose exec -T thrift-server bash -c '/spark/bin/beeline -u "jdbc:hive2://localhost:10000" -e "SHOW DATABASES;"'
 
       - name: "Pre-test: Drop ci schemas"
         run: |

diff --git a/.github/workflows/spark_deployment/Dockerfile b/.github/workflows/spark_deployment/Dockerfile
@@ -1,46 +1,34 @@
-# Use a multi-arch base image
-FROM --platform=$BUILDPLATFORM openjdk:11-jre-slim
+FROM openjdk:11-jre-slim
 
 # Set environment variables
-ARG BUILDPLATFORM
-ARG TARGETPLATFORM
-ARG SPARK_VERSION=3.5.1
-ARG HADOOP_VERSION=3.3.4
-ARG ICEBERG_VERSION=1.4.2
-ARG AWS_SDK_VERSION=1.12.581
+ENV SPARK_VERSION=3.5.1
+ENV HADOOP_VERSION=3.3.4
+ENV ICEBERG_VERSION=1.4.2
+ENV AWS_SDK_VERSION=1.12.581
 
 # Install necessary tools
 RUN apt-get update && apt-get install -y curl wget procps rsync ssh
 
 # Download and install Spark
 RUN wget https://downloads.apache.org/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz && \
     tar -xvzf spark-${SPARK_VERSION}-bin-hadoop3.tgz && \
-    mv spark-${SPARK_VERSION}-bin-hadoop3 /opt/spark && \
+    mv spark-${SPARK_VERSION}-bin-hadoop3 /spark && \
     rm spark-${SPARK_VERSION}-bin-hadoop3.tgz
 
 # Set Spark environment variables
-ENV SPARK_HOME=/opt/spark
+ENV SPARK_HOME=/spark
 ENV PATH=$PATH:$SPARK_HOME/bin:$SPARK_HOME/sbin
 
 # Download necessary JARs
-RUN mkdir -p /opt/spark/jars && \
-    wget https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-3.5_2.12/${ICEBERG_VERSION}/iceberg-spark-runtime-3.5_2.12-${ICEBERG_VERSION}.jar -O /opt/spark/jars/iceberg-spark-runtime.jar && \
-    wget https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar -O /opt/spark/jars/iceberg-aws-bundle.jar && \
-    wget https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/${HADOOP_VERSION}/hadoop-aws-${HADOOP_VERSION}.jar -O /opt/spark/jars/hadoop-aws.jar && \
-    wget https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/${AWS_SDK_VERSION}/aws-java-sdk-bundle-${AWS_SDK_VERSION}.jar -O /opt/spark/jars/aws-java-sdk-bundle.jar
+RUN mkdir -p /spark/jars && \
+    wget https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-3.5_2.12/${ICEBERG_VERSION}/iceberg-spark-runtime-3.5_2.12-${ICEBERG_VERSION}.jar -O /spark/jars/iceberg-spark-runtime.jar && \
+    wget https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar -O /spark/jars/iceberg-aws-bundle.jar && \
+    wget https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/${HADOOP_VERSION}/hadoop-aws-${HADOOP_VERSION}.jar -O /spark/jars/hadoop-aws.jar && \
+    wget https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/${AWS_SDK_VERSION}/aws-java-sdk-bundle-${AWS_SDK_VERSION}.jar -O /spark/jars/aws-java-sdk-bundle.jar
 
 # Create directory for Spark events
 RUN mkdir -p /tmp/spark-events
 
-# Create a non-root user to run Spark
-RUN useradd -ms /bin/bash spark
-RUN chown -R spark:spark /opt/spark /tmp/spark-events
-
-# Switch to non-root user
-USER spark
-WORKDIR /opt/spark
-
-# Expose Spark ports
-EXPOSE 8080 7077 10000
+WORKDIR /spark
 
 CMD ["bash"]
diff --git a/.github/workflows/spark_deployment/docker-compose.yml b/.github/workflows/spark_deployment/docker-compose.yml
@@ -6,9 +6,8 @@ networks:
 
 services:
   spark-master:
-    image: snowplow/spark-s3-iceberg:v2
-    platform: linux/amd64
-    command: ["/bin/bash", "-c", "/opt/spark/sbin/start-master.sh -h spark-master --properties-file /opt/spark/conf/spark-defaults.conf && tail -f /dev/null"]
+    build: .
+    command: ["/bin/bash", "-c", "/spark/sbin/start-master.sh -h spark-master --properties-file /spark/conf/spark-defaults.conf && tail -f /spark/logs/spark--org.apache.spark.deploy.master.Master-1-*.out"]
     hostname: spark-master
     ports:
       - '8080:8080'
@@ -23,19 +22,13 @@ services:
       - AWS_REGION=eu-west-1
       - AWS_DEFAULT_REGION=eu-west-1
     volumes:
-      - ./spark-defaults.conf:/opt/spark/conf/spark-defaults.conf
+      - ./spark-defaults.conf:/spark/conf/spark-defaults.conf
     networks:
       - spark-network
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8080"]
-      interval: 10s
-      timeout: 5s
-      retries: 3
 
   spark-worker:
-    image: snowplow/spark-s3-iceberg:v2
-    platform: linux/amd64
-    command: ["/bin/bash", "-c", "sleep 10 && /opt/spark/sbin/start-worker.sh spark://spark-master:7077 --properties-file /opt/spark/conf/spark-defaults.conf && tail -f /dev/null"]
+    build: .
+    command: ["/bin/bash", "-c", "sleep 10 && /spark/sbin/start-worker.sh spark://spark-master:7077 --properties-file /spark/conf/spark-defaults.conf && tail -f /spark/logs/spark--org.apache.spark.deploy.worker.Worker-*.out"]
     depends_on:
       - spark-master
     environment:
@@ -49,19 +42,13 @@ services:
       - AWS_REGION=eu-west-1
       - AWS_DEFAULT_REGION=eu-west-1
     volumes:
-      - ./spark-defaults.conf:/opt/spark/conf/spark-defaults.conf
+      - ./spark-defaults.conf:/spark/conf/spark-defaults.conf
     networks:
       - spark-network
-    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:8081"]
-      interval: 10s
-      timeout: 5s
-      retries: 3
 
   thrift-server:
-    image: snowplow/spark-s3-iceberg:v2
-    platform: linux/amd64
-    command: ["/bin/bash", "-c", "sleep 30 && /opt/spark/sbin/start-thriftserver.sh --master spark://spark-master:7077 --driver-memory 2g --executor-memory 3g --hiveconf hive.server2.thrift.port=10000 --hiveconf hive.server2.thrift.bind.host=0.0.0.0 --conf spark.sql.hive.thriftServer.async=true --conf spark.sql.hive.thriftServer.workerQueue.size=2000 --conf spark.sql.hive.thriftServer.maxWorkerThreads=100 --conf spark.sql.hive.thriftServer.minWorkerThreads=50 && tail -f /dev/null"]
+    build: .
+    command: ["/bin/bash", "-c", "sleep 30 && /spark/sbin/start-thriftserver.sh --master spark://spark-master:7077 --driver-memory 2g --executor-memory 3g --hiveconf hive.server2.thrift.port=10000 --hiveconf hive.server2.thrift.bind.host=0.0.0.0 --conf spark.sql.hive.thriftServer.async=true --conf spark.sql.hive.thriftServer.workerQueue.size=2000 --conf spark.sql.hive.thriftServer.maxWorkerThreads=100 --conf spark.sql.hive.thriftServer.minWorkerThreads=50 && tail -f /spark/logs/spark--org.apache.spark.sql.hive.thriftserver.HiveThriftServer2-*.out"]
     ports:
       - '10000:10000'
     depends_on:
@@ -74,11 +61,6 @@ services:
       - AWS_REGION=eu-west-1
       - AWS_DEFAULT_REGION=eu-west-1
     volumes:
-      - ./spark-defaults.conf:/opt/spark/conf/spark-defaults.conf
+      - ./spark-defaults.conf:/spark/conf/spark-defaults.conf
     networks:
-      - spark-network
-    healthcheck:
-      test: ["CMD", "nc", "-z", "localhost", "10000"]
-      interval: 10s
-      timeout: 5s
-      retries: 3
+      - spark-network