Update pr_tests_spark.yml

snowplow · Aug 7, 2024 · df1beb6 · df1beb6
1 parent 659920b
commit df1beb6
Showing 1 changed file with 9 additions and 1 deletion.
diff --git a/.github/workflows/pr_tests_spark.yml b/.github/workflows/pr_tests_spark.yml
@@ -77,21 +77,25 @@ jobs:
           sudo chmod +x /usr/local/bin/docker-compose
 
       - name: Build and start Spark cluster
+        working-directory: .github/workflows/spark_deployment
         run: |
           docker-compose build
           docker-compose up -d
           echo "Waiting for Spark services to start..."
           sleep 90
 
       - name: Check running containers
+        working-directory: .github/workflows/spark_deployment
         run: docker ps
 
       - name: Check Docker network
+        working-directory: .github/workflows/spark_deployment
         run: |
           docker network ls
           # docker network inspect spark-network
 
       - name: Print Docker logs
+        working-directory: .github/workflows/spark_deployment
         run: |
           echo "Docker logs for spark-master:"
           docker-compose logs --tail=1000 spark-master
@@ -101,6 +105,7 @@ jobs:
           docker-compose logs --tail=1000 thrift-server
 
       - name: Verify Spark configuration
+        working-directory: .github/workflows/spark_deployment
         run: |
           echo "Verifying Spark configuration..."
           docker-compose exec -T spark-master bash -c "cat /spark/conf/spark-defaults.conf"
@@ -111,12 +116,15 @@ jobs:
           sleep 60
 
       - name: Check ThriftServer Process
+        working-directory: .github/workflows/spark_deployment
         run: docker-compose exec -T thrift-server bash -c "ps aux | grep ThriftServer"
 
       - name: Check Latest ThriftServer Log
+        working-directory: .github/workflows/spark_deployment
         run: docker-compose exec -T thrift-server bash -c "tail -n 50 /spark/logs/\$(ls -t /spark/logs/ | grep thriftserver | head -n1)"
 
       - name: Test ThriftServer connection with Beeline
+        working-directory: .github/workflows/spark_deployment
         run: |
           docker-compose exec -T thrift-server bash -c '/spark/bin/beeline -u "jdbc:hive2://localhost:10000" -e "SHOW DATABASES;"'
 
@@ -129,4 +137,4 @@ jobs:
 
       - name: "Post-test: Drop ci schemas"
         run: |
-          dbt run-operation post_ci_cleanup --target spark
+          dbt run-operation post_ci_cleanup --target spark