pr_tests_spark.yml

name: pr_tests_spark

on:
  pull_request:

env:
  DBT_PROFILES_DIR: ./ci
  SPARK_MASTER_HOST: localhost
  SPARK_USER: spark
  SPARK_SCHEMA: default
  AWS_REGION: eu-west-1
  AWS_DEFAULT_REGION: eu-west-1
  DOCKER_PLATFORM: linux/amd64

jobs:
  pr_tests_spark:
    name: pr_tests_spark
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: ./integration_tests
    strategy:
      matrix:
        dbt_version:
          - 1.*
        warehouse:
          - spark
    steps:
      - name: Check out
        uses: actions/checkout@v3

      - name: Log in to Docker Hub
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Set SCHEMA_SUFFIX env
        run: >-
          echo "SCHEMA_SUFFIX=$(echo ${DBT_VERSION%.*} | tr . _)" >> $GITHUB_ENV
        env:
          DBT_VERSION: '${{ matrix.dbt_version }}'

      - name: Set DEFAULT_TARGET env
        run: |
          echo "DEFAULT_TARGET=${{ matrix.warehouse }}" >> $GITHUB_ENV

      - name: Python setup
        uses: actions/setup-python@v4
        with:
          python-version: 3.8.x

      - name: Install dependencies
        run: |
          pip install --upgrade pip wheel setuptools
          pip install -Iv "dbt-spark[PyHive]==${{ matrix.dbt_version }}" --upgrade
          pip install boto3 awscli
          dbt deps

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: eu-west-1

      - name: Install Docker Compose
        run: |
          sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
          sudo chmod +x /usr/local/bin/docker-compose

      - name: Start Spark cluster
        working-directory: .github/workflows/spark_deployment
        run: |
          docker-compose up -d
          echo "Waiting for Spark services to start..."
          sleep 30  # Reduced sleep time
          docker-compose ps
          docker-compose logs

      - name: Check running containers
        working-directory: .github/workflows/spark_deployment
        run: docker ps -a  # Show all containers, including stopped ones

      - name: Print Docker logs
        if: failure()
        working-directory: .github/workflows/spark_deployment
        run: |
          echo "Docker logs for spark-master:"
          docker-compose logs spark-master
          echo "Docker logs for spark-worker:"
          docker-compose logs spark-worker
          echo "Docker logs for thrift-server:"
          docker-compose logs thrift-server

      - name: "Pre-test: Drop ci schemas"
        run: |
          dbt run-operation post_ci_cleanup --target spark

      - name: Run tests
        run: ./.scripts/integration_tests.sh -d spark

      - name: "Post-test: Drop ci schemas"
        run: |
          dbt run-operation post_ci_cleanup --target spark