Update caffeine to 3.2.0 #1980
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Snapshot | |
| # Run workflow whenever a pull_request has been closed to the `master` and `branch-*` branches | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| types: [closed] | |
| branches: | |
| - master | |
| - branch-* | |
| jobs: | |
| snapshot: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Env | |
| run: | | |
| echo "SONATYPE_USERNAME=${{ secrets.SONATYPE_USERNAME }}" >> $GITHUB_ENV | |
| echo "SONATYPE_PASSWORD=${{ secrets.SONATYPE_PASSWORD }}" >> $GITHUB_ENV | |
| echo "GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" >> $GITHUB_ENV | |
| echo "REDSHIFT_DATABASE=${{ secrets.REDSHIFT_DATABASE }}" >> $GITHUB_ENV | |
| echo "REDSHIFT_HOST=${{ secrets.REDSHIFT_HOST }}" >> $GITHUB_ENV | |
| echo "REDSHIFT_PASSWORD=${{ secrets.REDSHIFT_PASSWORD }}" >> $GITHUB_ENV | |
| echo "REDSHIFT_USER=${{ secrets.REDSHIFT_USER }}" >> $GITHUB_ENV | |
| echo "REDSHIFT_ROLE=${{ secrets.REDSHIFT_ROLE }}" >> $GITHUB_ENV | |
| echo "SNOWFLAKE_ACCOUNT=${{ secrets.SNOWFLAKE_ACCOUNT }}" >> $GITHUB_ENV | |
| echo "SNOWFLAKE_DB=${{ secrets.SNOWFLAKE_DB }}" >> $GITHUB_ENV | |
| echo "SNOWFLAKE_PASSWORD=${{ secrets.SNOWFLAKE_PASSWORD }}" >> $GITHUB_ENV | |
| echo "SNOWFLAKE_USER=${{ secrets.SNOWFLAKE_USER }}" >> $GITHUB_ENV | |
| echo "SNOWFLAKE_WAREHOUSE=${{ secrets.SNOWFLAKE_WAREHOUSE }}" >> $GITHUB_ENV | |
| echo "AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }}" >> $GITHUB_ENV | |
| echo "AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }}" >> $GITHUB_ENV | |
| echo "AWS_ACCOUNT_ID=${{ secrets.AWS_ACCOUNT_ID }}" >> $GITHUB_ENV | |
| echo "GCP_PROJECT=${{ secrets.GCP_PROJECT }}" >> $GITHUB_ENV | |
| echo "TEMPORARY_GCS_BUCKET=${{ secrets.TEMPORARY_GCS_BUCKET }}" >> $GITHUB_ENV | |
| - uses: smartsquaregmbh/[email protected] | |
| with: | |
| version-pattern: "^\\d+\\.\\d+\\.\\d+-SNAPSHOT$" # Remove all previous SNAPSHOT versions | |
| keep: 0 | |
| names: | | |
| ai.starlake.starlake-core_2.12 | |
| - uses: actions/checkout@v4 | |
| - name: Install graphviz | |
| run: sudo apt-get install -y graphviz | |
| - name: Set up Zulu 11 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' # See 'Supported distributions' for available options | |
| java-version: '11' | |
| - name: Setup sbt launcher | |
| uses: sbt/setup-sbt@v1 | |
| - id: 'auth' | |
| uses: 'google-github-actions/auth@v1' | |
| with: | |
| credentials_json: '${{ secrets.GCP_SERVICE_ACCOUNT }}' | |
| cleanup_credentials: false | |
| - name: 'Set up Cloud SDK' | |
| uses: 'google-github-actions/setup-gcloud@v1' | |
| - name: Use gcloud CLI | |
| run: gcloud info | |
| - name: Run tests(integration.transform) | |
| run: SBT_OPTS="-Xss4M -Xms1g -Xmx4g" SL_SPARK_NO_CATALOG=true SL_SPARK_BIGQUERY_MATERIALIZATION_DATASET=SL_BQ_TEST_DS SL_ACCESS_POLICIES_PROJECT_ID=${{ env.GCP_PROJECT }} TEMPORARY_GCS_BUCKET=${{ env.TEMPORARY_GCS_BUCKET }} SL_REMOTE_TEST=true RELEASE_SONATYPE=true GITHUB_TOKEN=${{ env.GITHUB_TOKEN }} SONATYPE_USERNAME=${{ env.SONATYPE_USERNAME }} SONATYPE_PASSWORD=${{ env.SONATYPE_PASSWORD }} sbt ++2.13 "testOnly ai.starlake.integration.transform.*" | |
| - name: Run tests(schema) | |
| run: SBT_OPTS="-Xss4M -Xms1g -Xmx4g" SL_SPARK_NO_CATALOG=true SL_SPARK_BIGQUERY_MATERIALIZATION_DATASET=SL_BQ_TEST_DS SL_ACCESS_POLICIES_PROJECT_ID=${{ env.GCP_PROJECT }} TEMPORARY_GCS_BUCKET=${{ env.TEMPORARY_GCS_BUCKET }} SL_REMOTE_TEST=true RELEASE_SONATYPE=true GITHUB_TOKEN=${{ env.GITHUB_TOKEN }} SONATYPE_USERNAME=${{ env.SONATYPE_USERNAME }} SONATYPE_PASSWORD=${{ env.SONATYPE_PASSWORD }} sbt ++2.13 "testOnly ai.starlake.schema.*" | |
| - name: Run tests(job.load) | |
| run: SBT_OPTS="-Xss4M -Xms1g -Xmx4g" SL_SPARK_NO_CATALOG=true SL_SPARK_BIGQUERY_MATERIALIZATION_DATASET=SL_BQ_TEST_DS SL_ACCESS_POLICIES_PROJECT_ID=${{ env.GCP_PROJECT }} TEMPORARY_GCS_BUCKET=${{ env.TEMPORARY_GCS_BUCKET }} SL_REMOTE_TEST=true RELEASE_SONATYPE=true GITHUB_TOKEN=${{ env.GITHUB_TOKEN }} SONATYPE_USERNAME=${{ env.SONATYPE_USERNAME }} SONATYPE_PASSWORD=${{ env.SONATYPE_PASSWORD }} sbt ++2.13 "testOnly ai.starlake.job.load.*" | |
| - name: Run tests(job.metrics) | |
| run: SBT_OPTS="-Xss4M -Xms1g -Xmx4g" SL_SPARK_NO_CATALOG=true SL_SPARK_BIGQUERY_MATERIALIZATION_DATASET=SL_BQ_TEST_DS SL_ACCESS_POLICIES_PROJECT_ID=${{ env.GCP_PROJECT }} TEMPORARY_GCS_BUCKET=${{ env.TEMPORARY_GCS_BUCKET }} SL_REMOTE_TEST=true RELEASE_SONATYPE=true GITHUB_TOKEN=${{ env.GITHUB_TOKEN }} SONATYPE_USERNAME=${{ env.SONATYPE_USERNAME }} SONATYPE_PASSWORD=${{ env.SONATYPE_PASSWORD }} sbt ++2.13 "testOnly ai.starlake.job.metrics.*" | |
| - name: Run tests(job.sink) | |
| run: SBT_OPTS="-Xss4M -Xms1g -Xmx4g" SL_SPARK_NO_CATALOG=true SL_SPARK_BIGQUERY_MATERIALIZATION_DATASET=SL_BQ_TEST_DS SL_ACCESS_POLICIES_PROJECT_ID=${{ env.GCP_PROJECT }} TEMPORARY_GCS_BUCKET=${{ env.TEMPORARY_GCS_BUCKET }} SL_REMOTE_TEST=true RELEASE_SONATYPE=true GITHUB_TOKEN=${{ env.GITHUB_TOKEN }} SONATYPE_USERNAME=${{ env.SONATYPE_USERNAME }} SONATYPE_PASSWORD=${{ env.SONATYPE_PASSWORD }} sbt ++2.13 "testOnly ai.starlake.job.sink.*" | |
| - name: Run tests(job.strategies) | |
| run: SBT_OPTS="-Xss4M -Xms1g -Xmx4g" SL_SPARK_NO_CATALOG=true SL_SPARK_BIGQUERY_MATERIALIZATION_DATASET=SL_BQ_TEST_DS SL_ACCESS_POLICIES_PROJECT_ID=${{ env.GCP_PROJECT }} TEMPORARY_GCS_BUCKET=${{ env.TEMPORARY_GCS_BUCKET }} SL_REMOTE_TEST=true RELEASE_SONATYPE=true GITHUB_TOKEN=${{ env.GITHUB_TOKEN }} SONATYPE_USERNAME=${{ env.SONATYPE_USERNAME }} SONATYPE_PASSWORD=${{ env.SONATYPE_PASSWORD }} sbt ++2.13 "testOnly ai.starlake.job.strategies.*" | |
| - name: Run tests(job.transform) | |
| run: SBT_OPTS="-Xss4M -Xms1g -Xmx4g" SL_SPARK_NO_CATALOG=true SL_SPARK_BIGQUERY_MATERIALIZATION_DATASET=SL_BQ_TEST_DS SL_ACCESS_POLICIES_PROJECT_ID=${{ env.GCP_PROJECT }} TEMPORARY_GCS_BUCKET=${{ env.TEMPORARY_GCS_BUCKET }} SL_REMOTE_TEST=true RELEASE_SONATYPE=true GITHUB_TOKEN=${{ env.GITHUB_TOKEN }} SONATYPE_USERNAME=${{ env.SONATYPE_USERNAME }} SONATYPE_PASSWORD=${{ env.SONATYPE_PASSWORD }} sbt ++2.13 "testOnly ai.starlake.job.transform*" | |
| - name: Run tests(job.boostrap) | |
| run: SBT_OPTS="-Xss4M -Xms1g -Xmx4g" SL_SPARK_NO_CATALOG=true SL_SPARK_BIGQUERY_MATERIALIZATION_DATASET=SL_BQ_TEST_DS SL_ACCESS_POLICIES_PROJECT_ID=${{ env.GCP_PROJECT }} TEMPORARY_GCS_BUCKET=${{ env.TEMPORARY_GCS_BUCKET }} SL_REMOTE_TEST=true RELEASE_SONATYPE=true GITHUB_TOKEN=${{ env.GITHUB_TOKEN }} SONATYPE_USERNAME=${{ env.SONATYPE_USERNAME }} SONATYPE_PASSWORD=${{ env.SONATYPE_PASSWORD }} sbt ++2.13 "testOnly ai.starlake.job.bootstrap.*" | |
| - name: Run tests(job.connections) | |
| run: SBT_OPTS="-Xss4M -Xms1g -Xmx4g" SL_SPARK_NO_CATALOG=true SL_SPARK_BIGQUERY_MATERIALIZATION_DATASET=SL_BQ_TEST_DS SL_ACCESS_POLICIES_PROJECT_ID=${{ env.GCP_PROJECT }} TEMPORARY_GCS_BUCKET=${{ env.TEMPORARY_GCS_BUCKET }} SL_REMOTE_TEST=true RELEASE_SONATYPE=true GITHUB_TOKEN=${{ env.GITHUB_TOKEN }} SONATYPE_USERNAME=${{ env.SONATYPE_USERNAME }} SONATYPE_PASSWORD=${{ env.SONATYPE_PASSWORD }} sbt ++2.13 "testOnly ai.starlake.job.connections.*" | |
| - name: Run tests(job.convert) | |
| run: SBT_OPTS="-Xss4M -Xms1g -Xmx4g" SL_SPARK_NO_CATALOG=true SL_SPARK_BIGQUERY_MATERIALIZATION_DATASET=SL_BQ_TEST_DS SL_ACCESS_POLICIES_PROJECT_ID=${{ env.GCP_PROJECT }} TEMPORARY_GCS_BUCKET=${{ env.TEMPORARY_GCS_BUCKET }} SL_REMOTE_TEST=true RELEASE_SONATYPE=true GITHUB_TOKEN=${{ env.GITHUB_TOKEN }} SONATYPE_USERNAME=${{ env.SONATYPE_USERNAME }} SONATYPE_PASSWORD=${{ env.SONATYPE_PASSWORD }} sbt ++2.13 "testOnly ai.starlake.job.convert.*" | |
| - name: Run tests(job.infer) | |
| run: SBT_OPTS="-Xss4M -Xms1g -Xmx4g" SL_SPARK_NO_CATALOG=true SL_SPARK_BIGQUERY_MATERIALIZATION_DATASET=SL_BQ_TEST_DS SL_ACCESS_POLICIES_PROJECT_ID=${{ env.GCP_PROJECT }} TEMPORARY_GCS_BUCKET=${{ env.TEMPORARY_GCS_BUCKET }} SL_REMOTE_TEST=true RELEASE_SONATYPE=true GITHUB_TOKEN=${{ env.GITHUB_TOKEN }} SONATYPE_USERNAME=${{ env.SONATYPE_USERNAME }} SONATYPE_PASSWORD=${{ env.SONATYPE_PASSWORD }} sbt ++2.13 "testOnly ai.starlake.job.infer.*" | |
| - name: Run tests(job.ingest) | |
| run: SBT_OPTS="-Xss4M -Xms1g -Xmx4g" SL_SPARK_NO_CATALOG=true SL_SPARK_BIGQUERY_MATERIALIZATION_DATASET=SL_BQ_TEST_DS SL_ACCESS_POLICIES_PROJECT_ID=${{ env.GCP_PROJECT }} TEMPORARY_GCS_BUCKET=${{ env.TEMPORARY_GCS_BUCKET }} SL_REMOTE_TEST=true RELEASE_SONATYPE=true GITHUB_TOKEN=${{ env.GITHUB_TOKEN }} SONATYPE_USERNAME=${{ env.SONATYPE_USERNAME }} SONATYPE_PASSWORD=${{ env.SONATYPE_PASSWORD }} sbt ++2.13 "testOnly ai.starlake.job.ingest.*" | |
| - name: Run tests(job.kafka) | |
| run: SBT_OPTS="-Xss4M -Xms1g -Xmx4g" SL_SPARK_NO_CATALOG=true SL_SPARK_BIGQUERY_MATERIALIZATION_DATASET=SL_BQ_TEST_DS SL_ACCESS_POLICIES_PROJECT_ID=${{ env.GCP_PROJECT }} TEMPORARY_GCS_BUCKET=${{ env.TEMPORARY_GCS_BUCKET }} SL_REMOTE_TEST=true RELEASE_SONATYPE=true GITHUB_TOKEN=${{ env.GITHUB_TOKEN }} SONATYPE_USERNAME=${{ env.SONATYPE_USERNAME }} SONATYPE_PASSWORD=${{ env.SONATYPE_PASSWORD }} sbt ++2.13 "testOnly ai.starlake.job.kafka.*" | |
| - name: Run tests(config) | |
| run: SBT_OPTS="-Xss4M -Xms1g -Xmx4g" SL_SPARK_NO_CATALOG=true SL_SPARK_BIGQUERY_MATERIALIZATION_DATASET=SL_BQ_TEST_DS SL_ACCESS_POLICIES_PROJECT_ID=${{ env.GCP_PROJECT }} TEMPORARY_GCS_BUCKET=${{ env.TEMPORARY_GCS_BUCKET }} SL_REMOTE_TEST=true RELEASE_SONATYPE=true GITHUB_TOKEN=${{ env.GITHUB_TOKEN }} SONATYPE_USERNAME=${{ env.SONATYPE_USERNAME }} SONATYPE_PASSWORD=${{ env.SONATYPE_PASSWORD }} sbt ++2.13 "testOnly ai.starlake.config.*" | |
| - name: Run tests(extract) | |
| run: SBT_OPTS="-Xss4M -Xms1g -Xmx4g" SL_SPARK_NO_CATALOG=true SL_SPARK_BIGQUERY_MATERIALIZATION_DATASET=SL_BQ_TEST_DS SL_ACCESS_POLICIES_PROJECT_ID=${{ env.GCP_PROJECT }} TEMPORARY_GCS_BUCKET=${{ env.TEMPORARY_GCS_BUCKET }} SL_REMOTE_TEST=true RELEASE_SONATYPE=true GITHUB_TOKEN=${{ env.GITHUB_TOKEN }} SONATYPE_USERNAME=${{ env.SONATYPE_USERNAME }} SONATYPE_PASSWORD=${{ env.SONATYPE_PASSWORD }} sbt ++2.13 "testOnly ai.starlake.extract.*" | |
| - name: Run tests(integration.extract) | |
| run: SBT_OPTS="-Xss4M -Xms1g -Xmx4g" SL_SPARK_NO_CATALOG=true SL_SPARK_BIGQUERY_MATERIALIZATION_DATASET=SL_BQ_TEST_DS SL_ACCESS_POLICIES_PROJECT_ID=${{ env.GCP_PROJECT }} TEMPORARY_GCS_BUCKET=${{ env.TEMPORARY_GCS_BUCKET }} SL_REMOTE_TEST=true RELEASE_SONATYPE=true GITHUB_TOKEN=${{ env.GITHUB_TOKEN }} SONATYPE_USERNAME=${{ env.SONATYPE_USERNAME }} SONATYPE_PASSWORD=${{ env.SONATYPE_PASSWORD }} sbt ++2.13 "testOnly ai.starlake.integration.extract.*" | |
| - name: Run tests(integration.load) | |
| run: SBT_OPTS="-Xss4M -Xms1g -Xmx4g" SL_SPARK_NO_CATALOG=true SL_SPARK_BIGQUERY_MATERIALIZATION_DATASET=SL_BQ_TEST_DS SL_ACCESS_POLICIES_PROJECT_ID=${{ env.GCP_PROJECT }} TEMPORARY_GCS_BUCKET=${{ env.TEMPORARY_GCS_BUCKET }} SL_REMOTE_TEST=true RELEASE_SONATYPE=true GITHUB_TOKEN=${{ env.GITHUB_TOKEN }} SONATYPE_USERNAME=${{ env.SONATYPE_USERNAME }} SONATYPE_PASSWORD=${{ env.SONATYPE_PASSWORD }} sbt ++2.13 "testOnly ai.starlake.integration.load.*" | |
| - name: Run tests(integration.starbake) | |
| run: SBT_OPTS="-Xss4M -Xms1g -Xmx4g" SL_SPARK_NO_CATALOG=true SL_SPARK_BIGQUERY_MATERIALIZATION_DATASET=SL_BQ_TEST_DS SL_ACCESS_POLICIES_PROJECT_ID=${{ env.GCP_PROJECT }} TEMPORARY_GCS_BUCKET=${{ env.TEMPORARY_GCS_BUCKET }} SL_REMOTE_TEST=true RELEASE_SONATYPE=true GITHUB_TOKEN=${{ env.GITHUB_TOKEN }} SONATYPE_USERNAME=${{ env.SONATYPE_USERNAME }} SONATYPE_PASSWORD=${{ env.SONATYPE_PASSWORD }} sbt ++2.13 "testOnly ai.starlake.integration.starbake.*" | |
| - name: Run tests(integration.utils) | |
| run: SBT_OPTS="-Xss4M -Xms1g -Xmx4g" SL_SPARK_NO_CATALOG=true SL_SPARK_BIGQUERY_MATERIALIZATION_DATASET=SL_BQ_TEST_DS SL_ACCESS_POLICIES_PROJECT_ID=${{ env.GCP_PROJECT }} TEMPORARY_GCS_BUCKET=${{ env.TEMPORARY_GCS_BUCKET }} SL_REMOTE_TEST=true RELEASE_SONATYPE=true GITHUB_TOKEN=${{ env.GITHUB_TOKEN }} SONATYPE_USERNAME=${{ env.SONATYPE_USERNAME }} SONATYPE_PASSWORD=${{ env.SONATYPE_PASSWORD }} sbt ++2.13 "testOnly ai.starlake.integration.utils.*" | |
| - name: Run tests(privacy) | |
| run: SBT_OPTS="-Xss4M -Xms1g -Xmx4g" SL_SPARK_NO_CATALOG=true SL_SPARK_BIGQUERY_MATERIALIZATION_DATASET=SL_BQ_TEST_DS SL_ACCESS_POLICIES_PROJECT_ID=${{ env.GCP_PROJECT }} TEMPORARY_GCS_BUCKET=${{ env.TEMPORARY_GCS_BUCKET }} SL_REMOTE_TEST=true RELEASE_SONATYPE=true GITHUB_TOKEN=${{ env.GITHUB_TOKEN }} SONATYPE_USERNAME=${{ env.SONATYPE_USERNAME }} SONATYPE_PASSWORD=${{ env.SONATYPE_PASSWORD }} sbt ++2.13 "testOnly ai.starlake.privacy.*" | |
| - name: Run tests(serve) | |
| run: SBT_OPTS="-Xss4M -Xms1g -Xmx4g" SL_SPARK_NO_CATALOG=true SL_SPARK_BIGQUERY_MATERIALIZATION_DATASET=SL_BQ_TEST_DS SL_ACCESS_POLICIES_PROJECT_ID=${{ env.GCP_PROJECT }} TEMPORARY_GCS_BUCKET=${{ env.TEMPORARY_GCS_BUCKET }} SL_REMOTE_TEST=true RELEASE_SONATYPE=true GITHUB_TOKEN=${{ env.GITHUB_TOKEN }} SONATYPE_USERNAME=${{ env.SONATYPE_USERNAME }} SONATYPE_PASSWORD=${{ env.SONATYPE_PASSWORD }} sbt ++2.13 "testOnly ai.starlake.serve.*" | |
| - name: Run tests(sql) | |
| run: SBT_OPTS="-Xss4M -Xms1g -Xmx4g" SL_SPARK_NO_CATALOG=true SL_SPARK_BIGQUERY_MATERIALIZATION_DATASET=SL_BQ_TEST_DS SL_ACCESS_POLICIES_PROJECT_ID=${{ env.GCP_PROJECT }} TEMPORARY_GCS_BUCKET=${{ env.TEMPORARY_GCS_BUCKET }} SL_REMOTE_TEST=true RELEASE_SONATYPE=true GITHUB_TOKEN=${{ env.GITHUB_TOKEN }} SONATYPE_USERNAME=${{ env.SONATYPE_USERNAME }} SONATYPE_PASSWORD=${{ env.SONATYPE_PASSWORD }} sbt ++2.13 "testOnly ai.starlake.sql.*" | |
| - name: Run tests(udf) | |
| run: SBT_OPTS="-Xss4M -Xms1g -Xmx4g" SL_SPARK_NO_CATALOG=true SL_SPARK_BIGQUERY_MATERIALIZATION_DATASET=SL_BQ_TEST_DS SL_ACCESS_POLICIES_PROJECT_ID=${{ env.GCP_PROJECT }} TEMPORARY_GCS_BUCKET=${{ env.TEMPORARY_GCS_BUCKET }} SL_REMOTE_TEST=true RELEASE_SONATYPE=true GITHUB_TOKEN=${{ env.GITHUB_TOKEN }} SONATYPE_USERNAME=${{ env.SONATYPE_USERNAME }} SONATYPE_PASSWORD=${{ env.SONATYPE_PASSWORD }} sbt ++2.13 "testOnly ai.starlake.udf.*" | |
| - name: Run tests(utils) | |
| run: SBT_OPTS="-Xss4M -Xms1g -Xmx4g" SL_SPARK_NO_CATALOG=true SL_SPARK_BIGQUERY_MATERIALIZATION_DATASET=SL_BQ_TEST_DS SL_ACCESS_POLICIES_PROJECT_ID=${{ env.GCP_PROJECT }} TEMPORARY_GCS_BUCKET=${{ env.TEMPORARY_GCS_BUCKET }} SL_REMOTE_TEST=true RELEASE_SONATYPE=true GITHUB_TOKEN=${{ env.GITHUB_TOKEN }} SONATYPE_USERNAME=${{ env.SONATYPE_USERNAME }} SONATYPE_PASSWORD=${{ env.SONATYPE_PASSWORD }} sbt ++2.13 "testOnly ai.starlake.utils.*" | |
| - name: Run tests(workflow) | |
| run: SBT_OPTS="-Xss4M -Xms1g -Xmx4g" SL_SPARK_NO_CATALOG=true SL_SPARK_BIGQUERY_MATERIALIZATION_DATASET=SL_BQ_TEST_DS SL_ACCESS_POLICIES_PROJECT_ID=${{ env.GCP_PROJECT }} TEMPORARY_GCS_BUCKET=${{ env.TEMPORARY_GCS_BUCKET }} SL_REMOTE_TEST=true RELEASE_SONATYPE=true GITHUB_TOKEN=${{ env.GITHUB_TOKEN }} SONATYPE_USERNAME=${{ env.SONATYPE_USERNAME }} SONATYPE_PASSWORD=${{ env.SONATYPE_PASSWORD }} sbt ++2.13 "testOnly ai.starlake.workflow.*" | |
| - name: Publish Snapshot on Sonatype Spark 3 for scala 2.12 | |
| run: SBT_OPTS="-Xss4M -Xms1g -Xmx4g" SL_SPARK_NO_CATALOG=true SL_SPARK_BIGQUERY_MATERIALIZATION_DATASET=SL_BQ_TEST_DS SL_ACCESS_POLICIES_PROJECT_ID=${{ env.GCP_PROJECT }} TEMPORARY_GCS_BUCKET=${{ env.TEMPORARY_GCS_BUCKET }} SL_REMOTE_TEST=true RELEASE_SONATYPE=true GITHUB_TOKEN=${{ env.GITHUB_TOKEN }} SONATYPE_USERNAME=${{ env.SONATYPE_USERNAME }} SONATYPE_PASSWORD=${{ env.SONATYPE_PASSWORD }} sbt ++2.12 publish | |
| - name: Publish Snapshot on Sonatype Spark 3 for scala 2.13 | |
| run: SBT_OPTS="-Xss4M -Xms1g -Xmx4g" SL_SPARK_NO_CATALOG=true SL_SPARK_BIGQUERY_MATERIALIZATION_DATASET=SL_BQ_TEST_DS SL_ACCESS_POLICIES_PROJECT_ID=${{ env.GCP_PROJECT }} TEMPORARY_GCS_BUCKET=${{ env.TEMPORARY_GCS_BUCKET }} SL_REMOTE_TEST=true RELEASE_SONATYPE=true GITHUB_TOKEN=${{ env.GITHUB_TOKEN }} SONATYPE_USERNAME=${{ env.SONATYPE_USERNAME }} SONATYPE_PASSWORD=${{ env.SONATYPE_PASSWORD }} sbt ++2.13 publish | |
| - name: Trigger starlake-api workflow | |
| if: ${{ success() }} | |
| id: trigger-starlake-api | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.STARLAKE_API_TOKEN }} | |
| run: | | |
| curl -X POST -H "Authorization: token $GITHUB_TOKEN" \ | |
| -H "Accept: application/vnd.github+json" \ | |
| https://api.github.com/repos/starlake-ai/starlake-api/actions/workflows/docker-hub.yml/dispatches \ | |
| -d '{"ref":"main"}' | |
| docker-hub: | |
| needs: snapshot | |
| name: Snapshot Docker Image | |
| runs-on: ubuntu-latest | |
| permissions: | |
| packages: write | |
| contents: read | |
| attestations: write | |
| id-token: write | |
| strategy: | |
| fail-fast: false | |
| steps: | |
| - name: Checkout | |
| id: vcs | |
| uses: actions/checkout@v4 | |
| - name: Env | |
| run: | | |
| cat .versions >> $GITHUB_ENV | |
| source .versions | |
| FIRST_LINE=$(head -n 1 version.sbt) | |
| SL_VERSION=$(echo "$FIRST_LINE" | sed -E 's/.*version := "([0-9]+\.[0-9]+\.?.*)"/\1/') | |
| echo "SL_VERSION=${SL_VERSION}" >> $GITHUB_ENV | |
| SL_MAJOR_MINOR_VERSION=$(echo "${SL_VERSION}" | cut -d'.' -f1-2) | |
| echo "SL_MAJOR_MINOR_VERSION=${SL_MAJOR_MINOR_VERSION}" >> $GITHUB_ENV | |
| REGISTRY_IMAGE=starlakeai/starlake | |
| echo "REGISTRY_IMAGE=${REGISTRY_IMAGE}" >> $GITHUB_ENV | |
| REGISTRY_IMAGE_LATEST=${REGISTRY_IMAGE}:${SL_VERSION:-latest} | |
| echo "REGISTRY_IMAGE_LATEST=${REGISTRY_IMAGE_LATEST}" >> $GITHUB_ENV | |
| echo "DOCKERHUB_USERNAME=${{ secrets.DOCKERHUB_USERNAME }}" >> $GITHUB_ENV | |
| echo "DOCKERHUB_TOKEN=${{ secrets.DOCKERHUB_TOKEN }}" >> $GITHUB_ENV | |
| echo "GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" >> $GITHUB_ENV | |
| - name: Prepare | |
| run: | | |
| ./scripts/docker-prepare.sh | |
| - name: Docker meta | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{ env.REGISTRY_IMAGE }} | |
| labels: | | |
| org.opencontainers.image.vendor=starlakeai | |
| org.opencontainers.image.licenses=Apache-2.0 | |
| org.opencontainers.image.title=starlakeai/starlake | |
| org.opencontainers.image.description="A declarative text based tool that enables analysts and engineers to extract, load, transform and orchestrate their data pipelines." | |
| org.opencontainers.image.url=https://starlake.ai | |
| # generate Docker tags based on the following events/attributes | |
| tags: | | |
| type=schedule | |
| type=semver,pattern={{version}} | |
| type=semver,pattern={{major}}.{{minor}} | |
| type=semver,pattern={{major}} | |
| # type=sha | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ env.DOCKERHUB_USERNAME }} | |
| password: ${{ env.DOCKERHUB_TOKEN }} | |
| - name: Build and export to Docker | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: "./distrib/docker" | |
| pull: true | |
| load: true | |
| tags: ${{ steps.meta.outputs.tags }},${{ env.REGISTRY_IMAGE_LATEST }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| build-args: | | |
| BUILD_DATE=${{ env.BUILD_DATE }} | |
| VCS_REF=${{ steps.vcs.outputs.commit }} | |
| SL_VERSION=${{ env.SL_VERSION }} | |
| - name: Inspect image | |
| run: | | |
| docker image inspect --format='{{json .Config.Labels}}' ${{ env.REGISTRY_IMAGE_LATEST }} | |
| - name: Test | |
| run: | | |
| docker run --rm ${{ env.REGISTRY_IMAGE_LATEST }} help | |
| - name: Push image | |
| if: github.event_name != 'pull_request' | |
| id: push | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: "./distrib/docker" | |
| platforms: ${{ env.PLATFORMS }} | |
| pull: true | |
| push: true | |
| provenance: mode=max | |
| tags: ${{ steps.meta.outputs.tags }},${{ env.REGISTRY_IMAGE_LATEST }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| build-args: | | |
| BUILD_DATE=${{ env.BUILD_DATE }} | |
| VCS_REF=${{ steps.vcs.outputs.commit }} | |
| SL_VERSION=${{ env.SL_VERSION }} | |
| # - name: Generate artifact attestation | |
| # if: github.event_name != 'pull_request' | |
| # uses: actions/attest-build-provenance@v1 | |
| # with: | |
| # subject-name: ${{ env.REGISTRY_IMAGE }} | |
| # subject-digest: ${{ steps.push.outputs.digest }} | |
| # push-to-registry: true | |
| # github-token: ${{ env.GITHUB_TOKEN }} |