Adding the YML for running a pipeline on azure #54
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: PDT Azure CI | |
| # Controls when the workflow will run | |
| on: | |
| # Triggers the workflow on push or pull request events but only for the "develop" and "master" branch | |
| pull_request: | |
| branches: [ "develop", "master" ] | |
| # paths: | |
| # - '3rd-party-tools/zcall/**' | |
| # Allows you to run this workflow manually from the Actions tab | |
| workflow_dispatch: | |
| inputs: | |
| image_tag: | |
| description: 'Docker Image Tag (default: branch_name)' | |
| env: | |
| PROJECT_NAME: MultiSample SmartSeq2 Single Nucleus | |
| # Github repo name | |
| # REPOSITORY_NAME: ${{ github.event.repository.name }} | |
| # # Region-specific Google Docker repository where GOOGLE_PROJECT/REPOSITORY_NAME can be found | |
| # DOCKER_REGISTRY: us.gcr.io | |
| # GCR_PATH: broad-gotc-prod/zcall | |
| # ACR_PATH: zcall | |
| # The tag for the Docker imageAZURE_SUBSCRIPTION_ID="d44cb7d1-184a-4202-b1de-1ffe54218e99" | |
| AZURE_RESOURCE_GROUP_NAME: dsp-pipeline-dev | |
| AZURE_AKS_NAME: pdt-a1b950246124be | |
| AZURE_SUBSCRIPTION_ID: d44cb7d1-184a-4202-b1de-1ffe54218e99 | |
| TAG: ${{ github.event.inputs.image_tag || github.head_ref || github.ref_name }} | |
| # A workflow run is made up of one or more jobs that can run sequentially or in parallel | |
| jobs: | |
| # The job that builds our container | |
| # build-for-gcr: | |
| # runs-on: ubuntu-latest | |
| # defaults: | |
| # run: | |
| # working-directory: 3rd-party-tools/zcall | |
| # # Map a step output to a job output | |
| # outputs: | |
| # imagePath: ${{ steps.saveImagePath.outputs.url }} | |
| # steps: | |
| # # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it | |
| # - uses: actions/checkout@v3 | |
| # - name: Build the Docker image | |
| # run: docker build . --file Dockerfile --tag ${DOCKER_REGISTRY}/${GCR_PATH}:${TAG} | |
| # - name: Check working directory' | |
| # run: | | |
| # echo "Current directory: " | |
| # pwd | |
| # ls -lht | |
| # # Save the image path to an output | |
| # - id: 'saveImagePath' | |
| # run: echo "url=${DOCKER_REGISTRY}/${GCR_PATH}:${TAG}" >> $GITHUB_OUTPUT | |
| # # Log into the Google Docker registry | |
| # - id: 'Auth' | |
| # name: Login to GCR | |
| # uses: docker/login-action@v2 | |
| # with: | |
| # registry: ${{ env.DOCKER_REGISTRY }} | |
| # username: _json_key | |
| # password: ${{ secrets.GCR_CI_KEY }} | |
| # # Push the image to the Google Docker registry | |
| # - name: Push image | |
| # run: "docker push ${DOCKER_REGISTRY}/${GCR_PATH}:${TAG}" | |
| run-on-azure: | |
| runs-on: ubuntu-latest | |
| # defaults: | |
| # run: | |
| # working-directory: 3rd-party-tools/zcall | |
| steps: | |
| # checkout the repo | |
| - name: 'Checkout GitHub Action' | |
| uses: actions/checkout@v3 | |
| - name: 'Login via Azure CLI' | |
| uses: azure/login@v1 | |
| with: | |
| creds: ${{ secrets.AZURE_CREDENTIALS }} | |
| - name: 'Create Release Package' | |
| run: | | |
| pwd | |
| scripts/build_pipeline_release.sh -w MultiSampleSmartSeq2SingleNucleus.wdl -v 1.0 -o release -e dev | |
| ls -lhta release/MultiSampleSmartSeq2SingleNucleus | |
| tree release || true | |
| - name: 'Connect to Kube and Show Pods' | |
| run: | | |
| az aks get-credentials --resource-group ${AZURE_RESOURCE_GROUP_NAME} --name ${AZURE_AKS_NAME} --subscription ${AZURE_SUBSCRIPTION_ID} --file ~/.kube/config | |
| kubectl -n coa get svc | |
| - name: 'Set-up Port Forwarding and Run Cromshell' | |
| run: | | |
| #install cromshell | |
| pip install cromshell | |
| kubectl port-forward -n coa svc/cromwell 8000:8000 2>&1 & | |
| pid=$! | |
| echo pid: $pid | |
| while ! nc -vz localhost 8000 > /dev/null 2>&1 ; do | |
| # echo sleeping | |
| sleep 1 | |
| done | |
| echo "Cromwell is up!" | |
| #curl -s -X POST "http://localhost:8000/api/workflows/v1/query" -H "accept: application/json" -H "Content-Type: application/json" -d "[{\"status\":\"Running\"},{\"status\":\"Submitted\"}]" | jq | |
| # cromshell --help | |
| cromshell submit --help | |
| #Run cromshell | |
| export CROMWELL_URL=http://localhost:8000 | |
| echo $CROMWELL_URL is the cromwell url | |
| echo ZIP contents: | |
| #unzip -l release/MultiSampleSmartSeq2SingleNucleus/MultiSampleSmartSeq2SingleNucleus_1.0.zip || true | |
| unzip release/MultiSampleSmartSeq2SingleNucleus/MultiSampleSmartSeq2SingleNucleus_1.0.zip -d release/MultiSampleSmartSeq2SingleNucleus || true | |
| tree release/MultiSampleSmartSeq2SingleNucleus || true | |
| #echo WDL content: | |
| #cat release/MultiSampleSmartSeq2SingleNucleus/MultiSampleSmartSeq2SingleNucleus_1.0.wdl || true | |
| cp test_inputs/Plumbing/mouse_example_azure.json release/MultiSampleSmartSeq2SingleNucleus | |
| cd release/MultiSampleSmartSeq2SingleNucleus | |
| ls -lht ~/.cromshell || true | |
| tree ~/.cromshell || true | |
| cat ~/.cromshell/all.workflow.database.tsv || true | |
| CRUN=$(cromshell --no_turtle --cromwell_url http://localhost:8000 submit --no-validation -op MultiSampleSmartSeq2SingleNucleus_1.0.options.json_fail -d MultiSampleSmartSeq2SingleNucleus_1.0.zip MultiSampleSmartSeq2SingleNucleus_1.0.wdl mouse_example_azure.json) | |
| echo CRUN = $CRUN | |
| tree ~/.cromshell || true | |
| cromshell alias -- -1 ss_proto_1 || true | |
| cromshell list | |
| #wait for the job to start | |
| sleep 3 | |
| SECONDS=0 | |
| while true; do | |
| status_json=$(cromshell --no_turtle status ss_proto_1) | |
| # Use jq to extract the status from the JSON | |
| status=$(echo "$status_json" | jq -r '.status') | |
| if [[ $status != "Running" && $status != "Submitted" ]]; then | |
| break | |
| fi | |
| # Sleep for some time before checking again | |
| sleep 300 | |
| done | |
| t=$SECONDS | |
| echo "The status is now: $status" | |
| printf 'Time taken: %d days, %d minutes\n' "$(( t/86400 ))" "$(( t/60 - 1440*(t/86400) ))" | |
| CSTATUS=$status | |
| WID=$ss_proto_1 | |
| echo CSTATUS = $CSTATUS | |
| #CSTATUS=$(cromshell --cromwell_url http://localhost:8000 status 113556d4-ac73-492e-bf9d-03eb5edcf469) | |
| ALL_JOBS=$(curl -s -X POST "http://localhost:8000/api/workflows/v1/query" -H "accept: application/json" -H "Content-Type: application/json" -d "[{\"status\":\"Running\"},{\"status\":\"Submitted\"}]" | jq ) | |
| echo "### Summary of Azure Jobs! :rocket:" >> $GITHUB_STEP_SUMMARY | |
| echo "### Azure Cromwell Job Submitted $CRUN" >> $GITHUB_STEP_SUMMARY | |
| echo "### Job Summary " >> $GITHUB_STEP_SUMMARY | |
| echo "| Name | Value |" >> $GITHUB_STEP_SUMMARY | |
| echo "| ----------- | ----------- |" >> $GITHUB_STEP_SUMMARY | |
| echo "| WorkflowID | $WID |" >> $GITHUB_STEP_SUMMARY | |
| echo "| Status | $CSTATUS |" >> $GITHUB_STEP_SUMMARY | |
| echo "| Total Runtime (seconds) | $t |" >> $GITHUB_STEP_SUMMARY | |
| echo "### All Azure Jobs (Running/Pending) = $ALL_JOBS" >> $GITHUB_STEP_SUMMARY | |
| kill $pid | |
| #doesn't work because we need to pass env vars across steps, which we'll do in the non-prototype version of this | |
| # - name: 'Write the Summary' | |
| # run: | | |
| # echo "### Hello world! :rocket:" >> $GITHUB_STEP_SUMMARY | |
| # echo "### CSTATUS = $CSTATUS" >> $GITHUB_STEP_SUMMARY |