-
Notifications
You must be signed in to change notification settings - Fork 0
144 lines (126 loc) · 5.83 KB
/
run_pipeline_on_azure.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
name: PDT Azure CI
# Controls when the workflow will run
on:
# Triggers the workflow on push or pull request events but only for the "develop" and "master" branch
pull_request:
branches: [ "develop", "master" ]
# paths:
# - '3rd-party-tools/zcall/**'
# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:
inputs:
image_tag:
description: 'Docker Image Tag (default: branch_name)'
env:
PROJECT_NAME: MultiSample SmartSeq2 Single Nucleus
# Github repo name
# REPOSITORY_NAME: ${{ github.event.repository.name }}
# # Region-specific Google Docker repository where GOOGLE_PROJECT/REPOSITORY_NAME can be found
# DOCKER_REGISTRY: us.gcr.io
# GCR_PATH: broad-gotc-prod/zcall
# ACR_PATH: zcall
# The tag for the Docker imageAZURE_SUBSCRIPTION_ID="d44cb7d1-184a-4202-b1de-1ffe54218e99"
AZURE_RESOURCE_GROUP_NAME: dsp-pipeline-dev
AZURE_AKS_NAME: pdt-a1b950246124be
AZURE_SUBSCRIPTION_ID: d44cb7d1-184a-4202-b1de-1ffe54218e99
TAG: ${{ github.event.inputs.image_tag || github.head_ref || github.ref_name }}
# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
# The job that builds our container
# build-for-gcr:
# runs-on: ubuntu-latest
# defaults:
# run:
# working-directory: 3rd-party-tools/zcall
# # Map a step output to a job output
# outputs:
# imagePath: ${{ steps.saveImagePath.outputs.url }}
# steps:
# # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
# - uses: actions/checkout@v3
# - name: Build the Docker image
# run: docker build . --file Dockerfile --tag ${DOCKER_REGISTRY}/${GCR_PATH}:${TAG}
# - name: Check working directory'
# run: |
# echo "Current directory: "
# pwd
# ls -lht
# # Save the image path to an output
# - id: 'saveImagePath'
# run: echo "url=${DOCKER_REGISTRY}/${GCR_PATH}:${TAG}" >> $GITHUB_OUTPUT
# # Log into the Google Docker registry
# - id: 'Auth'
# name: Login to GCR
# uses: docker/login-action@v2
# with:
# registry: ${{ env.DOCKER_REGISTRY }}
# username: _json_key
# password: ${{ secrets.GCR_CI_KEY }}
# # Push the image to the Google Docker registry
# - name: Push image
# run: "docker push ${DOCKER_REGISTRY}/${GCR_PATH}:${TAG}"
run-on-azure:
runs-on: ubuntu-latest
# defaults:
# run:
# working-directory: 3rd-party-tools/zcall
steps:
# checkout the repo
- name: 'Checkout GitHub Action'
uses: actions/checkout@v3
- name: 'Login via Azure CLI'
uses: azure/login@v1
with:
creds: ${{ secrets.AZURE_CREDENTIALS }}
- name: 'Create Release Package'
run: |
pwd
scripts/build_pipeline_release.sh -w MultiSampleSmartSeq2SingleNucleus.wdl -v 1.0 -o release -e dev
ls -lhta release/MultiSampleSmartSeq2SingleNucleus
tree release || true
- name: 'Connect to Kube and Show Pods'
run: |
az aks get-credentials --resource-group ${AZURE_RESOURCE_GROUP_NAME} --name ${AZURE_AKS_NAME} --subscription ${AZURE_SUBSCRIPTION_ID} --file ~/.kube/config
kubectl -n coa get svc
- name: 'Set-up Port Forwarding and Run Cromshell'
run: |
#install cromshell
pip install cromshell
kubectl port-forward -n coa svc/cromwell 8000:8000 2>&1 &
pid=$!
echo pid: $pid
while ! nc -vz localhost 8000 > /dev/null 2>&1 ; do
# echo sleeping
sleep 1
done
echo "Cromwell is up!"
#curl -s -X POST "http://localhost:8000/api/workflows/v1/query" -H "accept: application/json" -H "Content-Type: application/json" -d "[{\"status\":\"Running\"},{\"status\":\"Submitted\"}]" | jq
# cromshell --help
cromshell submit --help
#Run cromshell
export CROMWELL_URL=http://localhost:8000
echo $CROMWELL_URL is the cromwell url
echo ZIP contents:
#unzip -l release/MultiSampleSmartSeq2SingleNucleus/MultiSampleSmartSeq2SingleNucleus_1.0.zip || true
unzip release/MultiSampleSmartSeq2SingleNucleus/MultiSampleSmartSeq2SingleNucleus_1.0.zip -d release/MultiSampleSmartSeq2SingleNucleus || true
tree release/MultiSampleSmartSeq2SingleNucleus || true
#echo WDL content:
#cat release/MultiSampleSmartSeq2SingleNucleus/MultiSampleSmartSeq2SingleNucleus_1.0.wdl || true
cp test_inputs/Plumbing/mouse_example_azure.json release/MultiSampleSmartSeq2SingleNucleus
cd release/MultiSampleSmartSeq2SingleNucleus
CRUN=$(cromshell --no_turtle --cromwell_url http://localhost:8000 submit --no-validation -op MultiSampleSmartSeq2SingleNucleus_1.0.options.json -d MultiSampleSmartSeq2SingleNucleus_1.0.zip MultiSampleSmartSeq2SingleNucleus_1.0.wdl mouse_example_azure.json)
echo CRUN = $CRUN
cat ~/.cromshell/${CROMWELL_URL}/ || true
cromshell alias -- -1 ss_proto_1 || true
CSTATUS=$(cromshell status ss_proto_1)
#CSTATUS=$(cromshell --cromwell_url http://localhost:8000 status 113556d4-ac73-492e-bf9d-03eb5edcf469)
curl -s -X POST "http://localhost:8000/api/workflows/v1/query" -H "accept: application/json" -H "Content-Type: application/json" -d "[{\"status\":\"Running\"},{\"status\":\"Submitted\"}]" | jq
echo "### Hello world! :rocket:" >> $GITHUB_STEP_SUMMARY
echo "### CRUN = $CRUN" >> $GITHUB_STEP_SUMMARY
echo "### CSTATUS = $CSTATUS" >> $GITHUB_STEP_SUMMARY
kill $pid
#doesn't work because we need to pass env vars across steps, which we'll do in the non-prototype version of this
# - name: 'Write the Summary'
# run: |
# echo "### Hello world! :rocket:" >> $GITHUB_STEP_SUMMARY
# echo "### CSTATUS = $CSTATUS" >> $GITHUB_STEP_SUMMARY