QA-2077: Proposed changes to Rawls e2e Azure tests using new infrastr…

…ucture stack (#2444) * AZ ece test * try ece-enabled workflow * Change to dev * Disable slack * revert to qa * Parameterize AZ coordinates * Fix job ref * Update job name * Refactored code make it cleaner * Refactored e2e workflow * Fix typo * Refactored users to use a map * Updated e2e flag to a string type * Pass in owner, no owner emails * Mock auth token * Use Mock GoogleCredential * Fix reference to billing project name * Refactor code * Refactored case class * Refactored fake google credential class * Updated delete billing project workflow path * Refactored code * Fix argument * Clean up * Clean up * Use v2 for delete billing project workflow * rename delete billing job to use v2 * keep BEE for debugging * use full path of workflow dispatch * Disable delete billing project for debugging * Updated comments. * Added a step to configure user subjects for testing * Echo test users * Updated comment. * Try default users * Break inputs to multiline for better readability * Use configured user subjects * Remove toJson * Remove single quote * Add back single quote * Fix syntax * Encode json string with toJson * Use toJson encode * Try multiple users * Restore delete BEE * Try capture an envvar * Rename variables * Parse envvar * Add circe generic dependency. * Test json env input * Get users metadata from json string * Refactored code to use envvars * Update user type to regular * Add logger * Clean up * Add B64 decode * Try B64 decode * Refactored code to use b64 * Clean up * Clean up * Refactored code * Refactor and clean up * Tidy up comment. Avoid leak of secrets * Clean up. * Refactored with dynamic envvar * Clean up comments * Refactor input user shape * Allow multiple users to be defined in input user shape * Try escape user input * Escape user input * Try another escape * No escape * No escape * Use input json no escape * Escape once * Use escape once * Fix missing } * Pass array on owners * Clean up * More refactoring * Clean up * Check boolean bee delete flag * Remove always() condition * Added script and template for running e2e tests locally. Useful for debugging. * Clean up * try removing pattern private_key * Add script to ggitallowed * Check delete-bee flag * Add always() * Add support for both scheduled and manual workflows. * use toJson to encode string * Use backslash escape * Use single and double quotes * Fix syntax * Use toJson * Make inputs optional for testing * Update student * Simulate schedule trigger * Updated comments * Updated student * Removed requirements to specify Azure IDs. * Reduce number of inputs * Add run-name * Updated run names * Refactored code * Add quotes * Don't wait-for-complete for bee-destroy * Include AZ coordinates in env * Updated comments * Updated comments * Update script * Revert to failure * Removed blank lines * Try without run-name * Put back run-name * Rename workflow * Update run-name * Open up delete billing project * Removed comments * Removed comments * Add silent-on-failure flag * Updated class name * Updated help * Simplified setup script * Updated comment * Change to main * Remove echo statement
broadinstitute · Sep 6, 2023 · 8a9fd0e · 8a9fd0e
1 parent 9eea4c8
commit 8a9fd0e
Show file tree

Hide file tree

Showing 10 changed files with 851 additions and 182 deletions.
diff --git a/.gitallowed b/.gitallowed
@@ -1,2 +1,3 @@
 core/src/test/scala/org/broadinstitute/dsde/rawls/model/ExecutionModelSpec.scala
 core/src/test/scala/org/broadinstitute/dsde/rawls/fastpass/MockFastPassService.scala
+automation/setup-local-env-for-e2e.sh
diff --git a/.github/workflows/rawls-run-azure-e2e-tests.yaml b/.github/workflows/rawls-run-azure-e2e-tests.yaml
@@ -1,5 +1,17 @@
 name: rawls-run-azure-e2e-tests
 
+# This workflow can be run on schedule OR dispatched (manually or by another dispatcher workflow)
+# When thw workflow is triggered on schedule, it will use the parameters specified on
+# the RHS of || in init-github-context.extract-inputs step. Otherwise, if the workflow is
+# triggered by a dispatch event, it will use the parameters specified on LHS of || in
+# the init-github-context.extract-inputs step.
+#
+# Ex.
+#
+# echo "student-subjects=${{ toJson(inputs.student-subjects || '["[email protected]","[email protected]"]') }}" >> "$GITHUB_OUTPUT"
+#
+# On schedule: student-subjects output will use '["[email protected]","[email protected]"]'
+# Dispatch event: student-subjects output will use inputs.student-subjects
 on:
   schedule:
     # run twice a day at 10:00 and 22:00 UTC every day of the week
@@ -16,23 +28,47 @@ on:
         required: true
         default: true
         type: boolean
+      owner-subject:
+        description: 'Owner subject (used for creating billing project in E2E testing)'
+        required: true
+        default: '[email protected]'
+        type: string
+      student-subjects:
+        description: 'A JSON array of Student subjects used for E2E testing'
+        required: true
+        default: '["[email protected]","[email protected]"]'
+        type: string
+      service-account:
+        description: 'Email address or unique identifier of the Google Cloud service account for which to generate credentials'
+        required: true
+        default: '[email protected]'
+        type: string
 
+# E2E_ENV is the name of a .env file that contains envvars for E2E tests
 env:
-  BEE_NAME: '${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt}}-dev'
+  RUN_NAME_SUFFIX: '${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt }}'
+  BEE_NAME: '${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt }}-dev'
   TOKEN: '${{ secrets.BROADBOT_TOKEN }}' # github token for access to kick off a job in the private repo
+  E2E_ENV: 'azure_e2e.env'
 
 jobs:
   init-github-context:
     runs-on: ubuntu-latest
     outputs:
       branch: ${{ steps.extract-inputs.outputs.branch }}
       delete-bee: ${{ steps.extract-inputs.outputs.delete-bee }}
+      student-subjects: ${{ steps.extract-inputs.outputs.student-subjects }}
+      owner-subject: ${{ steps.extract-inputs.outputs.owner-subject }}
+      service-account: ${{ steps.extract-inputs.outputs.service-account }}
     steps:
       - name: Get inputs or use defaults
         id: extract-inputs
         run: |
-          echo "branch=${{ inputs.branch || 'develop' }}" >> "$GITHUB_OUTPUT"
-          echo "delete-bee=${{ inputs.delete-bee || false }}" >> "$GITHUB_OUTPUT"
+          echo "branch=${{ inputs.branch                            || 'develop' }}" >> "$GITHUB_OUTPUT"
+          echo "delete-bee=${{ inputs.delete-bee                    || false }}" >> "$GITHUB_OUTPUT"
+          echo "owner-subject=${{ inputs.owner-subject              || '[email protected]' }}" >> "$GITHUB_OUTPUT"
+          echo "student-subjects=${{ toJson(inputs.student-subjects || '["[email protected]","[email protected]"]') }}" >> "$GITHUB_OUTPUT"
+          echo "service-account=${{ inputs.service-account          || '[email protected]' }}" >> "$GITHUB_OUTPUT"
 
   rawls-build-tag-publish-job:
     runs-on: ubuntu-latest
@@ -63,7 +99,11 @@ jobs:
           repo: broadinstitute/terra-github-workflows
           ref: refs/heads/main
           token: ${{ env.TOKEN }}
-          inputs: '{ "repository": "${{ github.event.repository.full_name }}", "ref": "refs/heads/${{ needs.init-github-context.outputs.branch }}", "rawls-release-tag": "${{ steps.tag.outputs.tag }}" }'
+          inputs: '{
+            "repository": "${{ github.event.repository.full_name }}",
+            "ref": "refs/heads/${{ needs.init-github-context.outputs.branch }}",
+            "rawls-release-tag": "${{ steps.tag.outputs.tag }}"
+          }'
 
       - name: Render Rawls version
         id: render-rawls-version
@@ -91,15 +131,57 @@ jobs:
           repo: broadinstitute/terra-github-workflows
           ref: refs/heads/main
           token: ${{ env.TOKEN }}
-          inputs: '{ "bee-name": "${{ env.BEE_NAME }}", "bee-template-name": "rawls-e2e-azure-tests", "version-template": "dev", "custom-version-json": "${{ needs.rawls-build-tag-publish-job.outputs.custom-version-json }}" }'
+          inputs: '{
+            "bee-name": "${{ env.BEE_NAME }}",
+            "bee-template-name": "rawls-e2e-azure-tests",
+            "version-template": "dev",
+            "custom-version-json": "${{ needs.rawls-build-tag-publish-job.outputs.custom-version-json }}"
+          }'
+
+  # This job can be used for generating parameters for E2E tests (e.g. a random project name).
+  params-gen:
+    runs-on: ubuntu-latest
+    outputs:
+      project-name: ${{ steps.gen.outputs.project_name }}
+    steps:
+      - uses: 'actions/checkout@v3'
+
+      - name: Generate a random billing project name
+        id: 'gen'
+        run: |
+          project_name=$(echo "tmp-billing-project-$(uuidgen)" | cut -c -30)
+          echo "project_name=${project_name}" >> $GITHUB_OUTPUT
+
+  # Azure Managed App Coordinates are defined in the following workflow:
+  #   https://github.com/broadinstitute/terra-github-workflows/blob/main/.github/workflows/attach-landing-zone-to-bee.yaml
+  attach-billing-project-to-landing-zone-workflow:
+    runs-on: ubuntu-latest
+    needs: [init-github-context, create-bee-workflow, params-gen]
+    steps:
+      - name: dispatch to terra-github-workflows
+        uses: broadinstitute/workflow-dispatch@v3
+        with:
+          workflow: attach-billing-project-to-landing-zone.yaml
+          repo: broadinstitute/terra-github-workflows
+          ref: refs/heads/main
+          token: ${{ env.TOKEN }}
+          inputs: '{
+            "run-name": "attach-billing-project-to-landing-zone-${{ env.RUN_NAME_SUFFIX }}",
+            "bee-name": "${{ env.BEE_NAME }}",
+            "billing-project": "${{ needs.params-gen.outputs.project-name }}",
+            "billing-project-creator": "${{ needs.init-github-context.outputs.owner-subject }}",
+            "service-account": "${{ needs.init-github-context.outputs.service-account }}"
+          }'
 
   rawls-swat-e2e-test-job:
     runs-on: ubuntu-latest
-    needs: [create-bee-workflow, init-github-context]
-    permissions:
-      contents: 'read'
-      id-token: 'write'
+    needs: [init-github-context, create-bee-workflow, params-gen, attach-billing-project-to-landing-zone-workflow]
     steps:
+      - name: Configure the user subjects for the test
+        run: |
+          escapedJSON=$(echo '${{ needs.init-github-context.outputs.student-subjects }}' | sed 's/"/\"/g')
+          echo "USER_SUBJECTS={\"service_account\":\"${{ needs.init-github-context.outputs.service-account }}\", \"owners\": [\"${{ needs.init-github-context.outputs.owner-subject }}\"], \"students\": $escapedJSON}" >> $GITHUB_ENV
+
       - name: dispatch to terra-github-workflows
         env:
           rawls_test_command: "testOnly -- -l ProdTest -l NotebooksCanaryTest -n org.broadinstitute.dsde.test.api.WorkspacesAzureTest"
@@ -109,15 +191,44 @@ jobs:
           repo: broadinstitute/terra-github-workflows
           ref: refs/heads/main
           token: ${{ env.TOKEN }}
-          inputs: '{ "bee-name": "${{ env.BEE_NAME }}", "ENV": "qa", "ref": "refs/heads/${{ needs.init-github-context.outputs.branch }}", "test-group-name": "workspaces_azure", "test-command": "${{ env.rawls_test_command }}", "java-version": "17" }'
+          inputs: '{
+            "run-name": "rawls-swat-tests-${{ env.RUN_NAME_SUFFIX }}",
+            "bee-name": "${{ env.BEE_NAME }}",
+            "ENV": "qa",
+            "ref": "refs/heads/${{ needs.init-github-context.outputs.branch }}",
+            "test-group-name": "workspaces_azure",
+            "test-command": "${{ env.rawls_test_command }}",
+            "java-version": "17",
+            "billing-project": "${{ needs.params-gen.outputs.project-name }}",
+            "e2e-env": "${{ env.E2E_ENV }}",
+            "user-subjects": ${{ toJson(env.USER_SUBJECTS) }}
+          }'
+
+  delete-billing-project-v2-from-bee-workflow:
+    runs-on: ubuntu-latest
+    needs: [init-github-context, params-gen, rawls-swat-e2e-test-job]
+    if: always()
+    steps:
+      - name: dispatch to terra-github-workflows
+        uses: broadinstitute/workflow-dispatch@v3
+        with:
+          workflow: .github/workflows/delete-billing-project-v2-from-bee.yaml
+          repo: broadinstitute/terra-github-workflows
+          ref: refs/heads/main
+          token: ${{ env.TOKEN }}
+          inputs: '{
+            "run-name": "delete-billing-project-v2-from-bee-${{ env.RUN_NAME_SUFFIX }}",
+            "bee-name": "${{ env.BEE_NAME }}",
+            "billing-project": "${{ needs.params-gen.outputs.project-name }}",
+            "billing-project-owner": "${{ needs.init-github-context.outputs.owner-subject }}",
+            "service-account": "${{ needs.init-github-context.outputs.service-account }}",
+            "silent-on-failure": "false"
+          }'
 
   destroy-bee-workflow:
     runs-on: ubuntu-latest
-    needs: [rawls-swat-e2e-test-job, init-github-context]
+    needs: [init-github-context, rawls-swat-e2e-test-job, delete-billing-project-v2-from-bee-workflow]
     if: ${{ needs.init-github-context.outputs.delete-bee && always() }} # always run to confirm bee is destroyed unless explicitly requested not to
-    permissions:
-      contents: 'read'
-      id-token: 'write'
     steps:
       - name: dispatch to terra-github-workflows
         uses: broadinstitute/workflow-dispatch@v3
@@ -127,6 +238,7 @@ jobs:
           ref: refs/heads/main
           token: ${{ env.TOKEN }}
           inputs: '{ "bee-name": "${{ env.BEE_NAME }}" }'
+          wait-for-completion: false
 
   notify-slack-on-failure:
     runs-on: ubuntu-latest

diff --git a/automation/application.bee.conf.template b/automation/application.bee.conf.template
@@ -0,0 +1,62 @@
+# To run against a local UI, set baseUrl = "http://local.broadinstitute.org/"
+fireCloud {
+  baseUrl = "https://firecloudui.${BEE_ENV}.bee.envs-terra.bio/"
+
+
+  orchApiUrl = "https://firecloudorch.${BEE_ENV}.bee.envs-terra.bio/"
+  rawlsApiUrl = "https://rawls.${BEE_ENV}.bee.envs-terra.bio/"
+  samApiUrl = "https://sam.${BEE_ENV}.bee.envs-terra.bio/"
+  thurloeApiUrl = "https://thurloe.${BEE_ENV}.bee.envs-terra.bio/"
+  workspaceManagerApiUrl = "https://workspace.${BEE_ENV}.bee.envs-terra.bio/"
+
+
+  fireCloudId = "${FC_ID}"
+  tcgaAuthDomain = "TCGA-dbGaP-Authorized"
+
+
+  # fiab integration tests should use TDR alpha environment
+  dataRepoApiUrl = "https://data.alpha.bee.envs-terra.bio/"
+
+
+
+  gpAllocApiUrl = "https://gpalloc-qa.dsp-techops.broadinstitute.org/api/"
+
+  waitForAccessDuration=3m
+}
+
+gcs {
+  appsDomain = "quality.firecloud.org"
+  qaEmail = "${QA_EMAIL}"
+  serviceProject = "broad-dsde-qa"
+  smoketestsProject = "broad-dsde-qa"
+  qaPemFile = "${SCRIPT_DIR}/src/test/resources/firecloud-account.pem"
+  qaJsonFile = "${SCRIPT_DIR}/src/test/resources/firecloud-account.json"
+  trialBillingPemFile = "${SCRIPT_DIR}/src/test/resources/trial-billing-account.pem"
+  trialBillingPemFileClientId = "${TRIAL_BILLING_CLIENT_ID}"
+  orchStorageSigningSA = "${ORCH_STORAGE_SIGNING_SA}"
+  billingAccount = "Broad Institute - 8201528"
+  billingAccountId = "${BILLING_ACCOUNT_ID}"
+  subEmail = "[email protected]"
+  googleAccessPolicy = "891321614892"
+}
+
+users {
+  notSoSecretPassword = "${AUTO_USERS_PASSWD}"
+
+  userDataPath = "${SCRIPT_DIR}/src/test/resources/users.json"
+  # for smoketests
+  smoketestpassword = "${USERS_PASSWD}"
+  smoketestuser = "[email protected]"
+}
+
+methods {
+  testMethod = "DO_NOT_CHANGE_test_method"
+  testMethodConfig = "DO_NOT_CHANGE_test1_config"
+  methodConfigNamespace = "automationmethods"
+  snapshotID = 1
+}
+
+chromeSettings {
+  chromedriverHost = "http://hub:4444/wd/hub"
+  chromedriverPath = "/usr/local/bin/chromedriver"
+}
diff --git a/automation/project/Dependencies.scala b/automation/project/Dependencies.scala
@@ -55,6 +55,7 @@ object Dependencies {
     "com.typesafe.akka"   %%  "akka-slf4j"          % akkaV,
     "org.specs2"          %%  "specs2-core"   % "4.15.0"  % Test,
     "org.scalatest"       %%  "scalatest"     % "3.2.2"   % Test,
+    "io.circe"            %%  "circe-generic" % "0.14.2"  % Test,
     "org.seleniumhq.selenium" % "selenium-java" % "3.8.1" % Test,
     "com.typesafe.scala-logging" %% "scala-logging" % "3.9.5",
     "org.broadinstitute.dsde"       %% "rawls-model"         % "0.1-3891e8393"