Try port-forward session

Sage-Bionetworks · Nov 6, 2023 · ba87896 · ba87896
1 parent 1be0aff
commit ba87896
Show file tree

Hide file tree

Showing 4 changed files with 69 additions and 56 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -42,10 +42,10 @@ jobs:
 
     strategy:
       matrix:
-        os: [ubuntu-20.04, macos-11, windows-2019]
+        os: [ubuntu-20.04]
 
         # if changing the below change the run-integration-tests versions and the check-deploy versions
-        python: [3.8, '3.9', '3.10', '3.11']
+        python: ['3.9']
 
     runs-on: ${{ matrix.os }}
 
@@ -95,6 +95,34 @@ jobs:
         shell: bash
         run: |
           pytest -sv tests/unit
+      - name: Check for Secret availability
+        id: secret-check
+        if: ${{ contains(fromJSON('["3.9"]'), matrix.python) }}
+        # perform secret check & put boolean result as an output
+        shell: bash
+        run: |
+          if [ -z "${{ secrets.encrypted_d17283647768_key }}" ]  || [ -z "${{ secrets.encrypted_d17283647768_iv }}" ]; then
+            echo "secrets_available=false" >> $GITHUB_OUTPUT;
+          else
+            echo "secrets_available=true" >> $GITHUB_OUTPUT;
+          fi
+      - name: Decrypt AWS Config
+        if: ${{ steps.secret-check.outputs.secrets_available == 'true' }}
+        # perform secret check & put boolean result as an output
+        shell: bash
+        run: |
+          openssl aes-256-cbc -K ${{ secrets.encrypted_d17283647768_key }} -iv ${{ secrets.encrypted_d17283647768_iv }} -in test.awsConfig.enc -out test.awsConfig -d
+          chmod +x tests/integration/synapse_creds.sh
+      - name: SSM Port Forward
+        if: ${{ steps.secret-check.outputs.secrets_available == 'true' }}
+        env:
+          AWS_REGION: "us-east-1"
+          AWS_CONFIG_FILE: "test.awsConfig"
+        uses: gian2dchris/[email protected]
+        with:
+          target-id: 'i-0ffcdecd1edf375ee'
+          portNumber: 4318
+          localPortNumber:  4318
 
       # run integration tests iff the decryption keys for the test configuration are available.
       # they will not be available in pull requests from forks.
@@ -105,53 +133,41 @@ jobs:
         shell: bash
 
         # keep versions consistent with the first and last from the strategy matrix
-        if: ${{ contains(fromJSON('["3.9"]'), matrix.python) }}
+        if: ${{ contains(fromJSON('["3.9"]'), matrix.python) && steps.secret-check.outputs.secrets_available == 'true'}}
         run: |
-          if [ -z "${{ secrets.encrypted_d17283647768_key }}" ]  || [ -z "${{ secrets.encrypted_d17283647768_key }}" ]; then
-            echo "No test configuration decryption keys available, skipping integration tests"
+          # decrypt the encrypted test synapse configuration
+          openssl aes-256-cbc -K ${{ secrets.encrypted_d17283647768_key }} -iv ${{ secrets.encrypted_d17283647768_iv }} -in test.synapseConfig.enc -out test.synapseConfig -d
+          mv test.synapseConfig ~/.synapseConfig
 
-          else
+          if [ "${{ startsWith(matrix.os, 'ubuntu') }}" == "true" ]; then
+            # on linux only we can build and run a docker container to serve as an SFTP host for our SFTP tests.
+            # Docker is not available on GH Action runners on Mac and Windows.
 
-            # decrypt the encrypted test synapse configuration
-            openssl aes-256-cbc -K ${{ secrets.encrypted_d17283647768_key }} -iv ${{ secrets.encrypted_d17283647768_iv }} -in test.synapseConfig.enc -out test.synapseConfig -d
-            mv test.synapseConfig ~/.synapseConfig
+            docker build -t sftp_tests - < tests/integration/synapseclient/core/upload/Dockerfile_sftp
+            docker run -d sftp_tests:latest
 
-            if [ "${{ startsWith(matrix.os, 'ubuntu') }}" == "true" ]; then
-              # on linux only we can build and run a docker container to serve as an SFTP host for our SFTP tests.
-              # Docker is not available on GH Action runners on Mac and Windows.
+            # get the internal IP address of the just launched container
+            export SFTP_HOST=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $(docker ps -q))
 
-              docker build -t sftp_tests - < tests/integration/synapseclient/core/upload/Dockerfile_sftp
-              docker run -d sftp_tests:latest
+            printf "[sftp://$SFTP_HOST]\nusername: test\npassword: test\n" >> ~/.synapseConfig
 
-              # get the internal IP address of the just launched container
-              export SFTP_HOST=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $(docker ps -q))
+            # add to known_hosts so the ssh connections can be made without any prompting/errors
+            mkdir -p ~/.ssh
+            ssh-keyscan -H $SFTP_HOST >> ~/.ssh/known_hosts
+          fi
 
-              printf "[sftp://$SFTP_HOST]\nusername: test\npassword: test\n" >> ~/.synapseConfig
+            # set env vars used in external bucket tests from secrets
+          export EXTERNAL_S3_BUCKET_NAME="${{secrets.EXTERNAL_S3_BUCKET_NAME}}"
+          export EXTERNAL_S3_BUCKET_AWS_ACCESS_KEY_ID="${{secrets.EXTERNAL_S3_BUCKET_AWS_ACCESS_KEY_ID}}"
+          export EXTERNAL_S3_BUCKET_AWS_SECRET_ACCESS_KEY="${{secrets.EXTERNAL_S3_BUCKET_AWS_SECRET_ACCESS_KEY}}"
+          export SYNAPSE_OTEL_INTEGRATION_TEST_PROVIDER="otlp"
 
-              # add to known_hosts so the ssh connections can be made without any prompting/errors
-              mkdir -p ~/.ssh
-              ssh-keyscan -H $SFTP_HOST >> ~/.ssh/known_hosts
-            fi
+            # use loadscope to avoid issues running tests concurrently that share scoped fixtures
+          pytest -sv tests/integration -n auto --ignore=tests/integration/synapseclient/test_command_line_client.py --dist loadscope
 
-             # set env vars used in external bucket tests from secrets
-            export EXTERNAL_S3_BUCKET_NAME="${{secrets.EXTERNAL_S3_BUCKET_NAME}}"
-            export EXTERNAL_S3_BUCKET_AWS_ACCESS_KEY_ID="${{secrets.EXTERNAL_S3_BUCKET_AWS_ACCESS_KEY_ID}}"
-            export EXTERNAL_S3_BUCKET_AWS_SECRET_ACCESS_KEY="${{secrets.EXTERNAL_S3_BUCKET_AWS_SECRET_ACCESS_KEY}}"
-            export SYNAPSE_OTEL_INTEGRATION_TEST_PROVIDER="file"
+          # Execute the CLI tests in a non-dist way because they were causing some test instability when being run concurrently
+          pytest -sv tests/integration/synapseclient/test_command_line_client.py
 
-             # use loadscope to avoid issues running tests concurrently that share scoped fixtures
-            pytest -sv tests/integration -n auto --ignore=tests/integration/synapseclient/test_command_line_client.py --dist loadscope
-
-            export SYNAPSE_OTEL_INTEGRATION_TEST_FILE_NAME="cli_traces.txt"
-            # Execute the CLI tests in a non-dist way because they were causing some test instability when being run concurrently
-            pytest -sv tests/integration/synapseclient/test_command_line_client.py
-          fi
-      - uses: actions/upload-artifact@v3
-        if: ${{ contains(fromJSON('["3.9"]'), matrix.python) }}
-        with:
-          name: opentelemetry_traces_${{ matrix.os }}
-          path: tests/integration/otel
-          retention-days: 7
 
   # on a GitHub release, build the pip package and upload it as a GitHub release asset
   package:

diff --git a/test.awsConfig.enc b/test.awsConfig.enc
diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
@@ -1,5 +1,6 @@
 import logging
 import platform
+import threading
 import uuid
 import os
 import sys
@@ -149,22 +150,5 @@ def setup_otel():
             trace.get_tracer_provider().add_span_processor(
                 BatchSpanProcessor(ConsoleSpanExporter())
             )
-        elif provider_type == "file":
-            file_name = os.environ.get(
-                "SYNAPSE_OTEL_INTEGRATION_TEST_FILE_NAME", "traces.txt"
-            )
-            path = f"tests/integration/otel/{file_name}"
-            utils.touch(path)
-            export_file = open(path, "w", encoding="utf-8")
-            span_processor = BatchSpanProcessor(ConsoleSpanExporter(out=export_file))
-            trace.get_tracer_provider().add_span_processor(span_processor)
     else:
         trace.set_tracer_provider(TracerProvider(sampler=ALWAYS_OFF))
-
-    yield
-
-    # Teardown
-    if export_file:
-        span_processor.force_flush()
-        span_processor.shutdown()
-        export_file.close()
diff --git a/tests/integration/synapse_creds.sh b/tests/integration/synapse_creds.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+# Inputs
+SC_ENDPOINT=$1   # i.e. https://sc.sageit.org
+SYNAPSE_PAT=$2   # The Synapse Personal Access Token
+
+# Endpoints
+STS_TOKEN_ENDPOINT="${SC_ENDPOINT}/ststoken"
+
+# Get Credentials
+AWS_STS_CREDS=$(curl --location-trusted --silent -H "Authorization:Bearer ${SYNAPSE_PAT}"  ${STS_TOKEN_ENDPOINT})
+
+echo ${AWS_STS_CREDS}