[SYNPY-1305] Collect trace data from integration tests (#1009)

* Collect trace data from integration tests

.github/workflows/build.yml

@@ -95,6 +95,53 @@ jobs:
         shell: bash
         run: |
           pytest -sv tests/unit
+      - name: Check for Secret availability
+        id: secret-check
+        if: ${{ contains(fromJSON('["3.9"]'), matrix.python) }}
+        # perform secret check & put boolean result as an output
+        shell: bash
+        run: |
+          if [ -z "${{ secrets.encrypted_d17283647768_key }}" ]  || [ -z "${{ secrets.encrypted_d17283647768_iv }}" ]; then
+            echo "secrets_available=false" >> $GITHUB_OUTPUT;
+          else
+            echo "secrets_available=true" >> $GITHUB_OUTPUT;
+          fi
+
+          if [ -z "${{ secrets.synapse_personal_access_token }}" ]; then
+            echo "synapse_pat_available=false" >> $GITHUB_OUTPUT;
+          else
+            echo "synapse_pat_available=true" >> $GITHUB_OUTPUT;
+          fi
+      - name: OpenTelemtry pre-check
+        id: otel-check
+        if: ${{ steps.secret-check.outputs.secrets_available == 'true' && steps.secret-check.outputs.synapse_pat_available == 'true' && !(startsWith(matrix.os, 'windows')) }}
+        shell: bash
+        run: |
+          echo "run_opentelemetry=true" >> $GITHUB_OUTPUT;
+
+          # AWS CLI is pre-installed on github hosted runners - Commented out for GH runs
+            # curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+            # unzip awscliv2.zip
+            # sudo ./aws/install
+            # curl "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/ubuntu_64bit/session-manager-plugin.deb" -o "session-manager-plugin.deb"
+            # sudo dpkg -i session-manager-plugin.deb
+      - name: Create AWS Config
+        if: ${{ steps.otel-check.outputs.run_opentelemetry == 'true' }}
+        shell: bash
+        run: |
+          touch test.awsConfig
+          printf "[default]\nregion = us-east-1\ncredential_process = \"tests/integration/synapse_creds.sh\" \"https://sc.sageit.org\" \"${{ secrets.synapse_personal_access_token }}\"\n" >> test.awsConfig
+          chmod +x tests/integration/synapse_creds.sh
+      - name: SSM Port Forward Start
+        if: ${{ steps.otel-check.outputs.run_opentelemetry == 'true' }}
+        shell: bash
+        env:
+          AWS_CONFIG_FILE: "test.awsConfig"
+        run: |
+          # Start a port-forwarding session in a non-interactive way. AWS will clean-up
+          # stale sessions after 20 minutes of inactivity
+          aws ssm start-session --target i-0ffcdecd1edf375ee --document-name AWS-StartPortForwardingSession  --parameters "portNumber"=["4318"],"localPortNumber"=["4318"] & disown
+          sleep 15
 
       # run integration tests iff the decryption keys for the test configuration are available.
       # they will not be available in pull requests from forks.
@@ -105,44 +152,43 @@ jobs:
         shell: bash
 
         # keep versions consistent with the first and last from the strategy matrix
-        if: ${{ contains(fromJSON('["3.9"]'), matrix.python) }}
+        if: ${{ contains(fromJSON('["3.9"]'), matrix.python) && steps.secret-check.outputs.secrets_available == 'true'}}
         run: |
-          if [ -z "${{ secrets.encrypted_d17283647768_key }}" ]  || [ -z "${{ secrets.encrypted_d17283647768_key }}" ]; then
-            echo "No test configuration decryption keys available, skipping integration tests"
+          # decrypt the encrypted test synapse configuration
+          openssl aes-256-cbc -K ${{ secrets.encrypted_d17283647768_key }} -iv ${{ secrets.encrypted_d17283647768_iv }} -in test.synapseConfig.enc -out test.synapseConfig -d
+          mv test.synapseConfig ~/.synapseConfig
 
-          else
+          if [ "${{ startsWith(matrix.os, 'ubuntu') }}" == "true" ]; then
+            # on linux only we can build and run a docker container to serve as an SFTP host for our SFTP tests.
+            # Docker is not available on GH Action runners on Mac and Windows.
 
-            # decrypt the encrypted test synapse configuration
-            openssl aes-256-cbc -K ${{ secrets.encrypted_d17283647768_key }} -iv ${{ secrets.encrypted_d17283647768_iv }} -in test.synapseConfig.enc -out test.synapseConfig -d
-            mv test.synapseConfig ~/.synapseConfig
+            docker build -t sftp_tests - < tests/integration/synapseclient/core/upload/Dockerfile_sftp
+            docker run -d sftp_tests:latest
 
-            if [ "${{ startsWith(matrix.os, 'ubuntu') }}" == "true" ]; then
-              # on linux only we can build and run a docker container to serve as an SFTP host for our SFTP tests.
-              # Docker is not available on GH Action runners on Mac and Windows.
+            # get the internal IP address of the just launched container
+            export SFTP_HOST=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $(docker ps -q))
 
-              docker build -t sftp_tests - < tests/integration/synapseclient/core/upload/Dockerfile_sftp
-              docker run -d sftp_tests:latest
+            printf "[sftp://$SFTP_HOST]\nusername: test\npassword: test\n" >> ~/.synapseConfig
 
-              # get the internal IP address of the just launched container
-              export SFTP_HOST=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $(docker ps -q))
+            # add to known_hosts so the ssh connections can be made without any prompting/errors
+            mkdir -p ~/.ssh
+            ssh-keyscan -H $SFTP_HOST >> ~/.ssh/known_hosts
+          fi
 
-              printf "[sftp://$SFTP_HOST]\nusername: test\npassword: test\n" >> ~/.synapseConfig
+            # set env vars used in external bucket tests from secrets
+          export EXTERNAL_S3_BUCKET_NAME="${{secrets.EXTERNAL_S3_BUCKET_NAME}}"
+          export EXTERNAL_S3_BUCKET_AWS_ACCESS_KEY_ID="${{secrets.EXTERNAL_S3_BUCKET_AWS_ACCESS_KEY_ID}}"
+          export EXTERNAL_S3_BUCKET_AWS_SECRET_ACCESS_KEY="${{secrets.EXTERNAL_S3_BUCKET_AWS_SECRET_ACCESS_KEY}}"
+          if [ ${{ steps.otel-check.outputs.run_opentelemetry }} == "true" ]; then
+            export SYNAPSE_OTEL_INTEGRATION_TEST_EXPORTER="otlp"
+          fi
 
-              # add to known_hosts so the ssh connections can be made without any prompting/errors
-              mkdir -p ~/.ssh
-              ssh-keyscan -H $SFTP_HOST >> ~/.ssh/known_hosts
-            fi
+          # use loadscope to avoid issues running tests concurrently that share scoped fixtures
+          pytest -sv tests/integration -n auto --ignore=tests/integration/synapseclient/test_command_line_client.py --dist loadscope
 
-             # set env vars used in external bucket tests from secrets
-            export EXTERNAL_S3_BUCKET_NAME="${{secrets.EXTERNAL_S3_BUCKET_NAME}}"
-            export EXTERNAL_S3_BUCKET_AWS_ACCESS_KEY_ID="${{secrets.EXTERNAL_S3_BUCKET_AWS_ACCESS_KEY_ID}}"
-            export EXTERNAL_S3_BUCKET_AWS_SECRET_ACCESS_KEY="${{secrets.EXTERNAL_S3_BUCKET_AWS_SECRET_ACCESS_KEY}}"
+          # Execute the CLI tests in a non-dist way because they were causing some test instability when being run concurrently
+          pytest -sv tests/integration/synapseclient/test_command_line_client.py
 
-             # use loadscope to avoid issues running tests concurrently that share scoped fixtures
-            pytest -sv tests/integration -n auto --ignore=tests/integration/synapseclient/test_command_line_client.py --dist loadscope
-            # Execute the CLI tests in a non-dist way because they were causing some test instability when being run concurrently
-            pytest -sv tests/integration/synapseclient/test_command_line_client.py
-          fi
 
   # on a GitHub release, build the pip package and upload it as a GitHub release asset
   package:

CONTRIBUTING.md

@@ -185,7 +185,10 @@ fileHandleEndpoint=https://repo-dev.dev.sagebase.org/file/v1
 ```
 
 #### Running OpenTelemetry in Integration Tests
-`tests/integration/conftest.py` is where we defining which trace provider to use. Set the `SYNAPSE_OTEL_INTEGRATION_TEST_PROVIDER` environment variable to `otlp` or `console` depending on your use case.
+`tests/integration/conftest.py` is where we defining which trace exporter to use. Set the `SYNAPSE_OTEL_INTEGRATION_TEST_EXPORTER` environment variable to `otlp` or `console` depending on your use case.
+
+When integration tests are ran in the Github CI/CD pipeline it will upload the trace data automatically using OLTP.
+
 
 #### Integration testing for external collaborators
 As an external collaborator you will not have access to a development account and environment to run the integration tests against. Either request that a Sage Bionetworks staff member run your integration tests via a pull request, or, contact us via the [Service Desk](https://sagebionetworks.jira.com/servicedesk/customer/portal/9) to requisition a development account for integration testing only.

tests/integration/conftest.py

@@ -1,4 +1,5 @@
 import logging
+import platform
 import uuid
 import os
 import sys
@@ -14,7 +15,7 @@
 from opentelemetry import trace
 from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter
-from opentelemetry.sdk.resources import SERVICE_NAME, Resource
+from opentelemetry.sdk.resources import SERVICE_NAME, OS_TYPE, OS_DESCRIPTION, Resource
 from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
 from opentelemetry.sdk.trace.sampling import ALWAYS_OFF
 
@@ -27,7 +28,7 @@
 
 @pytest.fixture(scope="session")
 @tracer.start_as_current_span("conftest::syn")
-def syn():
+def syn() -> Synapse:
     """
     Create a logged in Synapse instance that can be shared by all tests in the session.
     If xdist is being used a syn is created for each worker node.
@@ -48,7 +49,7 @@ def syn():
 
 @pytest.fixture(scope="session")
 @tracer.start_as_current_span("conftest::project")
-def project(request, syn):
+def project(request, syn: Synapse) -> Project:
     """
     Create a project to be shared by all tests in the session. If xdist is being used
     a project is created for each worker node.
@@ -72,7 +73,7 @@ def project_teardown():
 
 
 @pytest.fixture(scope="module")
-def schedule_for_cleanup(request, syn):
+def schedule_for_cleanup(request, syn: Synapse):
     """Returns a closure that takes an item that should be scheduled for cleanup.
     The cleanup will occur after the module tests finish to limit the residue left behind
     if a test session should be prematurely aborted for any reason."""
@@ -91,7 +92,7 @@ def cleanup_scheduled_items():
 
 
 @tracer.start_as_current_span("conftest::_cleanup")
-def _cleanup(syn, items):
+def _cleanup(syn: Synapse, items):
     """cleanup junk created during testing"""
     for item in reversed(items):
         if (
@@ -119,20 +120,34 @@ def _cleanup(syn, items):
             sys.stderr.write("Don't know how to clean: %s" % str(item))
 
 
-provider_type = os.environ.get("SYNAPSE_OTEL_INTEGRATION_TEST_PROVIDER", None)
-if provider_type:
-    trace.set_tracer_provider(
-        TracerProvider(
-            resource=Resource(attributes={SERVICE_NAME: "syn_int_tests"}),
-        )
-    )
-    if provider_type == "otlp":
-        trace.get_tracer_provider().add_span_processor(
-            BatchSpanProcessor(OTLPSpanExporter())
-        )
-    elif provider_type == "console":
-        trace.get_tracer_provider().add_span_processor(
-            BatchSpanProcessor(ConsoleSpanExporter())
+@pytest.fixture(scope="session", autouse=True)
+def setup_otel():
+    """
+    Handles setting up the OpenTelemetry tracer provider for integration tests.
+    Depending on the environment variables set, the provider will be configured
+    to export to the console, a file, or to an OTLP endpoint.
+    """
+    # Setup
+    exporter_type = os.environ.get("SYNAPSE_OTEL_INTEGRATION_TEST_EXPORTER", None)
+    if exporter_type:
+        trace.set_tracer_provider(
+            TracerProvider(
+                resource=Resource(
+                    attributes={
+                        SERVICE_NAME: "syn_int_tests",
+                        OS_DESCRIPTION: platform.release(),
+                        OS_TYPE: platform.system(),
+                    }
+                ),
+            )
         )
-else:
-    trace.set_tracer_provider(TracerProvider(sampler=ALWAYS_OFF))
+        if exporter_type == "otlp":
+            trace.get_tracer_provider().add_span_processor(
+                BatchSpanProcessor(OTLPSpanExporter())
+            )
+        elif exporter_type == "console":
+            trace.get_tracer_provider().add_span_processor(
+                BatchSpanProcessor(ConsoleSpanExporter())
+            )
+    else:
+        trace.set_tracer_provider(TracerProvider(sampler=ALWAYS_OFF))

tests/integration/synapse_creds.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+# Inputs
+SC_ENDPOINT=$1   # i.e. https://sc.sageit.org
+SYNAPSE_PAT=$2   # The Synapse Personal Access Token
+
+# Endpoints
+STS_TOKEN_ENDPOINT="${SC_ENDPOINT}/ststoken"
+
+# Get Credentials
+AWS_STS_CREDS=$(curl --location-trusted --silent -H "Authorization:Bearer ${SYNAPSE_PAT}"  ${STS_TOKEN_ENDPOINT})
+
+echo ${AWS_STS_CREDS}