diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a01ca61db..8ac00cbd6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -42,10 +42,10 @@ jobs: strategy: matrix: - os: [ubuntu-20.04, macos-11, windows-2019] + os: [ubuntu-20.04] # if changing the below change the run-integration-tests versions and the check-deploy versions - python: [3.8, '3.9', '3.10', '3.11'] + python: ['3.9'] runs-on: ${{ matrix.os }} @@ -95,6 +95,34 @@ jobs: shell: bash run: | pytest -sv tests/unit + - name: Check for Secret availability + id: secret-check + if: ${{ contains(fromJSON('["3.9"]'), matrix.python) }} + # perform secret check & put boolean result as an output + shell: bash + run: | + if [ -z "${{ secrets.encrypted_d17283647768_key }}" ] || [ -z "${{ secrets.encrypted_d17283647768_iv }}" ]; then + echo "secrets_available=false" >> $GITHUB_OUTPUT; + else + echo "secrets_available=true" >> $GITHUB_OUTPUT; + fi + - name: Decrypt AWS Config + if: ${{ steps.secret-check.outputs.secrets_available == 'true' }} + # perform secret check & put boolean result as an output + shell: bash + run: | + openssl aes-256-cbc -K ${{ secrets.encrypted_d17283647768_key }} -iv ${{ secrets.encrypted_d17283647768_iv }} -in test.awsConfig.enc -out test.awsConfig -d + chmod +x tests/integration/synapse_creds.sh + - name: SSM Port Forward + if: ${{ steps.secret-check.outputs.secrets_available == 'true' }} + env: + AWS_REGION: "us-east-1" + AWS_CONFIG_FILE: "test.awsConfig" + uses: gian2dchris/aws-ssm-port-forwarding-session-action@v1.0.0 + with: + target-id: 'i-0ffcdecd1edf375ee' + portNumber: 4318 + localPortNumber: 4318 # run integration tests iff the decryption keys for the test configuration are available. # they will not be available in pull requests from forks. @@ -105,53 +133,41 @@ jobs: shell: bash # keep versions consistent with the first and last from the strategy matrix - if: ${{ contains(fromJSON('["3.9"]'), matrix.python) }} + if: ${{ contains(fromJSON('["3.9"]'), matrix.python) && steps.secret-check.outputs.secrets_available == 'true'}} run: | - if [ -z "${{ secrets.encrypted_d17283647768_key }}" ] || [ -z "${{ secrets.encrypted_d17283647768_key }}" ]; then - echo "No test configuration decryption keys available, skipping integration tests" + # decrypt the encrypted test synapse configuration + openssl aes-256-cbc -K ${{ secrets.encrypted_d17283647768_key }} -iv ${{ secrets.encrypted_d17283647768_iv }} -in test.synapseConfig.enc -out test.synapseConfig -d + mv test.synapseConfig ~/.synapseConfig - else + if [ "${{ startsWith(matrix.os, 'ubuntu') }}" == "true" ]; then + # on linux only we can build and run a docker container to serve as an SFTP host for our SFTP tests. + # Docker is not available on GH Action runners on Mac and Windows. - # decrypt the encrypted test synapse configuration - openssl aes-256-cbc -K ${{ secrets.encrypted_d17283647768_key }} -iv ${{ secrets.encrypted_d17283647768_iv }} -in test.synapseConfig.enc -out test.synapseConfig -d - mv test.synapseConfig ~/.synapseConfig + docker build -t sftp_tests - < tests/integration/synapseclient/core/upload/Dockerfile_sftp + docker run -d sftp_tests:latest - if [ "${{ startsWith(matrix.os, 'ubuntu') }}" == "true" ]; then - # on linux only we can build and run a docker container to serve as an SFTP host for our SFTP tests. - # Docker is not available on GH Action runners on Mac and Windows. + # get the internal IP address of the just launched container + export SFTP_HOST=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $(docker ps -q)) - docker build -t sftp_tests - < tests/integration/synapseclient/core/upload/Dockerfile_sftp - docker run -d sftp_tests:latest + printf "[sftp://$SFTP_HOST]\nusername: test\npassword: test\n" >> ~/.synapseConfig - # get the internal IP address of the just launched container - export SFTP_HOST=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $(docker ps -q)) + # add to known_hosts so the ssh connections can be made without any prompting/errors + mkdir -p ~/.ssh + ssh-keyscan -H $SFTP_HOST >> ~/.ssh/known_hosts + fi - printf "[sftp://$SFTP_HOST]\nusername: test\npassword: test\n" >> ~/.synapseConfig + # set env vars used in external bucket tests from secrets + export EXTERNAL_S3_BUCKET_NAME="${{secrets.EXTERNAL_S3_BUCKET_NAME}}" + export EXTERNAL_S3_BUCKET_AWS_ACCESS_KEY_ID="${{secrets.EXTERNAL_S3_BUCKET_AWS_ACCESS_KEY_ID}}" + export EXTERNAL_S3_BUCKET_AWS_SECRET_ACCESS_KEY="${{secrets.EXTERNAL_S3_BUCKET_AWS_SECRET_ACCESS_KEY}}" + export SYNAPSE_OTEL_INTEGRATION_TEST_PROVIDER="otlp" - # add to known_hosts so the ssh connections can be made without any prompting/errors - mkdir -p ~/.ssh - ssh-keyscan -H $SFTP_HOST >> ~/.ssh/known_hosts - fi + # use loadscope to avoid issues running tests concurrently that share scoped fixtures + pytest -sv tests/integration -n auto --ignore=tests/integration/synapseclient/test_command_line_client.py --dist loadscope - # set env vars used in external bucket tests from secrets - export EXTERNAL_S3_BUCKET_NAME="${{secrets.EXTERNAL_S3_BUCKET_NAME}}" - export EXTERNAL_S3_BUCKET_AWS_ACCESS_KEY_ID="${{secrets.EXTERNAL_S3_BUCKET_AWS_ACCESS_KEY_ID}}" - export EXTERNAL_S3_BUCKET_AWS_SECRET_ACCESS_KEY="${{secrets.EXTERNAL_S3_BUCKET_AWS_SECRET_ACCESS_KEY}}" - export SYNAPSE_OTEL_INTEGRATION_TEST_PROVIDER="file" + # Execute the CLI tests in a non-dist way because they were causing some test instability when being run concurrently + pytest -sv tests/integration/synapseclient/test_command_line_client.py - # use loadscope to avoid issues running tests concurrently that share scoped fixtures - pytest -sv tests/integration -n auto --ignore=tests/integration/synapseclient/test_command_line_client.py --dist loadscope - - export SYNAPSE_OTEL_INTEGRATION_TEST_FILE_NAME="cli_traces.txt" - # Execute the CLI tests in a non-dist way because they were causing some test instability when being run concurrently - pytest -sv tests/integration/synapseclient/test_command_line_client.py - fi - - uses: actions/upload-artifact@v3 - if: ${{ contains(fromJSON('["3.9"]'), matrix.python) }} - with: - name: opentelemetry_traces_${{ matrix.os }} - path: tests/integration/otel - retention-days: 7 # on a GitHub release, build the pip package and upload it as a GitHub release asset package: diff --git a/test.awsConfig.enc b/test.awsConfig.enc new file mode 100644 index 000000000..2826704fb Binary files /dev/null and b/test.awsConfig.enc differ diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index acf01b4a8..2037be34c 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,5 +1,6 @@ import logging import platform +import threading import uuid import os import sys @@ -149,22 +150,5 @@ def setup_otel(): trace.get_tracer_provider().add_span_processor( BatchSpanProcessor(ConsoleSpanExporter()) ) - elif provider_type == "file": - file_name = os.environ.get( - "SYNAPSE_OTEL_INTEGRATION_TEST_FILE_NAME", "traces.txt" - ) - path = f"tests/integration/otel/{file_name}" - utils.touch(path) - export_file = open(path, "w", encoding="utf-8") - span_processor = BatchSpanProcessor(ConsoleSpanExporter(out=export_file)) - trace.get_tracer_provider().add_span_processor(span_processor) else: trace.set_tracer_provider(TracerProvider(sampler=ALWAYS_OFF)) - - yield - - # Teardown - if export_file: - span_processor.force_flush() - span_processor.shutdown() - export_file.close() diff --git a/tests/integration/synapse_creds.sh b/tests/integration/synapse_creds.sh new file mode 100644 index 000000000..83dcf0024 --- /dev/null +++ b/tests/integration/synapse_creds.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# Inputs +SC_ENDPOINT=$1 # i.e. https://sc.sageit.org +SYNAPSE_PAT=$2 # The Synapse Personal Access Token + +# Endpoints +STS_TOKEN_ENDPOINT="${SC_ENDPOINT}/ststoken" + +# Get Credentials +AWS_STS_CREDS=$(curl --location-trusted --silent -H "Authorization:Bearer ${SYNAPSE_PAT}" ${STS_TOKEN_ENDPOINT}) + +echo ${AWS_STS_CREDS}