diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 50c1f8f265..4de6c3fcac 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -519,13 +519,13 @@ jobs: test '::' - name: Compress Service Logs Before upload - if: ${{ failure() }} + if: failure() run: | ./tools/launchdev.sh stop # stop st2 before collecting logs tar cvzpf logs.tar.gz logs/* - name: Upload StackStorm services Logs - if: ${{ failure() }} + if: failure() uses: actions/upload-artifact@v4 with: name: logs-py${{ matrix.python-version }}-st2cluster-integration-tests @@ -539,6 +539,160 @@ jobs: path: .pants.d/pants.log if: always() # We want the log even on failures. + self-check: + name: 'Self-Check - Python ${{ matrix.python.version-short }}' + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + # NOTE: We need to use full Python version as part of Python deps cache key otherwise + # setup virtualenv step will fail. + python: + - {version-short: '3.8', version: '3.8.10'} + - {version-short: '3.9', version: '3.9.14'} + + services: + mongo: + image: mongo:7.0 + ports: + - 27017:27017 + + rabbitmq: + image: rabbitmq:3.8-management + options: >- + --name rabbitmq + ports: + - 5671:5671/tcp # AMQP SSL port + - 5672:5672/tcp # AMQP standard port + - 15672:15672/tcp # Management: HTTP, CLI + + redis: + # Docker Hub image + image: redis + # Set health checks to wait until redis has started + options: >- + --name "redis" + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 6379:6379/tcp + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + # a test uses a submodule, and pants needs access to it to calculate deps. + submodules: 'recursive' + # sadly, the submodule will only have fetch-depth=1, which is what we want + # for st2.git, but not for the submodules. We still want actions/checkout + # to do the initial checkout, however, so that it adds auth for fetching + # in the submodule. + + - name: Fetch repository submodules + run: | + git submodule status + git submodule foreach 'git fetch --all --tags' + git submodule foreach 'git tag' + + - name: 'Set up Python (${{ matrix.python.version }})' + id: python + uses: actions/setup-python@v5 + with: + python-version: '${{ matrix.python.version }}' + + - name: Cache and Install APT Dependencies + uses: ./.github/actions/apt-packages + + - name: Initialize Pants and its GHA caches + uses: ./.github/actions/init-pants + with: + # To ignore a bad cache, bump the cache* integer. + gha-cache-key: cache0-py${{ matrix.python.version }} + + - name: Export virtualenv to run Dev ST2 Cluster + run: > + pants + --python-bootstrap-search-path=[] + --python-bootstrap-search-path=${{ steps.python.outputs.python-path }} + export + --resolve=st2 + + - name: Add ST2 System User Key + env: + ST2_CI_USER: ${{ env.ST2TESTS_SYSTEM_USER }} + ST2_SYSTEM_USER__USER: ${{ env.ST2TESTS_SYSTEM_USER }} + ST2_SYSTEM_USER__SSH_KEY_FILE: /home/${{ env.ST2TESTS_SYSTEM_USER }}/.ssh/stanley_rsa + run: | + sudo -E ./scripts/ci/add-itest-user-key.sh + + - name: Launch Dev ST2 Cluster + env: + # NOTE: ST2_CONF defaults to ${GITHUB_WORKSPACE}/conf/st2.dev.conf + VIRTUALENV_DIR: ./dist/export/python/virtualenvs/st2/${{ steps.python.outputs.python-version }} + ST2_SYSTEM_USER__USER: ${{ env.ST2TESTS_SYSTEM_USER }} + ST2_SYSTEM_USER__SSH_KEY_FILE: /home/${{ env.ST2TESTS_SYSTEM_USER }}/.ssh/stanley_rsa + run: | + sudo -E ./scripts/github/prepare-integration.sh + + - name: Extend the path for upcoming tasks + # pants uses PEP 660 editable wheels to add our code to the virtualenv. + # But PEP 660 editable wheels do not include 'scripts'. + # https://peps.python.org/pep-0660/#limitations + # So, we need to include each bin dir in PATH instead of virtualenv/bin. + env: + VIRTUALENV_DIR: dist/export/python/virtualenvs/st2/${{ steps.python.outputs.python-version }} + run: | + for component_bin in ${GITHUB_WORKSPACE}/st2*/bin; do + echo ${component_bin} | tee -a $GITHUB_PATH + done + echo ${GITHUB_WORKSPACE}/${VIRTUALENV_DIR}/bin | tee -a $GITHUB_PATH + + - name: Create symlinks to find the binaries when running st2 actions + # st2 is actually a console_script entry point, not just a 'script' + # so it IS included in the virtualenv. But, st2-run-pack-tests might not be included. + env: + VIRTUALENV_DIR: dist/export/python/virtualenvs/st2/${{ steps.python.outputs.python-version }} + run: | + ln -s ${GITHUB_WORKSPACE}/${VIRTUALENV_DIR}/bin/st2 /usr/local/bin/st2 + ln -s ${GITHUB_WORKSPACE}/st2common/bin/st2-run-pack-tests /usr/local/bin/st2-run-pack-tests + + - name: Run st2-self-check + env: + # Space separated list of tests to be skipped if the self-check is running in GitHub Actions + TESTS_TO_SKIP: "tests.test_quickstart_rules tests.test_run_pack_tests_tool" + ST2_SYSTEM_USER__USER: ${{ env.ST2TESTS_SYSTEM_USER }} + ST2_SYSTEM_USER__SSH_KEY_FILE: /home/${{ env.ST2TESTS_SYSTEM_USER }}/.ssh/stanley_rsa + run: > + sudo + -E + ST2_AUTH_TOKEN=$(st2 auth testu -p 'testp' -t) + ST2_CONF=${GITHUB_WORKSPACE}/conf/st2.dev.conf + PATH=${PATH} + st2common/bin/st2-self-check + + - name: Compress Service Logs Before upload + if: failure() + run: | + ./tools/launchdev.sh stop # stop st2 before collecting logs + tar cvzpf logs.tar.gz logs/* + + - name: Upload StackStorm services Logs + if: failure() + uses: actions/upload-artifact@v4 + with: + name: logs-py${{ matrix.python-version }}-self-check + path: logs.tar.gz + retention-days: 7 + + - name: Upload pants log + uses: actions/upload-artifact@v4 + with: + name: pants-log-py${{ matrix.python.version }}-self-check + path: .pants.d/pants.log + if: always() # We want the log even on failures. + set_merge_ok: name: Set Merge OK (Tests) if: always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') @@ -548,6 +702,7 @@ jobs: - pack-tests - integration-tests - integration-st2cluster-tests + - self-check outputs: merge_ok: ${{ steps.set_merge_ok.outputs.merge_ok }} runs-on: ubuntu-latest diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 53a8456f44..4cc1791006 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -69,7 +69,7 @@ Added working on StackStorm, improve our security posture, and improve CI reliability thanks in part to pants' use of PEX lockfiles. This is not a user-facing addition. #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 #6229 #6240 #6241 #6244 #6251 #6253 - #6254 #6258 #6259 #6260 #6269 #6275 #6279 #6278 #6282 #6283 #6273 + #6254 #6258 #6259 #6260 #6269 #6275 #6279 #6278 #6282 #6283 #6273 #6287 Contributed by @cognifloyd * Build of ST2 EL9 packages #6153 Contributed by @amanda11 diff --git a/scripts/ci/add-itest-user-key.sh b/scripts/ci/add-itest-user-key.sh index c268e022b0..21c53316bc 100755 --- a/scripts/ci/add-itest-user-key.sh +++ b/scripts/ci/add-itest-user-key.sh @@ -20,24 +20,26 @@ chmod 0700 ~/.ssh chown -R "${ST2_CI_USER}:${ST2_CI_USER}" ~/.ssh +SYSTEM_USER=${ST2TESTS_SYSTEM_USER:-${ST2_SYSTEM_USER__USER:-stanley}} + # Create an SSH system user (default `stanley` user may be already created) -if (! id stanley 2>/dev/null); then - useradd stanley +if (! id "${SYSTEM_USER}" 2>/dev/null); then + useradd "${SYSTEM_USER}" fi -mkdir -p /home/stanley/.ssh +mkdir -p "/home/${SYSTEM_USER}/.ssh" # Generate ssh keys on StackStorm box and copy over public key into remote box. -ssh-keygen -f /home/stanley/.ssh/stanley_rsa -P "" +ssh-keygen -f "/home/${SYSTEM_USER}/.ssh/stanley_rsa" -P "" # Authorize key-base acces -sh -c 'cat /home/stanley/.ssh/stanley_rsa.pub >> /home/stanley/.ssh/authorized_keys' -chmod 0600 /home/stanley/.ssh/authorized_keys -chmod 0700 /home/stanley/.ssh -chown -R stanley:stanley /home/stanley +sh -c "cat /home/${SYSTEM_USER}/.ssh/stanley_rsa.pub >> /home/${SYSTEM_USER}/.ssh/authorized_keys" +chmod 0600 "/home/${SYSTEM_USER}/.ssh/authorized_keys" +chmod 0700 "/home/${SYSTEM_USER}/.ssh" +chown -R "${SYSTEM_USER}:${SYSTEM_USER}" "/home/${SYSTEM_USER}" # Apply sudo fix for GHA runner user sh -c 'echo "runner ALL=(ALL) NOPASSWD: SETENV: ALL" >> /etc/sudoers.d/st2' # Enable passwordless sudo for 'stanley' user -sh -c 'echo "stanley ALL=(ALL) NOPASSWD: SETENV: ALL" >> /etc/sudoers.d/st2' +sh -c "echo '${SYSTEM_USER} ALL=(ALL) NOPASSWD: SETENV: ALL' >> /etc/sudoers.d/st2" chmod 0440 /etc/sudoers.d/st2 diff --git a/st2common/bin/st2-self-check b/st2common/bin/st2-self-check index 1240b13fc7..d5f7e4337c 100755 --- a/st2common/bin/st2-self-check +++ b/st2common/bin/st2-self-check @@ -113,15 +113,18 @@ fi cp -R --preserve st2tests/packs/* /opt/stackstorm/packs/ echo "Installing asserts, fixtures, tests and examples packs." -st2 run packs.setup_virtualenv packs=examples,tests,asserts,fixtures,webui -st2ctl reload --register-all +st2 run packs.setup_virtualenv packs=examples,tests,asserts,fixtures,webui || ((ERRORS++)) +st2ctl reload --register-all || ((ERRORS++)) echo "Deleting 'st2tests' directory from `pwd`." rm -R st2tests/ popd -# Retrieve test action list -TEST_ACTION_LIST=`st2 action list --pack=tests -w 90 | awk '{ print $2 }' | grep -v "|" | grep -v "ref" | grep tests.test_` +TEST_ACTION_LIST="" +if [ $ERRORS -eq 0 ]; then + # Retrieve test action list + TEST_ACTION_LIST=`st2 action list --pack=tests -w 90 | awk '{ print $2 }' | grep -v "|" | grep -v "ref" | grep tests.test_` +fi # Run all the tests for TEST in $TEST_ACTION_LIST diff --git a/st2common/bin/st2ctl b/st2common/bin/st2ctl index a26f7bfb09..9ed12ada4b 100755 --- a/st2common/bin/st2ctl +++ b/st2common/bin/st2ctl @@ -141,7 +141,7 @@ function service_manager() { function reopen_component_log_files() { COM=${1} - PID=`ps ax | grep -v grep | grep -v st2ctl | grep -E "(${COM}\.wsgi)|(bin/${COM})|(hubot .*${COM})" | awk '{print $1}'` + PID=`ps axww | grep -v grep | grep -v st2ctl | grep -E "(${COM}\.wsgi)|(bin/${COM})|(hubot .*${COM})" | awk '{print $1}'` if [[ ! -z ${PID} ]]; then for p in ${PID}; do echo "Sending SIGUSR1 to ${COM} PID: ${p}" @@ -201,7 +201,7 @@ function getpids() { COMPONENTS=${COMPONENTS} for COM in ${COMPONENTS}; do - PID=`ps ax | grep -v grep | grep -v st2ctl | grep -E "(${COM}\.wsgi)|(bin/${COM})|(hubot .*${COM})" | awk '{print $1}'` + PID=`ps axww | grep -v grep | grep -v st2ctl | grep -E "(${COM}\.wsgi)|(bin/${COM})|(hubot .*${COM})" | awk '{print $1}'` if [[ ! -z ${PID} ]]; then for p in ${PID}; do diff --git a/st2reactor/tests/integration/test_sensor_watcher.py b/st2reactor/tests/integration/test_sensor_watcher.py index 2c062aee41..b7991dab42 100644 --- a/st2reactor/tests/integration/test_sensor_watcher.py +++ b/st2reactor/tests/integration/test_sensor_watcher.py @@ -31,6 +31,15 @@ class SensorWatcherTestCase(IntegrationTestCase): def setUpClass(cls): super(SensorWatcherTestCase, cls).setUpClass() + def setUp(self): + super().setUp() + # pre-condition: Make sure there is no test pollution + sw_queues = self._get_sensor_watcher_amqp_queues( + queue_name="st2.sensor.watch.covfefe" + ) + # TODO: Maybe just delete any leftover queues from previous failed test runs. + self.assertTrue(len(sw_queues) == 0) + def test_sensor_watch_queue_gets_deleted_on_stop(self): def create_handler(sensor_db): pass @@ -64,7 +73,8 @@ def delete_handler(sensor_db): ) self.assertTrue(len(sw_queues) == 0) - def _list_amqp_queues(self): + @staticmethod + def _list_amqp_queues(): rabbit_client = Client("localhost:15672", "guest", "guest") queues = [q["name"] for q in rabbit_client.get_queues()] return queues diff --git a/tools/launchdev.sh b/tools/launchdev.sh index c9426b04ea..321d3b20d6 100755 --- a/tools/launchdev.sh +++ b/tools/launchdev.sh @@ -140,7 +140,7 @@ function init() export ST2_SYSTEM_USER__USER="${ST2TESTS_SYSTEM_USER}" ST2VARS+=("ST2_SYSTEM_USER__USER") fi - if [ -n "${ST2TESTS_REDIS_HOST}" ] && [ -n "${ST2TESTS_REDIS_PORT}"]; then + if [ -n "${ST2TESTS_REDIS_HOST}" ] && [ -n "${ST2TESTS_REDIS_PORT}" ]; then export ST2_COORDINATION__URL="redis://${ST2TESTS_REDIS_HOST}:${ST2TESTS_REDIS_PORT}?namespace=_st2_dev" ST2VARS+=("ST2_COORDINATION__URL") fi