From 7dce40999b025bc7b1a5f7454390b4786598653a Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 11 Dec 2024 12:15:02 -0600 Subject: [PATCH 01/10] GHA: Drop unnecessary expression markers (${{}} on if keys) --- .github/workflows/test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 50c1f8f265..1193b04cf1 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -519,13 +519,13 @@ jobs: test '::' - name: Compress Service Logs Before upload - if: ${{ failure() }} + if: failure() run: | ./tools/launchdev.sh stop # stop st2 before collecting logs tar cvzpf logs.tar.gz logs/* - name: Upload StackStorm services Logs - if: ${{ failure() }} + if: failure() uses: actions/upload-artifact@v4 with: name: logs-py${{ matrix.python-version }}-st2cluster-integration-tests From 46854c4262b020a309c0379d7d9f3cbe9f44dda3 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 11 Dec 2024 19:53:16 -0600 Subject: [PATCH 02/10] tools/launchdev.sh: Fix bash syntax typo --- tools/launchdev.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/launchdev.sh b/tools/launchdev.sh index c9426b04ea..321d3b20d6 100755 --- a/tools/launchdev.sh +++ b/tools/launchdev.sh @@ -140,7 +140,7 @@ function init() export ST2_SYSTEM_USER__USER="${ST2TESTS_SYSTEM_USER}" ST2VARS+=("ST2_SYSTEM_USER__USER") fi - if [ -n "${ST2TESTS_REDIS_HOST}" ] && [ -n "${ST2TESTS_REDIS_PORT}"]; then + if [ -n "${ST2TESTS_REDIS_HOST}" ] && [ -n "${ST2TESTS_REDIS_PORT}" ]; then export ST2_COORDINATION__URL="redis://${ST2TESTS_REDIS_HOST}:${ST2TESTS_REDIS_PORT}?namespace=_st2_dev" ST2VARS+=("ST2_COORDINATION__URL") fi From 12af930845b7042053cc76f466a46ec0aa0cc5fe Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 12 Nov 2024 21:10:16 -0600 Subject: [PATCH 03/10] pants: add GHA job to run st2-self-check --- .github/workflows/test.yaml | 153 +++++++++++++++++++++++++++++++ scripts/ci/add-itest-user-key.sh | 20 ++-- 2 files changed, 164 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 1193b04cf1..ad7c2aa111 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -539,6 +539,158 @@ jobs: path: .pants.d/pants.log if: always() # We want the log even on failures. + self-check: + name: 'Self-Check - Python ${{ matrix.python.version-short }}' + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + # NOTE: We need to use full Python version as part of Python deps cache key otherwise + # setup virtualenv step will fail. + python: + - {version-short: '3.8', version: '3.8.10'} + - {version-short: '3.9', version: '3.9.14'} + + services: + mongo: + image: mongo:7.0 + ports: + - 27017:27017 + + rabbitmq: + image: rabbitmq:3.8-management + options: >- + --name rabbitmq + ports: + - 5671:5671/tcp # AMQP SSL port + - 5672:5672/tcp # AMQP standard port + - 15672:15672/tcp # Management: HTTP, CLI + + redis: + # Docker Hub image + image: redis + # Set health checks to wait until redis has started + options: >- + --name "redis" + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 6379:6379/tcp + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + # a test uses a submodule, and pants needs access to it to calculate deps. + submodules: 'recursive' + # sadly, the submodule will only have fetch-depth=1, which is what we want + # for st2.git, but not for the submodules. We still want actions/checkout + # to do the initial checkout, however, so that it adds auth for fetching + # in the submodule. + + - name: Fetch repository submodules + run: | + git submodule status + git submodule foreach 'git fetch --all --tags' + git submodule foreach 'git tag' + + - name: 'Set up Python (${{ matrix.python.version }})' + id: python + uses: actions/setup-python@v5 + with: + python-version: '${{ matrix.python.version }}' + + - name: Cache and Install APT Dependencies + uses: ./.github/actions/apt-packages + + - name: Initialize Pants and its GHA caches + uses: ./.github/actions/init-pants + with: + # To ignore a bad cache, bump the cache* integer. + gha-cache-key: cache0-py${{ matrix.python.version }} + + - name: Export virtualenv to run Dev ST2 Cluster + run: > + pants + --python-bootstrap-search-path=[] + --python-bootstrap-search-path=${{ steps.python.outputs.python-path }} + export + --resolve=st2 + + - name: Add ST2 System User Key + env: + ST2_CI_USER: ${{ env.ST2TESTS_SYSTEM_USER }} + ST2_SYSTEM_USER__USER: ${{ env.ST2TESTS_SYSTEM_USER }} + ST2_SYSTEM_USER__SSH_KEY_FILE: /home/${{ env.ST2TESTS_SYSTEM_USER }}/.ssh/stanley_rsa + run: | + sudo -E ./scripts/ci/add-itest-user-key.sh + + - name: Launch Dev ST2 Cluster + env: + VIRTUALENV_DIR: ./dist/export/python/virtualenvs/st2/${{ steps.python.outputs.python-version }} + ST2_SYSTEM_USER__USER: ${{ env.ST2TESTS_SYSTEM_USER }} + ST2_SYSTEM_USER__SSH_KEY_FILE: /home/${{ env.ST2TESTS_SYSTEM_USER }}/.ssh/stanley_rsa + run: | + sudo -E ./scripts/github/prepare-integration.sh + + - name: Extend the path for upcoming tasks + # pants uses PEP 660 editable wheels to add our code to the virtualenv. + # But PEP 660 editable wheels do not include 'scripts'. + # https://peps.python.org/pep-0660/#limitations + # So, we need to include each bin dir in PATH instead of virtualenv/bin. + env: + VIRTUALENV_DIR: dist/export/python/virtualenvs/st2/${{ steps.python.outputs.python-version }} + run: | + for component_bin in ${GITHUB_WORKSPACE}/st2*/bin; do + echo ${component_bin} | tee -a $GITHUB_PATH + done + echo ${GITHUB_WORKSPACE}/${VIRTUALENV_DIR}/bin | tee -a $GITHUB_PATH + + - name: Create symlinks to find the binaries when running st2 actions + # st2 is actually a console_script entry point, not just a 'script' + # so it IS included in the virtualenv. But, st2-run-pack-tests might not be included. + env: + VIRTUALENV_DIR: dist/export/python/virtualenvs/st2/${{ steps.python.outputs.python-version }} + run: | + ln -s ${GITHUB_WORKSPACE}/${VIRTUALENV_DIR}/bin/st2 /usr/local/bin/st2 + ln -s ${GITHUB_WORKSPACE}/st2common/bin/st2-run-pack-tests /usr/local/bin/st2-run-pack-tests + + - name: Run st2-self-check + env: + # Space separated list of tests to be skipped if the self-check is running in GitHub Actions + TESTS_TO_SKIP: "tests.test_quickstart_rules tests.test_run_pack_tests_tool" + ST2_SYSTEM_USER__USER: ${{ env.ST2TESTS_SYSTEM_USER }} + ST2_SYSTEM_USER__SSH_KEY_FILE: /home/${{ env.ST2TESTS_SYSTEM_USER }}/.ssh/stanley_rsa + run: > + sudo + -E + ST2_AUTH_TOKEN=$(st2 auth testu -p 'testp' -t) + PATH=${PATH} + st2common/bin/st2-self-check + + - name: Compress Service Logs Before upload + if: failure() + run: | + ./tools/launchdev.sh stop # stop st2 before collecting logs + tar cvzpf logs.tar.gz logs/* + + - name: Upload StackStorm services Logs + if: failure() + uses: actions/upload-artifact@v4 + with: + name: logs-py${{ matrix.python-version }}-self-check + path: logs.tar.gz + retention-days: 7 + + - name: Upload pants log + uses: actions/upload-artifact@v4 + with: + name: pants-log-py${{ matrix.python.version }}-self-check + path: .pants.d/pants.log + if: always() # We want the log even on failures. + set_merge_ok: name: Set Merge OK (Tests) if: always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') @@ -548,6 +700,7 @@ jobs: - pack-tests - integration-tests - integration-st2cluster-tests + - self-check outputs: merge_ok: ${{ steps.set_merge_ok.outputs.merge_ok }} runs-on: ubuntu-latest diff --git a/scripts/ci/add-itest-user-key.sh b/scripts/ci/add-itest-user-key.sh index c268e022b0..21c53316bc 100755 --- a/scripts/ci/add-itest-user-key.sh +++ b/scripts/ci/add-itest-user-key.sh @@ -20,24 +20,26 @@ chmod 0700 ~/.ssh chown -R "${ST2_CI_USER}:${ST2_CI_USER}" ~/.ssh +SYSTEM_USER=${ST2TESTS_SYSTEM_USER:-${ST2_SYSTEM_USER__USER:-stanley}} + # Create an SSH system user (default `stanley` user may be already created) -if (! id stanley 2>/dev/null); then - useradd stanley +if (! id "${SYSTEM_USER}" 2>/dev/null); then + useradd "${SYSTEM_USER}" fi -mkdir -p /home/stanley/.ssh +mkdir -p "/home/${SYSTEM_USER}/.ssh" # Generate ssh keys on StackStorm box and copy over public key into remote box. -ssh-keygen -f /home/stanley/.ssh/stanley_rsa -P "" +ssh-keygen -f "/home/${SYSTEM_USER}/.ssh/stanley_rsa" -P "" # Authorize key-base acces -sh -c 'cat /home/stanley/.ssh/stanley_rsa.pub >> /home/stanley/.ssh/authorized_keys' -chmod 0600 /home/stanley/.ssh/authorized_keys -chmod 0700 /home/stanley/.ssh -chown -R stanley:stanley /home/stanley +sh -c "cat /home/${SYSTEM_USER}/.ssh/stanley_rsa.pub >> /home/${SYSTEM_USER}/.ssh/authorized_keys" +chmod 0600 "/home/${SYSTEM_USER}/.ssh/authorized_keys" +chmod 0700 "/home/${SYSTEM_USER}/.ssh" +chown -R "${SYSTEM_USER}:${SYSTEM_USER}" "/home/${SYSTEM_USER}" # Apply sudo fix for GHA runner user sh -c 'echo "runner ALL=(ALL) NOPASSWD: SETENV: ALL" >> /etc/sudoers.d/st2' # Enable passwordless sudo for 'stanley' user -sh -c 'echo "stanley ALL=(ALL) NOPASSWD: SETENV: ALL" >> /etc/sudoers.d/st2' +sh -c "echo '${SYSTEM_USER} ALL=(ALL) NOPASSWD: SETENV: ALL' >> /etc/sudoers.d/st2" chmod 0440 /etc/sudoers.d/st2 From fe26f6c4ee4b69469d160331b9beb9696ce922db Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 11 Dec 2024 20:06:59 -0600 Subject: [PATCH 04/10] pants ci: Pass ST2_CONF to st2-self-check --- .github/workflows/test.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index ad7c2aa111..4de6c3fcac 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -629,6 +629,7 @@ jobs: - name: Launch Dev ST2 Cluster env: + # NOTE: ST2_CONF defaults to ${GITHUB_WORKSPACE}/conf/st2.dev.conf VIRTUALENV_DIR: ./dist/export/python/virtualenvs/st2/${{ steps.python.outputs.python-version }} ST2_SYSTEM_USER__USER: ${{ env.ST2TESTS_SYSTEM_USER }} ST2_SYSTEM_USER__SSH_KEY_FILE: /home/${{ env.ST2TESTS_SYSTEM_USER }}/.ssh/stanley_rsa @@ -667,6 +668,7 @@ jobs: sudo -E ST2_AUTH_TOKEN=$(st2 auth testu -p 'testp' -t) + ST2_CONF=${GITHUB_WORKSPACE}/conf/st2.dev.conf PATH=${PATH} st2common/bin/st2-self-check From 27a2f6075550a4ba32d54eed1bbc044ebb3212ce Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 11 Dec 2024 20:03:47 -0600 Subject: [PATCH 05/10] st2-self-check: Avoid false-positive success when setup_virtualenv st2ctl reload fails. --- st2common/bin/st2-self-check | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/st2common/bin/st2-self-check b/st2common/bin/st2-self-check index 1240b13fc7..d5f7e4337c 100755 --- a/st2common/bin/st2-self-check +++ b/st2common/bin/st2-self-check @@ -113,15 +113,18 @@ fi cp -R --preserve st2tests/packs/* /opt/stackstorm/packs/ echo "Installing asserts, fixtures, tests and examples packs." -st2 run packs.setup_virtualenv packs=examples,tests,asserts,fixtures,webui -st2ctl reload --register-all +st2 run packs.setup_virtualenv packs=examples,tests,asserts,fixtures,webui || ((ERRORS++)) +st2ctl reload --register-all || ((ERRORS++)) echo "Deleting 'st2tests' directory from `pwd`." rm -R st2tests/ popd -# Retrieve test action list -TEST_ACTION_LIST=`st2 action list --pack=tests -w 90 | awk '{ print $2 }' | grep -v "|" | grep -v "ref" | grep tests.test_` +TEST_ACTION_LIST="" +if [ $ERRORS -eq 0 ]; then + # Retrieve test action list + TEST_ACTION_LIST=`st2 action list --pack=tests -w 90 | awk '{ print $2 }' | grep -v "|" | grep -v "ref" | grep tests.test_` +fi # Run all the tests for TEST in $TEST_ACTION_LIST From ab5f7138ab8fd2938f432cca28378cb1c90df8e7 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 11 Dec 2024 21:25:18 -0600 Subject: [PATCH 06/10] st2ctl: Force ps to use wide output to avoid truncation At least in GHA in some cases, the ps output is getting truncated, possibly respecting our COLUMNS=120 env var. This adds ww to the ps command to make it have unlimited width. --- st2common/bin/st2ctl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/st2common/bin/st2ctl b/st2common/bin/st2ctl index a26f7bfb09..9ed12ada4b 100755 --- a/st2common/bin/st2ctl +++ b/st2common/bin/st2ctl @@ -141,7 +141,7 @@ function service_manager() { function reopen_component_log_files() { COM=${1} - PID=`ps ax | grep -v grep | grep -v st2ctl | grep -E "(${COM}\.wsgi)|(bin/${COM})|(hubot .*${COM})" | awk '{print $1}'` + PID=`ps axww | grep -v grep | grep -v st2ctl | grep -E "(${COM}\.wsgi)|(bin/${COM})|(hubot .*${COM})" | awk '{print $1}'` if [[ ! -z ${PID} ]]; then for p in ${PID}; do echo "Sending SIGUSR1 to ${COM} PID: ${p}" @@ -201,7 +201,7 @@ function getpids() { COMPONENTS=${COMPONENTS} for COM in ${COMPONENTS}; do - PID=`ps ax | grep -v grep | grep -v st2ctl | grep -E "(${COM}\.wsgi)|(bin/${COM})|(hubot .*${COM})" | awk '{print $1}'` + PID=`ps axww | grep -v grep | grep -v st2ctl | grep -E "(${COM}\.wsgi)|(bin/${COM})|(hubot .*${COM})" | awk '{print $1}'` if [[ ! -z ${PID} ]]; then for p in ${PID}; do From d46b106f3fce0928d5eb3562d6c2169a22bb2624 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 11 Dec 2024 19:45:19 -0600 Subject: [PATCH 07/10] update changelog entry --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 53a8456f44..4cc1791006 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -69,7 +69,7 @@ Added working on StackStorm, improve our security posture, and improve CI reliability thanks in part to pants' use of PEX lockfiles. This is not a user-facing addition. #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 #6229 #6240 #6241 #6244 #6251 #6253 - #6254 #6258 #6259 #6260 #6269 #6275 #6279 #6278 #6282 #6283 #6273 + #6254 #6258 #6259 #6260 #6269 #6275 #6279 #6278 #6282 #6283 #6273 #6287 Contributed by @cognifloyd * Build of ST2 EL9 packages #6153 Contributed by @amanda11 From 4067d83c81379cd3bf0f24430b4803d9c9d05269 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 11 Dec 2024 21:59:09 -0600 Subject: [PATCH 08/10] Deal with odd failure in CI itests --- st2reactor/tests/integration/test_sensor_watcher.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/st2reactor/tests/integration/test_sensor_watcher.py b/st2reactor/tests/integration/test_sensor_watcher.py index 2c062aee41..b7991dab42 100644 --- a/st2reactor/tests/integration/test_sensor_watcher.py +++ b/st2reactor/tests/integration/test_sensor_watcher.py @@ -31,6 +31,15 @@ class SensorWatcherTestCase(IntegrationTestCase): def setUpClass(cls): super(SensorWatcherTestCase, cls).setUpClass() + def setUp(self): + super().setUp() + # pre-condition: Make sure there is no test pollution + sw_queues = self._get_sensor_watcher_amqp_queues( + queue_name="st2.sensor.watch.covfefe" + ) + # TODO: Maybe just delete any leftover queues from previous failed test runs. + self.assertTrue(len(sw_queues) == 0) + def test_sensor_watch_queue_gets_deleted_on_stop(self): def create_handler(sensor_db): pass @@ -64,7 +73,8 @@ def delete_handler(sensor_db): ) self.assertTrue(len(sw_queues) == 0) - def _list_amqp_queues(self): + @staticmethod + def _list_amqp_queues(): rabbit_client = Client("localhost:15672", "guest", "guest") queues = [q["name"] for q in rabbit_client.get_queues()] return queues From 9e5b5190ca90507e1ca167bcf54b2263ec886ece Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 11 Dec 2024 22:09:51 -0600 Subject: [PATCH 09/10] Deal with odd failure in CI itests --- .github/workflows/test.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 4de6c3fcac..d1fe03fc3b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -406,6 +406,8 @@ jobs: ./scripts/github/configure-rabbitmq.sh - name: Integration Tests + env: + PANTS_TEST_ATTEMPTS_DEFAULT: 1 run: > pants --python-bootstrap-search-path=[] From 096d1224e5c7687bf6f4603bb94b3389d82410b0 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 11 Dec 2024 22:14:56 -0600 Subject: [PATCH 10/10] Revert "Deal with odd failure in CI itests" This reverts commit 9e5b5190ca90507e1ca167bcf54b2263ec886ece. --- .github/workflows/test.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d1fe03fc3b..4de6c3fcac 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -406,8 +406,6 @@ jobs: ./scripts/github/configure-rabbitmq.sh - name: Integration Tests - env: - PANTS_TEST_ATTEMPTS_DEFAULT: 1 run: > pants --python-bootstrap-search-path=[]