Preparation for tang test suite using common lib

* Preparation for tang test suite using common lib Tang test suite will be use partialy functions from common library due to decreasing duplicity of functions for similar test suites. Needed to clear actual function script and adjust each test to use ocpop lib. * Adjust README and update oc plan Adjust READM to describe how run test via tmt and update plan to add ENV var needed for succesfull run.
RedHat-SP-Security · Feb 5, 2024 · a024850 · a024850
1 parent e0fd4a7
commit a024850
Show file tree

Hide file tree

Showing 14 changed files with 328 additions and 787 deletions.
diff --git a/Plans/operator-oc.fmf b/Plans/operator-oc.fmf
@@ -1,5 +1,9 @@
 summary: test plan for running in oc
 
+environment:
+  TEST_OC_CLIENT: "oc"
+# Can be specified version of operator, by default is used latest
+  IMAGE_VERSION: "quay.io/sec-eng-special/tang-operator-bundle-container:latest"
 
 prepare:
   - how: shell

diff --git a/Plans/packit-ci.fmf b/Plans/packit-ci.fmf
@@ -13,12 +13,18 @@ environment:
   CI: "true"
 
 discover:
-  how: fmf
-  test:
-   - /Setup/setup_local_cluster
-   - /Setup/creating_test_namespace
-   - /Sanity
-   - /Setup/clean_cluster
+  - name: Configure_test_system
+    how: fmf
+    url: https://github.com/RedHat-SP-Security/common-cloud-orchestration
+    ref: main
+    test:
+      - /Setup/setup_local_cluster
+  - name: Run tests
+    how: fmf
+    test:
+      - /Setup/creating_test_namespace
+      - /Sanity
+      - /Setup/clean_cluster
 
 adjust:
   - when: distro == rhel-9 or distro == centos-stream-9 

diff --git a/README.md b/README.md
@@ -16,7 +16,7 @@ In order to execute tang-operator test suite, you will need some software instal
 - podman (for Malware Detection test execution)
 - clamav (for Malware Detection test execution)
 
-In case `helm`, `podman` or `clamav` does not exist, tests requiring its installation won't be executed.
+In case `helm`, `podman` or `clamav` does not exist, tests requiring its installation won't be executed. When is used Test Managament Tool (`tmt`) no requirements are needed.
 
 To execute the test suite, next steps must be followed:
 
@@ -50,3 +50,40 @@ In case it is necessary, a more verbose output of the test execution can be indi
 ```bash
 $ V=1 make
 ```
+Execute Test suite (through tmt command):
+
+To execute the test suite, next steps must be followed:
+
+Need to install tmt:
+
+```bash
+# dnf install -y tmt tmt-all
+```
+
+Clone Security Special Projects Test repository:
+```bash
+$ git clone https://github.com/RedHat-SP-Security/tang-operator-tests
+```
+
+1. Executing tests on minikube cluster ( setup of minikube is provided in test ).
+
+Execute localy via tmt:
+```bash
+# tmt -c distro=fedora-39 run plan --name packit-ci -vvv prepare discover provision -h local execute
+```
+
+Or running in virtual system via tmt:
+
+```bash
+# tmt -c distro=fedora-39 run plan --name packit-ci -vvv prepare discover provision -h virtual -i Fedora-39 -c system execute report finish
+```
+2. Executing test on OpenShift or Openshift-local(CRC).
+
+To execute the test suite, next steps must be followed:
+
+Have OpenShift cluster and be connected via oc cli.
+
+Execute localy via tmt:
+```bash
+# tmt -c distro=fedora-39 run plan --name operator-oc -vvv prepare discover provision -h local execute
+```
diff --git a/Sanity/DAST_test/runtest.sh b/Sanity/DAST_test/runtest.sh
@@ -32,7 +32,9 @@
 
 rlJournalStart
     rlPhaseStartSetup
+        rlRun 'rlImport "common-cloud-orchestration/ocpop-lib"' || rlDie "cannot import ocpop lib"
         rlRun ". ../../TestHelpers/functions.sh" || rlDie "cannot import function script"
+        TO_DAST_POD_COMPLETED=240 #seconds (DAST lasts around 120 seconds)
         if ! command -v helm &> /dev/null; then
             ARCH=$(case $(uname -m) in x86_64) echo -n amd64 ;; aarch64) echo -n arm64 ;; *) echo -n "$(uname -m)" ;; esac)
             OS=$(uname | awk '{print tolower($0)}')
@@ -49,7 +51,7 @@ rlJournalStart
     ############# DAST TESTS ##############
     rlPhaseStartTest "Dynamic Application Security Testing"
         # 1 - Log helm version
-        dumpVerbose "$(helm version)"
+        ocpopLogVerbose "$(helm version)"
 
         # 2 - clone rapidast code (development branch)
         tmpdir=$(mktemp -d)
@@ -72,9 +74,9 @@ rlJournalStart
         sed -i s@API_HOST_PORT_HERE@"${API_HOST_PORT}"@g tang_operator.yaml
         sed -i s@AUTH_TOKEN_HERE@"${DEFAULT_TOKEN}"@g tang_operator.yaml
         sed -i s@OPERATOR_NAMESPACE_HERE@"${OPERATOR_NAMESPACE}"@g tang_operator.yaml
-        dumpVerbose "API_HOST_PORT:[${API_HOST_PORT}]"
-        dumpVerbose "DEFAULT_TOKEN:[${DEFAULT_TOKEN}]"
-        dumpVerbose "OPERATOR_NAMESPACE provided to DAST:[${OPERATOR_NAMESPACE}]"
+        ocpopLogVerbose "API_HOST_PORT:[${API_HOST_PORT}]"
+        ocpopLogVerbose "DEFAULT_TOKEN:[${DEFAULT_TOKEN}]"
+        ocpopLogVerbose "OPERATOR_NAMESPACE provided to DAST:[${OPERATOR_NAMESPACE}]"
         rlAssertNotEquals "Checking token not empty" "${DEFAULT_TOKEN}" ""
 
         # 5 - adapt helm
@@ -85,26 +87,26 @@ rlJournalStart
 
         # 6 - run rapidast on adapted configuration file (via helm)
         rlRun -c "helm install rapidast ./helm/chart/ --set-file rapidastConfig=${tmpdir}/tang_operator.yaml 2>/dev/null" 0 "Installing rapidast helm chart"
-        pod_name=$(getPodNameWithPrefix "rapidast" "default" 5 1)
-        rlRun "checkPodState Completed ${TO_DAST_POD_COMPLETED} default ${pod_name}" 0 "Checking POD ${pod_name} in Completed state [Timeout=${TO_DAST_POD_COMPLETED} secs.]"
+        pod_name=$(ocpopGetPodNameWithPartialName "rapidast" "default" 5 1)
+        rlRun "ocpopCheckPodState Completed ${TO_DAST_POD_COMPLETED} default ${pod_name}" 0 "Checking POD ${pod_name} in Completed state [Timeout=${TO_DAST_POD_COMPLETED} secs.]"
 
         # 7 - extract results
         rlRun -c "bash ./helm/results.sh 2>/dev/null" 0 "Extracting DAST results"
 
         # 8 - parse results (do not have to ensure no previous results exist, as this is a temporary directory)
         # Check no alarm exist ...
         report_dir=$(ls -1d "${tmpdir}"/rapidast/tangservers/DAST*tangservers/ | head -1 | sed -e 's@/$@@g')
-        dumpVerbose "REPORT DIR:${report_dir}"
+        ocpopLogVerbose "REPORT DIR:${report_dir}"
 
         rlAssertNotEquals "Checking report_dir not empty" "${report_dir}" ""
 
         report_file="${report_dir}/zap/zap-report.json"
-        dumpVerbose "REPORT FILE:${report_file}"
+        ocpopLogVerbose "REPORT FILE:${report_file}"
 
         if [ -n "${report_dir}" ] && [ -f "${report_file}" ];
         then
             alerts=$(jq '.site[0].alerts | length' < "${report_dir}/zap/zap-report.json" )
-            dumpVerbose "Alerts:${alerts}"
+            ocpopLogVerbose "Alerts:${alerts}"
             for ((alert=0; alert<alerts; alert++));
             do
                 risk_desc=$(jq ".site[0].alerts[${alert}].riskdesc" < "${report_dir}/zap/zap-report.json" | awk '{print $1}' | tr -d '"' | tr -d " ")

diff --git a/Sanity/configuration_test/runtest.sh b/Sanity/configuration_test/runtest.sh
@@ -32,60 +32,61 @@
 rlJournalStart
     ########## CONFIGURATION TESTS #########
     rlPhaseStartTest "Minimal Configuration"
+        rlRun 'rlImport "common-cloud-orchestration/ocpop-lib"' || rlDie "cannot import ocpop lib"
         rlRun ". ../../TestHelpers/functions.sh" || rlDie "cannot import function script"
-        rlRun "${OC_CLIENT} apply -f ${FUNCTION_DIR}/reg_test/conf_test/minimal/" 0 "Creating minimal configuration"
-        rlRun "checkPodAmount 1 ${TO_POD_START} ${TEST_NAMESPACE}" 0 "Checking 1 POD is started [Timeout=${TO_POD_START} secs.]"
-        rlRun "checkServiceAmount 1 ${TO_SERVICE_START} ${TEST_NAMESPACE}" 0 "Checking 1 Service is started [Timeout=${TO_SERVICE_START} secs.]"
-        pod_name=$(getPodNameWithPrefix "tang" "${TEST_NAMESPACE}" 5)
+        rlRun "${OC_CLIENT} apply -f ${TANG_FUNCTION_DIR}/reg_test/conf_test/minimal/" 0 "Creating minimal configuration"
+        rlRun "ocpopCheckPodAmount 1 ${TO_POD_START} ${TEST_NAMESPACE}" 0 "Checking 1 POD is started [Timeout=${TO_POD_START} secs.]"
+        rlRun "ocpopCheckServiceAmount 1 ${TO_SERVICE_START} ${TEST_NAMESPACE}" 0 "Checking 1 Service is started [Timeout=${TO_SERVICE_START} secs.]"
+        pod_name=$(ocpopGetPodNameWithPartialName "tang" "${TEST_NAMESPACE}" 5)
         rlAssertNotEquals "Checking pod name not empty" "${pod_name}" ""
-        rlRun "checkPodState Running ${TO_POD_START} ${TEST_NAMESPACE} ${pod_name}" 0 "Checking POD in Running state [Timeout=${TO_POD_START} secs.]"
-        rlRun "${OC_CLIENT} delete -f ${FUNCTION_DIR}/reg_test/conf_test/minimal/" 0 "Deleting minimal configuration"
-        rlRun "checkPodAmount 0 ${TO_POD_STOP} ${TEST_NAMESPACE}" 0 "Checking no POD continues running [Timeout=${TO_POD_STOP} secs.]"
-        rlRun "checkServiceAmount 0 ${TO_SERVICE_STOP} ${TEST_NAMESPACE}" 0 "Checking no Services continue running [Timeout=${TO_SERVICE_STOP} secs.]"
+        rlRun "ocpopCheckPodState Running ${TO_POD_START} ${TEST_NAMESPACE} ${pod_name}" 0 "Checking POD in Running state [Timeout=${TO_POD_START} secs.]"
+        rlRun "${OC_CLIENT} delete -f ${TANG_FUNCTION_DIR}/reg_test/conf_test/minimal/" 0 "Deleting minimal configuration"
+        rlRun "ocpopCheckPodAmount 0 ${TO_POD_STOP} ${TEST_NAMESPACE}" 0 "Checking no POD continues running [Timeout=${TO_POD_STOP} secs.]"
+        rlRun "ocpopCheckServiceAmount 0 ${TO_SERVICE_STOP} ${TEST_NAMESPACE}" 0 "Checking no Services continue running [Timeout=${TO_SERVICE_STOP} secs.]"
     rlPhaseEnd
 
     rlPhaseStartTest "Main Configuration"
-        rlRun "${OC_CLIENT} apply -f ${FUNCTION_DIR}/reg_test/conf_test/main/" 0 "Creating main configuration"
-        rlRun "checkPodAmount 3 ${TO_POD_START} ${TEST_NAMESPACE}" 0 "Checking 3 PODs are started [Timeout=${TO_POD_START} secs.]"
-        rlRun "checkServiceAmount 1 ${TO_SERVICE_START} ${TEST_NAMESPACE}" 0 "Checking 1 Service is started [Timeout=${TO_SERVICE_START} secs.]"
-        pod1_name=$(getPodNameWithPrefix "tang" "${TEST_NAMESPACE}" 5 1)
-        pod2_name=$(getPodNameWithPrefix "tang" "${TEST_NAMESPACE}" 5 2)
-        pod3_name=$(getPodNameWithPrefix "tang" "${TEST_NAMESPACE}" 5 3)
+        rlRun "${OC_CLIENT} apply -f ${TANG_FUNCTION_DIR}/reg_test/conf_test/main/" 0 "Creating main configuration"
+        rlRun "ocpopCheckPodAmount 3 ${TO_POD_START} ${TEST_NAMESPACE}" 0 "Checking 3 PODs are started [Timeout=${TO_POD_START} secs.]"
+        rlRun "ocpopCheckServiceAmount 1 ${TO_SERVICE_START} ${TEST_NAMESPACE}" 0 "Checking 1 Service is started [Timeout=${TO_SERVICE_START} secs.]"
+        pod1_name=$(ocpopGetPodNameWithPartialName "tang" "${TEST_NAMESPACE}" 5 1)
+        pod2_name=$(ocpopGetPodNameWithPartialName "tang" "${TEST_NAMESPACE}" 5 2)
+        pod3_name=$(ocpopGetPodNameWithPartialName "tang" "${TEST_NAMESPACE}" 5 3)
         rlAssertNotEquals "Checking pod name not empty" "${pod1_name}" ""
         rlAssertNotEquals "Checking pod name not empty" "${pod2_name}" ""
         rlAssertNotEquals "Checking pod name not empty" "${pod3_name}" ""
-        rlRun "checkPodState Running ${TO_POD_START} ${TEST_NAMESPACE} ${pod1_name}" 0 "Checking POD:[$pod1_name] in Running state [Timeout=${TO_POD_START} secs.]"
-        rlRun "checkPodState Running ${TO_POD_START} ${TEST_NAMESPACE} ${pod2_name}" 0 "Checking POD:[$pod2_name] in Running state [Timeout=${TO_POD_START} secs.]"
-        rlRun "checkPodState Running ${TO_POD_START} ${TEST_NAMESPACE} ${pod3_name}" 0 "Checking POD:[$pod3_name] in Running state [Timeout=${TO_POD_START} secs.]"
-        rlRun "${OC_CLIENT} delete -f ${FUNCTION_DIR}/reg_test/conf_test/main/" 0 "Deleting main configuration"
-        rlRun "checkPodAmount 0 ${TO_POD_STOP} ${TEST_NAMESPACE}" 0 "Checking no PODs continue running [Timeout=${TO_POD_STOP} secs.]"
-        rlRun "checkServiceAmount 0 ${TO_SERVICE_STOP} ${TEST_NAMESPACE}" 0 "Checking no Services continue running [Timeout=${TO_SERVICE_STOP} secs.]"
+        rlRun "ocpopCheckPodState Running ${TO_POD_START} ${TEST_NAMESPACE} ${pod1_name}" 0 "Checking POD:[$pod1_name] in Running state [Timeout=${TO_POD_START} secs.]"
+        rlRun "ocpopCheckPodState Running ${TO_POD_START} ${TEST_NAMESPACE} ${pod2_name}" 0 "Checking POD:[$pod2_name] in Running state [Timeout=${TO_POD_START} secs.]"
+        rlRun "ocpopCheckPodState Running ${TO_POD_START} ${TEST_NAMESPACE} ${pod3_name}" 0 "Checking POD:[$pod3_name] in Running state [Timeout=${TO_POD_START} secs.]"
+        rlRun "${OC_CLIENT} delete -f ${TANG_FUNCTION_DIR}/reg_test/conf_test/main/" 0 "Deleting main configuration"
+        rlRun "ocpopCheckPodAmount 0 ${TO_POD_STOP} ${TEST_NAMESPACE}" 0 "Checking no PODs continue running [Timeout=${TO_POD_STOP} secs.]"
+        rlRun "ocpopCheckServiceAmount 0 ${TO_SERVICE_STOP} ${TEST_NAMESPACE}" 0 "Checking no Services continue running [Timeout=${TO_SERVICE_STOP} secs.]"
     rlPhaseEnd
 
     rlPhaseStartTest "Multiple Deployment Configuration"
-        rlRun "${OC_CLIENT} apply -f ${FUNCTION_DIR}/reg_test/conf_test/multi_deployment/" 0 "Creating multiple deployment configuration"
-        rlRun "checkPodAmount 5 ${TO_POD_START} ${TEST_NAMESPACE}" 0 "Checking 5 PODs are started [Timeout=${TO_POD_START} secs.]"
+        rlRun "${OC_CLIENT} apply -f ${TANG_FUNCTION_DIR}/reg_test/conf_test/multi_deployment/" 0 "Creating multiple deployment configuration"
+        rlRun "ocpopCheckPodAmount 5 ${TO_POD_START} ${TEST_NAMESPACE}" 0 "Checking 5 PODs are started [Timeout=${TO_POD_START} secs.]"
         rlRun "sleep 5" 0 "Waiting to ensure no more than expected replicas are started"
-        rlRun "checkPodAmount 5 ${TO_POD_START} ${TEST_NAMESPACE}" 0 "Checking 5 PODs continue running [Timeout=${TO_POD_START} secs.]"
-        rlRun "checkServiceAmount 2 ${TO_SERVICE_START} ${TEST_NAMESPACE}" 0 "Checking 2 Services are running [Timeout=${TO_SERVICE_START} secs.]"
-        pod1_name=$(getPodNameWithPrefix "tang" "${TEST_NAMESPACE}" 5 1)
-        pod2_name=$(getPodNameWithPrefix "tang" "${TEST_NAMESPACE}" 5 2)
-        pod3_name=$(getPodNameWithPrefix "tang" "${TEST_NAMESPACE}" 5 3)
-        pod4_name=$(getPodNameWithPrefix "tang" "${TEST_NAMESPACE}" 5 4)
-        pod5_name=$(getPodNameWithPrefix "tang" "${TEST_NAMESPACE}" 5 5)
+        rlRun "ocpopCheckPodAmount 5 ${TO_POD_START} ${TEST_NAMESPACE}" 0 "Checking 5 PODs continue running [Timeout=${TO_POD_START} secs.]"
+        rlRun "ocpopCheckServiceAmount 2 ${TO_SERVICE_START} ${TEST_NAMESPACE}" 0 "Checking 2 Services are running [Timeout=${TO_SERVICE_START} secs.]"
+        pod1_name=$(ocpopGetPodNameWithPartialName "tang" "${TEST_NAMESPACE}" 5 1)
+        pod2_name=$(ocpopGetPodNameWithPartialName "tang" "${TEST_NAMESPACE}" 5 2)
+        pod3_name=$(ocpopGetPodNameWithPartialName "tang" "${TEST_NAMESPACE}" 5 3)
+        pod4_name=$(ocpopGetPodNameWithPartialName "tang" "${TEST_NAMESPACE}" 5 4)
+        pod5_name=$(ocpopGetPodNameWithPartialName "tang" "${TEST_NAMESPACE}" 5 5)
         rlAssertNotEquals "Checking pod name not empty" "${pod1_name}" ""
         rlAssertNotEquals "Checking pod name not empty" "${pod2_name}" ""
         rlAssertNotEquals "Checking pod name not empty" "${pod3_name}" ""
         rlAssertNotEquals "Checking pod name not empty" "${pod4_name}" ""
         rlAssertNotEquals "Checking pod name not empty" "${pod5_name}" ""
-        rlRun "checkPodState Running ${TO_POD_START} ${TEST_NAMESPACE} ${pod1_name}" 0 "Checking POD:[$pod1_name] in Running state [Timeout=${TO_POD_START} secs.]"
-        rlRun "checkPodState Running ${TO_POD_START} ${TEST_NAMESPACE} ${pod2_name}" 0 "Checking POD:[$pod2_name] in Running state [Timeout=${TO_POD_START} secs.]"
-        rlRun "checkPodState Running ${TO_POD_START} ${TEST_NAMESPACE} ${pod3_name}" 0 "Checking POD:[$pod3_name] in Running state [Timeout=${TO_POD_START} secs.]"
-        rlRun "checkPodState Running ${TO_POD_START} ${TEST_NAMESPACE} ${pod4_name}" 0 "Checking POD:[$pod2_name] in Running state [Timeout=${TO_POD_START} secs.]"
-        rlRun "checkPodState Running ${TO_POD_START} ${TEST_NAMESPACE} ${pod5_name}" 0 "Checking POD:[$pod3_name] in Running state [Timeout=${TO_POD_START} secs.]"
-        rlRun "${OC_CLIENT} delete -f ${FUNCTION_DIR}/reg_test/conf_test/multi_deployment/" 0 "Deleting multiple deployment configuration"
-        rlRun "checkPodAmount 0 ${TO_POD_STOP} ${TEST_NAMESPACE}" 0 "Checking no PODs continue running [Timeout=${TO_POD_STOP} secs.]"
-        rlRun "checkServiceAmount 0 ${TO_SERVICE_STOP} ${TEST_NAMESPACE}" 0 "Checking no Services continue running [Timeout=${TO_SERVICE_STOP} secs.]"
+        rlRun "ocpopCheckPodState Running ${TO_POD_START} ${TEST_NAMESPACE} ${pod1_name}" 0 "Checking POD:[$pod1_name] in Running state [Timeout=${TO_POD_START} secs.]"
+        rlRun "ocpopCheckPodState Running ${TO_POD_START} ${TEST_NAMESPACE} ${pod2_name}" 0 "Checking POD:[$pod2_name] in Running state [Timeout=${TO_POD_START} secs.]"
+        rlRun "ocpopCheckPodState Running ${TO_POD_START} ${TEST_NAMESPACE} ${pod3_name}" 0 "Checking POD:[$pod3_name] in Running state [Timeout=${TO_POD_START} secs.]"
+        rlRun "ocpopCheckPodState Running ${TO_POD_START} ${TEST_NAMESPACE} ${pod4_name}" 0 "Checking POD:[$pod2_name] in Running state [Timeout=${TO_POD_START} secs.]"
+        rlRun "ocpopCheckPodState Running ${TO_POD_START} ${TEST_NAMESPACE} ${pod5_name}" 0 "Checking POD:[$pod3_name] in Running state [Timeout=${TO_POD_START} secs.]"
+        rlRun "${OC_CLIENT} delete -f ${TANG_FUNCTION_DIR}/reg_test/conf_test/multi_deployment/" 0 "Deleting multiple deployment configuration"
+        rlRun "ocpopCheckPodAmount 0 ${TO_POD_STOP} ${TEST_NAMESPACE}" 0 "Checking no PODs continue running [Timeout=${TO_POD_STOP} secs.]"
+        rlRun "ocpopCheckServiceAmount 0 ${TO_SERVICE_STOP} ${TEST_NAMESPACE}" 0 "Checking no Services continue running [Timeout=${TO_SERVICE_STOP} secs.]"
     rlPhaseEnd
     ######### /CONFIGURATION TESTS ########