Merge pull request #32 from VatsalJagani/integration-tests

Integration-tests

.github/workflows/py_unit_tests.yml

@@ -35,17 +35,18 @@ jobs:
             pip install -r tests/requirements.txt
 
         - name: Run Python Unit Tests
+          env:
+            SPLUNKBASE_USERNAME_FOR_TEST: ${{ secrets.SPLUNKBASE_USERNAME }}
+            SPLUNKBASE_PASSWORD_FOR_TEST: ${{ secrets.SPLUNKBASE_PASSWORD }}
           run: pytest tests --junitxml=junit/test-results.xml --cov=src --cov-config=tests/.coveragerc --cov-report=xml
 
         - name: Adding GitHub action step summary
-          uses: VatsalJagani/pytest-cov-action@v0.5
+          uses: VatsalJagani/pytest-cov-action@v0.6
           with:
             pytest_results_file: "junit/test-results.xml"
             pytest_cov_file: "coverage.xml"
-            pytest_cov_failure_threshold: 40
           if: ${{ always() }}
 
         - name: Linting with Ruff
-          run: ruff check --ignore=E721,E722,E401 --output-format=github .
+          run: ruff check --ignore=E721,E722,E401 --output-format=github --exclude=tests/test_app_repos . 
           # continue-on-error: true   # need to remove this later on to catch the errors anf fail the workflow on linting errors
-

.gitignore

@@ -1,10 +1,10 @@
 .DS_Store
 *.pyc
 __pycache__
-src/utilities/logger/props.conf_temp
-tests2
-.coverage
-coverage.xml
-.pytest_cache
-htmlcov
-junit
+/src/utilities/logger/props.conf_temp
+
+/.coverage
+/coverage.xml
+/.pytest_cache
+/htmlcov
+/junit

README.md

@@ -77,9 +77,9 @@
         ```
         - uses: VatsalJagani/splunk-app-action@v3
           env:
-            SPLUNK_APP_ACTION_1: "find . -type f -exec chmod 644 '{}' \;"
+            SPLUNK_APP_ACTION_1: "find . -type f -exec chmod 644 '{}' \\;"
             SPLUNK_APP_ACTION_2: "find . -type f -name '*.sh' -exec chmod +x '{}' \\;"
-            SPLUNK_APP_ACTION_3: "find . -type d -exec chmod 755 '{}' \;"
+            SPLUNK_APP_ACTION_3: "find . -type d -exec chmod 755 '{}' \\;"
           with:
             app_dir: "my_app"
         ```
@@ -212,11 +212,6 @@ def stream_events(input_script: smi.Script, inputs: smi.InputDefinition, event_w
 * required: false
 * default: ".", meaning root folder of the repository.
 
-#### is_generate_build
-* description: "Whether to generate the App Build or not."
-* required: false
-* default: true
-
 #### to_make_permission_changes
 * description: "Whether to apply file and folder permission changes according to Splunk App Inspect expectation before generating the build."
 * Before you add this parameter, read the instruction from [Avoid File and Folder Permission Issue on Your App Build](#-Avoid-File-and-Folder-Permission-Issue-on-Your-App-Build) section.
@@ -241,10 +236,6 @@ def stream_events(input_script: smi.Script, inputs: smi.InputDefinition, event_w
 * description: "Password required to call the Splunkbase API for App-Inspect. Required when is_app_inspect_check is set to true. Strongly recommend to use via GitHub secrets only and specify like `{{secrets.MY_SPLUNK_PASSWORD}}`."
 * required: false
 
-#### app_build_path
-* description: "Full App build path. Used only when is_generate_build is set to false."
-* required: false
-
 #### app_utilities
 * description: "Add comma separated list of utilities to use. You need to enable read and write permission for workflow to create Pull Requests. Valid options: whats_in_the_app, logger, splunk_python_sdk, common_js_utilities, ucc_additional_packaging"
 * required: false
@@ -290,6 +281,8 @@ def stream_events(input_script: smi.Script, inputs: smi.InputDefinition, event_w
 * `to_make_permission_changes` parameter's default value has been changed to `false`. Refer to `Avoid File and Folder Permission Issue on Your App Build` for more details.
     * Automatic file permission changes now also add executable permissions to following file extensions, `.msi`, `.exe`, `.cmd`, `.bat`, along with `.sh`.
 
+* The input parameters `is_generate_build` and `app_build_path` has been removed.
+
 
 ### (IN PROGRESS) Upgrade Guide from v3 to v4
 * From `v4` of the `splunk-app-action`, your user-defined custom command (Refer to `Running User Defined Commands Before Generating the final App Build` section) would run in a context of your App's folder instead of root folder.
@@ -315,6 +308,8 @@ def stream_events(input_script: smi.Script, inputs: smi.InputDefinition, event_w
 
 * `to_make_permission_changes` - default value has been changed to `false` from `true`. This means if you were to apply file and folder permission changes automatically, you have to explicitly add this parameter to your workflow. Refer to `Avoid File and Folder Permission Issue on Your App Build` for more details.
 
+* The input parameters `is_generate_build` and `app_build_path` has been removed from GitHub action from `v4` onwards, hence if you are using them then migrate the action config properly before version update.
+
 
 ### v3
 * use_ucc_gen parameter added to support UCC build Add-on support. (It uses `ucc-gen build` command to generate the build dynamically on the GitHub action directly.)

action.yml

@@ -11,11 +11,6 @@ inputs:
     required: false
     default: "."
 
-  is_generate_build:
-    description: "Whether to generate the App Build or not."
-    required: false
-    default: true
-
   to_make_permission_changes:
     description: "Whether to apply file and folder permission changes according to Splunk App Inspect expectation before generating the build."
     required: false
@@ -39,11 +34,6 @@ inputs:
     description: "Password required to call the Splunkbase API for App-Inspect. Required when is_app_inspect_check is set to true. Strongly recommend to use via GitHub secrets only and specify like `{{secrets.MY_SPLUNK_PASSWORD}}`."
     required: false
 
-  app_build_path:
-    description: "Full App build path. Used only when is_generate_build is set to false."
-    required: false
-    default: "NONE"
-
   app_utilities:
     description: "Add comma separated list of utilities to use. You need to enable read and write permission for workflow to create Pull Requests. Valid options: whats_in_the_app, logger, splunk_python_sdk, common_js_utilities, ucc_additional_packaging"
     required: false
@@ -92,13 +82,11 @@ runs:
         pwd
         echo "=== Inputs ==="
         echo "app_dir -> ${{inputs.app_dir}}"
-        echo "is_generate_build -> ${{inputs.is_generate_build}}"
         echo "use_ucc_gen -> ${{inputs.use_ucc_gen}}"
         echo "to_make_permission_changes -> ${{inputs.to_make_permission_changes}}"
         echo "is_app_inspect_check -> ${{inputs.is_app_inspect_check}}"
         echo "splunkbase_username -> ${{inputs.splunkbase_username}} (If using Github secret the value will be ***)"
         echo "splunkbase_password -> ${{inputs.splunkbase_password}} (If using Github secret the value will be ***)"
-        echo "app_build_path -> ${{inputs.app_build_path}}"
         echo "app_utilities -> ${{inputs.app_utilities}}"
         echo "my_github_token -> ${{inputs.my_github_token}} (If using Github secret the value will be ***)"
         echo "current_branch_name -> ${{steps.extract_branch.outputs.branch}}"
@@ -120,13 +108,11 @@ runs:
       shell: bash
       env:
         SPLUNK_app_dir: ${{inputs.app_dir}}
-        SPLUNK_is_generate_build: ${{inputs.is_generate_build}}
         SPLUNK_use_ucc_gen: ${{inputs.use_ucc_gen}}
         SPLUNK_to_make_permission_changes: ${{inputs.to_make_permission_changes}}
         SPLUNK_is_app_inspect_check: ${{inputs.is_app_inspect_check}}
         SPLUNK_splunkbase_username: ${{inputs.splunkbase_username}}
         SPLUNK_splunkbase_password: ${{inputs.splunkbase_password}}
-        SPLUNK_app_build_path: ${{inputs.app_build_path}}
         SPLUNK_app_utilities: ${{inputs.app_utilities}}
         GITHUB_TOKEN: ${{inputs.my_github_token}}
         SPLUNK_current_branch_name: ${{steps.extract_branch.outputs.branch}}

src/app_build_generate.py

@@ -22,14 +22,14 @@ def file_folder_permission_changes():
     if to_make_permission_changes:
         # Permission Changes
         utils.execute_system_command(
-            "find . -type f -exec chmod 644 '{}' \;")
+            "find . -type f -exec chmod 644 '{}' \\;")
 
         for file_ext in [".sh", ".exe", ".cmd", ".msi", ".bat"]:
             utils.execute_system_command(
-                f"find . -type f -name '*{file_ext}' -exec chmod 755 '{{}}' \;")
+                f"find . -type f -name '*{file_ext}' -exec chmod 755 '{{}}' \\;")
 
         utils.execute_system_command(
-            "find . -type d -exec chmod 755 '{}' \;")
+            "find . -type d -exec chmod 755 '{}' \\;")
 
 
 def run_custom_user_defined_commands():
@@ -46,24 +46,15 @@ def run_custom_user_defined_commands():
 def generate_build(app_build_dir_name, app_build_dir_path):
     utils.info(f"Generating the app build., app_dir_path={app_build_dir_path}, app_package_id={GlobalVariables.APP_PACKAGE_ID}, app_version_encoded={GlobalVariables.APP_VERSION_ENCODED}, app_build_number_encoded={GlobalVariables.APP_BUILD_NUMBER_ENCODED}")
 
-    is_generate_build = utils.str_to_boolean_default_true(
-        utils.get_input('is_generate_build'))
-    utils.info("is_generate_build: {}".format(is_generate_build))
-
-    direct_app_build_path = utils.get_input('app_build_path')
-    utils.info("app_build_path: {}".format(direct_app_build_path))
-
-    if not is_generate_build:
-        return direct_app_build_path
-
     os.chdir(GlobalVariables.ROOT_DIR_PATH)
     utils.execute_system_command(
             f'mv {app_build_dir_name} {GlobalVariables.APP_PACKAGE_ID}')
-    os.chdir(GlobalVariables.APP_PACKAGE_ID)
 
+    os.chdir(GlobalVariables.APP_PACKAGE_ID)
     remove_unwanted_files()
     run_custom_user_defined_commands()
     file_folder_permission_changes()
+    os.chdir(GlobalVariables.ROOT_DIR_PATH)
 
     # Generate Build
     build_name = f"{GlobalVariables.APP_PACKAGE_ID}_{GlobalVariables.APP_VERSION_ENCODED}_{GlobalVariables.APP_BUILD_NUMBER_ENCODED}.tgz"

src/app_inspect.py

@@ -22,17 +22,9 @@ class SplunkAppInspect:
     HTML_RESPONSE_URL = "{}/report".format(BASE_URL)
 
 
-    def __init__(self, app_build_path) -> None:
-        self.is_app_inspect_check = utils.str_to_boolean_default_true(
-            utils.get_input('is_app_inspect_check'))
-        utils.info("is_app_inspect_check: {}".format(
-            self.is_app_inspect_check))
-        if not self.is_app_inspect_check:
-            utils.info("Ignoring App-inspect checks.")
-            return
-
-        self.splunkbase_username = utils.get_input('splunkbase_username')
-        self.splunkbase_password = utils.get_input('splunkbase_password')
+    def __init__(self, app_build_path, splunkbase_username, splunkbase_password) -> None:
+        self.splunkbase_username = splunkbase_username
+        self.splunkbase_password = splunkbase_password
 
         if not self.splunkbase_username:
             msg = "splunkbase_username input is not provided."
@@ -65,7 +57,7 @@ def __init__(self, app_build_path) -> None:
 
         self._api_login()
 
-        os.chdir(utils.CommonDirPaths.MAIN_DIR)
+        os.chdir(GlobalVariables.ROOT_DIR_PATH)
 
 
     def _api_login(self):
@@ -233,8 +225,6 @@ def _perform_ssai_inspect_check(self):
 
     def run_all_checks(self):
         utils.info("Running the Splunk App inspect check.")
-        if not self.is_app_inspect_check:
-            return
 
         thread_app_inspect = Thread(target=self._perform_app_inspect_check)
         thread_app_inspect.start()

src/app_utilities.py

@@ -1,6 +1,7 @@
 
 import os
 import helpers.github_action_utils as utils
+from helpers.global_variables import GlobalVariables
 from utilities.whats_inside_app import WhatsInsideTheAppUtility
 from utilities.logger import LoggerUtility
 from utilities.splunk_sdk_python import SplunkPythonSDKUtility
@@ -21,7 +22,7 @@ def __init__(self, app_read_dir, app_write_dir, is_test=False) -> None:
             app_utilities = app_utilities.split(',')
             app_utilities = [u.strip() for u in app_utilities]
 
-        os.chdir(utils.CommonDirPaths.MAIN_DIR)
+        os.chdir(GlobalVariables.ROOT_DIR_PATH)
 
         self.add_utilities(app_utilities)
 
@@ -33,16 +34,16 @@ def add_utilities(self, app_utilities):
                 WhatsInsideTheAppUtility(self.app_read_dir, self.app_write_dir)
 
             elif utility == "logger":
-                LoggerUtility(self.app_read_dir, self.app_write_dir)
+                LoggerUtility(self.app_read_dir, self.app_write_dir).add()
 
             elif utility == "splunk_python_sdk":
-                SplunkPythonSDKUtility(self.app_read_dir, self.app_write_dir)
+                SplunkPythonSDKUtility(self.app_read_dir, self.app_write_dir).add()
 
             elif utility == "common_js_utilities":
-                CommonJSUtilitiesFile(self.app_read_dir, self.app_write_dir)
+                CommonJSUtilitiesFile(self.app_read_dir, self.app_write_dir).add()
 
             elif utility == "ucc_additional_packaging":
-                UCCAdditionalPackagingUtility(self.app_read_dir, self.app_write_dir)
+                UCCAdditionalPackagingUtility(self.app_read_dir, self.app_write_dir).add()
 
             else:
                 utils.error("utility={} is not supported.".format(utility))

src/helpers/file_manager.py

@@ -12,15 +12,21 @@ def __init__(self, input_file_path, output_file_path, words_for_replacement=dict
         self.words_for_replacement: dict = words_for_replacement
 
 
+    def text_words_replacement(self, content):
+        # do words replacement for file content
+        for word, replacement in self.words_for_replacement.items():
+            content = content.replace(word, replacement)
+
+        return content
+
+
+
     def get_input_file_content(self):
         input_content = None
         with open(self.input_file_path, 'r') as fr:
             input_content = fr.read()
 
-        for word, replacement in self.words_for_replacement.items():
-            input_content = input_content.replace(word, replacement)
-
-        return input_content
+        return self.text_words_replacement(input_content)
 
 
     def create_output_directory_path_if_not_exist(self):
@@ -45,8 +51,11 @@ def validate_config(self):
         input_parser = SplunkConfigParser(temp_file)
 
         self.create_output_directory_path_if_not_exist()
-        output_parser = SplunkConfigParser(self.output_file_path)
+        if not os.path.exists(self.output_file_path):
+            with open(self.output_file_path, 'w') as f:
+                pass   # writing empty file
 
+        output_parser = SplunkConfigParser(self.output_file_path)
         is_file_changed = output_parser.merge(input_parser)
 
         if is_file_changed:
@@ -79,6 +88,8 @@ def validate_file_content(self):
 class PartRawFileHandler(BaseFileHandler):
 
     def validate_file_content(self, new_content, start_markers, end_markers, start_marker_to_add='', end_marker_to_add=''):
+        new_content = self.text_words_replacement(new_content)
+
         content = ''
         lower_content = ''
         start_index = -1