From ee16a5f549db75e2c4b72ef04f6892eded07b4d7 Mon Sep 17 00:00:00 2001 From: Vatsal Jagani Date: Sat, 30 Mar 2024 10:58:40 +0530 Subject: [PATCH 1/6] Added splunk_python_sdk_install_path parameter. Added new test-cases for that as well. --- .github/workflows/py_unit_tests.yml | 4 - .vscode/settings.json | 5 + README.md | 21 ++++ action.yml | 7 ++ src/utilities/splunk_sdk_python/__init__.py | 7 +- tests/helper.py | 4 +- tests/test_utility_python_sdk.py | 102 +++++++++++++++++++- 7 files changed, 143 insertions(+), 7 deletions(-) create mode 100644 .vscode/settings.json diff --git a/.github/workflows/py_unit_tests.yml b/.github/workflows/py_unit_tests.yml index 36d5e2b..b81b9db 100644 --- a/.github/workflows/py_unit_tests.yml +++ b/.github/workflows/py_unit_tests.yml @@ -4,10 +4,6 @@ name: Running Python Unit Tests for Splunk-App-Action # Controls when the action will run. Triggers the workflow on push or pull request on: push: - branches: - - 'develop' - - pull_request: branches: - '*' - '*/*' diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..f401d7a --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "cSpell.words": [ + "splunklib" + ] +} \ No newline at end of file diff --git a/README.md b/README.md index 07b11da..c7fa3a0 100644 --- a/README.md +++ b/README.md @@ -188,6 +188,7 @@ graph TD #### `splunk_python_sdk` - Add Splunklib or Splunk SDK for Python and Auto Upgrades It * This utility adds the splunklib or Splunk SDK for Python to the App and auto upgrades it whenever new version is available. +Example-1 ``` - uses: VatsalJagani/splunk-app-action@v3 with: @@ -196,6 +197,17 @@ graph TD my_github_token: ${{ secrets.MY_GITHUB_TOKEN }} ``` +Example-2 +``` +- uses: VatsalJagani/splunk-app-action@v3 + with: + app_dir: "my_app" + app_utilities: "splunk_python_sdk" + my_github_token: ${{ secrets.MY_GITHUB_TOKEN }} + splunk_python_sdk_install_path: "bin/lib" +``` + + #### `common_js_utilities` - Add Common JavaScript Utilities File * This utility adds a JavaScript file that contains commonly used functionality for a JavaScript code for a Splunk App. @@ -287,6 +299,11 @@ def stream_events(input_script: smi.Script, inputs: smi.InputDefinition, event_w * description: "Sourcetype for the internal app logs. Required only for logger utility." * required: false +#### splunk_python_sdk_install_path +* description: "Path where you would like to install splunk-python-sdk (splunklib). Path is relative to App's root folder." +* required: false +* default: "bin" + ## Troubleshooting @@ -309,6 +326,10 @@ def stream_events(input_script: smi.Script, inputs: smi.InputDefinition, event_w ## Release Notes +### v4.1 +* Added `splunk_python_sdk_install_path` parameter for `splunk_python_sdk` utility. Default value is still `bin` folder, but now user can change based on their needs. + + ### v4 * Run the user-defined commands in the context of your App's root directory instead of Repo's root directory. Refer to `Running User Defined Commands Before Generating the final App Build` for more details. diff --git a/action.yml b/action.yml index 57718ea..1c2de3c 100644 --- a/action.yml +++ b/action.yml @@ -54,6 +54,11 @@ inputs: required: false default: "NONE" + splunk_python_sdk_install_path: + description: "Path where you would like to install splunk-python-sdk (splunklib). Path is relative to App's root folder." + required: false + default: "bin" + outputs: stdout: description: "Program stdout" @@ -92,6 +97,7 @@ runs: echo "current_branch_name -> ${{steps.extract_branch.outputs.branch}}" echo "logger_log_files_prefix -> ${{inputs.logger_log_files_prefix}}" echo "logger_sourcetype -> ${{inputs.logger_sourcetype}}" + echo "splunk_python_sdk_install_path -> ${{inputs.splunk_python_sdk_install_path}}" - name: "Install Python" uses: actions/setup-python@v5 @@ -118,6 +124,7 @@ runs: SPLUNK_current_branch_name: ${{steps.extract_branch.outputs.branch}} SPLUNK_logger_log_files_prefix: ${{inputs.logger_log_files_prefix}} SPLUNK_logger_sourcetype: ${{inputs.logger_sourcetype}} + SPLUNK_splunk_python_sdk_install_path: ${{inputs.splunk_python_sdk_install_path}} run: | python -u ${{ github.action_path }}/src/main.py diff --git a/src/utilities/splunk_sdk_python/__init__.py b/src/utilities/splunk_sdk_python/__init__.py index df63846..f86bf1c 100644 --- a/src/utilities/splunk_sdk_python/__init__.py +++ b/src/utilities/splunk_sdk_python/__init__.py @@ -22,8 +22,13 @@ def _get_splunklib_version(self, file_path): def implement_utility(self): utils.info("Adding SplunkPythonSDKUtility") + splunk_python_sdk_install_path = utils.get_input('splunk_python_sdk_install_path') + utils.info(f"splunk_python_sdk_install_path: {splunk_python_sdk_install_path}") + if not splunk_python_sdk_install_path or splunk_python_sdk_install_path == "NONE": + splunk_python_sdk_install_path = "bin" + folder_to_install_splunklib = os.path.join( - self.app_write_dir, 'bin') + self.app_write_dir, splunk_python_sdk_install_path) if not os.path.exists(folder_to_install_splunklib): os.mkdir(folder_to_install_splunklib) diff --git a/tests/helper.py b/tests/helper.py index 33e5665..4455ddf 100644 --- a/tests/helper.py +++ b/tests/helper.py @@ -17,7 +17,8 @@ def setup_action_yml(test_app_repo, my_github_token="NONE", current_branch="NONE", logger_log_files_prefix="NONE", - logger_sourcetype="NONE"): + logger_sourcetype="NONE", + splunk_python_sdk_install_path="bin"): app_dir_path = os.path.join(os.path.dirname(__file__), "test_app_repos", test_app_repo) print(f"TestIntegration.setup_action_yml_work -> app_dir_path={app_dir_path}") @@ -37,6 +38,7 @@ def setup_action_yml(test_app_repo, os.environ["SPLUNK_current_branch_name"] = current_branch os.environ["SPLUNK_logger_log_files_prefix"] = logger_log_files_prefix os.environ["SPLUNK_logger_sourcetype"] = logger_sourcetype + os.environ["SPLUNK_splunk_python_sdk_install_path"] = splunk_python_sdk_install_path try: yield diff --git a/tests/test_utility_python_sdk.py b/tests/test_utility_python_sdk.py index 7cbb769..fb7075b 100644 --- a/tests/test_utility_python_sdk.py +++ b/tests/test_utility_python_sdk.py @@ -1,6 +1,6 @@ import os from unittest.mock import patch -from .helper import get_temp_directory, stdout_capture +from .helper import get_temp_directory, setup_temporary_env_vars, stdout_capture from utilities.splunk_sdk_python import SplunkPythonSDKUtility from helpers import github_action_utils as utils @@ -22,6 +22,25 @@ def test_splunk_sdk_utility_installed_new(): assert os.path.exists(init_file) +def test_splunk_sdk_utility_installed_new_2(): + with get_temp_directory() as temp_dir: + with setup_temporary_env_vars({ + "SPLUNK_splunk_python_sdk_install_path": "lib", + }): + sdk_utility = SplunkPythonSDKUtility("nothing", temp_dir) + result = sdk_utility.implement_utility() + + # Validate the result + folder_path = os.path.join(temp_dir, 'lib') + splunklib_dir = os.path.join(folder_path, 'splunklib') + init_file = os.path.join(splunklib_dir, '__init__.py') + + print(f"init_file={init_file}") + + assert result == init_file + assert os.path.exists(init_file) + + def test_splunk_sdk_utility_upgraded_existing(): with get_temp_directory() as temp_dir: # Prepare the temporary directory @@ -49,6 +68,36 @@ def test_splunk_sdk_utility_upgraded_existing(): assert result == init_file +def test_splunk_sdk_utility_upgraded_existing_2(): + with get_temp_directory() as temp_dir: + with setup_temporary_env_vars({ + "SPLUNK_splunk_python_sdk_install_path": "bin/lib", + }): + # Prepare the temporary directory + folder_path = os.path.join(temp_dir, 'bin', 'lib') + os.makedirs(folder_path) + splunklib_dir = os.path.join(folder_path, 'splunklib') + os.makedirs(splunklib_dir) + init_file = os.path.join(splunklib_dir, '__init__.py') + with open(init_file, 'w') as f: + f.write('__version_info__ = "1.0.0"') + + # Initialize SplunkPythonSDKUtility instance + sdk_utility = SplunkPythonSDKUtility("nothing", temp_dir) + + with stdout_capture() as captured_stdout: + # Call implement_utility function + result = sdk_utility.implement_utility() + + # Get the captured stdout + output = captured_stdout.getvalue() + assert 'New splunklib version' in output.strip() + + # Validate the result + assert os.path.exists(init_file) + assert result == init_file + + def test_splunk_sdk_utility_skipped_due_to_existing_and_same_version(): with get_temp_directory() as temp_dir: # Prepare the temporary directory @@ -71,6 +120,31 @@ def test_splunk_sdk_utility_skipped_due_to_existing_and_same_version(): assert result is None +def test_splunk_sdk_utility_skipped_due_to_existing_and_same_version_2(): + with get_temp_directory() as temp_dir: + with setup_temporary_env_vars({ + "SPLUNK_splunk_python_sdk_install_path": "lib", + }): + # Prepare the temporary directory + folder_path = os.path.join(temp_dir, 'lib') + os.makedirs(folder_path) + splunklib_dir = os.path.join(folder_path, 'splunklib') + os.makedirs(splunklib_dir) + init_file = os.path.join(splunklib_dir, '__init__.py') + with open(init_file, 'w') as f: + f.write('__version_info__ = "1.2.3"') + + with patch.object(SplunkPythonSDKUtility, '_get_splunklib_version', return_value="1.2.3"): + # Initialize SplunkPythonSDKUtility instance + sdk_utility = SplunkPythonSDKUtility("nothing", temp_dir) + # Call implement_utility function + result = sdk_utility.implement_utility() + + # Validate the result + assert not os.path.exists(os.path.join(folder_path, 'splunk-sdk')) + assert result is None + + def test_splunk_sdk_utility_error_upgrading_existing(): with get_temp_directory() as temp_dir: # Prepare the temporary directory @@ -92,3 +166,29 @@ def test_splunk_sdk_utility_error_upgrading_existing(): # Validate the result assert result is None + + +def test_splunk_sdk_utility_error_upgrading_existing_2(): + with get_temp_directory() as temp_dir: + with setup_temporary_env_vars({ + "SPLUNK_splunk_python_sdk_install_path": "bin/lib", + }): + # Prepare the temporary directory + folder_path = os.path.join(temp_dir, 'bin', 'lib') + os.makedirs(folder_path) + splunklib_dir = os.path.join(folder_path, 'splunklib') + os.makedirs(splunklib_dir) + init_file = os.path.join(splunklib_dir, '__init__.py') + with open(init_file, 'w') as f: + f.write('__version_info__ = "1.0.0"') + + # Initialize SplunkPythonSDKUtility instance + sdk_utility = SplunkPythonSDKUtility("nothing", temp_dir) + + # Call implement_utility function with failing upgrade + with patch.object(SplunkPythonSDKUtility, '_get_splunklib_version', return_value="2.2.2"): + with patch.object(utils, 'execute_system_command', return_value=(1, '')): + result = sdk_utility.implement_utility() + + # Validate the result + assert result is None From 4052af4719b7ae5e29a408d6e6544d951fcf6215 Mon Sep 17 00:00:00 2001 From: Vatsal Jagani Date: Sat, 30 Mar 2024 11:19:45 +0530 Subject: [PATCH 2/6] Added code to remove pycache from splunklib installation directory before generating PR. --- README.md | 8 +++ action.yml | 8 +++ src/utilities/splunk_sdk_python/__init__.py | 19 +++++++ tests/helper.py | 4 +- tests/test_utility_python_sdk.py | 61 ++++++++++++++------- 5 files changed, 78 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index c7fa3a0..eb717a7 100644 --- a/README.md +++ b/README.md @@ -304,6 +304,13 @@ def stream_events(input_script: smi.Script, inputs: smi.InputDefinition, event_w * required: false * default: "bin" +#### is_remove_pyc_from_splunklib_dir +* description: "Remove `.pyc` files and `__pycache__` directory from splunk-python-sdk (splunklib) installation path before generating Pull Request. Do not turn this off unless you are facing any issues explicitly." +* required: false +* default: true + + + ## Troubleshooting @@ -328,6 +335,7 @@ def stream_events(input_script: smi.Script, inputs: smi.InputDefinition, event_w ### v4.1 * Added `splunk_python_sdk_install_path` parameter for `splunk_python_sdk` utility. Default value is still `bin` folder, but now user can change based on their needs. +* Avoid `.pyc` files and `__pycache__` files getting into `splunk_python_sdk` installation folder and getting into Pull Request by default. You can turn this off with `is_remove_pyc_from_splunklib_dir` parameter. ### v4 diff --git a/action.yml b/action.yml index 1c2de3c..5263ab3 100644 --- a/action.yml +++ b/action.yml @@ -59,6 +59,12 @@ inputs: required: false default: "bin" + is_remove_pyc_from_splunklib_dir: + description: "Remove `.pyc` files and `__pycache__` directory from splunk-python-sdk (splunklib) installation path before generating Pull Request. Do not turn this off unless you are facing any issues explicitly." + required: false + default: true + + outputs: stdout: description: "Program stdout" @@ -98,6 +104,7 @@ runs: echo "logger_log_files_prefix -> ${{inputs.logger_log_files_prefix}}" echo "logger_sourcetype -> ${{inputs.logger_sourcetype}}" echo "splunk_python_sdk_install_path -> ${{inputs.splunk_python_sdk_install_path}}" + echo "is_remove_pyc_from_splunklib_dir -> ${{inputs.is_remove_pyc_from_splunklib_dir}}" - name: "Install Python" uses: actions/setup-python@v5 @@ -125,6 +132,7 @@ runs: SPLUNK_logger_log_files_prefix: ${{inputs.logger_log_files_prefix}} SPLUNK_logger_sourcetype: ${{inputs.logger_sourcetype}} SPLUNK_splunk_python_sdk_install_path: ${{inputs.splunk_python_sdk_install_path}} + SPLUNK_is_remove_pyc_from_splunklib_dir: ${{inputs.is_remove_pyc_from_splunklib_dir}} run: | python -u ${{ github.action_path }}/src/main.py diff --git a/src/utilities/splunk_sdk_python/__init__.py b/src/utilities/splunk_sdk_python/__init__.py index f86bf1c..191d3ee 100644 --- a/src/utilities/splunk_sdk_python/__init__.py +++ b/src/utilities/splunk_sdk_python/__init__.py @@ -1,6 +1,7 @@ import os import re +import shutil import helpers.github_action_utils as utils from utilities.base_utility import BaseUtility @@ -19,6 +20,16 @@ def _get_splunklib_version(self, file_path): utils.info("Error with getting the splunklib version.") + def remove_pycache(self, directory): + for root, dirs, files in os.walk(directory): + for file in files: + if file.endswith(".pyc"): + os.remove(os.path.join(root, file)) + for dir in dirs: + if dir == "__pycache__": + shutil.rmtree(os.path.join(root, dir)) + + def implement_utility(self): utils.info("Adding SplunkPythonSDKUtility") @@ -27,6 +38,10 @@ def implement_utility(self): if not splunk_python_sdk_install_path or splunk_python_sdk_install_path == "NONE": splunk_python_sdk_install_path = "bin" + is_remove_pyc_from_splunklib_dir = utils.str_to_boolean_default_true( + utils.get_input('is_remove_pyc_from_splunklib_dir')) + utils.info(f"is_remove_pyc_from_splunklib_dir: {is_remove_pyc_from_splunklib_dir}") + folder_to_install_splunklib = os.path.join( self.app_write_dir, splunk_python_sdk_install_path) @@ -60,6 +75,10 @@ def implement_utility(self): utils.execute_system_command( f'pip install splunk-sdk --target "{folder_to_install_splunklib}"') + # Removing .pyc and __pycache__ + if is_remove_pyc_from_splunklib_dir: + self.remove_pycache(folder_to_install_splunklib) + new_version = self._get_splunklib_version(init_file) utils.info(f"New splunklib version = {new_version}") diff --git a/tests/helper.py b/tests/helper.py index 4455ddf..2ae10e5 100644 --- a/tests/helper.py +++ b/tests/helper.py @@ -18,7 +18,8 @@ def setup_action_yml(test_app_repo, current_branch="NONE", logger_log_files_prefix="NONE", logger_sourcetype="NONE", - splunk_python_sdk_install_path="bin"): + splunk_python_sdk_install_path="bin", + is_remove_pyc_from_splunklib_dir="true"): app_dir_path = os.path.join(os.path.dirname(__file__), "test_app_repos", test_app_repo) print(f"TestIntegration.setup_action_yml_work -> app_dir_path={app_dir_path}") @@ -39,6 +40,7 @@ def setup_action_yml(test_app_repo, os.environ["SPLUNK_logger_log_files_prefix"] = logger_log_files_prefix os.environ["SPLUNK_logger_sourcetype"] = logger_sourcetype os.environ["SPLUNK_splunk_python_sdk_install_path"] = splunk_python_sdk_install_path + os.environ["SPLUNK_is_remove_pyc_from_splunklib_dir"] = is_remove_pyc_from_splunklib_dir try: yield diff --git a/tests/test_utility_python_sdk.py b/tests/test_utility_python_sdk.py index fb7075b..e5bde9e 100644 --- a/tests/test_utility_python_sdk.py +++ b/tests/test_utility_python_sdk.py @@ -6,6 +6,17 @@ from helpers import github_action_utils as utils +def check_no_pycache(folder_path): + for root, dirs, files in os.walk(folder_path): + for file in files: + if file.endswith(".pyc"): + return False + for dir in dirs: + if dir == "__pycache__": + return False + return True + + def test_splunk_sdk_utility_installed_new(): with get_temp_directory() as temp_dir: sdk_utility = SplunkPythonSDKUtility("nothing", temp_dir) @@ -16,16 +27,16 @@ def test_splunk_sdk_utility_installed_new(): splunklib_dir = os.path.join(folder_path, 'splunklib') init_file = os.path.join(splunklib_dir, '__init__.py') - print(f"init_file={init_file}") - assert result == init_file assert os.path.exists(init_file) + assert check_no_pycache(folder_path) def test_splunk_sdk_utility_installed_new_2(): with get_temp_directory() as temp_dir: with setup_temporary_env_vars({ - "SPLUNK_splunk_python_sdk_install_path": "lib", + "SPLUNK_splunk_python_sdk_install_path": "lib", + "SPLUNK_is_remove_pyc_from_splunklib_dir": "true" }): sdk_utility = SplunkPythonSDKUtility("nothing", temp_dir) result = sdk_utility.implement_utility() @@ -35,10 +46,9 @@ def test_splunk_sdk_utility_installed_new_2(): splunklib_dir = os.path.join(folder_path, 'splunklib') init_file = os.path.join(splunklib_dir, '__init__.py') - print(f"init_file={init_file}") - assert result == init_file assert os.path.exists(init_file) + assert check_no_pycache(folder_path) def test_splunk_sdk_utility_upgraded_existing(): @@ -66,6 +76,7 @@ def test_splunk_sdk_utility_upgraded_existing(): # Validate the result assert os.path.exists(init_file) assert result == init_file + assert check_no_pycache(folder_path) def test_splunk_sdk_utility_upgraded_existing_2(): @@ -96,6 +107,7 @@ def test_splunk_sdk_utility_upgraded_existing_2(): # Validate the result assert os.path.exists(init_file) assert result == init_file + assert check_no_pycache(folder_path) def test_splunk_sdk_utility_skipped_due_to_existing_and_same_version(): @@ -118,6 +130,7 @@ def test_splunk_sdk_utility_skipped_due_to_existing_and_same_version(): # Validate the result assert not os.path.exists(os.path.join(folder_path, 'splunk-sdk')) assert result is None + assert check_no_pycache(folder_path) def test_splunk_sdk_utility_skipped_due_to_existing_and_same_version_2(): @@ -143,29 +156,34 @@ def test_splunk_sdk_utility_skipped_due_to_existing_and_same_version_2(): # Validate the result assert not os.path.exists(os.path.join(folder_path, 'splunk-sdk')) assert result is None + assert check_no_pycache(folder_path) def test_splunk_sdk_utility_error_upgrading_existing(): with get_temp_directory() as temp_dir: - # Prepare the temporary directory - folder_path = os.path.join(temp_dir, 'bin') - os.makedirs(folder_path) - splunklib_dir = os.path.join(folder_path, 'splunklib') - os.makedirs(splunklib_dir) - init_file = os.path.join(splunklib_dir, '__init__.py') - with open(init_file, 'w') as f: - f.write('__version_info__ = "1.0.0"') + with setup_temporary_env_vars({ + "SPLUNK_is_remove_pyc_from_splunklib_dir": "true", + }): + # Prepare the temporary directory + folder_path = os.path.join(temp_dir, 'bin') + os.makedirs(folder_path) + splunklib_dir = os.path.join(folder_path, 'splunklib') + os.makedirs(splunklib_dir) + init_file = os.path.join(splunklib_dir, '__init__.py') + with open(init_file, 'w') as f: + f.write('__version_info__ = "1.0.0"') - # Initialize SplunkPythonSDKUtility instance - sdk_utility = SplunkPythonSDKUtility("nothing", temp_dir) + # Initialize SplunkPythonSDKUtility instance + sdk_utility = SplunkPythonSDKUtility("nothing", temp_dir) - # Call implement_utility function with failing upgrade - with patch.object(SplunkPythonSDKUtility, '_get_splunklib_version', return_value="2.2.2"): - with patch.object(utils, 'execute_system_command', return_value=(1, '')): - result = sdk_utility.implement_utility() + # Call implement_utility function with failing upgrade + with patch.object(SplunkPythonSDKUtility, '_get_splunklib_version', return_value="2.2.2"): + with patch.object(utils, 'execute_system_command', return_value=(1, '')): + result = sdk_utility.implement_utility() - # Validate the result - assert result is None + # Validate the result + assert result is None + assert check_no_pycache(folder_path) def test_splunk_sdk_utility_error_upgrading_existing_2(): @@ -192,3 +210,4 @@ def test_splunk_sdk_utility_error_upgrading_existing_2(): # Validate the result assert result is None + assert check_no_pycache(folder_path) From 6b147dd96a1b09e17c053ada4933b2b1c281e89e Mon Sep 17 00:00:00 2001 From: Vatsal Jagani Date: Sat, 30 Mar 2024 11:30:34 +0530 Subject: [PATCH 3/6] Readme file improvements. --- README.md | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index eb717a7..aab8077 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ graph TD * The action automatically generates build artifact from GitHub repo. ``` -- uses: VatsalJagani/splunk-app-action@v3 +- uses: VatsalJagani/splunk-app-action@v4 with: app_dir: "my_app" ``` @@ -57,11 +57,11 @@ graph TD * Supports multiple Apps/Add-ons in single repository. ``` - - uses: VatsalJagani/splunk-app-action@v3 + - uses: VatsalJagani/splunk-app-action@v4 with: app_dir: "my_splunk_app" - - uses: VatsalJagani/splunk-app-action@v3 + - uses: VatsalJagani/splunk-app-action@v4 with: app_dir: "my_splunk_add-on" ``` @@ -87,7 +87,7 @@ graph TD * Alternatively you can use `Running User Defined Commands Before Generating the final App Build` section here to see how you can assign right permission for your App/Add-on. * You can add `to_make_permission_changes: true` parameter to fix the issues with file and folder permissions to avoid App-inspect check automatically. ``` - - uses: VatsalJagani/splunk-app-action@v3 + - uses: VatsalJagani/splunk-app-action4 with: app_dir: "my_app" to_make_permission_changes: true @@ -109,7 +109,7 @@ graph TD * If you wish to run the user defined linux commands before generating the App build, set the environment variables `SPLUNK_APP_ACTION_`. ``` - - uses: VatsalJagani/splunk-app-action@v3 + - uses: VatsalJagani/splunk-app-action@v4 env: SPLUNK_APP_ACTION_1: "find . -type f -exec chmod 644 '{}' \\;" SPLUNK_APP_ACTION_2: "find . -type f -name '*.sh' -exec chmod +x '{}' \\;" @@ -125,7 +125,7 @@ graph TD * It allows you to run command before building the App build. * This could be useful if you wish to remove some files that you don't want in the build, change permission of some files before running the rest of the app build or app-inspect check. ``` - - uses: VatsalJagani/splunk-app-action@v3 + - uses: VatsalJagani/splunk-app-action4 env: SPLUNK_APP_ACTION_1: "rm -rf extra_test_folder" with: @@ -148,7 +148,7 @@ graph TD * It requires to set inputs: splunkbase_username and splunkbase_password. ``` -- uses: VatsalJagani/splunk-app-action@v3 +- uses: VatsalJagani/splunk-app-action4 with: app_dir: "my_app" splunkbase_username: ${{ secrets.SPLUNKBASE_USERNAME }} @@ -165,7 +165,7 @@ graph TD #### `whats_in_the_app` - Utility that adds information about the App inside the README.md file * The splunk-app-action has utility which automatically adds information about the App, like how many alerts does it have, how many dashboards does it have, etc inside the App's README.md file. ``` -- uses: VatsalJagani/splunk-app-action@v3 +- uses: VatsalJagani/splunk-app-action4 with: app_dir: "my_app" app_utilities: "whats_in_the_app" @@ -176,7 +176,7 @@ graph TD * Auto adds python logger manager, including python file necessary, props.conf to assign right sourcetype for it under the internal logs. ``` -- uses: VatsalJagani/splunk-app-action@v3 +- uses: VatsalJagani/splunk-app-action4 with: app_dir: "my_app" app_utilities: "logger" @@ -190,7 +190,7 @@ graph TD Example-1 ``` -- uses: VatsalJagani/splunk-app-action@v3 +- uses: VatsalJagani/splunk-app-action@v4 with: app_dir: "my_app" app_utilities: "splunk_python_sdk" @@ -199,7 +199,7 @@ Example-1 Example-2 ``` -- uses: VatsalJagani/splunk-app-action@v3 +- uses: VatsalJagani/splunk-app-action@v4 with: app_dir: "my_app" app_utilities: "splunk_python_sdk" @@ -212,7 +212,7 @@ Example-2 * This utility adds a JavaScript file that contains commonly used functionality for a JavaScript code for a Splunk App. ``` -- uses: VatsalJagani/splunk-app-action@v3 +- uses: VatsalJagani/splunk-app-action@v4 with: app_dir: "my_app" app_utilities: "common_js_utilities" @@ -223,7 +223,7 @@ Example-2 * This utility adds additional_packaging.py file that contains code to better generate input handler python file to easily re-generate code on change, rather than making manual changes. ``` -- uses: VatsalJagani/splunk-app-action@v3 +- uses: VatsalJagani/splunk-app-action@v4 with: app_dir: "." use_ucc_gen: true @@ -351,7 +351,7 @@ def stream_events(input_script: smi.Script, inputs: smi.InputDefinition, event_w * From `v4` of the `splunk-app-action`, your user-defined custom command (Refer to `Running User Defined Commands Before Generating the final App Build` section) would run in a context of your App's folder instead of root folder. * So you need to change the code from this: ``` - - uses: VatsalJagani/splunk-app-action@v3 + - uses: VatsalJagani/splunk-app-action@v4 env: SPLUNK_APP_ACTION_1: "rm -rf my_app/extra_test_folder" SPLUNK_APP_ACTION_2: "cat 'abc,123' >> my_app/lookups/my_custom_lookup.csv" @@ -360,7 +360,7 @@ def stream_events(input_script: smi.Script, inputs: smi.InputDefinition, event_w ``` * to ``` - - uses: VatsalJagani/splunk-app-action@v3 + - uses: VatsalJagani/splunk-app-action4 env: SPLUNK_APP_ACTION_1: "rm -rf extra_test_folder" SPLUNK_APP_ACTION_2: "cat 'abc,123' >> lookups/my_custom_lookup.csv" From 8b45dc8605b2389f947fe912b9d2ff7ea1a48ff7 Mon Sep 17 00:00:00 2001 From: Vatsal Jagani Date: Sat, 30 Mar 2024 11:35:00 +0530 Subject: [PATCH 4/6] Fixed app-build test-cases. --- DEV_README.md | 23 +++++-------- tests/test_build_generation.py | 63 +++++++++++++++++++++------------- 2 files changed, 47 insertions(+), 39 deletions(-) diff --git a/DEV_README.md b/DEV_README.md index 31ea651..b14d33a 100644 --- a/DEV_README.md +++ b/DEV_README.md @@ -1,17 +1,11 @@ -# README for Developers +# README for Developers of this action * Fork the project and submit a Pull Request if you would like to contribute to the project. * Automated tests for the GitHub action present in the separate repo here [splunk-app-action-test-app](https://github.com/VatsalJagani/splunk-app-action-test-app) -## How to Run Locally -``` -python3 src/main.py local_test -``` - - -# Notes for author +## How to generate tag * Create a new tag and push the tag from git: ``` @@ -33,13 +27,12 @@ python3 src/main.py local_test ``` -* Run below command to install all node JS dependencies. - * `npm install` - -* Run below command if you make any changes to JS file before committing the code to repo. - * `ncc build index.js --license LICENSE` - * This do not require if there is change in python file or action.yml file. - +## How to run python test-cases locally +``` +python -m pip install --upgrade pip +python -m pip install -r tests/requirements.txt +pytest tests --junitxml=junit/test-results.xml --cov=src --cov-config=tests/.coveragerc --cov-report=xml +``` ## Note from actions/upload-artifact - Zipped Artifact Downloads diff --git a/tests/test_build_generation.py b/tests/test_build_generation.py index de1d7ec..8fcc132 100644 --- a/tests/test_build_generation.py +++ b/tests/test_build_generation.py @@ -58,6 +58,13 @@ def get_file_permissions(filepath): return file_permissions +def remove_ds_store_files(directory): + for root, dirs, files in os.walk(directory): + for file in files: + if file == ".DS_Store": + os.remove(os.path.join(root, file)) + + class TestAppBuild(unittest.TestCase): def extract_app_build(self, tgz_file): @@ -70,10 +77,11 @@ def extract_app_build(self, tgz_file): with tarfile.open(tgz_file, 'r:gz') as tar: tar.extractall(extract_dir) + remove_ds_store_files(extract_dir) + # Initialize counters - file_count = 0 - folder_count = 0 all_files = [] + all_folders = [] is_root = True # Walk through the extracted directory @@ -82,19 +90,20 @@ def extract_app_build(self, tgz_file): is_root = False continue - folder_count += len(dirs) - file_count += len(files) - - # Print relative paths of files for file in files: relative_path = os.path.relpath(os.path.join(root, file), extract_dir) # print("DEBUG: File:", relative_path) all_files.append(relative_path) - # print("DEBUG: Total Files:", file_count) - # print("DEBUG: Total Folders:", folder_count) - # print(f"All Files: {all_files}") - return file_count, folder_count, all_files + for dir in dirs: + relative_path = os.path.relpath(os.path.join(root, dir), extract_dir) + all_folders.append(relative_path) + + print("DEBUG: Total Files:", len(all_files)) + print("DEBUG: Total Folders:", len(all_folders)) + print(f"DEBUG: All Files: {all_files}") + print(f"DEBUG: All Folders: {all_folders}") + return len(all_files), len(all_folders), all_files, all_folders finally: # Cleanup: Remove the temporary extraction directory @@ -115,7 +124,7 @@ def test_build_1_regular(self): app_build_name = "my_app_1_1_1_2_1.tgz" assert os.path.isfile(app_build_name) - file_count, folder_count, all_files = self.extract_app_build(app_build_name) + file_count, folder_count, all_files, _ = self.extract_app_build(app_build_name) assert folder_count == 7 assert file_count == 10 assert "my_app_1/static/appIconAlt.png" in all_files @@ -130,7 +139,7 @@ def test_build_repo_root_as_app_dir(self): app_build_name = "my_app_1_1_1_2_1.tgz" assert os.path.isfile(app_build_name) - file_count, folder_count, all_files = self.extract_app_build(app_build_name) + file_count, folder_count, all_files, _ = self.extract_app_build(app_build_name) assert folder_count == 7 assert file_count == 10 assert "my_app_1/static/appIconAlt.png" in all_files @@ -151,9 +160,7 @@ def test_ucc_build_1_regular(self): app_build_name = app_build_files[0] assert os.path.isfile(app_build_name) - file_count, folder_count, all_files = self.extract_app_build(app_build_name) - assert folder_count == 57 - assert file_count == 362 + _, _, all_files, all_folders = self.extract_app_build(app_build_name) assert "my_app_ucc_1/default/app.conf" in all_files assert "my_app_ucc_1/appserver/static/js/build/globalConfig.json" in all_files assert "my_app_ucc_1/lib/splunklib/client.py" in all_files @@ -164,6 +171,11 @@ def test_ucc_build_1_regular(self): assert all("my_app_ucc_1/globalConfig.json" not in s for s in all_files), "'globalConfig.json' file shouldn't be in the root of the App." assert all("additional_packaging.py" not in s for s in all_files), "'additional_packaging.py' file shouldn't be part of the App build." + assert "my_app_ucc_1/lib/splunklib/modularinput" in all_folders + assert "my_app_ucc_1/lib/solnlib" in all_folders + assert "my_app_ucc_1/appserver/static/js/build" in all_folders + assert "my_app_ucc_1/lib/splunktaucclib/common" in all_folders + def test_ucc_build_repo_root_as_app_dir(self): with setup_action_yml("repo_ucc_2_repo_root_as_app_dir", app_dir=".", use_ucc_gen="true", is_app_inspect_check="false"): @@ -178,9 +190,7 @@ def test_ucc_build_repo_root_as_app_dir(self): app_build_name = app_build_files[0] assert os.path.isfile(app_build_name) - file_count, folder_count, all_files = self.extract_app_build(app_build_name) - assert folder_count == 57 - assert file_count == 362 + _, _, all_files, all_folders = self.extract_app_build(app_build_name) assert "my_app_ucc_1/default/app.conf" in all_files assert "my_app_ucc_1/appserver/static/js/build/globalConfig.json" in all_files assert "my_app_ucc_1/lib/splunklib/client.py" in all_files @@ -191,6 +201,11 @@ def test_ucc_build_repo_root_as_app_dir(self): assert all("my_app_ucc_1/globalConfig.json" not in s for s in all_files), "'globalConfig.json' file shouldn't be in the root of the App." assert all("additional_packaging.py" not in s for s in all_files), "'additional_packaging.py' file shouldn't be part of the App build." + assert "my_app_ucc_1/lib/splunklib/modularinput" in all_folders + assert "my_app_ucc_1/lib/solnlib" in all_folders + assert "my_app_ucc_1/appserver/static/js/build" in all_folders + assert "my_app_ucc_1/lib/splunktaucclib/common" in all_folders + def test_file_permission_check_no_change(self): with setup_action_yml("repo_file_permission", app_dir="my_app_2", is_app_inspect_check="false"): @@ -199,7 +214,7 @@ def test_file_permission_check_no_change(self): app_build_name = "my_app_2_1_1_2_1.tgz" assert os.path.isfile(app_build_name) - _, _, all_files = self.extract_app_build(app_build_name) + _, _, all_files, all_folders = self.extract_app_build(app_build_name) assert "my_app_2/bin/file1.sh" in all_files assert get_file_permissions("my_app_2/bin/file1.sh") == "rwxr-xr-x" assert "my_app_2/bin/file2.sh" in all_files @@ -217,7 +232,7 @@ def test_file_auto_change_permission(self): app_build_name = "my_app_2_1_1_2_1.tgz" assert os.path.isfile(app_build_name) - file_count, folder_count, all_files = self.extract_app_build(app_build_name) + file_count, folder_count, all_files, all_folders = self.extract_app_build(app_build_name) assert "my_app_2/bin/file1.sh" in all_files assert get_file_permissions("my_app_2/bin/file1.sh") == "rwxr-xr-x" assert "my_app_2/bin/file2.sh" in all_files @@ -235,7 +250,7 @@ def test_file_permission_check_no_change_2(self): app_build_name = "my_app_2_1_1_2_1.tgz" assert os.path.isfile(app_build_name) - _, _, all_files = self.extract_app_build(app_build_name) + _, _, all_files, all_folders = self.extract_app_build(app_build_name) assert "my_app_2/bin/file1.sh" in all_files assert get_file_permissions("my_app_2/bin/file1.sh") == "rwxr-xr-x" assert "my_app_2/bin/file2.sh" in all_files @@ -253,7 +268,7 @@ def test_file_auto_change_permission_2(self): app_build_name = "my_app_2_1_1_2_1.tgz" assert os.path.isfile(app_build_name) - _, _, all_files = self.extract_app_build(app_build_name) + _, _, all_files, all_folders = self.extract_app_build(app_build_name) assert "my_app_2/bin/file1.sh" in all_files assert get_file_permissions("my_app_2/bin/file1.sh") == "rwxr-xr-x" assert "my_app_2/bin/file2.sh" in all_files @@ -279,7 +294,7 @@ def test_file_permission_change_via_user_commands(self): app_build_name = "my_app_2_1_1_2_1.tgz" assert os.path.isfile(app_build_name) - _, _, all_files = self.extract_app_build(app_build_name) + _, _, all_files, all_folders = self.extract_app_build(app_build_name) assert "my_app_2/bin/file1.sh" in all_files assert get_file_permissions("my_app_2/bin/file1.sh") == "rwxr-xr-x" assert "my_app_2/bin/file2.sh" in all_files @@ -305,7 +320,7 @@ def test_file_permission_change_via_user_commands_root_dir(self): app_build_name = "my_app_2_1_1_2_1.tgz" assert os.path.isfile(app_build_name) - _, _, all_files = self.extract_app_build(app_build_name) + _, _, all_files, all_folders = self.extract_app_build(app_build_name) assert "my_app_2/bin/file1.sh" in all_files assert get_file_permissions("my_app_2/bin/file1.sh") == "rwxr-xr-x" assert "my_app_2/bin/file2.sh" in all_files From 901477204853acfdd9808130e1583ba3ed23539f Mon Sep 17 00:00:00 2001 From: Vatsal Jagani Date: Sat, 30 Mar 2024 11:56:24 +0530 Subject: [PATCH 5/6] Minor python code improvement. --- tests/test_build_generation.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_build_generation.py b/tests/test_build_generation.py index 8fcc132..c207168 100644 --- a/tests/test_build_generation.py +++ b/tests/test_build_generation.py @@ -214,7 +214,7 @@ def test_file_permission_check_no_change(self): app_build_name = "my_app_2_1_1_2_1.tgz" assert os.path.isfile(app_build_name) - _, _, all_files, all_folders = self.extract_app_build(app_build_name) + _, _, all_files, _ = self.extract_app_build(app_build_name) assert "my_app_2/bin/file1.sh" in all_files assert get_file_permissions("my_app_2/bin/file1.sh") == "rwxr-xr-x" assert "my_app_2/bin/file2.sh" in all_files @@ -232,7 +232,7 @@ def test_file_auto_change_permission(self): app_build_name = "my_app_2_1_1_2_1.tgz" assert os.path.isfile(app_build_name) - file_count, folder_count, all_files, all_folders = self.extract_app_build(app_build_name) + _, _, all_files, _ = self.extract_app_build(app_build_name) assert "my_app_2/bin/file1.sh" in all_files assert get_file_permissions("my_app_2/bin/file1.sh") == "rwxr-xr-x" assert "my_app_2/bin/file2.sh" in all_files @@ -250,7 +250,7 @@ def test_file_permission_check_no_change_2(self): app_build_name = "my_app_2_1_1_2_1.tgz" assert os.path.isfile(app_build_name) - _, _, all_files, all_folders = self.extract_app_build(app_build_name) + _, _, all_files, _ = self.extract_app_build(app_build_name) assert "my_app_2/bin/file1.sh" in all_files assert get_file_permissions("my_app_2/bin/file1.sh") == "rwxr-xr-x" assert "my_app_2/bin/file2.sh" in all_files @@ -268,7 +268,7 @@ def test_file_auto_change_permission_2(self): app_build_name = "my_app_2_1_1_2_1.tgz" assert os.path.isfile(app_build_name) - _, _, all_files, all_folders = self.extract_app_build(app_build_name) + _, _, all_files, _ = self.extract_app_build(app_build_name) assert "my_app_2/bin/file1.sh" in all_files assert get_file_permissions("my_app_2/bin/file1.sh") == "rwxr-xr-x" assert "my_app_2/bin/file2.sh" in all_files @@ -294,7 +294,7 @@ def test_file_permission_change_via_user_commands(self): app_build_name = "my_app_2_1_1_2_1.tgz" assert os.path.isfile(app_build_name) - _, _, all_files, all_folders = self.extract_app_build(app_build_name) + _, _, all_files, _ = self.extract_app_build(app_build_name) assert "my_app_2/bin/file1.sh" in all_files assert get_file_permissions("my_app_2/bin/file1.sh") == "rwxr-xr-x" assert "my_app_2/bin/file2.sh" in all_files @@ -320,7 +320,7 @@ def test_file_permission_change_via_user_commands_root_dir(self): app_build_name = "my_app_2_1_1_2_1.tgz" assert os.path.isfile(app_build_name) - _, _, all_files, all_folders = self.extract_app_build(app_build_name) + _, _, all_files, _ = self.extract_app_build(app_build_name) assert "my_app_2/bin/file1.sh" in all_files assert get_file_permissions("my_app_2/bin/file1.sh") == "rwxr-xr-x" assert "my_app_2/bin/file2.sh" in all_files From 823656ddc71c0088b20c9ef4639277dab2411058 Mon Sep 17 00:00:00 2001 From: Vatsal Jagani Date: Sat, 30 Mar 2024 11:59:09 +0530 Subject: [PATCH 6/6] Readme typo fix. --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index aab8077..6f4cbe0 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ graph TD * Alternatively you can use `Running User Defined Commands Before Generating the final App Build` section here to see how you can assign right permission for your App/Add-on. * You can add `to_make_permission_changes: true` parameter to fix the issues with file and folder permissions to avoid App-inspect check automatically. ``` - - uses: VatsalJagani/splunk-app-action4 + - uses: VatsalJagani/splunk-app-action@v4 with: app_dir: "my_app" to_make_permission_changes: true @@ -125,7 +125,7 @@ graph TD * It allows you to run command before building the App build. * This could be useful if you wish to remove some files that you don't want in the build, change permission of some files before running the rest of the app build or app-inspect check. ``` - - uses: VatsalJagani/splunk-app-action4 + - uses: VatsalJagani/splunk-app-action@v4 env: SPLUNK_APP_ACTION_1: "rm -rf extra_test_folder" with: @@ -148,7 +148,7 @@ graph TD * It requires to set inputs: splunkbase_username and splunkbase_password. ``` -- uses: VatsalJagani/splunk-app-action4 +- uses: VatsalJagani/splunk-app-action@v4 with: app_dir: "my_app" splunkbase_username: ${{ secrets.SPLUNKBASE_USERNAME }} @@ -165,7 +165,7 @@ graph TD #### `whats_in_the_app` - Utility that adds information about the App inside the README.md file * The splunk-app-action has utility which automatically adds information about the App, like how many alerts does it have, how many dashboards does it have, etc inside the App's README.md file. ``` -- uses: VatsalJagani/splunk-app-action4 +- uses: VatsalJagani/splunk-app-action@v4 with: app_dir: "my_app" app_utilities: "whats_in_the_app" @@ -176,7 +176,7 @@ graph TD * Auto adds python logger manager, including python file necessary, props.conf to assign right sourcetype for it under the internal logs. ``` -- uses: VatsalJagani/splunk-app-action4 +- uses: VatsalJagani/splunk-app-action@v4 with: app_dir: "my_app" app_utilities: "logger" @@ -360,7 +360,7 @@ def stream_events(input_script: smi.Script, inputs: smi.InputDefinition, event_w ``` * to ``` - - uses: VatsalJagani/splunk-app-action4 + - uses: VatsalJagani/splunk-app-action@v4 env: SPLUNK_APP_ACTION_1: "rm -rf extra_test_folder" SPLUNK_APP_ACTION_2: "cat 'abc,123' >> lookups/my_custom_lookup.csv"