From 98e66b065f07d57dee7aa6ca841ab2701f3bedf5 Mon Sep 17 00:00:00 2001 From: Gabe Date: Mon, 23 Dec 2024 20:21:09 +0100 Subject: [PATCH 01/13] Make backup_appdata work if container needs file binds In rare cases a container image requires binding a file directly, which the script takes as source_dir for the backup. This adds a check, if the directory the scripts grabs is a filename and strips the filename. i.e. ./backup_appdata.sh: line 185: cd: /appdata/container/container.conf/..: Not a directory stripes the filename (container.conf), so the script backups /appdata/container/ and its content properly. --- extra-scripts/backup_appdata.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/extra-scripts/backup_appdata.sh b/extra-scripts/backup_appdata.sh index 8d02dfe..c68f7bc 100755 --- a/extra-scripts/backup_appdata.sh +++ b/extra-scripts/backup_appdata.sh @@ -182,6 +182,10 @@ create_backup() { # Create the backup file name backup_file="$(realpath -s "$destination_dir")/$(date +%F)@$now/$container_name" # Go to the source directory + # Check if source_dir is a directory, if not, strip the filename + if [ ! -d "$source_dir" ]; then + source_dir=$(dirname "$source_dir") + fi cd "$source_dir"/.. || return # Get the name of the source directory source_dir=$(basename "$source_dir") @@ -980,4 +984,4 @@ main() { fi } -main \ No newline at end of file +main From 56d50f4d3a0303fda1ffea05582e5ce5f0797dc3 Mon Sep 17 00:00:00 2001 From: Matthew van der Hoorn Date: Fri, 10 Jan 2025 18:15:40 +0100 Subject: [PATCH 02/13] Fix file write encoding errors You got UnicodeException when writing tables to the log file. This fixes that by explicitly setting the encoding to utf-8. --- util/logger.py | 1 + 1 file changed, 1 insertion(+) diff --git a/util/logger.py b/util/logger.py index 136ea22..eda657c 100644 --- a/util/logger.py +++ b/util/logger.py @@ -67,6 +67,7 @@ def setup_logger(log_level, script_name, max_logs=9): # Create a RotatingFileHandler for log files handler = RotatingFileHandler(log_file, delay=True, mode="w", backupCount=max_logs) + handler.encoding = "utf-8" handler.setFormatter(formatter) # Add the file handler to the logger From 063834578929af719cf8b842eeba6a8a3be81c32 Mon Sep 17 00:00:00 2001 From: Matthew van der Hoorn Date: Fri, 10 Jan 2025 18:54:00 +0100 Subject: [PATCH 03/13] Fixed Logging Rotation Issue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When getting to max logs, you would get an exception where it tries to rename a file into an existing file. I have fixed this by replacing the entire rotation logic with my own, inspired by my own logging framework in Python. This is proven to work. ⚠️ I have not tested if this works in a Docker configuration. ⚠️ --- scripts/nohl_bash.sh | 2 +- util/hoorn_lib_common/__init__.py | 0 util/hoorn_lib_common/file_handler.py | 88 +++++++++++++++++++++++++++ util/hoorn_lib_common/log_rotator.py | 81 ++++++++++++++++++++++++ util/logger.py | 28 +++------ 5 files changed, 180 insertions(+), 19 deletions(-) create mode 100644 util/hoorn_lib_common/__init__.py create mode 100644 util/hoorn_lib_common/file_handler.py create mode 100644 util/hoorn_lib_common/log_rotator.py diff --git a/scripts/nohl_bash.sh b/scripts/nohl_bash.sh index f945f41..777269f 100755 --- a/scripts/nohl_bash.sh +++ b/scripts/nohl_bash.sh @@ -87,7 +87,7 @@ log_file() { # remove trailing slash from source_dir if it exists source_dir=${source_dir%%/} - log_file=$log_dir/nohl_bash/nohl.log + log_file=$log_dir/nohl_bash/log_0.txt echo "Log directory: $log_dir" echo "Log file: $log_file" diff --git a/util/hoorn_lib_common/__init__.py b/util/hoorn_lib_common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/util/hoorn_lib_common/file_handler.py b/util/hoorn_lib_common/file_handler.py new file mode 100644 index 0000000..edcf4e4 --- /dev/null +++ b/util/hoorn_lib_common/file_handler.py @@ -0,0 +1,88 @@ +# See: https://github.com/LordMartron94/py-common/blob/main/py_common/handlers/file_handler.py + +import os +from pathlib import Path +from typing import List, Union + + +class FileHandler: + def is_file_of_type(self, file: Union[Path, str], extension: str) -> bool: + return Path(file).is_file() and str(file).lower().endswith(extension) + + def get_file_modified_date(self, file_path: str) -> float: + """Get the modification time of a file.""" + return os.path.getmtime(file_path) + + def get_number_of_files_in_dir(self, directory: Path, extension: str = "*") -> int: + if not directory.is_dir(): + raise ValueError("The provided path is not a valid directory.") + + # Count the number of files in the folder + if extension == "*": + num_files = sum(1 for _ in directory.iterdir() if _.is_file()) + else: + num_files = sum(1 for _ in directory.iterdir() if self.is_file_of_type(_, extension)) + + return num_files + + def get_children_paths(self, directory: Path, extension: str = "*", recursive=False) -> List[Path]: + """ + Gets all the files with the given extension in the directory. + + Args: + directory: The directory to search in. + extension: The extension of the files to search for. + Defaults to '*', to search for all files. + recursive: Whether to search subfolders recursively. + + Returns: + A list of paths to the files with the given extension. + + Raises: + ValueError: If the provided path is not a valid directory. + """ + if not directory.is_dir(): + raise ValueError("The provided path is not a valid directory.") + + paths: List[Path] = [] + items = [directory] # Initialize with the starting directory + + num_processed_files: int = 0 + + while items: + current_item = items.pop() + if current_item.is_dir(): + if recursive or num_processed_files == 0: + # Add subdirectories to the list for iterative processing + items.extend(current_item.iterdir()) + elif extension == "*" or (current_item.suffix == extension): + paths.append(current_item) + + num_processed_files += 1 + + return paths + + def save_dict_to_file(self, data: dict, file_path: Path, header: str = None): + with open(file_path, 'w') as file: + if header is not None: + file.write(f"{header}\n\n") + + for key, value in data.items(): + file.write(f"{key}: {value}\n") + + def get_children_directories(self, root_dir: Path, recursive:bool = False) -> List[Path]: + if not root_dir.is_dir(): + raise ValueError("The provided path is not a valid directory.") + + directories: List[Path] = [] + items = [] + items.extend(root_dir.iterdir()) + + while items: + current_item = items.pop() + if current_item.is_dir(): + directories.append(current_item) + if recursive: + items.extend(current_item.iterdir()) + + return directories \ No newline at end of file diff --git a/util/hoorn_lib_common/log_rotator.py b/util/hoorn_lib_common/log_rotator.py new file mode 100644 index 0000000..612a7bb --- /dev/null +++ b/util/hoorn_lib_common/log_rotator.py @@ -0,0 +1,81 @@ +# Written by Mr. Hoorn to fix the weird and problematic log rotation logic in the original code. +# Inspired by the FileHoornLogOutput of Mr. Hoorn's common library. +# See: https://github.com/LordMartron94/py-common/blob/main/py_common/logging/output/file_hoorn_log_output.py +import os +from pathlib import Path +from typing import List + +from util.hoorn_lib_common.file_handler import FileHandler + + +class LogRotator: + def __init__(self, log_directory: Path, max_logs_to_keep: int = 3, create_directory: bool = True): + """ + Rotates logs around based on the max logs to keep. + + :param log_directory: The base directory for logs. + :param max_logs_to_keep: The max number of logs to keep (per directory). + :param create_directory: Whether to initialize the creation of log directories if they don't exist. + """ + + self._file_handler: FileHandler = FileHandler() + + self._root_log_directory: Path = log_directory + self._max_logs_to_keep: int = max_logs_to_keep + + self._validate_directory(self._root_log_directory, create_directory) + + def _validate_directory(self, directory: Path, create_directory: bool): + if not directory.exists(): + if create_directory: + directory.mkdir(parents=True, exist_ok=True) + return + + raise FileNotFoundError(f"Log directory {directory} does not exist") + + def increment_logs(self) -> None: + """ + Increments the log number by 1 and removes old logs if necessary. + + :return: None + """ + + children = self._file_handler.get_children_paths(self._root_log_directory, ".txt", recursive=True) + children.sort(reverse=True) + + organized_by_separator: List[List[Path]] = self._organize_logs_by_subdirectory(children) + + for directory_logs in organized_by_separator: + self._increment_logs_in_directory(directory_logs) + + def _increment_logs_in_directory(self, log_files: List[Path]) -> None: + for i in range(len(log_files)): + child = log_files[i] + number = int(child.stem.split("_")[-1]) + if number + 1 > self._max_logs_to_keep: + os.remove(child) + continue + + os.rename(child, Path.joinpath(child.parent.absolute(), f"log_{number + 1}.txt")) + + def _organize_logs_by_subdirectory(self, log_paths: List[Path]) -> List[List[Path]]: + """ + Organizes a list of log file paths into a list of lists, + where each sublist contains logs from the same subdirectory. + + Args: + log_paths: A list of WindowsPath objects representing log file paths. + + Returns: + A list of lists, where each sublist contains log paths from the same subdirectory. + """ + log_groups = {} + for log_path in log_paths: + parent_dir = log_path.parent.name + if parent_dir not in log_groups: + log_groups[parent_dir] = [] + log_groups[parent_dir].append(log_path) + return list(log_groups.values()) + + def get_log_file(self) -> str: + return f"{self._root_log_directory}/log_0.txt" diff --git a/util/logger.py b/util/logger.py index eda657c..5689f4d 100644 --- a/util/logger.py +++ b/util/logger.py @@ -1,12 +1,13 @@ import os -import time import logging -import pathlib from logging.handlers import RotatingFileHandler +from pathlib import Path + +from util.hoorn_lib_common.log_rotator import LogRotator from util.version import get_version from util.utility import create_bar -def setup_logger(log_level, script_name, max_logs=9): +def setup_logger(log_level, script_name, max_logs=10): """ Setup the logger. @@ -21,9 +22,9 @@ def setup_logger(log_level, script_name, max_logs=9): if os.environ.get('DOCKER_ENV'): config_dir = os.getenv('CONFIG_DIR', '/config') - log_dir = f"{config_dir}/logs/{script_name}" + log_dir: str = f"{config_dir}/logs/{script_name}" else: - log_dir = f"{os.path.join(pathlib.Path(__file__).parents[1], 'logs', script_name)}" + log_dir = f"{os.path.join(Path(__file__).parents[1], 'logs', script_name)}" if log_level not in ['debug', 'info', 'critical']: log_level = 'info' @@ -32,19 +33,10 @@ def setup_logger(log_level, script_name, max_logs=9): # Create the log directory if it doesn't exist if not os.path.exists(log_dir): os.makedirs(log_dir) - - - # Define the log file path with the current date - log_file = f"{log_dir}/{script_name}.log" - - # Check if log file already exists - if os.path.isfile(log_file): - for i in range(max_logs - 1, 0, -1): - old_log = f"{log_dir}/{script_name}.log.{i}" - new_log = f"{log_dir}/{script_name}.log.{i + 1}" - if os.path.exists(old_log): - os.rename(old_log, new_log) - os.rename(log_file, f"{log_dir}/{script_name}.log.1") + + rotator: LogRotator = LogRotator(log_directory=Path(log_dir), max_logs_to_keep=max_logs, create_directory=True) + rotator.increment_logs() + log_file = rotator.get_log_file() # Create a logger object with the script name logger = logging.getLogger(script_name) From d8ebf6e91c5ff38a93da9d6cfb102d0beedfbb28 Mon Sep 17 00:00:00 2001 From: Matthew van der Hoorn Date: Fri, 10 Jan 2025 19:01:38 +0100 Subject: [PATCH 04/13] Removed unnecessary function call --- main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/main.py b/main.py index 505be8b..498ef5c 100755 --- a/main.py +++ b/main.py @@ -155,7 +155,6 @@ def main(): old_schedule = None running_scripts = {} waiting_message_shown = False - scripts_schedules=load_schedule() if len(sys.argv) > 1: for input_name in sys.argv[1:]: if input_name in list_of_bash_scripts or input_name in list_of_python_scripts: From c748891d82333fa17fbfa76df856dbf5e1b60abb Mon Sep 17 00:00:00 2001 From: Matthew van der Hoorn Date: Fri, 10 Jan 2025 19:02:55 +0100 Subject: [PATCH 05/13] Optimize Imports Automatic Jetbrains IDEA Ultimate run to remove unnecessary imports (useful for decluttering) --- modules/bash_scripts.py | 9 +++++---- modules/border_replacerr.py | 9 +++------ modules/health_checkarr.py | 5 +---- modules/labelarr.py | 10 ++++------ modules/nohl.py | 5 +---- modules/poster_cleanarr.py | 6 +----- modules/poster_renamerr.py | 9 +++------ modules/renameinatorr.py | 4 +--- modules/sync_gdrive.py | 7 +++---- modules/unmatched_assets.py | 5 ++--- modules/upgradinatorr.py | 3 +-- util/arrpy.py | 4 +--- util/call_script.py | 4 +++- util/config.py | 7 +++---- util/discord.py | 6 ++++-- util/logger.py | 5 +++-- util/scheduler.py | 1 + util/utility.py | 8 +++----- util/version.py | 4 +++- 19 files changed, 46 insertions(+), 65 deletions(-) diff --git a/modules/bash_scripts.py b/modules/bash_scripts.py index cd73f11..3172e61 100755 --- a/modules/bash_scripts.py +++ b/modules/bash_scripts.py @@ -1,12 +1,13 @@ -import shlex import json +import pathlib +import shlex import sys -from util.config import Config + from util.call_script import * from util.discord import get_discord_data, discord_check -from util.utility import create_bar from util.logger import setup_logger -import pathlib +from util.utility import create_bar + def set_cmd_args(settings, bash_script_file, logger, script_name): """ diff --git a/modules/border_replacerr.py b/modules/border_replacerr.py index 7141507..b28646c 100755 --- a/modules/border_replacerr.py +++ b/modules/border_replacerr.py @@ -14,17 +14,14 @@ # License: MIT License # ================================================================================= -import os -import json -import re -import logging import filecmp +import logging import shutil import sys -from util.utility import * -from util.scheduler import check_schedule from util.logger import setup_logger +from util.scheduler import check_schedule +from util.utility import * try: from tqdm import tqdm diff --git a/modules/health_checkarr.py b/modules/health_checkarr.py index 998b49b..0af0d40 100644 --- a/modules/health_checkarr.py +++ b/modules/health_checkarr.py @@ -14,14 +14,11 @@ # License: MIT License # =================================================================================================== -import json -import re import sys from util.arrpy import StARR -from util.utility import * -from util.discord import discord from util.logger import setup_logger +from util.utility import * try: from tqdm import tqdm diff --git a/modules/labelarr.py b/modules/labelarr.py index 65baa9f..fb1a42a 100755 --- a/modules/labelarr.py +++ b/modules/labelarr.py @@ -12,16 +12,14 @@ # License: MIT License # ====================================================================================== -import json -import time import sys +import time -from util.discord import discord, discord_check from util.arrpy import StARR -from util.utility import * +from util.discord import discord, discord_check from util.logger import setup_logger -import re - +from util.utility import * + try: from plexapi.server import PlexServer from plexapi.exceptions import BadRequest diff --git a/modules/nohl.py b/modules/nohl.py index cfbe794..553e5b1 100755 --- a/modules/nohl.py +++ b/modules/nohl.py @@ -16,15 +16,12 @@ # License: MIT License # =================================================================================================== -import os -import re import sys -import json from util.arrpy import StARR from util.discord import discord, discord_check -from util.utility import * from util.logger import setup_logger +from util.utility import * try: from tqdm import tqdm diff --git a/modules/poster_cleanarr.py b/modules/poster_cleanarr.py index 64e97a3..a82eb59 100755 --- a/modules/poster_cleanarr.py +++ b/modules/poster_cleanarr.py @@ -16,16 +16,12 @@ # License: MIT License # =========================================================================================================== -import os -import re -import json -import logging import shutil import sys -from util.utility import * from util.arrpy import StARR from util.logger import setup_logger +from util.utility import * try: from plexapi.server import PlexServer diff --git a/modules/poster_renamerr.py b/modules/poster_renamerr.py index 44e2201..db95a5e 100755 --- a/modules/poster_renamerr.py +++ b/modules/poster_renamerr.py @@ -14,18 +14,15 @@ # License: MIT License # =================================================================================================== -import os -import sys -import re -import json import filecmp import shutil +import sys import time -from util.utility import * -from util.discord import discord, discord_check from util.arrpy import StARR +from util.discord import discord, discord_check from util.logger import setup_logger +from util.utility import * try: from plexapi.server import PlexServer diff --git a/modules/renameinatorr.py b/modules/renameinatorr.py index b20c829..a05f2a3 100755 --- a/modules/renameinatorr.py +++ b/modules/renameinatorr.py @@ -14,15 +14,13 @@ # License: MIT License # =================================================================================================== -import json -import re import sys import time from util.arrpy import StARR -from util.utility import * from util.discord import discord, discord_check from util.logger import setup_logger +from util.utility import * try: from tqdm import tqdm diff --git a/modules/sync_gdrive.py b/modules/sync_gdrive.py index e7a833f..83c3379 100755 --- a/modules/sync_gdrive.py +++ b/modules/sync_gdrive.py @@ -1,12 +1,11 @@ -import shlex import json import os +import shlex +import sys from util.call_script import call_script -from util.utility import create_bar from util.logger import setup_logger -import sys - +from util.utility import create_bar script_name = "sync_gdrive" diff --git a/modules/unmatched_assets.py b/modules/unmatched_assets.py index d1b29c3..f7ae735 100755 --- a/modules/unmatched_assets.py +++ b/modules/unmatched_assets.py @@ -17,12 +17,11 @@ # License: MIT License # =========================================================================================================== -import json -import os import sys -from util.utility import * + from util.arrpy import StARR from util.logger import setup_logger +from util.utility import * try: from plexapi.server import PlexServer diff --git a/modules/upgradinatorr.py b/modules/upgradinatorr.py index 661fda1..fb769fa 100755 --- a/modules/upgradinatorr.py +++ b/modules/upgradinatorr.py @@ -14,14 +14,13 @@ # License: MIT License # =================================================================================================== -import json import sys import time from util.arrpy import StARR from util.discord import discord, discord_check -from util.utility import * from util.logger import setup_logger +from util.utility import * script_name = "upgradinatorr" diff --git a/util/arrpy.py b/util/arrpy.py index 712950d..f3cef16 100755 --- a/util/arrpy.py +++ b/util/arrpy.py @@ -1,7 +1,5 @@ -import sys -import time -import json import logging +import time try: import requests diff --git a/util/call_script.py b/util/call_script.py index 17eda43..61e2332 100755 --- a/util/call_script.py +++ b/util/call_script.py @@ -1,9 +1,11 @@ -from subprocess import PIPE, STDOUT, CalledProcessError, CompletedProcess, Popen +from subprocess import Popen, PIPE, STDOUT, CompletedProcess, CalledProcessError +from util.utility import redact_sensitive_info from subprocess import Popen, PIPE, STDOUT, CompletedProcess, CalledProcessError from util.utility import redact_sensitive_info + def call_script(command, logger): """ Run a bash script diff --git a/util/config.py b/util/config.py index 9c59c1e..ad735aa 100755 --- a/util/config.py +++ b/util/config.py @@ -1,9 +1,8 @@ -import pathlib +import time + import yaml -import os -from pathlib import Path + from util.utility import * -import time # Set the config file path if os.environ.get('DOCKER_ENV'): diff --git a/util/discord.py b/util/discord.py index a69f821..4322227 100755 --- a/util/discord.py +++ b/util/discord.py @@ -5,10 +5,12 @@ print("Please install the required modules with 'pip install -r requirements.txt'") exit(1) -import requests -import random import json +import random from datetime import datetime + +import requests + from util.config import Config config = Config(script_name="discord") diff --git a/util/logger.py b/util/logger.py index 5689f4d..b65b514 100644 --- a/util/logger.py +++ b/util/logger.py @@ -1,11 +1,12 @@ -import os import logging +import os from logging.handlers import RotatingFileHandler from pathlib import Path from util.hoorn_lib_common.log_rotator import LogRotator -from util.version import get_version from util.utility import create_bar +from util.version import get_version + def setup_logger(log_level, script_name, max_logs=10): """ diff --git a/util/scheduler.py b/util/scheduler.py index ea9f089..ca2e214 100755 --- a/util/scheduler.py +++ b/util/scheduler.py @@ -1,4 +1,5 @@ from datetime import datetime + from croniter import croniter from dateutil import tz diff --git a/util/utility.py b/util/utility.py index 1b5ddd1..a077dca 100755 --- a/util/utility.py +++ b/util/utility.py @@ -1,10 +1,8 @@ -import re -import os -import json -from pathlib import Path -import subprocess import math +import os import pathlib +import re +import subprocess try: import html diff --git a/util/version.py b/util/version.py index 8cbe7ae..84af4ec 100755 --- a/util/version.py +++ b/util/version.py @@ -1,8 +1,10 @@ -import requests import os import pathlib from util.discord import discord +import os +import pathlib +from util.discord import discord try: import requests From cd2a8fb0bf0d838eeae8630a460a93e47584babb Mon Sep 17 00:00:00 2001 From: Matthew van der Hoorn Date: Fri, 10 Jan 2025 20:35:36 +0100 Subject: [PATCH 06/13] Fix the fix Apparently my own solution did not work for log files above 9... I worked this out now. --- util/logger.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/util/logger.py b/util/logger.py index b65b514..3083ce5 100644 --- a/util/logger.py +++ b/util/logger.py @@ -3,6 +3,7 @@ from logging.handlers import RotatingFileHandler from pathlib import Path +from util.constants import DOCKER_ENV from util.hoorn_lib_common.log_rotator import LogRotator from util.utility import create_bar from util.version import get_version @@ -21,7 +22,7 @@ def setup_logger(log_level, script_name, max_logs=10): A logger object for logging messages. """ - if os.environ.get('DOCKER_ENV'): + if DOCKER_ENV: config_dir = os.getenv('CONFIG_DIR', '/config') log_dir: str = f"{config_dir}/logs/{script_name}" else: From 671eaaf525af2125905f2ac4d9b95c2c63c4e4c1 Mon Sep 17 00:00:00 2001 From: Matthew van der Hoorn Date: Fri, 10 Jan 2025 22:31:38 +0100 Subject: [PATCH 07/13] Added Windows Support for Syncing Posters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ⚠️ Your conf name through rclone has to be "daps" in order for this to work correctly. ⚠️ --- modules/sync_gdrive.py | 255 +++++++++++++++++------ requirements.txt | 1 + util/call_script.py | 10 +- util/constants.py | 34 +++ util/get_sync_file.py | 18 ++ util/hoorn_lib_common/command_handler.py | 148 +++++++++++++ util/hoorn_lib_common/log_rotator.py | 25 ++- 7 files changed, 407 insertions(+), 84 deletions(-) create mode 100644 util/constants.py create mode 100644 util/get_sync_file.py create mode 100644 util/hoorn_lib_common/command_handler.py diff --git a/modules/sync_gdrive.py b/modules/sync_gdrive.py index 83c3379..51b12f1 100755 --- a/modules/sync_gdrive.py +++ b/modules/sync_gdrive.py @@ -2,14 +2,19 @@ import os import shlex import sys +from pathlib import Path +from typing import Union, List, Optional, Dict + +import pydantic from util.call_script import call_script +from util.constants import OS_NAME +from util.get_sync_file import SyncFileGetter, OsName +from util.hoorn_lib_common.command_handler import CommandHelper from util.logger import setup_logger from util.utility import create_bar -script_name = "sync_gdrive" - -bash_script_file = os.path.realpath(os.path.dirname(os.path.realpath(__file__)) + '/../scripts/rclone.sh') +SCRIPT_NAME = "sync_gdrive" def output_debug_info(cmd, settings): client_id = settings.get('client_id', None) @@ -26,18 +31,82 @@ def output_debug_info(cmd, settings): return debug_cmd -def set_cmd_args(settings, logger): - cmds = [] - cmd = [bash_script_file] - sync_list = [] +class SyncArgContext(pydantic.BaseModel): + gdrive_sa_location: Optional[str] = None + client_id: Optional[str] = None + client_secret: Optional[str] = None + token: Optional[Dict] = None + gdrive_sync: Optional[List] = None + gdrive_okay: bool + + +def get_sync_args(sync_item: Dict, logger, context: SyncArgContext) -> Union[List, None]: + # TODO - Note to Mr. Hoorn -> Refactor for SOLID/Clean code & maintainability + logger.debug(f"Syncing: {sync_item}") + sync_location = sync_item['location'] + sync_id = sync_item['id'] + + sync_cmd = [] + if not context.gdrive_sa_location: + if context.client_id: + sync_cmd.append('-i') + sync_cmd.append(shlex.quote(context.client_id)) + else: + logger.error("No client id provided") + return + + if context.client_secret: + sync_cmd.append('-s') + sync_cmd.append(shlex.quote(context.client_secret)) + else: + logger.error("No client secret provided") + return + + if context.gdrive_sync: + if sync_location != '' and os.path.exists(sync_location): + sync_cmd.append('-l') + sync_cmd.append(shlex.quote(sync_item['location'])) + else: + if not os.path.exists(sync_location): + logger.error(f"Sync location {sync_location} does not exist") + # Create the directory if it doesn't exist + try: + os.makedirs(sync_location) + logger.info(f"Created {sync_location}") + sync_cmd.append('-l') + sync_cmd.append(shlex.quote(sync_item['location'])) + except Exception as e: + logger.error(f"Exception occurred while creating {sync_location}: {e}") + return + else: + logger.error("No sync location provided") + return + if sync_id != '': + sync_cmd.append('-f') + sync_cmd.append(shlex.quote(sync_item['id'])) + else: + logger.error("No gdrive id provided") + return + + if context.token: + sync_cmd.append('-t') + sync_cmd.append(json.dumps(context.token)) + + if context.gdrive_okay: + sync_cmd.append('-g') + sync_cmd.append(shlex.quote(context.gdrive_sa_location)) + + return sync_cmd + +def get_sync_arg_context(settings, logger) -> SyncArgContext: + # TODO - Note to Mr. Hoorn -> Refactor for SOLID/Clean code & maintainability + client_id = settings.get('client_id', None) client_secret = settings.get('client_secret', None) token = settings.get('token', None) gdrive_sa_location = settings.get('gdrive_sa_location', None) gdrive_sync = settings.get('gdrive_sync', None) - sync_list = gdrive_sync if isinstance(gdrive_sync, list) else [gdrive_sync] - if gdrive_sa_location and os.path.isfile(gdrive_sa_location): gdrive_okay = True elif gdrive_sa_location and not os.path.isfile(gdrive_sa_location): @@ -46,63 +115,99 @@ def set_cmd_args(settings, logger): else: gdrive_okay = False + return SyncArgContext( + gdrive_sa_location=gdrive_sa_location, + client_id=client_id, + client_secret=client_secret, + token=token, + gdrive_sync=gdrive_sync, + gdrive_okay=gdrive_okay, + ) + +def get_cmds(settings, logger, base_cmd=None, windows: bool = False) -> Union[List[List], None]: + # TODO - Note to Mr. Hoorn -> Refactor for SOLID/Clean code & maintainability + + if base_cmd is None: + base_cmd = [] + cmds = [] + + context = get_sync_arg_context(settings, logger) + + sync_list: List = context.gdrive_sync if isinstance(context.gdrive_sync, list) else [context.gdrive_sync] + logger.debug(f"Sync list: {sync_list}") + for sync_item in sync_list: - logger.debug(f"Syncing: {sync_item}") - sync_location = sync_item['location'] - sync_id = sync_item['id'] - - sync_cmd = cmd.copy() - if not gdrive_sa_location: - if client_id: - sync_cmd.append('-i') - sync_cmd.append(shlex.quote(client_id)) - else: - logger.error("No client id provided") - return + if not windows: + sync_cmd = get_sync_args(sync_item, logger, context) + if sync_cmd: + if base_cmd: + cmds.append(base_cmd + sync_cmd) + else: + cmds.append(sync_cmd) + if windows: + use_client: bool = context.client_id is not None and context.client_secret is not None and context.token is not None + use_saf: bool = context.gdrive_sa_location is not None - if client_secret: - sync_cmd.append('-s') - sync_cmd.append(shlex.quote(client_secret)) - else: - logger.error("No client secret provided") + if not use_client and not use_saf: + logger.error("No (client id, client secret), or service account file provided") return - if gdrive_sync: - if sync_location != '' and os.path.exists(sync_location): - sync_cmd.append('-l') - sync_cmd.append(shlex.quote(sync_item['location'])) - else: - if not os.path.exists(sync_location): - logger.error(f"Sync location {sync_location} does not exist") - # Create the directory if it doesn't exist - try: - os.makedirs(sync_location) - logger.info(f"Created {sync_location}") - sync_cmd.append('-l') - sync_cmd.append(shlex.quote(sync_item['location'])) - except Exception as e: - logger.error(f"Exception occurred while creating {sync_location}: {e}") - return - else: - logger.error("No sync location provided") - return - if sync_id != '': - sync_cmd.append('-f') - sync_cmd.append(shlex.quote(sync_item['id'])) - else: - logger.error("No gdrive id provided") + if use_client and use_saf: + logger.error("Both (client id, client secret), and service account file provided") return - - if token: - sync_cmd.append('-t') - sync_cmd.append(json.dumps(token)) - if gdrive_okay: - sync_cmd.append('-g') - sync_cmd.append(shlex.quote(gdrive_sa_location)) + cmd = [ + "rclone sync" + ] + + + if use_client: + cmd.extend([ + "--drive-client-id", shlex.quote(context.client_id), + "--drive-client-secret", shlex.quote(context.client_secret), + # "--drive-token", shlex.quote(json.dumps(context.token)), + ]) - cmds.append(sync_cmd) + if use_saf: + print("Using service account") + cmd.extend([ + "--drive-service-account-file", shlex.quote(context.gdrive_sa_location), + ]) + + cmd.extend([ + "--drive-root-folder-id", shlex.quote(sync_item['id']), + "--fast-list", + "--tpslimit=5", + "--no-update-modtime", + "--drive-use-trash=false", + "--drive-chunk-size=512M", + "--exclude=**.partial", + "--check-first", + "--bwlimit=80M", + "--size-only", + "--delete-after", + "--cache-db-purge", + "--dump-bodies", # Added this to be sure something is happening... Otherwise it fools me into thinking it's done. + "-vv", + "daps:", shlex.quote(sync_item['location']) + ]) + cmds.append(cmd) + + return cmds + +def set_cmd_args(settings, logger): + get_sync_file: SyncFileGetter = SyncFileGetter(logger) + path: Union[Path, None] = get_sync_file.get_sync_file() + + if path is not None: + file_path = str(path.absolute()) + else: + logger.error("set_cmd_args called for wrong OS version!") + return + + cmd = [file_path] + cmds = get_cmds(settings, logger, cmd) return cmds @@ -114,25 +219,47 @@ def run_rclone(cmd, settings, logger): call_script(cmd, logger) logger.debug(f"RClone command with args: {debug_cmd} --> Success") except Exception as e: - logger.error(f"Exception occurred while running rclone.sh: {e}") + # Note by Mr. Hoorn on 10 January 2025: If there is an exception that the programme returns with exit code 1 (non-0), it will print out the original command without redactions, this is a security issue. + # I will leave it to someone else to potentially fix this. + logger.error(f"Exception occurred while running rclone for OS version: {OS_NAME.value}: {e}") + + # This one works as expected. 10 January 2025. logger.error(f"RClone command with args: {debug_cmd} --> Failed") pass # Main function -def main(config, logger=None): +def main(config): """ Main function. """ global dry_run settings = config.script_config log_level = config.log_level - logger = setup_logger(log_level, script_name) - name = script_name.replace("_", " ").upper() + logger = setup_logger(log_level, SCRIPT_NAME) + name = SCRIPT_NAME.replace("_", " ").upper() try: + # TODO - Note to Mr. Hoorn (or maybe someone else wants to pick this up): + # Integrate executions into a single flow, maybe with strategy pattern, + # instead of having different run processes. logger.info(create_bar(f"START {name}")) - for cmd in set_cmd_args(settings, logger): - run_rclone(cmd, settings, logger) + if OS_NAME == OsName.LINUX or OS_NAME == OS_NAME.DOCKER: + for cmd in set_cmd_args(settings, logger): + run_rclone(cmd, settings, logger) + elif OS_NAME == OsName.WINDOWS: + cmd_helper: CommandHelper = CommandHelper(logger) + cmds: List[List] = get_cmds(settings, logger, windows=True) + refresh_path_cmd = '$env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User")' + # create_posters_cmd = "rclone config create posters drive config_is_local=false" + + for cmd in cmds: + # Combine the refresh command and the normal command into a single string + combined_cmd = [f"{refresh_path_cmd}; {', '.join(cmd)}"] + + powershell_path = Path(r"C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe") + cmd_helper.open_application(powershell_path, combined_cmd, new_window=False, keep_open=False) + else: + raise Exception("Unsupported OS") except KeyboardInterrupt: print("Keyboard Interrupt detected. Exiting...") sys.exit() diff --git a/requirements.txt b/requirements.txt index cc56dcf..d003ffc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,3 +16,4 @@ tqdm==4.66.2; python_version >= '3.7' unidecode==1.3.8; python_version >= '3.5' urllib3==2.2.1; python_version >= '3.8' wcwidth==0.2.13 +pydantic diff --git a/util/call_script.py b/util/call_script.py index 61e2332..d951923 100755 --- a/util/call_script.py +++ b/util/call_script.py @@ -17,16 +17,8 @@ def call_script(command, logger): Returns: CompletedProcess: The completed process """ - # Print the command being executed + # Note 10 January 2025: Removed printing redacted command because it is already being logged elsewhere - # Redact command secrets - redacted_command = str(' '.join(command)) - # Redact random strings of characters - - redacted_command = redact_sensitive_info(redacted_command) - - print(f"\nRunning command:\n\n{redacted_command}\n\n") - # Execute the command and capture the output with Popen(command, text=True, stdout=PIPE, stderr=STDOUT) as process: for line in process.stdout: diff --git a/util/constants.py b/util/constants.py new file mode 100644 index 0000000..8286a37 --- /dev/null +++ b/util/constants.py @@ -0,0 +1,34 @@ +import os +import platform +from enum import Enum +from pathlib import Path + +ROOT: Path = Path(os.path.dirname(__file__)).parent + +def get_os_type(): + """Returns the operating system type.""" + + return platform.system() + + +class OsName(Enum): + DOCKER = "docker", + WINDOWS = "windows" + LINUX = "linux" + MACOS = "mac" + +if os.environ.get('DOCKER_ENV'): + DOCKER_ENV: bool = True + OS_NAME = OsName.DOCKER +else: + DOCKER_ENV = False + os_name = get_os_type() + + if os_name == 'Windows': + OS_NAME = OsName.WINDOWS + elif os_name == 'Darwin': + OS_NAME = OsName.MACOS + elif os_name == 'Linux': + OS_NAME = OsName.LINUX + else: + print("WARNING: Unsupported File System") diff --git a/util/get_sync_file.py b/util/get_sync_file.py new file mode 100644 index 0000000..b51b4ba --- /dev/null +++ b/util/get_sync_file.py @@ -0,0 +1,18 @@ +# Written to make the process for executing the correct executable for the OS straightforward +from logging import Logger +from pathlib import Path +from typing import Union + +from util.constants import ROOT, OsName, OS_NAME + + +class SyncFileGetter: + def __init__(self, logger: Logger): + self._logger = logger + + def get_sync_file(self) -> Union[Path, None]: + if OS_NAME == OsName.DOCKER or OS_NAME == OsName.LINUX: + return ROOT.joinpath("scripts/rclone.sh") + else: + self._logger.warning("Unsupported OS for getting sync file: %s" % OS_NAME) + return None diff --git a/util/hoorn_lib_common/command_handler.py b/util/hoorn_lib_common/command_handler.py new file mode 100644 index 0000000..5b8a442 --- /dev/null +++ b/util/hoorn_lib_common/command_handler.py @@ -0,0 +1,148 @@ +# See: https://github.com/LordMartron94/py-common/blob/main/py_common/command_handling/command_helper.py +# Made for Windows specifically, MIGHT work on other OS, but untested. + +import asyncio +import os +import shutil +import subprocess +from logging import Logger +from pathlib import Path +from typing import Union, List + + +class CommandHelper: + """ + Helper class meant to streamline the execution of a command in command prompt. Enjoy. + """ + + def __init__(self, logger: Logger): + """ + Initializes the command helper with the provided Logger and module separator. + :param logger: Logger instance. + """ + + self._logger: Logger = logger + + def _format_error(self, stderr: str) -> str: + formatted = "Error executing command:\n" + + for line in stderr.split('\n'): + formatted += f" {line}\n" + + return formatted + + def execute_command(self, command: list, output_override: bool = False) -> subprocess.CompletedProcess: + """ + Executes a given command. + Prints errors in all cases. + """ + if output_override: + self._logger.info(f"Executing command: {command}") + self._logger.info(f"Stringified: {' '.join(command)}") + + result = subprocess.run(command, capture_output=True) + if result.returncode != 0: + error_message = self._format_error(result.stderr.decode('utf-8')) + self._logger.error(error_message) + self._logger.info(f"Command causing error: {command}") + + return result + + def execute_command_v2(self, executable: Union[Path, str], command: list, shell: bool, hide_console: bool = True, keep_open: bool = False) -> None: + """Use this if `execute_command` does not work.""" + + self._logger.debug(f"Executing {' '.join(command)} with executable {executable}") + + bat_file_path = Path(__file__).parent.joinpath("temp.bat") + + with open(bat_file_path, 'w') as bat_file: + bat_file.write("@echo off\n") + bat_file.write(f'"{executable}" {" ".join(command)}\n') + + if not keep_open: + bat_file.write(f'exit\n') + else: bat_file.write(f'pause\n') + + print(bat_file_path) + + if hide_console: + subprocess.run(['start', '/b', os.environ["COMSPEC"], '/c', f"{bat_file_path}"], shell=shell) + return + + subprocess.run(['start', os.environ["COMSPEC"], '/k', f"{bat_file_path}"], shell=shell) + + async def execute_command_v2_async(self, executable: Union[Path, str], command: list, hide_console: bool = True, keep_open: bool = False) -> None: + """Use this if `execute_command` does not work. Async version.""" + + self._logger.debug(f"Executing {' '.join(command)} with executable {executable}") + + bat_file_path = Path(__file__).parent.joinpath("temp.bat") + + executable_path = shutil.which(executable.name if type(executable) == Path else executable) + + with open(bat_file_path, 'w') as bat_file: + bat_file.write(f'"{executable_path}" {" ".join(command)}\n') + + if not keep_open: + bat_file.write(f'exit\n') + else: bat_file.write(f'pause\nexit\n') + + print(bat_file_path) + + if hide_console: + proc = await asyncio.create_subprocess_exec( + os.environ["COMSPEC"], + '/k', + str(bat_file_path), + shell=False + ) + await proc.wait() + return + + proc = await asyncio.create_subprocess_exec( + os.environ["COMSPEC"], + '/k', + str(bat_file_path), + shell=False + ) + await proc.wait() + return + + def open_python_module_with_custom_interpreter(self, interpreter_path: Path, working_directory: Path, module_name: str, args: list[str]): + """ + Opens a python module with a custom interpreter. + """ + + self._logger.debug(f"Opening module {module_name} with interpreter {interpreter_path}") + + bat_file_path = Path(__file__).parent.joinpath("temp.bat") + + with open(bat_file_path, 'w') as bat_file: + bat_file.write(f'cd "{working_directory}"\n') + bat_file.write(f'"{interpreter_path}" -m {module_name} {" ".join(args)}\n') + bat_file.write(f'pause\n') + + print(bat_file_path) + subprocess.run(['start', os.environ["COMSPEC"], '/k', f"{bat_file_path}"], shell=True) + + def open_application(self, exe: Path, args: List[str], new_window: bool = True, keep_open: bool = False): + """ + Opens an application with the provided arguments. + """ + self._logger.info(f"Opening application {exe}") + + if new_window: + creationflags = subprocess.CREATE_NEW_CONSOLE + else: + creationflags = 0 + + try: + if keep_open: + # Add a command to pause execution and keep the window open + args.append("&& pause") + + subprocess.Popen([str(exe)] + args, creationflags=creationflags) + except PermissionError as e: + self._logger.error(f"Permission denied to execute {exe}. Please ensure you have the necessary permissions.\n{e}") + except Exception as e: + self._logger.error(f"Error executing {exe}.\n{e}") \ No newline at end of file diff --git a/util/hoorn_lib_common/log_rotator.py b/util/hoorn_lib_common/log_rotator.py index 612a7bb..3161981 100644 --- a/util/hoorn_lib_common/log_rotator.py +++ b/util/hoorn_lib_common/log_rotator.py @@ -3,7 +3,8 @@ # See: https://github.com/LordMartron94/py-common/blob/main/py_common/logging/output/file_hoorn_log_output.py import os from pathlib import Path -from typing import List +from sys import path_hooks +from typing import List, Dict, Tuple from util.hoorn_lib_common.file_handler import FileHandler @@ -41,22 +42,24 @@ def increment_logs(self) -> None: """ children = self._file_handler.get_children_paths(self._root_log_directory, ".txt", recursive=True) - children.sort(reverse=True) - organized_by_separator: List[List[Path]] = self._organize_logs_by_subdirectory(children) + # Map file_paths to associated numbers + matched: List[Tuple[Path, int]] = [(path, int(path.stem.split("_")[1])) for path in children] - for directory_logs in organized_by_separator: - self._increment_logs_in_directory(directory_logs) + # Sort by number in reverse + matched.sort(key=lambda x: x[1], reverse=True) - def _increment_logs_in_directory(self, log_files: List[Path]) -> None: - for i in range(len(log_files)): - child = log_files[i] - number = int(child.stem.split("_")[-1]) + self._increment_logs_in_directory(matched) + + def _increment_logs_in_directory(self, matched_logs: List[Tuple[Path, int]]) -> None: + for i in range(len(matched_logs)): + path = matched_logs[i][0] + number = matched_logs[i][1] if number + 1 > self._max_logs_to_keep: - os.remove(child) + os.remove(path) continue - os.rename(child, Path.joinpath(child.parent.absolute(), f"log_{number + 1}.txt")) + os.rename(path, Path.joinpath(path.parent.absolute(), f"log_{number + 1}.txt")) def _organize_logs_by_subdirectory(self, log_paths: List[Path]) -> List[List[Path]]: """ From 86c0178da36e39b87dd8944cd0e5d4ffec4e3f37 Mon Sep 17 00:00:00 2001 From: Matthew van der Hoorn Date: Sat, 11 Jan 2025 15:59:26 +0100 Subject: [PATCH 08/13] Fixed missing import in Renaminatorr --- modules/renameinatorr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/renameinatorr.py b/modules/renameinatorr.py index a05f2a3..46e143f 100755 --- a/modules/renameinatorr.py +++ b/modules/renameinatorr.py @@ -13,7 +13,7 @@ # Requirements: requests, pyyaml # License: MIT License # =================================================================================================== - +import json import sys import time From 37587952a95e51138c5f3b17be8e4126a204e167 Mon Sep 17 00:00:00 2001 From: Matthew van der Hoorn Date: Sat, 11 Jan 2025 16:01:48 +0100 Subject: [PATCH 09/13] Fixed missing import in poster_renamerr Apparently the automatic optimization of imports by Jetbrains failed for the first time and deleted some necessary json imports, lol --- modules/poster_renamerr.py | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/poster_renamerr.py b/modules/poster_renamerr.py index db95a5e..7bef161 100755 --- a/modules/poster_renamerr.py +++ b/modules/poster_renamerr.py @@ -15,6 +15,7 @@ # =================================================================================================== import filecmp +import json import shutil import sys import time From ede2d24ab57f466f01deed969b8deff6ee5b847b Mon Sep 17 00:00:00 2001 From: Matthew van der Hoorn Date: Sat, 11 Jan 2025 16:04:23 +0100 Subject: [PATCH 10/13] Fixed redundant argument error --- modules/poster_renamerr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/poster_renamerr.py b/modules/poster_renamerr.py index 7bef161..c2eea4a 100755 --- a/modules/poster_renamerr.py +++ b/modules/poster_renamerr.py @@ -599,7 +599,7 @@ def main(config): from modules.sync_gdrive import main as gdrive_main from util.config import Config gdrive_config = Config("sync_gdrive") - gdrive_main(gdrive_config, logger) + gdrive_main(gdrive_config) logger.info(f"Finished running sync_gdrive") else: logger.debug(f"Sync posters is disabled. Skipping...") From 479763886d84503d21b9aa5516b99f92d05432b6 Mon Sep 17 00:00:00 2001 From: Matthew van der Hoorn Date: Sat, 11 Jan 2025 16:48:37 +0100 Subject: [PATCH 11/13] Deleted command helper and used call script This is for now better for flow control. Later on I will work on creating a more efficient command executor. (also added some debug log statements) --- modules/poster_renamerr.py | 2 + modules/sync_gdrive.py | 8 +- util/hoorn_lib_common/command_handler.py | 148 ----------------------- 3 files changed, 4 insertions(+), 154 deletions(-) delete mode 100644 util/hoorn_lib_common/command_handler.py diff --git a/modules/poster_renamerr.py b/modules/poster_renamerr.py index c2eea4a..cd52162 100755 --- a/modules/poster_renamerr.py +++ b/modules/poster_renamerr.py @@ -56,10 +56,12 @@ def get_assets_files(source_dirs, logger): # Iterate through each source directory for source_dir in source_dirs: + logger.debug(f"Getting asset files for: {source_dir}") new_assets = categorize_files(source_dir) if new_assets: # Merge new_assets with final_assets for new in new_assets: + logger.debug(f"Processing asset: {new}") found_match = False for final in final_assets: if final['normalized_title'] == new['normalized_title'] and final['year'] == new['year']: diff --git a/modules/sync_gdrive.py b/modules/sync_gdrive.py index 51b12f1..dbf9ba5 100755 --- a/modules/sync_gdrive.py +++ b/modules/sync_gdrive.py @@ -10,7 +10,6 @@ from util.call_script import call_script from util.constants import OS_NAME from util.get_sync_file import SyncFileGetter, OsName -from util.hoorn_lib_common.command_handler import CommandHelper from util.logger import setup_logger from util.utility import create_bar @@ -166,7 +165,6 @@ def get_cmds(settings, logger, base_cmd=None, windows: bool = False) -> Union[Li cmd.extend([ "--drive-client-id", shlex.quote(context.client_id), "--drive-client-secret", shlex.quote(context.client_secret), - # "--drive-token", shlex.quote(json.dumps(context.token)), ]) if use_saf: @@ -247,17 +245,15 @@ def main(config): for cmd in set_cmd_args(settings, logger): run_rclone(cmd, settings, logger) elif OS_NAME == OsName.WINDOWS: - cmd_helper: CommandHelper = CommandHelper(logger) cmds: List[List] = get_cmds(settings, logger, windows=True) refresh_path_cmd = '$env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User")' # create_posters_cmd = "rclone config create posters drive config_is_local=false" for cmd in cmds: # Combine the refresh command and the normal command into a single string - combined_cmd = [f"{refresh_path_cmd}; {', '.join(cmd)}"] - powershell_path = Path(r"C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe") - cmd_helper.open_application(powershell_path, combined_cmd, new_window=False, keep_open=False) + combined_cmd = [powershell_path, f"{refresh_path_cmd}; {', '.join(cmd)}"] + call_script(combined_cmd, logger) else: raise Exception("Unsupported OS") except KeyboardInterrupt: diff --git a/util/hoorn_lib_common/command_handler.py b/util/hoorn_lib_common/command_handler.py deleted file mode 100644 index 5b8a442..0000000 --- a/util/hoorn_lib_common/command_handler.py +++ /dev/null @@ -1,148 +0,0 @@ -# See: https://github.com/LordMartron94/py-common/blob/main/py_common/command_handling/command_helper.py -# Made for Windows specifically, MIGHT work on other OS, but untested. - -import asyncio -import os -import shutil -import subprocess -from logging import Logger -from pathlib import Path -from typing import Union, List - - -class CommandHelper: - """ - Helper class meant to streamline the execution of a command in command prompt. Enjoy. - """ - - def __init__(self, logger: Logger): - """ - Initializes the command helper with the provided Logger and module separator. - :param logger: Logger instance. - """ - - self._logger: Logger = logger - - def _format_error(self, stderr: str) -> str: - formatted = "Error executing command:\n" - - for line in stderr.split('\n'): - formatted += f" {line}\n" - - return formatted - - def execute_command(self, command: list, output_override: bool = False) -> subprocess.CompletedProcess: - """ - Executes a given command. - Prints errors in all cases. - """ - if output_override: - self._logger.info(f"Executing command: {command}") - self._logger.info(f"Stringified: {' '.join(command)}") - - result = subprocess.run(command, capture_output=True) - if result.returncode != 0: - error_message = self._format_error(result.stderr.decode('utf-8')) - self._logger.error(error_message) - self._logger.info(f"Command causing error: {command}") - - return result - - def execute_command_v2(self, executable: Union[Path, str], command: list, shell: bool, hide_console: bool = True, keep_open: bool = False) -> None: - """Use this if `execute_command` does not work.""" - - self._logger.debug(f"Executing {' '.join(command)} with executable {executable}") - - bat_file_path = Path(__file__).parent.joinpath("temp.bat") - - with open(bat_file_path, 'w') as bat_file: - bat_file.write("@echo off\n") - bat_file.write(f'"{executable}" {" ".join(command)}\n') - - if not keep_open: - bat_file.write(f'exit\n') - else: bat_file.write(f'pause\n') - - print(bat_file_path) - - if hide_console: - subprocess.run(['start', '/b', os.environ["COMSPEC"], '/c', f"{bat_file_path}"], shell=shell) - return - - subprocess.run(['start', os.environ["COMSPEC"], '/k', f"{bat_file_path}"], shell=shell) - - async def execute_command_v2_async(self, executable: Union[Path, str], command: list, hide_console: bool = True, keep_open: bool = False) -> None: - """Use this if `execute_command` does not work. Async version.""" - - self._logger.debug(f"Executing {' '.join(command)} with executable {executable}") - - bat_file_path = Path(__file__).parent.joinpath("temp.bat") - - executable_path = shutil.which(executable.name if type(executable) == Path else executable) - - with open(bat_file_path, 'w') as bat_file: - bat_file.write(f'"{executable_path}" {" ".join(command)}\n') - - if not keep_open: - bat_file.write(f'exit\n') - else: bat_file.write(f'pause\nexit\n') - - print(bat_file_path) - - if hide_console: - proc = await asyncio.create_subprocess_exec( - os.environ["COMSPEC"], - '/k', - str(bat_file_path), - shell=False - ) - await proc.wait() - return - - proc = await asyncio.create_subprocess_exec( - os.environ["COMSPEC"], - '/k', - str(bat_file_path), - shell=False - ) - await proc.wait() - return - - def open_python_module_with_custom_interpreter(self, interpreter_path: Path, working_directory: Path, module_name: str, args: list[str]): - """ - Opens a python module with a custom interpreter. - """ - - self._logger.debug(f"Opening module {module_name} with interpreter {interpreter_path}") - - bat_file_path = Path(__file__).parent.joinpath("temp.bat") - - with open(bat_file_path, 'w') as bat_file: - bat_file.write(f'cd "{working_directory}"\n') - bat_file.write(f'"{interpreter_path}" -m {module_name} {" ".join(args)}\n') - bat_file.write(f'pause\n') - - print(bat_file_path) - subprocess.run(['start', os.environ["COMSPEC"], '/k', f"{bat_file_path}"], shell=True) - - def open_application(self, exe: Path, args: List[str], new_window: bool = True, keep_open: bool = False): - """ - Opens an application with the provided arguments. - """ - self._logger.info(f"Opening application {exe}") - - if new_window: - creationflags = subprocess.CREATE_NEW_CONSOLE - else: - creationflags = 0 - - try: - if keep_open: - # Add a command to pause execution and keep the window open - args.append("&& pause") - - subprocess.Popen([str(exe)] + args, creationflags=creationflags) - except PermissionError as e: - self._logger.error(f"Permission denied to execute {exe}. Please ensure you have the necessary permissions.\n{e}") - except Exception as e: - self._logger.error(f"Error executing {exe}.\n{e}") \ No newline at end of file From ca1301f41edfbb0356c13e06efa7a27f678011f1 Mon Sep 17 00:00:00 2001 From: Matthew van der Hoorn Date: Sat, 11 Jan 2025 16:49:27 +0100 Subject: [PATCH 12/13] Added a JSON metadata check for asset folders That way it doesn't need to iterate over all directories multiple times, I noticed a slow-down here, so I implemented this to prevent that. For now will only work on my machine since I am the only one with such a json file. Feel free to standardize this into the libraries themselves. --- util/utility.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/util/utility.py b/util/utility.py index a077dca..e326956 100755 --- a/util/utility.py +++ b/util/utility.py @@ -1,9 +1,12 @@ +import json import math import os import pathlib import re import subprocess +from pathlib import Path + try: import html from unidecode import unidecode @@ -152,10 +155,20 @@ def _is_asset_folders(folder_path): Returns: bool: True if the folder contains asset folders, False otherwise """ - if not os.path.exists(folder_path): + folder_path = Path(folder_path) + + if not folder_path.exists(): return False else: + # Added this myself on 11 January 2025 (Mr. Hoorn) to massively speed up the check. + associated_json_path = folder_path.parent.joinpath(f"{folder_path.stem}.json") + if associated_json_path.exists() and associated_json_path.is_file(): + with open(associated_json_path, 'r') as file: + json_data = json.load(file) + return json_data['contains_asset_folders'] + for item in os.listdir(folder_path): + print(f'Checking {item}') if item.startswith('.') or item.startswith('@') or item == "tmp": continue if os.path.isdir(os.path.join(folder_path, item)): @@ -168,12 +181,16 @@ def categorize_files(folder_path): Args: folder_path (str): The path to the folder to sort - asset_folders (bool): Whether or not to sort by folders Returns: list: A list of dictionaries containing the sorted files """ + # TODO - Clean up and refactor for maintainability and readability + # Seriously, damn, this is so complex. + # 1675% cognitive complexity according to: + # https://plugins.jetbrains.com/plugin/index?xmlId=com.github.nikolaikopernik.codecomplexity&utm_source=product&utm_medium=link&utm_campaign=IU&utm_content=2024.2 + asset_folders = _is_asset_folders(folder_path) assets_dict = [] @@ -186,13 +203,20 @@ def categorize_files(folder_path): if not asset_folders: # Get list of files in the folder try: - files = [f.name for f in os.scandir(folder_path) if f.is_file()] + files = [] + for f in os.scandir(folder_path): + print(f"Scanning: {f.name}") + if f.is_file(): + files.append(f.name) except FileNotFoundError: return None + print("Starting sort files") files = sorted(files, key=lambda x: x.lower()) # Sort files alphabetically + print("End sort files") if files: # Loop through each file in the folder for file in tqdm(files, desc=f"Processing '{base_name}' folder", total=len(files), disable=None, leave=True): + print(f"Processing file {file}") if file.startswith('.') or "(N-A)" in file: continue # Skip hidden files or files with "(N-A)" in the name From c0836b11c2e42bfd02f5e1cfb195aac5655a29ec Mon Sep 17 00:00:00 2001 From: Matthew van der Hoorn Date: Mon, 13 Jan 2025 18:45:18 +0100 Subject: [PATCH 13/13] Fix Windows not Syncing multiple libraries --- modules/sync_gdrive.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/modules/sync_gdrive.py b/modules/sync_gdrive.py index dbf9ba5..a2913ad 100755 --- a/modules/sync_gdrive.py +++ b/modules/sync_gdrive.py @@ -3,6 +3,7 @@ import shlex import sys from pathlib import Path +from pprint import pprint from typing import Union, List, Optional, Dict import pydantic @@ -247,13 +248,18 @@ def main(config): elif OS_NAME == OsName.WINDOWS: cmds: List[List] = get_cmds(settings, logger, windows=True) refresh_path_cmd = '$env:Path = [System.Environment]::GetEnvironmentVariable("Path","Machine") + ";" + [System.Environment]::GetEnvironmentVariable("Path","User")' - # create_posters_cmd = "rclone config create posters drive config_is_local=false" + powershell_path = Path(r"C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe") + + cmd_string = "" for cmd in cmds: - # Combine the refresh command and the normal command into a single string - powershell_path = Path(r"C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe") - combined_cmd = [powershell_path, f"{refresh_path_cmd}; {', '.join(cmd)}"] - call_script(combined_cmd, logger) + cmd_string += ', '.join(cmd) + cmd_string += '; ' + + combined_cmd = [powershell_path, f"{refresh_path_cmd}; {cmd_string}"] + # print(combined_cmd) + + call_script(combined_cmd, logger) else: raise Exception("Unsupported OS") except KeyboardInterrupt: