Merge pull request #9 from bbtufty/dev

Cleaned up logging

CHANGES.rst

@@ -7,6 +7,7 @@ Fixes
 General
 ~~~~~~~
 
+- Overhauled logging system to avoid unnecessary file bloat and I/O. Speed ups of about a factor 3
 - Overhauled how config files are read in to avoid unneccesary I/O. Speed ups of about a factor 2
 
 0.0.4 (2024-05-09)

pyproject.toml

@@ -33,6 +33,7 @@ classifiers = [
 ]
 
 dependencies = [
+    "colorlog",
     "discordwebhook",
     "numpy",
     "packaging",

romsearch/modules/datparser.py

@@ -1,13 +1,11 @@
 import glob
-import json
 import os
-from datetime import datetime
 from urllib.request import urlopen
 
 import xmltodict
 
 import romsearch
-from ..util import load_yml, setup_logger, create_bar, unzip_file, save_json
+from ..util import load_yml, setup_logger, unzip_file, save_json
 
 ALLOWED_GROUPS = [
     "No-Intro",
@@ -44,6 +42,7 @@ def __init__(self,
                  config_file=None,
                  config=None,
                  platform_config=None,
+                 logger=None,
                  ):
         """Parser for dat files from Redump or No-Intro
 
@@ -56,18 +55,19 @@ def __init__(self,
             config_file (str, optional): Configuration file. Defaults to None
             config (dict, optional): Configuration dictionary. Defaults to None
             platform_config (dict, optional): Platform configuration dictionary. Defaults to None
+            logger (logging.Logger, optional): Logger instance. Defaults to None
         """
 
         if platform is None:
             raise ValueError("platform must be specified")
         self.platform = platform
 
-        logger_add_dir = str(os.path.join(platform))
-
-        self.logger = setup_logger(log_level="info",
-                                   script_name=f"DATParser",
-                                   additional_dir=logger_add_dir,
-                                   )
+        if logger is None:
+            logger = setup_logger(log_level="info",
+                                  script_name=f"DATParser",
+                                  additional_dir=platform,
+                                  )
+        self.logger = logger
 
         if config_file is None and config is None:
             raise ValueError("config_file or config must be specified")
@@ -108,8 +108,6 @@ def __init__(self,
 
     def run(self):
 
-        self.logger.info(create_bar(f"START DATParser"))
-
         run_datparser = True
 
         if self.dat_dir is None:
@@ -125,8 +123,6 @@ def run(self):
         if run_datparser:
             self.run_datparser()
 
-        self.logger.info(create_bar(f"FINISH DATParser"))
-
         return True
 
     def run_datparser(self):
@@ -146,6 +142,8 @@ def run_datparser(self):
         if dat is None:
             return False
 
+        self.logger.info(f"Using dat file {os.path.split(dat_file_name)[-1]}")
+
         rom_dict = format_dat(dat)
 
         self.save_rom_dict(rom_dict)

romsearch/modules/dupeparser.py

@@ -5,7 +5,6 @@
 
 import romsearch
 from ..util import (setup_logger,
-                    create_bar,
                     load_yml,
                     get_parent_name,
                     get_short_name,
@@ -27,6 +26,7 @@ def __init__(self,
                  config=None,
                  default_config=None,
                  regex_config=None,
+                 logger=None,
                  ):
         """Tool for figuring out a list of dupes
 
@@ -36,6 +36,7 @@ def __init__(self,
             config (dict, optional): Configuration dictionary. Defaults to None
             default_config (dict, optional): Default configuration dictionary. Defaults to None
             regex_config (dict, optional): Configuration dictionary for regex search. Defaults to None
+            logger (logging.Logger, optional): Logger instance. Defaults to None
 
         TODO:
             - At some point, we might want to consider adding in the retool supersets
@@ -45,12 +46,12 @@ def __init__(self,
             raise ValueError("platform must be specified")
         self.platform = platform
 
-        logger_add_dir = str(os.path.join(platform))
-
-        self.logger = setup_logger(log_level="info",
-                                   script_name=f"DupeParser",
-                                   additional_dir=logger_add_dir,
-                                   )
+        if logger is None:
+            logger = setup_logger(log_level="info",
+                                  script_name=f"DupeParser",
+                                  additional_dir=platform,
+                                  )
+        self.logger = logger
 
         if config_file is None and config is None:
             raise ValueError("config_file or config must be specified")
@@ -95,16 +96,12 @@ def run(self):
             self.logger.warning("retool config for the platform needs to be present if using retool")
             return False
 
-        self.logger.info(create_bar(f"START DupeParser"))
-
         dupe_dict = self.get_dupe_dict()
 
         # Save out the dupe dict
         out_file = os.path.join(self.dupe_dir, f"{self.platform} (dupes).json")
         save_json(dupe_dict, out_file)
 
-        self.logger.info(create_bar(f"FINISH DupeParser"))
-
         return True
 
     def get_dupe_dict(self):
@@ -114,10 +111,8 @@ def get_dupe_dict(self):
 
         # Prefer retool dupes first
         if self.use_retool:
-            self.logger.info("Gettings dupes from retool file")
             dupe_dict = self.get_retool_dupes(dupe_dict)
         if self.use_dat:
-            self.logger.info("Gettings dupes from dat file")
             dupe_dict = self.get_dat_dupes(dupe_dict)
 
         dupe_dict = dict(sorted(dupe_dict.items()))
@@ -209,11 +204,6 @@ def get_retool_dupes(self, dupe_dict=None):
                             for f in retool_dupe["titles"]]
             priorities = [f.get("priority", 1) for f in retool_dupe["titles"]]
 
-            # Parse down to the game name here
-            # if "(" in group:
-            #     group_parsed = get_game_name(group)
-            # else:
-            #     group_parsed = copy.deepcopy(group)
             group_parsed = get_short_name(group,
                                           default_config=self.default_config,
                                           regex_config=self.regex_config,

romsearch/modules/gamefinder.py

@@ -5,7 +5,12 @@
 import numpy as np
 
 import romsearch
-from ..util import setup_logger, load_yml, get_parent_name, get_short_name, create_bar, load_json
+from ..util import (setup_logger,
+                    load_yml,
+                    get_parent_name,
+                    get_short_name,
+                    load_json
+                    )
 
 
 def get_all_games(files,
@@ -49,6 +54,7 @@ def __init__(self,
                  config=None,
                  default_config=None,
                  regex_config=None,
+                 logger=None,
                  ):
         """Tool to find games within a list of files
 
@@ -61,12 +67,15 @@ def __init__(self,
             config (dict, optional): Configuration dictionary. Defaults to None.
             default_config (dict, optional): Default configuration dictionary. Defaults to None.
             regex_config (dict, optional): Dictionary of regex config. Defaults to None.
+            logger (logging.Logger, optional): Logger instance. Defaults to None.
         """
 
-        self.logger = setup_logger(log_level="info",
-                                   script_name=f"GameFinder",
-                                   additional_dir=platform,
-                                   )
+        if logger is None:
+            logger = setup_logger(log_level="info",
+                                  script_name=f"GameFinder",
+                                  additional_dir=platform,
+                                  )
+        self.logger = logger
 
         if platform is None:
             raise ValueError("platform must be specified")
@@ -104,14 +113,12 @@ def run(self,
             files,
             ):
 
-        self.logger.info(create_bar(f"START GameFinder"))
-
         games_dict = self.get_game_dict(files)
         games_dict = dict(sorted(games_dict.items()))
 
-        self.logger.info(f"Found {len(games_dict)} games:")
+        self.logger.debug(f"Found {len(games_dict)} games:")
         for g in games_dict:
-            self.logger.info(f"{g}: {games_dict[g]}")
+            self.logger.debug(f"{g}: {games_dict[g]}")
 
         return games_dict
 

romsearch/modules/romchooser.py

@@ -8,7 +8,6 @@
 
 import romsearch
 from ..util import (setup_logger,
-                    create_bar,
                     load_yml,
                     )
 
@@ -302,6 +301,7 @@ def __init__(self,
                  config=None,
                  default_config=None,
                  regex_config=None,
+                 logger=None,
                  ):
         """ROM choose tool
 
@@ -314,18 +314,20 @@ def __init__(self,
             config (dict, optional): Configuration dictionary. Defaults to None.
             default_config (dict, optional): Default configuration dictionary. Defaults to None.
             regex_config (dict, optional): Configuration dictionary. Defaults to None.
+            logger (logging.Logger, optional): Logger instance. Defaults to None.
         """
 
         if platform is None:
             raise ValueError("platform must be specified")
         self.platform = platform
 
-        logger_add_dir = str(os.path.join(platform, game))
-
-        self.logger = setup_logger(log_level="info",
-                                   script_name=f"ROMChooser",
-                                   additional_dir=logger_add_dir,
-                                   )
+        if logger is None:
+            logger_add_dir = str(os.path.join(platform, game))
+            logger = setup_logger(log_level="info",
+                                  script_name=f"ROMChooser",
+                                  additional_dir=logger_add_dir,
+                                  )
+        self.logger = logger
 
         if config_file is None and config is None:
             raise ValueError("config_file or config must be specified")
@@ -394,12 +396,8 @@ def run(self,
             rom_dict):
         """Run the ROM chooser"""
 
-        self.logger.info(create_bar(f"START ROMChooser"))
-
         rom_dict = self.run_chooser(rom_dict)
 
-        self.logger.info(create_bar(f"FINISH ROMChooser"))
-
         return rom_dict
 
     def run_chooser(self,

romsearch/modules/romdownloader.py

@@ -5,11 +5,10 @@
 
 import romsearch
 from ..util import (setup_logger,
-                    create_bar,
                     load_yml,
                     get_file_pattern,
                     discord_push,
-split,
+                    split,
                     )
 
 
@@ -40,6 +39,7 @@ def add_rclone_filter(pattern=None,
 
     return cmd
 
+
 def get_tidy_files(glob_pattern):
     """Get a tidy list of files from a glob pattern.
 
@@ -62,6 +62,7 @@ def __init__(self,
                  config_file=None,
                  config=None,
                  platform_config=None,
+                 logger=None,
                  ):
         """Downloader tool via rclone
 
@@ -72,16 +73,19 @@ def __init__(self,
             config (str, optional): Configuration file. Defaults to None
             config (dict, optional): Configuration dictionary. Defaults to None
             platform_config (dict, optional): Platform configuration dictionary. Defaults to None
+            logger (logging.Logger, optional): Logger instance. Defaults to None
         """
 
         if platform is None:
             raise ValueError("platform must be specified")
         self.platform = platform
 
-        self.logger = setup_logger(log_level="info",
-                                   script_name=f"ROMDownloader",
-                                   additional_dir=platform,
-                                   )
+        if logger is None:
+            logger = setup_logger(log_level="info",
+                                  script_name=f"ROMDownloader",
+                                  additional_dir=platform,
+                                  )
+        self.logger = logger
 
         if config_file is None and config is None:
             raise ValueError("config_file or config must be specified")
@@ -138,8 +142,6 @@ def run(self,
             ):
         """Run Rclone sync tool"""
 
-        self.logger.info(create_bar(f"START ROMDownloader"))
-
         start_files = get_tidy_files(os.path.join(str(self.out_dir), "*"))
 
         self.rclone_sync(ftp_dir=self.ftp_dir,
@@ -149,7 +151,6 @@ def run(self,
         end_files = get_tidy_files(os.path.join(str(self.out_dir), "*"))
 
         if self.discord_url is not None:
-
             name = f"ROMDownloader: {self.platform}"
             self.post_to_discord(start_files,
                                  end_files,
@@ -179,8 +180,6 @@ def run(self,
                                          name=name
                                          )
 
-        self.logger.info(create_bar(f"FINISH ROMDownloader"))
-
     def rclone_sync(self,
                     ftp_dir,
                     out_dir=None,