Merge pull request #8 from bbtufty/dev

Config I/O overhaul

CHANGES.rst

@@ -1,3 +1,14 @@
+0.0.5 (Unreleased)
+==================
+
+Fixes
+-----
+
+General
+~~~~~~~
+
+- Overhauled how config files are read in to avoid unneccesary I/O. Speed ups of about a factor 2
+
 0.0.4 (2024-05-09)
 ==================
 

romsearch/modules/datparser.py

@@ -40,18 +40,27 @@ def format_dat(dat):
 class DATParser:
 
     def __init__(self,
-                 config_file,
-                 platform,
+                 platform=None,
+                 config_file=None,
+                 config=None,
+                 platform_config=None,
                  ):
         """Parser for dat files from Redump or No-Intro
 
         For Redump dats, we can download directly from the site.
         Users will have to provide their own files for No-Intro,
         since there's no good way to scrape them automatically
+
+        Args:
+            platform (str, optional): Platform name. Defaults to None, which will throw a ValueError
+            config_file (str, optional): Configuration file. Defaults to None
+            config (dict, optional): Configuration dictionary. Defaults to None
+            platform_config (dict, optional): Platform configuration dictionary. Defaults to None
         """
 
-        self.config_file = config_file
-        config = load_yml(self.config_file)
+        if platform is None:
+            raise ValueError("platform must be specified")
+        self.platform = platform
 
         logger_add_dir = str(os.path.join(platform))
 
@@ -60,15 +69,25 @@ def __init__(self,
                                    additional_dir=logger_add_dir,
                                    )
 
-        self.dat_dir = config.get("dat_dir", None)
-        self.parsed_dat_dir = config.get("parsed_dat_dir", None)
+        if config_file is None and config is None:
+            raise ValueError("config_file or config must be specified")
+
+        if config is None:
+            config = load_yml(config_file)
+        self.config = config
+
+        self.dat_dir = self.config.get("dat_dir", None)
+        self.parsed_dat_dir = self.config.get("parsed_dat_dir", None)
 
         self.platform = platform
 
         # Read in the specific platform configuration
         mod_dir = os.path.dirname(romsearch.__file__)
-        platform_config_file = os.path.join(mod_dir, "configs", "platforms", f"{self.platform}.yml")
-        self.platform_config = load_yml(platform_config_file)
+
+        if platform_config is None:
+            platform_config_file = os.path.join(mod_dir, "configs", "platforms", f"{platform}.yml")
+            platform_config = load_yml(platform_config_file)
+        self.platform_config = platform_config
 
         self.group = self.platform_config.get("group", None)
         if self.group is None:

romsearch/modules/dupeparser.py

@@ -22,37 +22,54 @@
 class DupeParser:
 
     def __init__(self,
-                 config_file,
-                 platform,
+                 platform=None,
+                 config_file=None,
+                 config=None,
+                 default_config=None,
+                 regex_config=None,
                  ):
         """Tool for figuring out a list of dupes
 
+        Args:
+            platform (str, optional): Platform name. Defaults to None, which will throw a ValueError.
+            config_file (str, optional): Path to config file. Defaults to None
+            config (dict, optional): Configuration dictionary. Defaults to None
+            default_config (dict, optional): Default configuration dictionary. Defaults to None
+            regex_config (dict, optional): Configuration dictionary for regex search. Defaults to None
+
         TODO:
             - At some point, we might want to consider adding in the retool supersets
         """
 
+        if platform is None:
+            raise ValueError("platform must be specified")
+        self.platform = platform
+
         logger_add_dir = str(os.path.join(platform))
 
         self.logger = setup_logger(log_level="info",
                                    script_name=f"DupeParser",
                                    additional_dir=logger_add_dir,
                                    )
 
-        config = load_yml(config_file)
+        if config_file is None and config is None:
+            raise ValueError("config_file or config must be specified")
 
-        self.use_dat = config.get("dupeparser", {}).get("use_dat", True)
-        self.use_retool = config.get("dupeparser", {}).get('use_retool', True)
+        if config is None:
+            config = load_yml(config_file)
+        self.config = config
 
-        self.parsed_dat_dir = config.get("parsed_dat_dir", None)
+        self.use_dat = self.config.get("dupeparser", {}).get("use_dat", True)
+        self.use_retool = self.config.get("dupeparser", {}).get('use_retool', True)
+
+        self.parsed_dat_dir = self.config.get("parsed_dat_dir", None)
         if self.use_dat and self.parsed_dat_dir is None:
             raise ValueError("Must specify parsed_dat_dir if using dat files")
 
-        self.dupe_dir = config.get("dupe_dir", None)
+        self.dupe_dir = self.config.get("dupe_dir", None)
         if self.dupe_dir is None:
             raise ValueError("dupe_dir should be specified in config file")
 
-        self.platform = platform
-
         # Pull in platform config that we need
         mod_dir = os.path.dirname(romsearch.__file__)
         retool_config_file = os.path.join(mod_dir, "configs", "clonelists", f"retool.yml")
@@ -61,11 +78,15 @@ def __init__(self,
         self.retool_url = retool_config.get("url", None)
         self.retool_platform_file = retool_config.get(platform, None)
 
-        default_file = os.path.join(mod_dir, "configs", "defaults.yml")
-        self.default_config = load_yml(default_file)
+        if default_config is None:
+            default_file = os.path.join(mod_dir, "configs", "defaults.yml")
+            default_config = load_yml(default_file)
+        self.default_config = default_config
 
-        regex_file = os.path.join(mod_dir, "configs", "regex.yml")
-        self.regex_config = load_yml(regex_file)
+        if regex_config is None:
+            regex_file = os.path.join(mod_dir, "configs", "regex.yml")
+            regex_config = load_yml(regex_file)
+        self.regex_config = regex_config
 
     def run(self):
         """Run the dupe parser"""

romsearch/modules/gamefinder.py

@@ -44,13 +44,23 @@ def get_priority(dupe_dict, parent_name, game_name):
 class GameFinder:
 
     def __init__(self,
-                 config_file,
                  platform,
+                 config_file=None,
+                 config=None,
+                 default_config=None,
+                 regex_config=None,
                  ):
         """Tool to find games within a list of files
 
         Will parse through files to get a unique list of games, then pull
         out potential aliases and optionally remove things from user excluded list
+        
+        Args:
+            platform (str): Platform name
+            config_file (str, optional): Path to config file. Defaults to None.
+            config (dict, optional): Configuration dictionary. Defaults to None.
+            default_config (dict, optional): Default configuration dictionary. Defaults to None.
+            regex_config (dict, optional): Dictionary of regex config. Defaults to None.
         """
 
         self.logger = setup_logger(log_level="info",
@@ -62,19 +72,29 @@ def __init__(self,
             raise ValueError("platform must be specified")
         self.platform = platform
 
-        config = load_yml(config_file)
+        if config_file is None and config is None:
+            raise ValueError("config_file or config must be specified")
+
+        if config is None:
+            config = load_yml(config_file)
+        self.config = config
 
         # Pull in specifics to include/exclude
-        self.include_games = config.get("include_games", None)
-        self.exclude_games = config.get("exclude_games", None)
+        self.include_games = self.config.get("include_games", None)
+        self.exclude_games = self.config.get("exclude_games", None)
 
         # Pull in defaults for finding short game names
         mod_dir = os.path.dirname(romsearch.__file__)
-        default_file = os.path.join(mod_dir, "configs", "defaults.yml")
-        self.default_config = load_yml(default_file)
 
-        regex_file = os.path.join(mod_dir, "configs", "regex.yml")
-        self.regex_config = load_yml(regex_file)
+        if default_config is None:
+            default_file = os.path.join(mod_dir, "configs", "defaults.yml")
+            default_config = load_yml(default_file)
+        self.default_config = default_config
+
+        if regex_config is None:
+            regex_file = os.path.join(mod_dir, "configs", "regex.yml")
+            regex_config = load_yml(regex_file)
+        self.regex_config = regex_config
 
         # Info for dupes
         self.dupe_dir = config.get("dupe_dir", None)

romsearch/modules/romchooser.py

@@ -154,6 +154,7 @@ def get_best_rom_per_region(rom_dict,
 
     return rom_dict
 
+
 def remove_unwanted_roms(rom_dict, key_to_check, check_type="include"):
     """Remove unwanted ROMs from the dict
 
@@ -295,13 +296,24 @@ def filter_by_list(rom_dict,
 class ROMChooser:
 
     def __init__(self,
-                 config_file,
                  platform,
-                 game
+                 game,
+                 config_file=None,
+                 config=None,
+                 default_config=None,
+                 regex_config=None,
                  ):
         """ROM choose tool
 
         This works per-game, per-platform, so must be specified here
+
+        Args:
+            platform (str): Platform name
+            game (str): Game name
+            config_file (str, optional): Path to config file. Defaults to None.
+            config (dict, optional): Configuration dictionary. Defaults to None.
+            default_config (dict, optional): Default configuration dictionary. Defaults to None.
+            regex_config (dict, optional): Configuration dictionary. Defaults to None.
         """
 
         if platform is None:
@@ -315,18 +327,27 @@ def __init__(self,
                                    additional_dir=logger_add_dir,
                                    )
 
-        config = load_yml(config_file)
+        if config_file is None and config is None:
+            raise ValueError("config_file or config must be specified")
+
+        if config is None:
+            config = load_yml(config_file)
+        self.config = config
 
         mod_dir = os.path.dirname(romsearch.__file__)
 
-        default_config_file = os.path.join(mod_dir, "configs", "defaults.yml")
-        self.default_config = load_yml(default_config_file)
+        if default_config is None:
+            default_file = os.path.join(mod_dir, "configs", "defaults.yml")
+            default_config = load_yml(default_file)
+        self.default_config = default_config
 
-        platform_config_file = os.path.join(mod_dir, "configs", "platforms", f"{platform}.yml")
-        self.platform_config = load_yml(platform_config_file)
+        if regex_config is None:
+            regex_file = os.path.join(mod_dir, "configs", "regex.yml")
+            regex_config = load_yml(regex_file)
+        self.regex_config = regex_config
 
         # Region preference (usually set USA for retroachievements, can also be a list to fall back to)
-        region_preferences = config.get("region_preferences", self.default_config["default_region"])
+        region_preferences = self.config.get("region_preferences", self.default_config["default_region"])
         if isinstance(region_preferences, str):
             region_preferences = [region_preferences]
 
@@ -337,7 +358,7 @@ def __init__(self,
         self.region_preferences = region_preferences
 
         # Language preference (usually set En, can also be a list to fall back to)
-        language_preferences = config.get("language_preferences", self.default_config["default_language"])
+        language_preferences = self.config.get("language_preferences", self.default_config["default_language"])
         if isinstance(language_preferences, str):
             language_preferences = [language_preferences]
 
@@ -348,7 +369,7 @@ def __init__(self,
         self.language_preferences = language_preferences
 
         # Various filters. First are the boolean ones
-        bool_filters = config.get("romchooser", {}).get("bool_filters", "all_but_games")
+        bool_filters = self.config.get("romchooser", {}).get("bool_filters", "all_but_games")
         if "all" in bool_filters:
             all_bool_filters = copy.deepcopy(BOOL_FILTERS)
             if "all_but" in bool_filters:
@@ -362,12 +383,12 @@ def __init__(self,
             all_bool_filters = bool_filters
         self.bool_filters = all_bool_filters
 
-        self.filter_regions = config.get("romchooser", {}).get("filter_regions", True)
-        self.filter_languages = config.get("romchooser", {}).get("filter_languages", True)
-        self.allow_multiple_regions = config.get("romchooser", {}).get("allow_multiple_regions", False)
-        self.use_best_version = config.get("romchooser", {}).get("use_best_version", True)
+        self.filter_regions = self.config.get("romchooser", {}).get("filter_regions", True)
+        self.filter_languages = self.config.get("romchooser", {}).get("filter_languages", True)
+        self.allow_multiple_regions = self.config.get("romchooser", {}).get("allow_multiple_regions", False)
+        self.use_best_version = self.config.get("romchooser", {}).get("use_best_version", True)
 
-        self.dry_run = config.get("romchooser", {}).get("dry_run", False)
+        self.dry_run = self.config.get("romchooser", {}).get("dry_run", False)
 
     def run(self,
             rom_dict):