diff --git a/CHANGES.rst b/CHANGES.rst index 3971c89..58bef0f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,6 +4,8 @@ Features -------- +- Initial support for ``retool`` filters +- Added ROMAssociator, which simplifies the association of ROM files to games - Includes initial support for ``retool`` compilations - Added Game Boy Advance - ROMPatcher now supports RomPatcher.js @@ -11,6 +13,11 @@ Features Fixes ----- +DupeParser +~~~~~~~~~~ + +- Removed dat parsing, as this can cause issues. Now rely on ``retool`` filters + ROMChooser ~~~~~~~~~~ diff --git a/docs/configs/config.rst b/docs/configs/config.rst index 14e6509..e056757 100644 --- a/docs/configs/config.rst +++ b/docs/configs/config.rst @@ -62,7 +62,6 @@ Syntax: :: # suggests caching this aggressively. Defaults to 30 dupeparser: # DupeParser specific options - use_dat: true # OPTIONAL. Whether to use .dat files or not. Defaults to true use_retool: true # OPTIONAL. Whether to use retool clonelists or not. Defaults to true gamefinder: # GameFinder specific options diff --git a/docs/modules.rst b/docs/modules.rst index 929a868..6383c3c 100644 --- a/docs/modules.rst +++ b/docs/modules.rst @@ -16,6 +16,7 @@ the available modules. modules/datparser modules/dupeparser modules/gamefinder + modules/romassociator modules/romparser modules/romchooser modules/rommover diff --git a/docs/modules/dupeparser.rst b/docs/modules/dupeparser.rst index 4a932d4..18ec433 100644 --- a/docs/modules/dupeparser.rst +++ b/docs/modules/dupeparser.rst @@ -2,8 +2,8 @@ DupeParser ########## -DupeParser generates a list of potential duplicate files based on name. It either does this via curated clonelists -(currently only retool), or via information in parsed .dat files. +DupeParser generates a list of potential duplicate files based on name. It does this via curated clonelists +(currently only retool). If priorities are present in the retool clonelist, it will use these to prioritise particular release versions versus others diff --git a/docs/modules/romassociator.rst b/docs/modules/romassociator.rst new file mode 100644 index 0000000..355c40c --- /dev/null +++ b/docs/modules/romassociator.rst @@ -0,0 +1,20 @@ +############# +ROMAssociator +############# + +The ROMAssociator matches up files to games, potentially de-duping and assigning retool filters. It does this by +checking the filename against potential game names, and assigning them to an overall association. If the retool dupe +list has more complicated filters (see their description at +`https://unexpectedpanda.github.io/retool/contribute-clone-lists-variants-filters/ +`_), then it will also parse through +these as appropriate. + +ROMAssociator has no user-configurable arguments. + +API +=== + +.. autoclass:: romsearch.ROMAssociator + :no-index: + :members: + :undoc-members: \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 5a5ab38..21abe04 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,7 @@ classifiers = [ dependencies = [ "colorlog == 6.9.0", "discordwebhook == 1.0.3", + "iso639-lang == 2.5.1", "numpy == 2.2.0", "packaging == 24.2", "pathvalidate == 3.2.1", diff --git a/romsearch/__init__.py b/romsearch/__init__.py index f109371..5184cef 100644 --- a/romsearch/__init__.py +++ b/romsearch/__init__.py @@ -8,6 +8,7 @@ DupeParser, GameFinder, RAHasher, + ROMAssociator, ROMDownloader, ROMChooser, ROMCleaner, @@ -22,6 +23,7 @@ "DupeParser", "GameFinder", "RAHasher", + "ROMAssociator", "ROMDownloader", "ROMChooser", "ROMCleaner", diff --git a/romsearch/configs/sample_config.yml b/romsearch/configs/sample_config.yml index 6381837..8a862cb 100644 --- a/romsearch/configs/sample_config.yml +++ b/romsearch/configs/sample_config.yml @@ -40,7 +40,6 @@ romdownloader: dry_run: false dupeparser: - use_dat: true use_retool: true gamefinder: diff --git a/romsearch/gui/gui_config.py b/romsearch/gui/gui_config.py index be442b8..2ada8d0 100644 --- a/romsearch/gui/gui_config.py +++ b/romsearch/gui/gui_config.py @@ -221,7 +221,7 @@ def __init__( } self.all_dupeparser_options = { - "use_dat": self.ui.checkBoxConfigDupeParserUseDat, + # "use_dat": self.ui.checkBoxConfigDupeParserUseDat, "use_retool": self.ui.checkBoxConfigDupeParserUseRetool, } diff --git a/romsearch/gui/layout_romsearch.py b/romsearch/gui/layout_romsearch.py index 3ed6150..f03c0fa 100644 --- a/romsearch/gui/layout_romsearch.py +++ b/romsearch/gui/layout_romsearch.py @@ -848,12 +848,6 @@ def setupUi(self, RomSearch): self.verticalLayoutConfigDupeParserMiddle.addWidget(self.lineConfigDupeParserUseDatTop) - self.checkBoxConfigDupeParserUseDat = QCheckBox(self.tabConfigDupeParser) - self.checkBoxConfigDupeParserUseDat.setObjectName(u"checkBoxConfigDupeParserUseDat") - self.checkBoxConfigDupeParserUseDat.setChecked(True) - - self.verticalLayoutConfigDupeParserMiddle.addWidget(self.checkBoxConfigDupeParserUseDat) - self.checkBoxConfigDupeParserUseRetool = QCheckBox(self.tabConfigDupeParser) self.checkBoxConfigDupeParserUseRetool.setObjectName(u"checkBoxConfigDupeParserUseRetool") self.checkBoxConfigDupeParserUseRetool.setChecked(True) @@ -1758,10 +1752,6 @@ def retranslateUi(self, RomSearch): self.lineEditConfigRAHasherCachePeriod.setText(QCoreApplication.translate("RomSearch", u"30", None)) self.lineEditConfigRAHasherCachePeriod.setPlaceholderText(QCoreApplication.translate("RomSearch", u"30", None)) self.tabWidgetConfig.setTabText(self.tabWidgetConfig.indexOf(self.tabConfigRAHasher), QCoreApplication.translate("RomSearch", u"RAHasher", None)) -#if QT_CONFIG(statustip) - self.checkBoxConfigDupeParserUseDat.setStatusTip(QCoreApplication.translate("RomSearch", u"Whether to use the dat file to figure out dupes. Default checked", None)) -#endif // QT_CONFIG(statustip) - self.checkBoxConfigDupeParserUseDat.setText(QCoreApplication.translate("RomSearch", u"Use .dat", None)) #if QT_CONFIG(statustip) self.checkBoxConfigDupeParserUseRetool.setStatusTip(QCoreApplication.translate("RomSearch", u"Whether to use the retool clonelist to figure out dupes. Default checked", None)) #endif // QT_CONFIG(statustip) diff --git a/romsearch/gui/layout_romsearch.ui b/romsearch/gui/layout_romsearch.ui index bd06f1b..0419b52 100644 --- a/romsearch/gui/layout_romsearch.ui +++ b/romsearch/gui/layout_romsearch.ui @@ -1459,19 +1459,6 @@ - - - - Whether to use the dat file to figure out dupes. Default checked - - - Use .dat - - - true - - - @@ -2715,7 +2702,7 @@ - + diff --git a/romsearch/modules/__init__.py b/romsearch/modules/__init__.py index 651ef32..ffc7f6a 100644 --- a/romsearch/modules/__init__.py +++ b/romsearch/modules/__init__.py @@ -2,6 +2,7 @@ from .dupeparser import DupeParser from .gamefinder import GameFinder from .rahasher import RAHasher +from .romassociator import ROMAssociator from .romchooser import ROMChooser from .romcleaner import ROMCleaner from .romdownloader import ROMDownloader @@ -15,6 +16,7 @@ "DupeParser", "GameFinder", "RAHasher", + "ROMAssociator", "ROMChooser", "ROMCleaner", "ROMDownloader", diff --git a/romsearch/modules/dupeparser.py b/romsearch/modules/dupeparser.py index 326b28b..1cc0fa2 100644 --- a/romsearch/modules/dupeparser.py +++ b/romsearch/modules/dupeparser.py @@ -72,12 +72,12 @@ def __init__( ) self.logger = logger - self.use_dat = self.config.get("dupeparser", {}).get("use_dat", True) + # self.use_dat = self.config.get("dupeparser", {}).get("use_dat", True) self.use_retool = self.config.get("dupeparser", {}).get("use_retool", True) self.parsed_dat_dir = self.config.get("dirs", {}).get("parsed_dat_dir", None) - if self.use_dat and self.parsed_dat_dir is None: - raise ValueError("Must specify parsed_dat_dir if using dat files") + if self.use_retool and self.parsed_dat_dir is None: + raise ValueError("Must specify parsed_dat_dir if using retool files") self.dupe_dir = self.config.get("dirs", {}).get("dupe_dir", None) if self.dupe_dir is None: @@ -140,16 +140,16 @@ def run(self): return dupe_dict, retool_dict def get_dupe_dict(self): - """Loop through potentially both the dat files and the retool config file to get out dupes""" + """Loop through potentially the retool file to get out dupes""" dupe_dict = {} - # Prefer retool dupes first + # Retool dupes retool_dict = None if self.use_retool: dupe_dict, retool_dict = self.get_retool_dupes(dupe_dict) - if self.use_dat: - dupe_dict = self.get_dat_dupes(dupe_dict) + # if self.use_dat: + # dupe_dict = self.get_dat_dupes(dupe_dict) dupe_dict = dict(sorted(dupe_dict.items())) @@ -278,9 +278,11 @@ def get_retool_dupes(self, dupe_dict=None): for title in retool_dupe["titles"]: title_g = title["searchTerm"] priority = title.get("priority", 1) + filters = title.get("filters", None) dupe_dict[found_parent_name][title_g] = { "priority": priority, + "filters": filters, } # Next, check for compilations. If we have them, pull them out and potentially the title position @@ -295,11 +297,13 @@ def get_retool_dupes(self, dupe_dict=None): comp_g = compilation["searchTerm"] title_pos = compilation.get("titlePosition", None) priority = compilation.get("priority", 1) + filters = compilation.get("filters", None) dupe_dict[found_parent_name][comp_g] = { "is_compilation": True, "priority": priority, "title_pos": title_pos, + "filters": filters, } return dupe_dict, retool_dupes diff --git a/romsearch/modules/gamefinder.py b/romsearch/modules/gamefinder.py index 6d2073d..cc7a6bd 100644 --- a/romsearch/modules/gamefinder.py +++ b/romsearch/modules/gamefinder.py @@ -15,7 +15,12 @@ load_json, ) -DUPE_DEFAULT = {"is_compilation": False, "priority": 1, "title_pos": None} +DUPE_DEFAULT = { + "is_compilation": False, + "priority": 1, + "title_pos": None, + "filters": None, +} def get_all_games( @@ -23,7 +28,13 @@ def get_all_games( default_config=None, regex_config=None, ): - """Get all unique game names from a list of game files.""" + """Get a list of short game names. + + Args: + files: List of files to parse + default_config: Default config to use + regex_config: Regex settings to use + """ games = [ get_short_name(f, default_config=default_config, regex_config=regex_config) @@ -192,6 +203,14 @@ def get_game_dict( self, files, ): + """Get a game dictionary out. + + From a list of files, parse out dupes, apply includes and excludes, + and return a game dictionary. + + Args: + files (list): List of files to associate + """ games = get_all_games( files, @@ -212,7 +231,9 @@ def get_game_dict( for game in games: game_dict[game] = { - "priority": 1, + game: { + "priority": 1, + } } # Remove any excluded files @@ -239,7 +260,6 @@ def get_game_dict( filtered_game_dict[g] = game_dict[g] game_dict = copy.deepcopy(filtered_game_dict) - return game_dict def get_game_matches( @@ -247,7 +267,7 @@ def get_game_matches( game_dict, games_to_match, ): - """Get files that match an input dictionary (so as to properly handle dupes + """Get files that match an input dictionary (to properly handle dupes) Args: - game_dict (dict): Dictionary of games to match against @@ -328,43 +348,89 @@ def get_filter_dupes(self, games): # Loop over games, and the dupes dictionary. Also pull out various other important info for g in games: - # Because we have compilations, these can be lists - found_parent_names = get_parent_name( - game_name=g, - dupe_dict=self.dupe_dict, + # Look at the short names + game_dict = self.filter_by_short_name( + game=g, + game_dict=game_dict, ) - for found_parent_name in found_parent_names: + return game_dict - found_parent_name_lower = found_parent_name.lower() - game_dict_keys = [key for key in game_dict.keys()] - game_dict_keys_lower = [key.lower() for key in game_dict.keys()] + def filter_by_short_name( + self, + game, + game_dict=None, + ): + """Add entries to game dict based on short name - if found_parent_name_lower not in game_dict_keys_lower: - game_dict[found_parent_name] = {} - final_parent_name = copy.deepcopy(found_parent_name) - else: - final_parent_idx = game_dict_keys_lower.index( - found_parent_name_lower - ) - final_parent_name = game_dict_keys[final_parent_idx] + Will find possible parents, then add those to the game dict - dupe_entry = get_dupe_entry( - dupe_dict=self.dupe_dict, - parent_name=found_parent_name, - game_name=g, - ) + Args: + game (str): Short game name + game_dict (dict): Dictionary of games to match against. Defaults + to None, which will create an empty dict + """ + + if game_dict is None: + game_dict = {} + + found_parent_names = get_parent_name( + game_name=game, + dupe_dict=self.dupe_dict, + ) + + for found_parent_name in found_parent_names: + + game_dict = self.add_dupe_entry_to_game_dict( + game, + game_dict=game_dict, + parent_name=found_parent_name, + ) + + return game_dict + + def add_dupe_entry_to_game_dict( + self, + game, + game_dict, + parent_name, + ): + """Add a dupe entry to the game dict + + Args: + game (str): Game name + game_dict (dict): Dictionary of games. Defaults + to None, which will create an empty dict + parent_name (str): Parent name for the game + """ + + parent_name_lower = parent_name.lower() + game_dict_keys = [key for key in game_dict.keys()] + game_dict_keys_lower = [key.lower() for key in game_dict.keys()] + + if parent_name_lower not in game_dict_keys_lower: + game_dict[parent_name] = {} + final_parent_name = copy.deepcopy(parent_name) + else: + final_parent_idx = game_dict_keys_lower.index(parent_name_lower) + final_parent_name = game_dict_keys[final_parent_idx] + + dupe_entry = get_dupe_entry( + dupe_dict=self.dupe_dict, + parent_name=parent_name, + game_name=game, + ) - # We want to make sure we also don't duplicate on the names being upper/lowercase - g_names = [g_dict for g_dict in game_dict[final_parent_name]] - g_names_lower = [g_name.lower() for g_name in g_names] - if g.lower() in g_names_lower: - g_idx = g_names_lower.index(g.lower()) - g = g_names[g_idx] + # We want to make sure we also don't duplicate on the names being upper/lowercase + g_names = [g_dict for g_dict in game_dict[final_parent_name]] + g_names_lower = [g_name.lower() for g_name in g_names] + if game.lower() in g_names_lower: + g_idx = g_names_lower.index(game.lower()) + game = g_names[g_idx] - if g not in game_dict[final_parent_name]: - game_dict[final_parent_name][g] = {} + if game not in game_dict[final_parent_name]: + game_dict[final_parent_name][game] = {} - game_dict[final_parent_name][g].update(dupe_entry) + game_dict[final_parent_name][game].update(dupe_entry) return game_dict diff --git a/romsearch/modules/romassociator.py b/romsearch/modules/romassociator.py new file mode 100644 index 0000000..2a4c756 --- /dev/null +++ b/romsearch/modules/romassociator.py @@ -0,0 +1,483 @@ +import copy +import os +import re +from iso639 import Lang + +import romsearch +from .romparser import ROMParser +from ..util import ( + setup_logger, + load_yml, + centred_string, +) + + +def check_regions( + filter_regions, + rom_regions, +): + """Check condition regions against parsed regions + + Here, we only check that any of the ROM regions match any of the + filter regions + + Args: + filter_regions (list): List of regions to match from the filter + rom_regions (list): Parsed ROM regions + """ + + s_f = set(filter_regions) + s_r = set(rom_regions) + + s_i = s_r.intersection(s_f) + + # If we've got no matches, then this hasn't been satisfied + if len(s_i) == 0: + return False + + return True + + +def check_languages( + filter_langs, + rom_langs, +): + """Check condition languages against parsed languages + + Here, we only check that any of the ROM languages match any of the + filter languages. N.B. retool here uses ISO 639-1, so we need to + parse them + + Args: + filter_langs (list): List of languages to match from the filter + rom_langs (list): Parsed ROM languages + """ + + long_filter_langs = [Lang(fl.lower()).name for fl in filter_langs] + + s_f = set(long_filter_langs) + s_r = set(rom_langs) + + s_i = s_r.intersection(s_f) + + # If we've got no matches, then this hasn't been satisfied + if len(s_i) == 0: + return False + + return True + + +def check_string( + regex_str, + file_name, +): + """Check filename for regex string + + Args: + regex_str (str): String to check + file_name (str): Name of file to check + """ + + match = re.search(regex_str, file_name) + + if match is None: + return False + + return True + + +def check_region_order( + region_order, + user_region_preferences, + all_regions=None, +): + """Check region order against parsed regions + + Here, we only check that *any* of the higher regions are + higher in user preference than *all* of the lower regions + + Args: + region_order (dict): Dictionary of form {"higherRegions": [], "lowerRegions": []} + user_region_preferences (list): Ordered list of user region preferences + all_regions (list): List of all available regions + """ + + if all_regions is None: + all_regions = [] + + higher_regions = region_order["higherRegions"] + lower_regions = region_order["lowerRegions"] + + # If we've got an 'all other regions' here, then + # pull them out + updated_higher_regions = copy.deepcopy(higher_regions) + if higher_regions == ["All other regions"]: + updated_higher_regions = copy.deepcopy(all_regions) + for reg in lower_regions: + updated_higher_regions.remove(reg) + + updated_lower_regions = copy.deepcopy(lower_regions) + if lower_regions == ["All other regions"]: + updated_lower_regions = copy.deepcopy(all_regions) + for reg in higher_regions: + updated_lower_regions.remove(reg) + + higher_regions = copy.deepcopy(updated_higher_regions) + lower_regions = copy.deepcopy(updated_lower_regions) + + high_prio_region = 99 + for reg in higher_regions: + + # If we have the region in the preferences, note the location + if reg in user_region_preferences: + + # Take the highest priority + high_prio_region = min( + [high_prio_region, user_region_preferences.index(reg)] + ) + + # If we still have a high priority of 99, we haven't found anything so jump out + if high_prio_region == 99: + return False + + # Now look through the low priorities, and find the highest priority of those + low_prio_region = 99 + for reg in lower_regions: + + # If we have the region in the preferences, note the location + if reg in user_region_preferences: + + # Take the highest priority + low_prio_region = min([low_prio_region, user_region_preferences.index(reg)]) + + # If we're at a low priority of 99, then we haven't found anything so we're good to + # return a True + if low_prio_region == 99: + return True + + # If *any* of the high priority regions are higher than *all* of the low + # priority, then return True. Else, False + if low_prio_region > high_prio_region: + return False + + return True + + +def apply_results( + game, + game_dict, + results, +): + """Apply results to a filtered match + + Args: + game: Game name + game_dict: Dictionary of game properties + results: Dictionary of results to apply to the game/game dict + """ + + for r in results: + + # If we're changing the name, that gets pulled out here + if r == "group": + game = copy.deepcopy(results[r]) + + # If we're changing priority, edit the game dict + elif r == "priority": + game_dict[r] = results[r] + + # Ignore local names, since we won't use them + elif r == "localNames": + continue + + else: + print(game, game_dict) + raise ValueError(f"Unsure how to deal with result type {r}") + + return game, game_dict + + +class ROMAssociator: + + def __init__( + self, + platform=None, + dat=None, + retool=None, + ra_hashes=None, + config_file=None, + config=None, + platform_config=None, + regex_config=None, + default_config=None, + logger=None, + log_line_sep="=", + log_line_length=100, + ): + """Tool for associating ROMs to games + + This will primarily use a list of associations, although there are also more granular + options for filtering based on specific retool conditions + + Args: + platform (str, optional): Platform name. Defaults to None, which will throw a ValueError. + dat (dict): Parsed dat dictionary. Defaults to None, which will try to load the dat file if it exists + retool (dict): Retool dictionary. Defaults to None, which will try to load the file if it exists + ra_hashes (dict): RA hash dictionary. Defaults to None, which will try to load the file if it exists + config_file (str, optional): path to config file. Defaults to None. + config (dict, optional): configuration dictionary. Defaults to None. + platform_config (dict, optional): platform configuration dictionary. Defaults to None. + regex_config (dict, optional): regex configuration dictionary. Defaults to None. + default_config (dict, optional): default configuration dictionary. Defaults to None. + logger (logging.Logger, optional): logger instance. Defaults to None. + log_line_length (int, optional): Line length of log. Defaults to 100 + """ + + if config_file is None and config is None: + raise ValueError("config_file or config must be specified") + + if config is None: + config = load_yml(config_file) + self.config = config + + self.platform = platform + + # Pull in platform config that we need + mod_dir = os.path.dirname(romsearch.__file__) + + if default_config is None: + default_file = os.path.join(mod_dir, "configs", "defaults.yml") + default_config = load_yml(default_file) + self.default_config = default_config + + if regex_config is None: + regex_file = os.path.join(mod_dir, "configs", "regex.yml") + regex_config = load_yml(regex_file) + self.regex_config = regex_config + + self.dat = dat + self.retool = retool + self.ra_hashes = ra_hashes + self.platform_config = platform_config + + if logger is None: + log_dir = self.config.get("dirs", {}).get( + "log_dir", os.path.join(os.getcwd(), "logs") + ) + log_level = self.config.get("logger", {}).get("level", "info") + logger = setup_logger( + log_level=log_level, + script_name=f"ROMAssociator", + log_dir=log_dir, + additional_dir=platform, + ) + self.logger = logger + + self.log_line_sep = log_line_sep + self.log_line_length = log_line_length + + def run( + self, + files, + games, + ): + """Run the ROM associator""" + + self.logger.info(f"{self.log_line_sep * self.log_line_length}") + self.logger.info( + centred_string("Running ROMAssociator", total_length=self.log_line_length) + ) + self.logger.info(f"{self.log_line_sep * self.log_line_length}") + + associations = self.run_associator(files, games) + + self.logger.info(f"{self.log_line_sep * self.log_line_length}") + + return associations + + def run_associator(self, files, games): + """Associate files with games + + Args: + files: List of files to associate + games: List of games to associate + """ + + self.logger.info( + centred_string( + f"Associating {len(files)} files to {len(games)} games", + total_length=self.log_line_length, + ) + ) + + associations = {} + + # Loop over each file + for f in files: + + # Loop over each game + for game in games: + + # We check by a lowercase version of the short name + f_lower = files[f]["short_name"].lower() + for g in games[game]: + + g_lower = g.lower() + + if f_lower == g_lower: + + # Pull out the particular dictionary and a copy + # of the game name. We'll need these separately + # as they may get updated by filters + final_game = copy.deepcopy(game) + final_game_dict = copy.deepcopy(games[game][g]) + + # Pull out potential filters + filters = games[game][g].get("filters", None) + if filters is not None: + final_game, final_game_dict = self.apply_filters( + game=final_game, + game_dict=final_game_dict, + file=f, + file_dict=files[f], + filters=filters, + ) + + # Make sure we match by lower case, just to be sure + all_associations = [k for k in associations] + all_associations_lower = [k.lower() for k in all_associations] + + if final_game.lower() not in all_associations_lower: + associations[final_game] = {} + else: + idx = all_associations.index(final_game) + final_game = all_associations[idx] + + # Update the dictionary as appropriate + if f not in associations[final_game]: + associations[final_game][f] = {} + associations[final_game][f].update(final_game_dict) + + # If we're duplicating a match, and it's not part of a compilation, freak out + is_compilation = final_game_dict.get("is_compilation", False) + if files[f]["matched"] and not is_compilation: + self.logger.warning( + centred_string( + f"{f} has already been matched! " + f"This should not generally happen", + total_length=self.log_line_length, + ) + ) + + files[f]["matched"] = True + + # Log out any unmatched files + self.log_unmatched_files(files) + + return associations + + def apply_filters( + self, + file, + file_dict, + game, + game_dict, + filters, + ): + """Apply filters to game if conditions are met + + Args: + file: Filename for ROM + file_dict: Dictionary of file properties + game: Game name + game_dict: Dictionary of game properties + filters: Filters with conditions and results to apply + """ + + # Parse out the filename + file_to_parse = {file: file_dict} + rp = ROMParser( + platform=self.platform, + game=game, + config=self.config, + dat=self.dat, + retool=self.retool, + ra_hashes=self.ra_hashes, + default_config=self.default_config, + regex_config=self.regex_config, + logger=self.logger, + ) + rom_parsed = rp.run(file_to_parse) + rom_parsed = rom_parsed.get(file, None) + + if rom_parsed is None: + raise ValueError("ROM parsing failed") + + # Having parsed the file, now loop over the conditions + for filt in filters: + + conditions_met = [] + for c in filt["conditions"]: + if c == "matchLanguages": + condition_met = check_languages( + filt["conditions"][c], rom_parsed["languages"] + ) + elif c == "matchRegions": + condition_met = check_regions( + filt["conditions"][c], rom_parsed["regions"] + ) + elif c == "matchString": + condition_met = check_string(filt["conditions"][c], file) + elif c == "regionOrder": + condition_met = check_region_order( + filt["conditions"][c], + self.config["region_preferences"], + all_regions=list(self.default_config["regions"]), + ) + else: + raise ValueError(f"Unsure how to deal with condition {c}") + conditions_met.append(condition_met) + + # If we've met the conditions, then apply the results + if all(conditions_met): + game, game_dict = apply_results( + game, + game_dict, + filt["results"], + ) + + return game, game_dict + + def log_unmatched_files(self, files): + """Log out files we haven't matched + + Args: + files: Dictionary of files with a flag for whether they've + been matched + """ + + unmatched = [f for f in files if not files[f]["matched"]] + n_unmatched = len(unmatched) + + if n_unmatched > 0: + self.logger.warning( + centred_string( + f"Failed to associate {n_unmatched} files", + total_length=self.log_line_length, + ) + ) + else: + self.logger.info( + centred_string( + f"All files associated", total_length=self.log_line_length + ) + ) + + self.logger.debug(f"{'-' * self.log_line_length}") + self.logger.debug( + centred_string("Unmatched files:", total_length=self.log_line_length) + ) + self.logger.debug(f"{'-' * self.log_line_length}") + for f in unmatched: + self.logger.debug(centred_string(f"{f}", total_length=self.log_line_length)) + self.logger.debug(f"{'-' * self.log_line_length}") diff --git a/romsearch/modules/romcleaner.py b/romsearch/modules/romcleaner.py index 1e906a0..86c7623 100644 --- a/romsearch/modules/romcleaner.py +++ b/romsearch/modules/romcleaner.py @@ -115,9 +115,10 @@ def run( self.logger.info(f"{self.log_line_sep * self.log_line_length}") # Join these up into a dictionary - cleaned = {"ROMs": roms_cleaned, - "Cache": cache_cleaned, - } + cleaned = { + "ROMs": roms_cleaned, + "Cache": cache_cleaned, + } return cleaned diff --git a/romsearch/modules/romsearch.py b/romsearch/modules/romsearch.py index 615f22e..591f610 100644 --- a/romsearch/modules/romsearch.py +++ b/romsearch/modules/romsearch.py @@ -9,6 +9,7 @@ from .dupeparser import DupeParser from .gamefinder import GameFinder from .rahasher import RAHasher +from .romassociator import ROMAssociator from .romchooser import ROMChooser from .romcleaner import ROMCleaner from .romdownloader import ROMDownloader @@ -291,38 +292,30 @@ def run( all_roms_moved = [] all_roms_dict = {} - for i, game in enumerate(all_games): - - rom_files = {} - - # We check by a lowercase version of the short name - for f in all_file_dict: - f_lower = all_file_dict[f]["short_name"].lower() - for g in all_games[game]: - - g_lower = g.lower() + # Associate files to games + associator = ROMAssociator( + platform=platform, + dat=dat_dict, + retool=retool_dict, + ra_hashes=ra_hash_dict, + config=self.config, + platform_config=platform_config, + regex_config=self.regex_config, + default_config=self.default_config, + logger=self.logger, + log_line_length=log_line_length, + ) - if f_lower == g_lower: + associations = associator.run( + files=all_file_dict, + games=all_games, + ) - # Update the dictionary as appropriate - if f not in rom_files: - rom_files[f] = {} - rom_files[f].update(all_games[game][g]) + continue - # If we're duplicating a match, and it's not part of a compilation, freak out - is_compilation = all_games[game][g].get( - "is_compilation", False - ) - if all_file_dict[f]["matched"] and not is_compilation: - self.logger.warning( - centred_string( - f"{f} has already been matched! " - f"This should not generally happen", - total_length=log_line_length, - ) - ) + for game in associations: - all_file_dict[f]["matched"] = True + rom_files = associations[game] parse = ROMParser( platform=platform, @@ -358,18 +351,6 @@ def run( # Save to a big dictionary, since we'll move all at once all_roms_dict[game] = rom_dict - self.logger.debug(f"{log_line_sep * log_line_length}") - self.logger.debug( - centred_string("Unmatched files:", total_length=log_line_length) - ) - self.logger.debug(f"{'-' * log_line_length}") - for f in all_file_dict: - if not all_file_dict[f]["matched"]: - self.logger.debug( - centred_string(f"{f}", total_length=log_line_length) - ) - self.logger.debug(f"{log_line_sep * log_line_length}") - if self.dry_run: self.logger.info(f"{log_line_sep * log_line_length}") self.logger.info( diff --git a/romsearch/util/general.py b/romsearch/util/general.py index 04d8d3e..cdbe24e 100644 --- a/romsearch/util/general.py +++ b/romsearch/util/general.py @@ -20,6 +20,7 @@ def split(full_list, chunk_size=10): def get_parent_name( game_name, dupe_dict, + return_if_not_found=False, ): """Get the parent name(s) recursively searching through a dupe dict @@ -28,6 +29,8 @@ def get_parent_name( Args: game_name (str): game name to find parents for dupe_dict (dict): dupe dict to search through + return_if_not_found (bool, optional): If we don't find a dupe, return a None. + Defaults to False. """ # We do this by lowercase checking @@ -58,7 +61,11 @@ def get_parent_name( found_dupe = True if not found_dupe: - found_parent_names = copy.deepcopy(game_name) + + if return_if_not_found: + return None + else: + found_parent_names = copy.deepcopy(game_name) if found_parent_names is None: raise ValueError("Could not find a parent name!")