Add in ROMAssociator

- Add ROMAssociator
- Initial support for retool filters
- Removed dat dupe parsing from DupeParser

CHANGES.rst

@@ -4,13 +4,20 @@
 Features
 --------
 
+- Initial support for ``retool`` filters
+- Added ROMAssociator, which simplifies the association of ROM files to games
 - Includes initial support for ``retool`` compilations
 - Added Game Boy Advance
 - ROMPatcher now supports RomPatcher.js
 
 Fixes
 -----
 
+DupeParser
+~~~~~~~~~~
+
+- Removed dat parsing, as this can cause issues. Now rely on ``retool`` filters
+
 ROMChooser
 ~~~~~~~~~~
 

docs/configs/config.rst

@@ -62,7 +62,6 @@ Syntax: ::
                                         # suggests caching this aggressively. Defaults to 30
 
     dupeparser:                         # DupeParser specific options
-      use_dat: true                     # OPTIONAL. Whether to use .dat files or not. Defaults to true
       use_retool: true                  # OPTIONAL. Whether to use retool clonelists or not. Defaults to true
 
     gamefinder:                         # GameFinder specific options

docs/modules.rst

@@ -16,6 +16,7 @@ the available modules.
   modules/datparser
   modules/dupeparser
   modules/gamefinder
+  modules/romassociator
   modules/romparser
   modules/romchooser
   modules/rommover

docs/modules/dupeparser.rst

@@ -2,8 +2,8 @@
 DupeParser
 ##########
 
-DupeParser generates a list of potential duplicate files based on name. It either does this via curated clonelists
-(currently only retool), or via information in parsed .dat files.
+DupeParser generates a list of potential duplicate files based on name. It does this via curated clonelists
+(currently only retool).
 
 If priorities are present in the retool clonelist, it will use these to prioritise particular release versions versus
 others

docs/modules/romassociator.rst

@@ -0,0 +1,20 @@
+#############
+ROMAssociator
+#############
+
+The ROMAssociator matches up files to games, potentially de-duping and assigning retool filters. It does this by
+checking the filename against potential game names, and assigning them to an overall association. If the retool dupe
+list has more complicated filters (see their description at
+`https://unexpectedpanda.github.io/retool/contribute-clone-lists-variants-filters/
+<https://unexpectedpanda.github.io/retool/contribute-clone-lists-variants-filters/>`_), then it will also parse through
+these as appropriate.
+
+ROMAssociator has no user-configurable arguments.
+
+API
+===
+
+.. autoclass:: romsearch.ROMAssociator
+    :no-index:
+    :members:
+    :undoc-members:

pyproject.toml

@@ -33,6 +33,7 @@ classifiers = [
 dependencies = [
     "colorlog == 6.9.0",
     "discordwebhook == 1.0.3",
+    "iso639-lang == 2.5.1",
     "numpy == 2.2.0",
     "packaging == 24.2",
     "pathvalidate == 3.2.1",

romsearch/__init__.py

@@ -8,6 +8,7 @@
     DupeParser,
     GameFinder,
     RAHasher,
+    ROMAssociator,
     ROMDownloader,
     ROMChooser,
     ROMCleaner,
@@ -22,6 +23,7 @@
     "DupeParser",
     "GameFinder",
     "RAHasher",
+    "ROMAssociator",
     "ROMDownloader",
     "ROMChooser",
     "ROMCleaner",

romsearch/configs/sample_config.yml

@@ -40,7 +40,6 @@ romdownloader:
   dry_run: false
 
 dupeparser:
-  use_dat: true
   use_retool: true
 
 gamefinder:

romsearch/gui/gui_config.py

@@ -221,7 +221,7 @@ def __init__(
         }
 
         self.all_dupeparser_options = {
-            "use_dat": self.ui.checkBoxConfigDupeParserUseDat,
+            # "use_dat": self.ui.checkBoxConfigDupeParserUseDat,
             "use_retool": self.ui.checkBoxConfigDupeParserUseRetool,
         }
 

romsearch/gui/layout_romsearch.py

@@ -848,12 +848,6 @@ def setupUi(self, RomSearch):
 
         self.verticalLayoutConfigDupeParserMiddle.addWidget(self.lineConfigDupeParserUseDatTop)
 
-        self.checkBoxConfigDupeParserUseDat = QCheckBox(self.tabConfigDupeParser)
-        self.checkBoxConfigDupeParserUseDat.setObjectName(u"checkBoxConfigDupeParserUseDat")
-        self.checkBoxConfigDupeParserUseDat.setChecked(True)
-
-        self.verticalLayoutConfigDupeParserMiddle.addWidget(self.checkBoxConfigDupeParserUseDat)
-
         self.checkBoxConfigDupeParserUseRetool = QCheckBox(self.tabConfigDupeParser)
         self.checkBoxConfigDupeParserUseRetool.setObjectName(u"checkBoxConfigDupeParserUseRetool")
         self.checkBoxConfigDupeParserUseRetool.setChecked(True)
@@ -1758,10 +1752,6 @@ def retranslateUi(self, RomSearch):
         self.lineEditConfigRAHasherCachePeriod.setText(QCoreApplication.translate("RomSearch", u"30", None))
         self.lineEditConfigRAHasherCachePeriod.setPlaceholderText(QCoreApplication.translate("RomSearch", u"30", None))
         self.tabWidgetConfig.setTabText(self.tabWidgetConfig.indexOf(self.tabConfigRAHasher), QCoreApplication.translate("RomSearch", u"RAHasher", None))
-#if QT_CONFIG(statustip)
-        self.checkBoxConfigDupeParserUseDat.setStatusTip(QCoreApplication.translate("RomSearch", u"Whether to use the dat file to figure out dupes. Default checked", None))
-#endif // QT_CONFIG(statustip)
-        self.checkBoxConfigDupeParserUseDat.setText(QCoreApplication.translate("RomSearch", u"Use .dat", None))
 #if QT_CONFIG(statustip)
         self.checkBoxConfigDupeParserUseRetool.setStatusTip(QCoreApplication.translate("RomSearch", u"Whether to use the retool clonelist to figure out dupes. Default checked", None))
 #endif // QT_CONFIG(statustip)

romsearch/gui/layout_romsearch.ui

@@ -1459,19 +1459,6 @@
                   </property>
                  </widget>
                 </item>
-                <item>
-                 <widget class="QCheckBox" name="checkBoxConfigDupeParserUseDat">
-                  <property name="statusTip">
-                   <string>Whether to use the dat file to figure out dupes. Default checked</string>
-                  </property>
-                  <property name="text">
-                   <string>Use .dat</string>
-                  </property>
-                  <property name="checked">
-                   <bool>true</bool>
-                  </property>
-                 </widget>
-                </item>
                 <item>
                  <widget class="QCheckBox" name="checkBoxConfigDupeParserUseRetool">
                   <property name="statusTip">
@@ -2715,7 +2702,7 @@
  <resources/>
  <connections/>
  <buttongroups>
-  <buttongroup name="radioButtonConfigRomsearchMethod"/>
   <buttongroup name="radioButtonConfigLoggerLevel"/>
+  <buttongroup name="radioButtonConfigRomsearchMethod"/>
  </buttongroups>
 </ui>

romsearch/modules/__init__.py

@@ -2,6 +2,7 @@
 from .dupeparser import DupeParser
 from .gamefinder import GameFinder
 from .rahasher import RAHasher
+from .romassociator import ROMAssociator
 from .romchooser import ROMChooser
 from .romcleaner import ROMCleaner
 from .romdownloader import ROMDownloader
@@ -15,6 +16,7 @@
     "DupeParser",
     "GameFinder",
     "RAHasher",
+    "ROMAssociator",
     "ROMChooser",
     "ROMCleaner",
     "ROMDownloader",

romsearch/modules/dupeparser.py

@@ -72,12 +72,12 @@ def __init__(
             )
         self.logger = logger
 
-        self.use_dat = self.config.get("dupeparser", {}).get("use_dat", True)
+        # self.use_dat = self.config.get("dupeparser", {}).get("use_dat", True)
         self.use_retool = self.config.get("dupeparser", {}).get("use_retool", True)
 
         self.parsed_dat_dir = self.config.get("dirs", {}).get("parsed_dat_dir", None)
-        if self.use_dat and self.parsed_dat_dir is None:
-            raise ValueError("Must specify parsed_dat_dir if using dat files")
+        if self.use_retool and self.parsed_dat_dir is None:
+            raise ValueError("Must specify parsed_dat_dir if using retool files")
 
         self.dupe_dir = self.config.get("dirs", {}).get("dupe_dir", None)
         if self.dupe_dir is None:
@@ -140,16 +140,16 @@ def run(self):
         return dupe_dict, retool_dict
 
     def get_dupe_dict(self):
-        """Loop through potentially both the dat files and the retool config file to get out dupes"""
+        """Loop through potentially the retool file to get out dupes"""
 
         dupe_dict = {}
 
-        # Prefer retool dupes first
+        # Retool dupes
         retool_dict = None
         if self.use_retool:
             dupe_dict, retool_dict = self.get_retool_dupes(dupe_dict)
-        if self.use_dat:
-            dupe_dict = self.get_dat_dupes(dupe_dict)
+        # if self.use_dat:
+        #     dupe_dict = self.get_dat_dupes(dupe_dict)
 
         dupe_dict = dict(sorted(dupe_dict.items()))
 
@@ -278,9 +278,11 @@ def get_retool_dupes(self, dupe_dict=None):
                     for title in retool_dupe["titles"]:
                         title_g = title["searchTerm"]
                         priority = title.get("priority", 1)
+                        filters = title.get("filters", None)
 
                         dupe_dict[found_parent_name][title_g] = {
                             "priority": priority,
+                            "filters": filters,
                         }
 
             # Next, check for compilations. If we have them, pull them out and potentially the title position
@@ -295,11 +297,13 @@ def get_retool_dupes(self, dupe_dict=None):
                         comp_g = compilation["searchTerm"]
                         title_pos = compilation.get("titlePosition", None)
                         priority = compilation.get("priority", 1)
+                        filters = compilation.get("filters", None)
 
                         dupe_dict[found_parent_name][comp_g] = {
                             "is_compilation": True,
                             "priority": priority,
                             "title_pos": title_pos,
+                            "filters": filters,
                         }
 
         return dupe_dict, retool_dupes