Improved game association

- GameFinder has been significantly overhauled
- Normalise games with disc names in them
- Initial support for retool filters
- Removed dat dupe parsing from DupeParser
- Include (Pirate) tag in short name
- Include Disc # tag in short name
- Added some more regex tags
- Increase RAHasher sleep time to 0.5s
- Update dev scripts

CHANGES.rst

@@ -4,13 +4,26 @@
 Features
 --------
 
+- GameFinder has been significantly overhauled
+- Normalise games with disc names in them
+- Initial support for ``retool`` filters
 - Includes initial support for ``retool`` compilations
 - Added Game Boy Advance
 - ROMPatcher now supports RomPatcher.js
 
 Fixes
 -----
 
+DupeParser
+~~~~~~~~~~
+
+- Removed dat parsing, as this can cause issues. Now rely on ``retool`` filters
+
+RAHasher
+~~~~~~~~
+
+- Increase sleep time to 0.5s to avoid API errors
+
 ROMChooser
 ~~~~~~~~~~
 
@@ -39,6 +52,8 @@ General
 - Added known issue for long filenames
 - RAPatch checks now includes modern/improved/demoted versions
 - Language parsing can now handle languages formatted like "En+De" (and test updated)
+- Updated regex
+- Updated dev scripts
 
 0.1.0 (2024-12-04)
 ==================

docs/configs/config.rst

@@ -62,7 +62,6 @@ Syntax: ::
                                         # suggests caching this aggressively. Defaults to 30
 
     dupeparser:                         # DupeParser specific options
-      use_dat: true                     # OPTIONAL. Whether to use .dat files or not. Defaults to true
       use_retool: true                  # OPTIONAL. Whether to use retool clonelists or not. Defaults to true
 
     gamefinder:                         # GameFinder specific options

docs/modules/dupeparser.rst

@@ -2,8 +2,8 @@
 DupeParser
 ##########
 
-DupeParser generates a list of potential duplicate files based on name. It either does this via curated clonelists
-(currently only retool), or via information in parsed .dat files.
+DupeParser generates a list of potential duplicate files based on name. It does this via curated clonelists
+(currently only retool).
 
 If priorities are present in the retool clonelist, it will use these to prioritise particular release versions versus
 others

pyproject.toml

@@ -33,6 +33,7 @@ classifiers = [
 dependencies = [
     "colorlog == 6.9.0",
     "discordwebhook == 1.0.3",
+    "iso639-lang == 2.5.1",
     "numpy == 2.2.0",
     "packaging == 24.2",
     "pathvalidate == 3.2.1",

romsearch/configs/defaults.yml

@@ -210,6 +210,51 @@ dat_checksums:
   - "sha1"
   - "sha256"
 
+disc_rename:
+  (Disc 0: "(Disc "
+  (Disco : "(Disc "
+  (Disk : "(Disc "
+  (CD : "(Disc "
+  (Disque : "(Disc "
+  (Lato : "(Disc "
+  (Seite : "(Disc "
+  (Teil : "(Disc "
+  (Disc ): "(Disc 0)"
+  (Disc A): "(Disc 1)"
+  (Disc B): "(Disc 2)"
+  (Disc C): "(Disc 3)"
+  (Disc D): "(Disc 4)"
+  (Disc E): "(Disc 5)"
+  (Disc F): "(Disc 6)"
+  (Disc G): "(Disc 7)"
+  (Disc H): "(Disc 8)"
+  (Disc I): "(Disc 1)"
+  (Disc II): "(Disc 2)"
+  (Disc III): "(Disc 3)"
+  (Disc IV): "(Disc 4)"
+  (Disc V): "(Disc 5)"
+  (Disc VI): "(Disc 6)"
+  (Disc VII): "(Disc 7)"
+  (Disc VIII): "(Disc 8)"
+  (Disc One): "(Disc 1)"
+  (Disc Two): "(Disc 2)"
+  (Disc Three): "(Disc 3)"
+  (Disc Four): "(Disc 4)"
+  (Disc Five): "(Disc 5)"
+  (Disc Six): "(Disc 6)"
+  (Disc Seven): "(Disc 7)"
+  (Disc Eight): "(Disc 8)"
+  (Disc Uno): "(Disc 1)"
+  (Disc Due): "(Disc 2)"
+  (Disc Tre): "(Disc 3)"
+  (Disc Quattro): "(Disc 4)"
+  (Disc Cinque): "(Disc 5)"
+  (Disc Sei): "(Disc 6)"
+  (Disc Sette): "(Disc 7)"
+  (Disc Otto): "(Disc 8)"
+  (Disc-: "(Disc "
+  (Disc Writer): "(Disk Writer)"
+
 ra_file_exts:
   - ".bin"
   - ".bs"

romsearch/configs/regex.yml

@@ -10,6 +10,7 @@ languages:
 
 multi_disc:
   pattern: "\\((Dis[ck]|Seite)[-\\s][0-9A-Z]\\)"
+  include_in_short_name: true
 
 date:
   pattern: "\\s?\\((?:\\d{8}|\\d{4}-\\d{2}-\\d{2}|\\d{2}-\\d{2}-\\d{4}|\\d{2}-\\d{2}-\\d{2}|\\d{4}-\\d{2}-\\d{2}T\\d{6}|~?\\d{4}-(?:\\d{2}|xx)-xx|(?:January|February|March|April|May|June|July|August|September|October|November|December),\\s?\\d{4})\\)"
@@ -466,6 +467,10 @@ collection_of_mana:
   pattern: "\\(Collection of Mana\\)"
   group: "modern_version"
 
+collection_of_saga:
+  pattern: "\\(Collection of SaGa\\)"
+  group: "modern_version"
+
 columbus_circle:
   pattern: "\\(Columbus Circle\\)"
   group: "modern_version"
@@ -753,6 +758,15 @@ whirlwind_manu:
 
 # GENERALLY IGNORE
 
+l_tags:
+  pattern: "\\s?\\(L[ABCDEFGH]\\d{2,2}\\)"
+
+knv_st_tags:
+  pattern: "\\s?\\([KNV][ST]-\\d{3,4}[A-Z]?\\)"
+
+jy_tags:
+  pattern: "\\s?\\(JY-?\\d{3,3}\\)"
+
 60_pin_cart:
   pattern: "\\(60 pin cart\\)"
 
@@ -773,7 +787,7 @@ ntsc:
   flags: "NOFLAG"
 
 pal:
-  pattern: "([?:-][\\s])?[(]?PAL(?: [a-zA-Z]+| 50[Hh]z)?(?:(?!-)\\)?| (?=\\())"
+  pattern: "([?:-][\\s])?[(]?PAL(?: [a-zA-Z]+| 50[Hh]z)?(?:\\)?| (?=\\())"
   flags: "NOFLAG"
 
 supervision:

romsearch/configs/sample_config.yml

@@ -40,7 +40,6 @@ romdownloader:
   dry_run: false
 
 dupeparser:
-  use_dat: true
   use_retool: true
 
 gamefinder:

romsearch/dev/gamefinder_dev.py

@@ -3,7 +3,14 @@
 
 import romsearch
 from romsearch import GameFinder
-from ..util import get_short_name, load_json, load_yml
+from ..util import (
+    get_short_name,
+    load_json,
+    load_yml,
+    get_directory_name,
+    normalize_name,
+    get_region_free_name,
+)
 
 
 def check_regex_parsing(dat_filename):
@@ -32,14 +39,29 @@ def check_regex_parsing(dat_filename):
     # Parse all the names in the dat file
     all_file_dict = {}
     for f in all_files:
-        short_name = get_short_name(
+
+        dir_name = get_directory_name(f)
+        full_name = normalize_name(
             f,
+            disc_rename=default_config["disc_rename"],
+        )
+        short_name = get_short_name(
+            full_name,
+            regex_config=regex_config,
+            default_config=default_config,
+        )
+        region_free_name = get_region_free_name(
+            full_name,
             regex_config=regex_config,
             default_config=default_config,
         )
 
         all_file_dict[f] = {
+            "original_name": f,
+            "dir_name": dir_name,
+            "full_name": full_name,
             "short_name": short_name,
+            "region_free_name": region_free_name,
             "matched": False,
         }
 

romsearch/gui/gui_config.py

@@ -221,7 +221,7 @@ def __init__(
         }
 
         self.all_dupeparser_options = {
-            "use_dat": self.ui.checkBoxConfigDupeParserUseDat,
+            # "use_dat": self.ui.checkBoxConfigDupeParserUseDat,
             "use_retool": self.ui.checkBoxConfigDupeParserUseRetool,
         }
 

romsearch/gui/layout_romsearch.py

@@ -848,12 +848,6 @@ def setupUi(self, RomSearch):
 
         self.verticalLayoutConfigDupeParserMiddle.addWidget(self.lineConfigDupeParserUseDatTop)
 
-        self.checkBoxConfigDupeParserUseDat = QCheckBox(self.tabConfigDupeParser)
-        self.checkBoxConfigDupeParserUseDat.setObjectName(u"checkBoxConfigDupeParserUseDat")
-        self.checkBoxConfigDupeParserUseDat.setChecked(True)
-
-        self.verticalLayoutConfigDupeParserMiddle.addWidget(self.checkBoxConfigDupeParserUseDat)
-
         self.checkBoxConfigDupeParserUseRetool = QCheckBox(self.tabConfigDupeParser)
         self.checkBoxConfigDupeParserUseRetool.setObjectName(u"checkBoxConfigDupeParserUseRetool")
         self.checkBoxConfigDupeParserUseRetool.setChecked(True)
@@ -1758,10 +1752,6 @@ def retranslateUi(self, RomSearch):
         self.lineEditConfigRAHasherCachePeriod.setText(QCoreApplication.translate("RomSearch", u"30", None))
         self.lineEditConfigRAHasherCachePeriod.setPlaceholderText(QCoreApplication.translate("RomSearch", u"30", None))
         self.tabWidgetConfig.setTabText(self.tabWidgetConfig.indexOf(self.tabConfigRAHasher), QCoreApplication.translate("RomSearch", u"RAHasher", None))
-#if QT_CONFIG(statustip)
-        self.checkBoxConfigDupeParserUseDat.setStatusTip(QCoreApplication.translate("RomSearch", u"Whether to use the dat file to figure out dupes. Default checked", None))
-#endif // QT_CONFIG(statustip)
-        self.checkBoxConfigDupeParserUseDat.setText(QCoreApplication.translate("RomSearch", u"Use .dat", None))
 #if QT_CONFIG(statustip)
         self.checkBoxConfigDupeParserUseRetool.setStatusTip(QCoreApplication.translate("RomSearch", u"Whether to use the retool clonelist to figure out dupes. Default checked", None))
 #endif // QT_CONFIG(statustip)

romsearch/gui/layout_romsearch.ui

@@ -1459,19 +1459,6 @@
                   </property>
                  </widget>
                 </item>
-                <item>
-                 <widget class="QCheckBox" name="checkBoxConfigDupeParserUseDat">
-                  <property name="statusTip">
-                   <string>Whether to use the dat file to figure out dupes. Default checked</string>
-                  </property>
-                  <property name="text">
-                   <string>Use .dat</string>
-                  </property>
-                  <property name="checked">
-                   <bool>true</bool>
-                  </property>
-                 </widget>
-                </item>
                 <item>
                  <widget class="QCheckBox" name="checkBoxConfigDupeParserUseRetool">
                   <property name="statusTip">
@@ -2715,7 +2702,7 @@
  <resources/>
  <connections/>
  <buttongroups>
-  <buttongroup name="radioButtonConfigRomsearchMethod"/>
   <buttongroup name="radioButtonConfigLoggerLevel"/>
+  <buttongroup name="radioButtonConfigRomsearchMethod"/>
  </buttongroups>
 </ui>