Implement 'deduplicating' and 'post-processing' states

Implement LRUCache for file hash caching
Fix insane CPU spike generated by BufferedReader::readLine()

README.md

@@ -1,6 +1,6 @@
 # GDownloader
 
-A GUI Wrapper for [YT-DLP](https://github.com/yt-dlp/yt-dlp) written in Java.
+A GUI Wrapper for [yt-dlp](https://github.com/yt-dlp/yt-dlp) and [gallery-dl](https://github.com/mikf/gallery-dl) written in Java.
 
 ## Overview
 
@@ -10,10 +10,10 @@ It supports various platforms such as Crunchyroll, Twitch, X/Twitter, and all ot
 ## Features
 
 - Batch download videos and playlists
-- Supports multiple video platforms
-- Embeds thumbnails and subtitles in the resulting videos, when available
+- Supports multiple sites and content types
+- Embeds thumbnails and subtitles in the resulting media files, when available
 - Automatic FFMPEG setup for Windows upon first boot
-- Keeps yt-dlp always updated and ready go
+- Keeps yt-dlp and gallery-dl always updated and ready go
 - Multiple customizable settings to best suit your usage style
 
 ## Requirements
@@ -57,5 +57,7 @@ We welcome any feedback you may have to improve the user experience.
 ## Atributions
 
 - Icons by [IconsDB.com](https://www.iconsdb.com)
-- FFMpeg builds by [GyanD](https://github.com/GyanD/codexffmpeg)
-- YT-DLP builds by [YT-DLP](https://github.com/yt-dlp/yt-dlp)
+- FFMpeg builds by [GyanD/codexffmpeg](https://github.com/GyanD/codexffmpeg)
+- yt-dlp builds by [yt-dlp/yt-dlp](https://github.com/yt-dlp/yt-dlp)
+- gallery-dl builds by [mikf/gallery-dl](https://github.com/mikf/gallery-dl)
+

core/src/main/java/net/brlns/gdownloader/downloader/DownloadManager.java

@@ -717,6 +717,8 @@ public void processQueue() {
                                 fireListeners();
                                 return;
                             } else if (!entry.getCancelHook().get() && FLAG_SUCCESS.isSet(flags)) {
+                                entry.updateStatus(DownloadStatusEnum.POST_PROCESSING, l10n("gui.download_status.processing_media_files"));
+
                                 Map<String, IMenuEntry> rightClickOptions = downloader.processMediaFiles(entry);
 
                                 Map<String, IMenuEntry> controlOptions = new LinkedHashMap<>();

core/src/main/java/net/brlns/gdownloader/downloader/GalleryDlDownloader.java

@@ -222,6 +222,8 @@ protected Map<String, IMenuEntry> processMediaFiles(QueueEntry entry) {
                     new RunnableMenuEntry(() -> main.open(deepestDirectory)));
 
                 if (main.getConfig().isGalleryDlDeduplication()) {
+                    entry.updateStatus(DownloadStatusEnum.DEDUPLICATING, l10n("gui.deduplication.deduplicating"));
+
                     DirectoryDeduplicator.deduplicateDirectory(deepestDirectory);
                 }
             }
@@ -258,13 +260,15 @@ protected Pair<Integer, String> processDownload(QueueEntry entry, List<String> a
                     throw new InterruptedException("Download interrupted");
                 }
 
-                if ((line = reader.readLine()) != null) {
-                    lastOutput = line;
+                if (reader.ready()) {
+                    if ((line = reader.readLine()) != null) {
+                        lastOutput = line;
 
-                    processProgress(entry, lastOutput);
+                        processProgress(entry, lastOutput);
+                    }
+                } else {
+                    Thread.sleep(100);
                 }
-
-                Thread.sleep(100);
             }
 
             entry.getDownloadStarted().set(false);

core/src/main/java/net/brlns/gdownloader/downloader/QueueEntry.java

@@ -292,7 +292,7 @@ public void updateStatus(DownloadStatusEnum status) {
                     mediaCard.setPercentage(100);
                     mediaCard.setProgressBarTextAndColors(status.getDisplayName(), Color.MAGENTA);
                 }
-                case PROCESSING -> {
+                case PROCESSING, POST_PROCESSING, DEDUPLICATING -> {
                     mediaCard.setPercentage(100);
                     mediaCard.setProgressBarTextAndColors(status.getDisplayName(), Color.ORANGE);
                 }

core/src/main/java/net/brlns/gdownloader/downloader/enums/DownloadStatusEnum.java

@@ -31,6 +31,8 @@ public enum DownloadStatusEnum implements ISettingsEnum {
     RETRYING("enums.download_status.retrying"),
     PREPARING("enums.download_status.preparing"),
     PROCESSING("enums.download_status.processing"),
+    POST_PROCESSING("enums.download_status.post_processing"),
+    DEDUPLICATING("enums.download_status.deduplicating"),
     DOWNLOADING("enums.download_status.downloading"),
     COMPLETE("enums.download_status.complete"),
     FAILED("enums.download_status.failed"),

core/src/main/java/net/brlns/gdownloader/util/DirectoryDeduplicator.java

@@ -22,9 +22,7 @@
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.util.HashSet;
-import java.util.Map;
 import java.util.Set;
-import java.util.WeakHashMap;
 import lombok.extern.slf4j.Slf4j;
 
 /**
@@ -33,7 +31,9 @@
 @Slf4j
 public class DirectoryDeduplicator {
 
-    private static final Map<File, String> HASH_CACHE = new WeakHashMap<>();
+    // We're pretty light-weight on memory so far, even while using native Java collections.
+    // We're safe to push this one a bit.
+    private static final LRUCache<File, String> HASH_CACHE = new LRUCache<>(2000);
 
     /**
      * Deduplicates the specified directory using SHA-256.

core/src/main/java/net/brlns/gdownloader/util/LRUCache.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright (C) 2024 hstr0100
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package net.brlns.gdownloader.util;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * @author Gabriel / hstr0100 / vertx010
+ */
+public class LRUCache<K, V> {
+
+    private final int capacity;
+    private final LinkedHashMap<K, V> backingMap;
+
+    public LRUCache(int capacityIn) {
+        capacity = capacityIn;
+
+        backingMap = new LinkedHashMap<K, V>(capacity, 0.75f, true) {
+            @Override
+            protected boolean removeEldestEntry(Map.Entry<K, V> eldest) {
+                return size() > capacity;
+            }
+        };
+    }
+
+    public V get(K key) {
+        return backingMap.getOrDefault(key, null);
+    }
+
+    public void put(K key, V value) {
+        backingMap.put(key, value);
+    }
+
+    @Override
+    public String toString() {
+        return backingMap.toString();
+    }
+}

core/src/main/java/net/brlns/gdownloader/util/StringUtils.java

@@ -48,9 +48,17 @@ public static String getHumanReadableFileSize(long bytes) {
     }
 
     public static String getStringAfterLastSeparator(String filePath) {
-        if (filePath == null || filePath.isEmpty() || !filePath.contains(File.separator)) {
+        if (filePath == null || filePath.isEmpty()) {
             return filePath;
         } else {
+            while (filePath.endsWith(File.separator)) {
+                filePath = filePath.substring(0, filePath.length() - 1);
+            }
+
+            if (!filePath.contains(File.separator)) {
+                return filePath;
+            }
+
             String fileName = filePath.substring(filePath.lastIndexOf(File.separator) + 1);
 
             return !fileName.isEmpty() ? fileName : filePath;

core/src/main/resources/lang/language_en.properties

@@ -152,3 +152,6 @@ gui.deduplication.deduplicate_downloads_directory=Deduplicate files in download
 gui.deduplication.notification_title=Deduplicator
 gui.deduplication.deduplicating=Deduplicating files...
 gui.deduplication.deduplicated=Deduplication complete
+gui.download_status.processing_media_files=Processing media files
+enums.download_status.deduplicating=DEDUPLICATING
+enums.download_status.post_processing=POST PROCESSING

core/src/main/resources/lang/language_es_MX.properties

@@ -152,3 +152,6 @@ gui.deduplication.deduplicate_downloads_directory=Eliminar duplicados en el dire
 gui.deduplication.notification_title=Eliminador de duplicados
 gui.deduplication.deduplicating=Eliminando duplicados...
 gui.deduplication.deduplicated=Eliminaci\u00f3n de duplicados completa
+gui.download_status.processing_media_files=Procesando archivos multimedia
+enums.download_status.deduplicating=ELIMINANDO DUPLICADOS
+enums.download_status.post_processing=POSPROCESANDO

core/src/main/resources/lang/language_pt_BR.properties

@@ -152,3 +152,6 @@ gui.deduplication.deduplicate_downloads_directory=Remover duplicatas no diret\u0
 gui.deduplication.notification_title=Removedor de duplicatas
 gui.deduplication.deduplicating=Removendo duplicatas...
 gui.deduplication.deduplicated=Remo\u00e7\u00e3o de duplicatas conclu\u00edda
+gui.download_status.processing_media_files=Processando arquivos de m\u00eddia
+enums.download_status.deduplicating=REMOVENDO DUPLICATAS
+enums.download_status.post_processing=P\u00d3S-PROCESSAMENTO