Experimental gallery-dl integration

README.md

@@ -1,6 +1,6 @@
 # GDownloader
 
-A GUI Wrapper for [YT-DLP](https://github.com/yt-dlp/yt-dlp) written in Java.
+A user-friendly GUI wrapper for [yt-dlp](https://github.com/yt-dlp/yt-dlp) and [gallery-dl](https://github.com/mikf/gallery-dl) written in Java.
 
 ## Overview
 
@@ -10,10 +10,10 @@ It supports various platforms such as Crunchyroll, Twitch, X/Twitter, and all ot
 ## Features
 
 - Batch download videos and playlists
-- Supports multiple video platforms
-- Embeds thumbnails and subtitles in the resulting videos, when available
+- Supports multiple sites and content types
+- Embeds thumbnails and subtitles in the resulting media files, when available
 - Automatic FFMPEG setup for Windows upon first boot
-- Keeps yt-dlp always updated and ready go
+- Keeps yt-dlp and gallery-dl always updated and ready go
 - Multiple customizable settings to best suit your usage style
 
 ## Requirements
@@ -39,12 +39,12 @@ Download the latest version for your platform from the [releases page](https://g
    ```bash
    cd GDownloader
    ```
-   
+
 3. Build the project using Gradle:
    ```bash
    ./gradlew clean build jpackage
    ```
-   
+
 ## Screenshots
 
 <img src="screenshot1.png" alt="Screenshot1" width="500"/>
@@ -57,5 +57,7 @@ We welcome any feedback you may have to improve the user experience.
 ## Atributions
 
 - Icons by [IconsDB.com](https://www.iconsdb.com)
-- FFMpeg builds by [GyanD](https://github.com/GyanD/codexffmpeg)
-- YT-DLP builds by [YT-DLP](https://github.com/yt-dlp/yt-dlp)
+- FFMpeg builds by [GyanD/codexffmpeg](https://github.com/GyanD/codexffmpeg)
+- yt-dlp builds by [yt-dlp/yt-dlp](https://github.com/yt-dlp/yt-dlp)
+- gallery-dl builds by [mikf/gallery-dl](https://github.com/mikf/gallery-dl)
+

build.gradle

@@ -1,4 +1,4 @@
-rootProject.version = '1.1.5'
+rootProject.version = '1.2.0'
 
 subprojects {
 	repositories {

core/src/main/java/module-info.java

@@ -40,12 +40,16 @@
 
     exports net.brlns.gdownloader;
     exports net.brlns.gdownloader.clipboard;
+    exports net.brlns.gdownloader.downloader;
+    exports net.brlns.gdownloader.downloader.enums;
+    exports net.brlns.gdownloader.downloader.structs;
     exports net.brlns.gdownloader.settings;
     exports net.brlns.gdownloader.settings.enums;
     exports net.brlns.gdownloader.settings.filters;
     exports net.brlns.gdownloader.ui;
     exports net.brlns.gdownloader.ui.custom;
     exports net.brlns.gdownloader.ui.dnd;
+    exports net.brlns.gdownloader.ui.menu;
     exports net.brlns.gdownloader.ui.themes;
     exports net.brlns.gdownloader.updater;
     exports net.brlns.gdownloader.util;

core/src/main/java/net/brlns/gdownloader/GDownloader.java

@@ -47,26 +47,69 @@
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 import net.brlns.gdownloader.clipboard.ClipboardManager;
+import net.brlns.gdownloader.downloader.DownloadManager;
+import net.brlns.gdownloader.lang.Language;
 import net.brlns.gdownloader.settings.Settings;
 import net.brlns.gdownloader.settings.enums.BrowserEnum;
 import net.brlns.gdownloader.ui.GUIManager;
 import net.brlns.gdownloader.ui.GUIManager.MessageType;
 import net.brlns.gdownloader.ui.themes.ThemeProvider;
-import net.brlns.gdownloader.updater.AbstractGitUpdater;
-import net.brlns.gdownloader.updater.FFMpegUpdater;
-import net.brlns.gdownloader.updater.SelfUpdater;
-import net.brlns.gdownloader.updater.UpdaterBootstrap;
-import net.brlns.gdownloader.updater.YtDlpUpdater;
-import net.brlns.gdownloader.util.DirectoryUtils;
-import net.brlns.gdownloader.util.LoggerUtils;
-import net.brlns.gdownloader.util.NoFallbackAvailableException;
-import net.brlns.gdownloader.util.Nullable;
-import net.brlns.gdownloader.util.PriorityThreadPoolExecutor;
+import net.brlns.gdownloader.updater.*;
+import net.brlns.gdownloader.util.*;
 import org.slf4j.helpers.FormattingTuple;
 import org.slf4j.helpers.MessageFormatter;
 
-import static net.brlns.gdownloader.Language.*;
-
+import static net.brlns.gdownloader.lang.Language.*;
+
+// TODO media converter
+// TODO implement CD Ripper
+// TODO d&d files for conversion to different formats, we already have ffmpeg anyway
+//
+// TODO max simultaneous downloads should be independent per website
+// TODO silence unnecessary debug messages
+// TODO investigate adding AppImage build
+// TODO scale on resolution DPI
+// TODO save last window size in config
+// TODO keep older versions of ytdlp and retry failed downloads against them
+// TODO Advanced users can edit the config.json directly to add extra yt-dlp arguments like proxy settings. but maybe expose those settings to the ui.
+// TODO verify checksums during updates, add bouncycastle, check signatures
+// TODO write a component factory for GUIManager
+// TODO git actions build for different platforms
+// FEEDBACK Should choose to download video and audio independently on each card
+// TODO maybe add notifications for each toggled toolbar option
+// DROPPED check updates on a timer, but do not ever restart when anything is in the queue.
+// TODO individual 'retry failed download' button
+// TODO --no-playlist when single video option is active
+// TODO Artifacting seems to be happening on the scroll pane with AMD video cards
+// TODO open a window asking which videos in a playlist to download or not
+// TODO RearrangeableDeque's offerLast should be linked to the cards in the UI
+// TODO Better visual eye candy for when dragging cards
+// TODO Add setting to allow the user to manually specify the target codec for audio transcoding? currently it defaults to aac.
+// TODO Add 'Clear Completed Downloads' button.
+// TODO Javadoc, a whole lot of it.
+// TODO Refactor this very class. Separate some logic into different methods.
+// TODO Twitch settings purposefully default to suboptimal quality due to huge file sizes. Maybe consider adding a warning about this in the GUI.
+// TODO Split GUI into a different subproject from core logic.
+// TODO Investigate screen reader support (https:// www.nvaccess.org/download/)
+// TODO Send notifications when a NO_METHOD is triggered, explaining why it was triggered.
+// TODO Test downloading sections of a livestream (currently it gets stuck on status PREPARING). Note: it also leaves a zombie ffmpeg process behind dealing with the hls stream.
+// TODO The issue above is a yt-dlp bug https:// github.com/yt-dlp/yt-dlp/issues/7927
+// TODO Implement rate-limiting options internally; the way it's currently implemented does not account for concurrent or non-playlist downloads.
+// TODO Notify the user whenever a setting that requires restart was changed.
+// TODO Quit lingering ffmpeg processes spawned by yt-dlp
+// TODO Verify which exceptions are important to display to the user via GDownloader::handleException
+// TODO Add an url ignore list / Allow filters to be disabled
+// TODO Add option to clear all installed updates and start fresh. (Tackling certain issues where failed updates could break downloads)
+// TODO Optional curl/wget integration
+// TODO NTFS File path length workaround for gallery-dl
+// TODO Split main window from GUIManager
+// TODO Card multi-select support / Undo last action / try with downloader X
+// TODO Debug no console output from gallery-dl on Windows when using channels
+// TODO gallery-dl does not accept an argument specifying yt-dlp/ffmpeg location, figure out a workaround to pass the correct path to it
+// TODO Fastutil collections
+// TODO Proxy settings should be add to the UI, fields should be validated on the fly
+// TODO Tabs in settings for the different downloaders
+// TODO Crawl for valid links that can be consumed by direct-http 
 /**
  * GDownloader - GUI wrapper for yt-dlp
  *
@@ -87,6 +130,9 @@ public final class GDownloader {
 
     public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
 
+    @Getter
+    private static GDownloader instance;
+
     private static String launcher;
 
     @Getter
@@ -104,7 +150,7 @@ public final class GDownloader {
     private ClipboardManager clipboardManager;
 
     @Getter
-    private YtDlpDownloader downloadManager;
+    private DownloadManager downloadManager;
 
     @Getter
     private List<AbstractGitUpdater> updaters = new ArrayList<>();
@@ -187,7 +233,7 @@ public GDownloader() {
             clipboardManager = new ClipboardManager(this);
             clipboardManager.init();
 
-            downloadManager = new YtDlpDownloader(this);
+            downloadManager = new DownloadManager(this);
 
             guiManager = new GUIManager(this);
 
@@ -210,6 +256,10 @@ public GDownloader() {
             updaters.add(new YtDlpUpdater(this));
             updaters.add(new FFMpegUpdater(this));
 
+            if (config.isGalleryDlEnabled()) {
+                updaters.add(new GalleryDlUpdater(this));
+            }
+
             if (config.isDebugMode()) {
                 for (AbstractGitUpdater updater : updaters) {
                     updater.registerListener((status, progress) -> {
@@ -361,7 +411,7 @@ public boolean checkForUpdates(boolean userInitiated) {
                 }
             }
 
-            if (downloadManager.getYtDlpPath() == null || !downloadManager.getYtDlpPath().exists()) {
+            if (!downloadManager.isMainDownloaderInitialized()) {
                 log.error("Failed to initialize YT-DLP, the program cannot continue. Exitting...");
 
                 if (!userInitiated) {
@@ -421,6 +471,10 @@ public void openDownloadsDirectory() {
         open(getOrCreateDownloadsDirectory());
     }
 
+    public void openLogFile() {
+        open(LoggerUtils.getLogFile());
+    }
+
     public void open(File file) {
         try {
             Desktop desktop = Desktop.getDesktop();
@@ -518,9 +572,9 @@ public BrowserEnum getBrowserForCookies() {
                     }
                 } else if (os.contains("mac")) {
                     List<String> output = readOutput(
-                            "bash",
-                            "-c",
-                            "defaults read ~/Library/Preferences/com.apple.LaunchServices/com.apple.launchservices.secure | awk -F '\"' '/http;/{print window[(NR)-1]}{window[NR]=$2}'");
+                        "bash",
+                        "-c",
+                        "defaults read ~/Library/Preferences/com.apple.LaunchServices/com.apple.launchservices.secure | awk -F '\"' '/http;/{print window[(NR)-1]}{window[NR]=$2}'");
 
                     log.info("Default browser: {}", output);
 
@@ -844,6 +898,29 @@ public void updateStartupStatus() {
         }
     }
 
+    public void deduplicateDownloadsDirectory() {
+        guiManager.showMessage(
+            l10n("gui.deduplication.notification_title"),
+            l10n("gui.deduplication.deduplicating"),
+            1500,
+            GUIManager.MessageType.INFO,
+            false);
+
+        globalThreadPool.submitWithPriority(() -> {
+            File directory = getDownloadsDirectory();
+            if (directory.exists()) {
+                DirectoryDeduplicator.deduplicateDirectory(directory);
+            }
+
+            guiManager.showMessage(
+                l10n("gui.deduplication.notification_title"),
+                l10n("gui.deduplication.deduplicated"),
+                2000,
+                GUIManager.MessageType.INFO,
+                false);
+        }, 0);
+    }
+
     public File getOrCreateDownloadsDirectory() {
         return getDownloadsDirectory(true);
     }
@@ -950,15 +1027,15 @@ private void printDebugInformation() {
         log.info("Number of available processor cores: {}", cores);
     }
 
-    public final void handleException(Throwable e) {
+    public static final void handleException(Throwable e) {
         handleException(e, true);
     }
 
-    public final void handleException(Throwable e, boolean displayToUser) {
+    public static final void handleException(Throwable e, boolean displayToUser) {
         log.error("An exception has been caught", e);
 
         if (displayToUser) {
-            guiManager.showMessage(
+            instance.getGuiManager().showMessage(
                 l10n("gui.error_popup_title"),
                 l10n("gui.error_popup", e.getLocalizedMessage()),
                 4000,
@@ -993,6 +1070,12 @@ public static boolean isFromJar() {
         return getJarLocation() != null;
     }
 
+    public static boolean isFromJpackage() {
+        String appPath = System.getProperty("jpackage.app-path");
+
+        return appPath != null;
+    }
+
     @Nullable
     private static String getJarLocation() {
         try {
@@ -1169,6 +1252,7 @@ public static void main(String[] args) {
             }
 
             GDownloader instance = new GDownloader();
+            GDownloader.instance = instance;
 
             if (!noGui) {
                 instance.initUi();
@@ -1193,10 +1277,10 @@ public static void main(String[] args) {
             }));
 
             Thread.setDefaultUncaughtExceptionHandler((Thread t, Throwable e) -> {
-                instance.handleException(e);
+                GDownloader.handleException(e);
             });
         } else {
-            System.err.println("System tray not supported???? did you run this on a calculator?");
+            log.error("System tray not supported??? did you run this on a calculator?");
         }
     }