Skip to content

Commit

Permalink
Add ability to select a downloader to start downloading with
Browse files Browse the repository at this point in the history
  • Loading branch information
@hstr0100
hstr0100 committed Nov 20, 2024
1 parent c6e97a4 commit 4ce2146
Show file tree
Hide file tree
Showing 10 changed files with 139 additions and 73 deletions.
2 changes: 2 additions & 0 deletions core/src/main/java/net/brlns/gdownloader/downloader/AbstractDownloader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ public AbstractDownloader(DownloadManager managerIn) {
manager = managerIn;
}

public abstract boolean isEnabled();

protected abstract boolean canConsumeUrl(String inputUrl);

protected abstract boolean tryQueryVideo(QueueEntry queueEntry);
Expand Down
16 changes: 11 additions & 5 deletions core/src/main/java/net/brlns/gdownloader/downloader/DirectHttpDownloader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,11 @@ public DirectHttpDownloader(DownloadManager managerIn) {
super(managerIn);
}

@Override
public boolean isEnabled() {
return main.getConfig().isDirectHttpEnabled();
}

@Override
public DownloaderIdEnum getDownloaderId() {
return DownloaderIdEnum.DIRECT_HTTP;
Expand All @@ -98,7 +103,7 @@ public boolean isMainDownloader() {

@Override
protected boolean canConsumeUrl(String inputUrl) {
return main.getConfig().isDirectHttpEnabled()
return isEnabled()
&& !(inputUrl.contains("ytimg")
|| inputUrl.contains("ggpht")
|| inputUrl.endsWith("youtube.com/"));
Expand Down Expand Up @@ -153,12 +158,13 @@ protected DownloadResult tryDownload(QueueEntry entry) throws Exception {
});

lastOutput = PREFIX + "Download complete";

entry.getDownloadStarted().set(false);
entry.updateStatus(DownloadStatusEnum.DOWNLOADING, lastOutput);
} catch (Exception e) {
lastOutput = PREFIX + e.getMessage();

success = false;
} finally {
entry.getDownloadStarted().set(false);
}

log.info(lastOutput);
Expand Down Expand Up @@ -381,7 +387,7 @@ private boolean downloadFile(QueueEntry queueEntry, ProgressUpdater progressCall
future.get();
}
} catch (Exception e) {
throw new IOException("Failed to download a chunk: " + fileUrl, e);
throw new IOException("Failed to download a chunk: " + fileUrl + ": " + e.getMessage(), e);
}

if (downloadedBytes.get() != totalBytes) {
Expand Down Expand Up @@ -485,7 +491,7 @@ private boolean downloadChunk(ChunkData chunkData) throws IOException {

if (attempt == MAX_CHUNK_RETRIES) {
chunkData.getAbortHook().set(true);
throw new IOException("Failed to download file after " + MAX_CHUNK_RETRIES + " attempts.");
throw new IOException("Failed to download file after " + MAX_CHUNK_RETRIES + " attempts: " + e.getMessage(), e);
}
} finally {
if (connection != null) {
Expand Down
39 changes: 27 additions & 12 deletions core/src/main/java/net/brlns/gdownloader/downloader/DownloadManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import lombok.AccessLevel;
Expand Down Expand Up @@ -73,7 +74,6 @@ public class DownloadManager {
private final List<Consumer<DownloadManager>> listeners = new ArrayList<>();

private final List<AbstractDownloader> downloaders = new ArrayList<>();

private final Set<String> capturedLinks = Collections.synchronizedSet(new HashSet<>());
private final Set<String> capturedPlaylists = Collections.synchronizedSet(new HashSet<>());

Expand All @@ -93,6 +93,8 @@ public class DownloadManager {
private final AtomicBoolean downloadsRunning = new AtomicBoolean(false);
private final AtomicBoolean downloadsManuallyStarted = new AtomicBoolean(false);

private final AtomicReference<DownloaderIdEnum> suggestedDownloaderId = new AtomicReference<>(null);

private final ExpiringSet<String> urlIgnoreSet = new ExpiringSet<>(TimeUnit.SECONDS, 5);

private final ExecutorService forcefulExecutor = Executors.newCachedThreadPool();// No limits, power to ya
Expand Down Expand Up @@ -135,14 +137,8 @@ public DownloadManager(GDownloader mainIn) {
});

downloaders.add(new YtDlpDownloader(this));

if (main.getConfig().isGalleryDlEnabled()) {
downloaders.add(new GalleryDlDownloader(this));
}

if (main.getConfig().isDebugMode()) {
downloaders.add(new DirectHttpDownloader(this));
}
downloaders.add(new GalleryDlDownloader(this));
downloaders.add(new DirectHttpDownloader(this));
}

public boolean isBlocked() {
Expand Down Expand Up @@ -187,6 +183,12 @@ private List<AbstractDownloader> getCompatibleDownloaders(String inputUrl) {
.collect(Collectors.toUnmodifiableList());
}

public List<AbstractDownloader> getEnabledDownloaders() {
return downloaders.stream()
.filter(downloader -> downloader.isEnabled())
.collect(Collectors.toUnmodifiableList());
}

public CompletableFuture<Boolean> captureUrl(@Nullable String inputUrl, boolean force) {
return captureUrl(inputUrl, force, main.getConfig().getPlaylistDownloadOption());
}
Expand Down Expand Up @@ -395,7 +397,7 @@ public CompletableFuture<Boolean> captureUrl(@Nullable String inputUrl, boolean
enqueueLast(queueEntry);

if (main.getConfig().isAutoDownloadStart() && !downloadsRunning.get()) {
startDownloads();
startDownloads(suggestedDownloaderId.get());
}

future.complete(true);
Expand Down Expand Up @@ -441,16 +443,24 @@ public void toggleDownloads() {
}

public void startDownloads() {
startDownloads(null);
}

public void startDownloads(@Nullable DownloaderIdEnum downloaderId) {
if (!downloadsBlocked.get()) {
downloadsRunning.set(true);
downloadsManuallyStarted.set(true);
suggestedDownloaderId.set(downloaderId);

fireListeners();
}
}

public void stopDownloads() {
downloadsRunning.set(false);
downloadsManuallyStarted.set(false);
suggestedDownloaderId.set(null);

fireListeners();
}

Expand Down Expand Up @@ -482,7 +492,7 @@ public void retryFailedDownloads() {
restartDownload(entry, false);
}

startDownloads();
startDownloads(suggestedDownloaderId.get());
fireListeners();
}

Expand Down Expand Up @@ -699,6 +709,9 @@ private void submitDownloadTask(QueueEntry entry, boolean force) {
inProgressDownloads.offer(entry);

DownloaderIdEnum forcedDownloader = entry.getForcedDownloader();
if (forcedDownloader == null) {
forcedDownloader = suggestedDownloaderId.get();
}

int maxRetries = !main.getConfig().isAutoDownloadRetry() ? 1 : MAX_DOWNLOAD_RETRIES;
String lastOutput = "";
Expand Down Expand Up @@ -743,11 +756,13 @@ private void submitDownloadTask(QueueEntry entry, boolean force) {
entry.logError(lastOutput);

if (disabled || unsupported || entry.getRetryCounter().get() >= maxRetries) {
entry.updateStatus(DownloadStatusEnum.FAILED, lastOutput);
log.error("Download of {} failed on {}: {} supported downloader: {}",
entry.getUrl(), downloaderId, lastOutput, !unsupported);

entry.blackListDownloader(downloaderId);
} else {
entry.updateStatus(DownloadStatusEnum.RETRYING, lastOutput);
log.warn("Download of {} failed with {}, retrying ({}/{}): {}",
entry.getUrl(),
downloaderId,
Expand Down Expand Up @@ -885,7 +900,7 @@ private void tryStopProcess(Process process) throws InterruptedException {
public void close() {
stopDownloads();

clearQueue(RUNNING, true);
clearQueue(RUNNING, false);
clearQueue();

for (AbstractDownloader downloader : downloaders) {
Expand Down
17 changes: 11 additions & 6 deletions core/src/main/java/net/brlns/gdownloader/downloader/GalleryDlDownloader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,12 @@ public GalleryDlDownloader(DownloadManager managerIn) {
super(managerIn);
}

@Override
public boolean isEnabled() {
return getExecutablePath().isPresent()
&& main.getConfig().isGalleryDlEnabled();
}

@Override
public DownloaderIdEnum getDownloaderId() {
return DownloaderIdEnum.GALLERY_DL;
Expand All @@ -77,8 +83,7 @@ public boolean isMainDownloader() {

@Override
protected boolean canConsumeUrl(String inputUrl) {
return getExecutablePath().isPresent()
&& main.getConfig().isGalleryDlEnabled()
return isEnabled()
&& !(inputUrl.contains("ytimg")
|| inputUrl.contains("ggpht")
|| inputUrl.endsWith("youtube.com/"));
Expand Down Expand Up @@ -256,7 +261,7 @@ private Pair<Integer, String> processDownload(QueueEntry entry, List<String> arg
String line;
while (manager.isRunning() && !entry.getCancelHook().get() && process.isAlive()) {
if (Thread.currentThread().isInterrupted()) {
process.destroy();
process.destroyForcibly();
throw new InterruptedException("Download interrupted");
}

Expand All @@ -271,13 +276,11 @@ private Pair<Integer, String> processDownload(QueueEntry entry, List<String> arg
Thread.sleep(100);
} catch (InterruptedException e) {
log.debug("Sleep interrupted, closing process");
process.destroy();
process.destroyForcibly();
}
}
}

entry.getDownloadStarted().set(false);

long stopped = System.currentTimeMillis() - start;

if (!manager.isRunning() || entry.getCancelHook().get()) {
Expand All @@ -299,6 +302,8 @@ private Pair<Integer, String> processDownload(QueueEntry entry, List<String> arg

return null;
} finally {
entry.getDownloadStarted().set(false);

// Our ProcessMonitor will take care of closing the underlying process.
}
}
Expand Down
15 changes: 10 additions & 5 deletions core/src/main/java/net/brlns/gdownloader/downloader/YtDlpDownloader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,11 @@ public YtDlpDownloader(DownloadManager managerIn) {
super(managerIn);
}

@Override
public boolean isEnabled() {
return getExecutablePath().isPresent();// TODO: allow disabling?
}

@Override
public DownloaderIdEnum getDownloaderId() {
return DownloaderIdEnum.YT_DLP;
Expand All @@ -81,7 +86,7 @@ public boolean isMainDownloader() {

@Override
protected boolean canConsumeUrl(String inputUrl) {
return getExecutablePath().isPresent()
return isEnabled()
&& !(inputUrl.contains("ytimg")
|| inputUrl.contains("ggpht")
|| inputUrl.endsWith("youtube.com/")
Expand Down Expand Up @@ -344,7 +349,7 @@ private Pair<Integer, String> processDownload(QueueEntry entry, List<String> arg
while (manager.isRunning() && !entry.getCancelHook().get() && process.isAlive()) {
if (Thread.currentThread().isInterrupted()) {
log.debug("Process is closing");
process.destroy();
process.destroyForcibly();
throw new InterruptedException("Download interrupted");
}

Expand Down Expand Up @@ -372,12 +377,10 @@ private Pair<Integer, String> processDownload(QueueEntry entry, List<String> arg
Thread.sleep(100);
} catch (InterruptedException e) {
log.debug("Sleep interrupted, closing process");
process.destroy();
process.destroyForcibly();
}
}

entry.getDownloadStarted().set(false);

long stopped = System.currentTimeMillis() - start;

if (!manager.isRunning() || entry.getCancelHook().get()) {
Expand All @@ -399,6 +402,8 @@ private Pair<Integer, String> processDownload(QueueEntry entry, List<String> arg

return null;
} finally {
entry.getDownloadStarted().set(false);

// Our ProcessMonitor will take care of closing the underlying process.
}
}
Expand Down
Loading

0 comments on commit 4ce2146

Please sign in to comment.