Feat[downloader]: downloader improvements (#6428)

- Add download size queries through a HEAD request
- Use the file size for progress instead of file count when all file size are available
- Add download speed meter

app_pojavlauncher/src/main/java/net/kdt/pojavlaunch/mirrors/DownloadMirror.java

@@ -43,7 +43,7 @@ public static void downloadFileMirrored(int downloadClass, String urlInput, File
             return;
         }catch (FileNotFoundException e) {
             Log.w("DownloadMirror", "Cannot find the file on the mirror", e);
-            Log.i("DownloadMirror", "Failling back to default source");
+            Log.i("DownloadMirror", "Falling back to default source");
         }
         DownloadUtils.downloadFileMonitored(urlInput, outputFile, buffer, monitor);
     }
@@ -63,11 +63,30 @@ public static void downloadFileMirrored(int downloadClass, String urlInput, File
             return;
         }catch (FileNotFoundException e) {
             Log.w("DownloadMirror", "Cannot find the file on the mirror", e);
-            Log.i("DownloadMirror", "Failling back to default source");
+            Log.i("DownloadMirror", "Falling back to default source");
         }
         DownloadUtils.downloadFile(urlInput, outputFile);
     }
 
+    /**
+     * Get the content length of a file on the current mirror. If the file is missing on the mirror,
+     * or the mirror does not give out the length, request the length from the original source
+     * @param downloadClass Class of the download. Can either be DOWNLOAD_CLASS_LIBRARIES,
+     *                      DOWNLOAD_CLASS_METADATA or DOWNLOAD_CLASS_ASSETS
+     * @param urlInput The original (Mojang) URL for the download
+     * @return the length of the file denoted by the URL in bytes, or -1 if not available
+     */
+    public static long getContentLengthMirrored(int downloadClass, String urlInput) throws IOException {
+        long length = DownloadUtils.getContentLength(getMirrorMapping(downloadClass, urlInput));
+        if(length < 1) {
+            Log.w("DownloadMirror", "Unable to get content length from mirror");
+            Log.i("DownloadMirror", "Falling back to default source");
+            return DownloadUtils.getContentLength(urlInput);
+        }else {
+            return length;
+        }
+    }
+
     /**
      * Check if the current download source is a mirror and not an official source.
      * @return true if the source is a mirror, false otherwise

app_pojavlauncher/src/main/java/net/kdt/pojavlaunch/tasks/MinecraftDownloader.java

@@ -37,14 +37,18 @@
 import java.util.concurrent.atomic.AtomicReference;
 
 public class MinecraftDownloader {
+    private static final double ONE_MEGABYTE = (1024d * 1024d);
     public static final String MINECRAFT_RES = "https://resources.download.minecraft.net/";
     private AtomicReference<Exception> mDownloaderThreadException;
     private ArrayList<DownloaderTask> mScheduledDownloadTasks;
-    private AtomicLong mDownloadFileCounter;
-    private AtomicLong mDownloadSizeCounter;
-    private long mDownloadFileCount;
+    private AtomicLong mProcessedFileCounter;
+    private AtomicLong mProcessedSizeCounter; // Total bytes of processed files (passed SHA1 or downloaded)
+    private AtomicLong mInternetUsageCounter; // How many bytes downloaded over Internet
+    private long mTotalFileCount;
+    private long mTotalSize;
     private File mSourceJarFile; // The source client JAR picked during the inheritance process
     private File mTargetJarFile; // The destination client JAR to which the source will be copied to.
+    private boolean mUseFileCounter; // Whether a file counter or a size counter should be used for progress
 
     private static final ThreadLocal<byte[]> sThreadLocalDownloadBuffer = new ThreadLocal<>();
 
@@ -80,12 +84,15 @@ private void downloadGame(Activity activity, JMinecraftVersionList.Version verIn
         // Put up a dummy progress line, for the activity to start the service and do all the other necessary
         // work to keep the launcher alive. We will replace this line when we will start downloading stuff.
         ProgressLayout.setProgress(ProgressLayout.DOWNLOAD_MINECRAFT, 0, R.string.newdl_starting);
+        SpeedCalculator speedCalculator = new SpeedCalculator();
 
         mTargetJarFile = createGameJarPath(versionName);
         mScheduledDownloadTasks = new ArrayList<>();
-        mDownloadFileCounter = new AtomicLong(0);
-        mDownloadSizeCounter = new AtomicLong(0);
+        mProcessedFileCounter = new AtomicLong(0);
+        mProcessedSizeCounter = new AtomicLong(0);
+        mInternetUsageCounter = new AtomicLong(0);
         mDownloaderThreadException = new AtomicReference<>(null);
+        mUseFileCounter = false;
 
         if(!downloadAndProcessMetadata(activity, verInfo, versionName)) {
             throw new RuntimeException(activity.getString(R.string.exception_failed_to_unpack_jre17));
@@ -104,11 +111,9 @@ private void downloadGame(Activity activity, JMinecraftVersionList.Version verIn
         try {
             while (mDownloaderThreadException.get() == null &&
                     !downloaderPool.awaitTermination(33, TimeUnit.MILLISECONDS)) {
-                long dlFileCounter = mDownloadFileCounter.get();
-                int progress = (int)((dlFileCounter * 100L) / mDownloadFileCount);
-                ProgressLayout.setProgress(ProgressLayout.DOWNLOAD_MINECRAFT, progress,
-                        R.string.newdl_downloading_game_files, dlFileCounter,
-                        mDownloadFileCount, (double)mDownloadSizeCounter.get() / (1024d * 1024d));
+                double speed = speedCalculator.feed(mInternetUsageCounter.get()) / ONE_MEGABYTE;
+                if(mUseFileCounter) reportProgressFileCounter(speed);
+                else reportProgressSizeCounter(speed);
             }
             Exception thrownException = mDownloaderThreadException.get();
             if(thrownException != null) {
@@ -123,6 +128,23 @@ private void downloadGame(Activity activity, JMinecraftVersionList.Version verIn
         }
     }
 
+    private void reportProgressFileCounter(double speed) {
+        long dlFileCounter = mProcessedFileCounter.get();
+        int progress = (int)((dlFileCounter * 100L) / mTotalFileCount);
+        ProgressLayout.setProgress(ProgressLayout.DOWNLOAD_MINECRAFT, progress,
+                R.string.newdl_downloading_game_files, dlFileCounter,
+                mTotalFileCount, speed);
+    }
+
+    private void reportProgressSizeCounter(double speed) {
+        long dlFileSize = mProcessedSizeCounter.get();
+        double dlSizeMegabytes = (double) dlFileSize / ONE_MEGABYTE;
+        double dlTotalMegabytes = (double) mTotalSize / ONE_MEGABYTE;
+        int progress = (int)((dlFileSize * 100L) / mTotalSize);
+        ProgressLayout.setProgress(ProgressLayout.DOWNLOAD_MINECRAFT, progress,
+                R.string.newdl_downloading_game_files_size, dlSizeMegabytes, dlTotalMegabytes, speed);
+    }
+
     private File createGameJsonPath(String versionId) {
         return new File(Tools.DIR_HOME_VERSION, versionId + File.separator + versionId + ".json");
     }
@@ -233,7 +255,19 @@ private void growDownloadList(int addedElementCount) {
     private void scheduleDownload(File targetFile, int downloadClass, String url, String sha1,
                                   long size, boolean skipIfFailed) throws IOException {
         FileUtils.ensureParentDirectory(targetFile);
-        mDownloadFileCount++;
+        mTotalFileCount++;
+        if(size < 0) {
+            size = DownloadMirror.getContentLengthMirrored(downloadClass, url);
+        }
+        if(size < 0) {
+            // If we were unable to get the content length ourselves, we automatically fall back
+            // to tracking the progress using the file counter.
+            size = 0;
+            mUseFileCounter = true;
+            Log.i("MinecraftDownloader", "Failed to determine size of "+targetFile.getName()+", switching to file counter");
+        }else {
+            mTotalSize += size;
+        }
         mScheduledDownloadTasks.add(
                 new DownloaderTask(targetFile, downloadClass, url, sha1, size, skipIfFailed)
         );
@@ -401,18 +435,20 @@ private void downloadFile() throws Exception {
             }catch (Exception e) {
                 if(!mSkipIfFailed) throw e;
             }
-            mDownloadFileCounter.incrementAndGet();
+            mProcessedFileCounter.incrementAndGet();
         }
 
         private void finishWithoutDownloading() {
-            mDownloadFileCounter.incrementAndGet();
-            mDownloadSizeCounter.addAndGet(mDownloadSize);
+            mProcessedFileCounter.incrementAndGet();
+            mProcessedSizeCounter.addAndGet(mDownloadSize);
         }
 
         @Override
         public void updateProgress(int curr, int max) {
-           mDownloadSizeCounter.addAndGet(curr - mLastCurr);
-           mLastCurr = curr;
+            int delta = curr - mLastCurr;
+            mProcessedSizeCounter.addAndGet(delta);
+            mInternetUsageCounter.addAndGet(delta);
+            mLastCurr = curr;
         }
     }
 }

app_pojavlauncher/src/main/java/net/kdt/pojavlaunch/tasks/SpeedCalculator.java

@@ -0,0 +1,44 @@
+package net.kdt.pojavlaunch.tasks;
+
+/**
+ * A simple class to calculate the average Internet speed using a simple moving average.
+ */
+public class SpeedCalculator {
+    private long mLastMillis;
+    private long mLastBytes;
+    private int mIndex;
+    private final double[] mPreviousInputs;
+    private double mSum;
+
+    public SpeedCalculator() {
+        this(64);
+    }
+
+    public SpeedCalculator(int averageDepth) {
+        mPreviousInputs = new double[averageDepth];
+    }
+
+    private double addToAverage(double speed) {
+        mSum -= mPreviousInputs[mIndex];
+        mSum += speed;
+        mPreviousInputs[mIndex] = speed;
+        if(++mIndex == mPreviousInputs.length) mIndex = 0;
+        double dLength = mPreviousInputs.length;
+        return (mSum + (dLength / 2d)) / dLength;
+    }
+
+    /**
+     * Update the current amount of bytes downloaded.
+     * @param bytes the new amount of bytes downloaded
+     * @return the current download speed in bytes per second
+     */
+    public double feed(long bytes) {
+        long millis = System.currentTimeMillis();
+        long deltaBytes = bytes - mLastBytes;
+        long deltaMillis = millis - mLastMillis;
+        mLastBytes = bytes;
+        mLastMillis = millis;
+        double speed = (double)deltaBytes / ((double)deltaMillis / 1000d);
+        return addToAverage(speed);
+    }
+}

app_pojavlauncher/src/main/java/net/kdt/pojavlaunch/utils/DownloadUtils.java

@@ -155,6 +155,23 @@ public static <T> T ensureSha1(File outputFile, @Nullable String sha1, Callable<
         return result;
     }
 
+    /**
+     * Get the content length for a given URL.
+     * @param url the URL to get the length for
+     * @return the length in bytes or -1 if not available
+     * @throws IOException if an I/O error occurs.
+     */
+    public static long getContentLength(String url) throws IOException {
+        HttpURLConnection urlConnection = (HttpURLConnection) new URL(url).openConnection();
+        urlConnection.setRequestMethod("HEAD");
+        urlConnection.setDoInput(false);
+        urlConnection.setDoOutput(false);
+        urlConnection.connect();
+        int responseCode = urlConnection.getResponseCode();
+        if(responseCode >= 200 && responseCode <= 299) return urlConnection.getContentLength();
+        return -1;
+    }
+
     public interface ParseCallback<T> {
         T process(String input) throws ParseException;
     }

app_pojavlauncher/src/main/res/values/strings.xml

@@ -374,7 +374,8 @@
     <string name="exception_failed_to_unpack_jre17">Failed to install JRE 17</string>
     <string name="newdl_starting">Reading game metadata…</string>
     <string name="newdl_downloading_metadata">Downloading game metadata (%s)</string>
-    <string name="newdl_downloading_game_files">Downloading game files… (%d/%d, %.2f MB)</string>
+    <string name="newdl_downloading_game_files">Downloading game… (%d/%d, %.2f MB/s)</string>
+    <string name="newdl_downloading_game_files_size">Downloading game… (%.2f/%.2f MB, %.2f MB/s)</string>
     <string name="cropper_title">Select image region</string>
     <string name="cropper_lock_vertical">V. lock</string>
     <string name="cropper_lock_horizontal">H. lock</string>