feat(ivy): Added a progress bar to cleanup_and_fetch_binaries (#26812)

To display the status of downloading of binaries.
Also had to move tqdm from optional to requirements as a result.

ivy/utils/binaries.py

@@ -3,6 +3,7 @@
 import json
 from pip._vendor.packaging import tags
 from urllib import request
+from tqdm import tqdm
 
 
 def _get_paths_from_binaries(binaries, root_dir=""):
@@ -68,38 +69,44 @@ def cleanup_and_fetch_binaries(clean=True):
 
         # clean up existing binaries
         if clean:
-            print("Cleaning up existing binaries...")
+            print("Cleaning up existing binaries", end="\r")
             for root, _, files in os.walk(folder_path, topdown=True):
                 for file in files:
                     if file.split(".")[-1] in binaries_exts:
                         os.remove(os.path.join(root, file))
+            print("Cleaning up existing binaries --> done")
 
-        print("Downloading new binaries...")
+        print("Downloading new binaries")
         all_tags = list(tags.sys_tags())
         version = os.environ["VERSION"] if "VERSION" in os.environ else "main"
         terminate = False
 
         # download binaries for the tag with highest precedence
-        for tag in all_tags:
-            if terminate:
-                break
-            for path in binaries_paths:
-                module = path[len(folder_path) :][1:].split(os.sep)[1]
-                if os.path.exists(path) or str(tag) not in available_configs[module]:
-                    continue
-                folders = path.split(os.sep)
-                _, file_path = os.sep.join(folders[:-1]), folders[-1]
-                file_name = f"{file_path[:-3]}_{tag}.so"
-                search_path = f"{module}/{file_name}"
-                try:
-                    response = request.urlopen(
-                        "https://github.com/unifyai/binaries/raw/"
-                        f"{version}/{search_path}",
-                        timeout=40,
-                    )
-                    os.makedirs(os.path.dirname(path), exist_ok=True)
-                    with open(path, "wb") as f:
-                        f.write(response.read())
-                    terminate = path == binaries_paths[-1]
-                except request.HTTPError:
+        with tqdm(total=len(binaries_paths)) as pbar:
+            for tag in all_tags:
+                if terminate:
                     break
+                for path in binaries_paths:
+                    module = path[len(folder_path) :][1:].split(os.sep)[1]
+                    if (
+                        os.path.exists(path)
+                        or str(tag) not in available_configs[module]
+                    ):
+                        continue
+                    folders = path.split(os.sep)
+                    _, file_path = os.sep.join(folders[:-1]), folders[-1]
+                    file_name = f"{file_path[:-3]}_{tag}.so"
+                    search_path = f"{module}/{file_name}"
+                    try:
+                        response = request.urlopen(
+                            "https://github.com/unifyai/binaries/raw/"
+                            f"{version}/{search_path}",
+                            timeout=40,
+                        )
+                        os.makedirs(os.path.dirname(path), exist_ok=True)
+                        with open(path, "wb") as f:
+                            f.write(response.read())
+                        terminate = path == binaries_paths[-1]
+                        pbar.update(1)
+                    except request.HTTPError:
+                        break

requirements/optional.txt

@@ -17,7 +17,6 @@ scipy # unpinned
 dm-haiku # unpinned mod_name=haiku
 flax
 pydriller
-tqdm
 coverage
 scikit-learn # mod_name=sklearn
 pandas

requirements/optional_apple_silicon_2.txt

@@ -5,7 +5,6 @@ dm-haiku  # mod_name=haiku
 flax
 protobuf
 pydriller
-tqdm
 coverage
 scikit-learn  # mod_name=sklearn
 pandas

requirements/optional_apple_silicon_gpu_2.txt

@@ -5,7 +5,6 @@ dm-haiku  # mod_name=haiku
 flax
 protobuf
 pydriller
-tqdm
 coverage
 scikit-learn  # mod_name=sklearn
 pandas

requirements/optional_gpu.txt

@@ -16,7 +16,6 @@ scipy # unpinned
 dm-haiku # unpinned mod_name=haiku
 flax
 pydriller
-tqdm
 coverage
 scikit-learn # mod_name=sklearn
 pandas

requirements/requirements.txt

@@ -15,3 +15,4 @@ astunparse
 ml-dtypes # mod_name=ml_dtypes
 cloudpickle
 gast
+tqdm