Clean CIs and set up model exportation ZIP

.github/workflows/export_notebook.yml

@@ -0,0 +1,30 @@
+name: Update the manifest with DL4MicEverywhere information
+on:
+  # At 05:30 AM of every day
+  # schedule:
+    # - cron: "30 5 * * *"
+  workflow_dispatch:
+    inputs:
+      notebook_name: # the variable you can use in place of a matrix
+        description: 'Name of the notebook you want to export. You need to provide the 'id' from manifest.bioimage.io.yaml.'
+        type: string
+
+jobs:
+  update-manifest:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Clone ZeroCostDL4Mic repository
+        uses: actions/checkout@v4
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.9'
+      - name: Install requirements
+        run: pip install -r Tools/CI_requirements.txt
+      - name: Export the notebook and create a ZIP file with it
+        run: python3 Tools/export_bmz_notebook.py -n "${{ inputs.notebook_name }}" -o "./tmp"
+      - name: Upload exported ZIP file
+        uses: actions/upload-artifact@v4
+        with:
+          name: '${{ inputs.notebook_name }}.zip'
+          path: './tmp/${{ inputs.notebook_name }}.zip'

.github/workflows/mamba.yml

@@ -13,11 +13,10 @@ jobs:
       - uses: actions/checkout@v2
       - uses: conda-incubator/setup-miniconda@v2
         with:
-          # mamba-version: "*"
+          miniconda-version: "latest"
           auto-update-conda: true
           environment-file: .binder/environment.yml
           python-version: ${{ matrix.python-version }}
-          # use-mamba: true
       - name: Conda info
         shell: bash -el {0}
         run: conda info

.github/workflows/update_manifest_dl4miceverywhere.yml

@@ -18,6 +18,10 @@ jobs:
         with:
             repository: HenriquesLab/DL4MicEverywhere
             path: DL4MicEverywhere
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.9'
       - name: Install requirements
         working-directory: ./ZeroCostDL4Mic
         run: pip install -r Tools/CI_requirements.txt

.github/workflows/update_manifest_versioning.yml

@@ -1,4 +1,7 @@
-name: Update versioning on the manifest
+# Updates the version from all the notebooks on manifest.bioimage.io.yaml 
+#using the versions from Latest_Notebook_versions.csv
+
+name: Update versions on the manifest
 on:
   workflow_dispatch:
 

.github/workflows/validate-collection.yml

@@ -7,10 +7,16 @@ on:
     branches: [master]
 jobs:
   run:
+    name: Validate manifest.bioimage.io.yaml 
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
-    - name: install bioimageio.spec
+    - name: Clone ZeroCostDL4Mic
+      uses: actions/checkout@v4
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.9'
+    - name: Install bioimageio.spec
       run: pip install bioimageio.spec lxml
-    - name: validate collection
+    - name: Validate collection manifest
       run: bioimageio validate-partner-collection manifest.bioimage.io.yaml

Colab_notebooks/Latest_Notebook_versions.csv

@@ -16,7 +16,7 @@ U-Net (2D),2.2.1
 U-Net (3D),2.2.1
 U-Net (2D) multilabel,2.1.4
 Kaibu,1.13.2
-MaskRCNN,1.14.1
+MaskRCNN,1.14.2
 Noise2Void (2D),1.16.2
 Noise2Void (3D),1.16.2
 pix2pix,1.17.3

Tools/check_outdated_notebooks.py

@@ -0,0 +1,47 @@
+from ruamel.yaml import YAML
+import pandas as pd
+
+from parser_dicts_variables import dict_manifest_to_version
+
+def check_outdated_notebooks():
+    # Read the information from the manifest
+    with open('manifest.bioimage.io.yaml', 'r', encoding='utf8') as f:
+        yaml = YAML()
+        yaml.preserve_quotes = True
+        manifest_data = yaml.load(f)
+
+    # Read the versions of the notebooks
+    all_notebook_versions = pd.read_csv('Colab_notebooks/Latest_Notebook_versions.csv', dtype=str)
+
+    list_notebook_need_update = []
+    # We are going to check the elements in the collection from the manifest
+    for element in manifest_data['collection']:
+        # In case it is an application, we need to update the version
+        if element['type'] == 'application' and element['id'] != 'notebook_preview': 
+            # Get the different versions of the notebooks (latest_version and manifest)
+            notebook_version = all_notebook_versions[all_notebook_versions["Notebook"] == dict_manifest_to_version[element['id']]]['Version'].iloc[0]
+            manifest_version = element['version']
+
+            # Check if it is the same or different
+            if notebook_version != manifest_version:
+                # In case its different, add its id to the list
+                list_notebook_need_update.append(element['id'])
+
+    return list_notebook_need_update
+
+def main():
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Checks the notebooks that need to be updated and returns a list with its IDs, no arguments required.",
+                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    args = vars(parser.parse_args())
+
+    list_notebook_need_update = check_outdated_notebooks()
+
+    if len(list_notebook_need_update) == 0:
+        print('')
+    else:
+        print(' '.join(list_notebook_need_update))
+
+if __name__ == "__main__":
+    main()

Tools/export_bmz_notebook.py

@@ -0,0 +1,86 @@
+from ruamel.yaml import YAML
+import tempfile
+import zipfile
+import os
+
+def export_bmz_notebook(notebook_id, output_path):
+    # Read the information from the manifest
+    with open("manifest.bioimage.io.yaml", 'r', encoding='utf8') as f:
+        yaml = YAML()
+        yaml.preserve_quotes = True
+        manifest_data = yaml.load(f)
+
+    wanted_notebook = None
+    # We are going to check the elements in the collection from the manifest
+    for element in manifest_data['collection']:
+        # We are only looking for the selected notebook
+        if element['id'] == notebook_id: 
+            wanted_notebook = element.copy()
+            break
+
+    # Check if the notebook was found, otherwise raise an Error
+    if wanted_notebook is None:
+        raise ValueError(f"Sorry, we were not able to find a notebook with the name: {notebook_id}")
+
+    # Get the path to the notebook
+    notebook_url = wanted_notebook['config']['dl4miceverywhere']['notebook_url']
+    notebook_name = os.path.basename(notebook_url)
+    notebook_local_path = os.path.join("Colab_notebooks", notebook_name)
+
+    # Get the path to the requirements
+    requirements_url = wanted_notebook['config']['dl4miceverywhere']['requirements_url']
+    requirements_name = os.path.basename(requirements_url)
+    requirements_local_path = os.path.join("requirements_files", requirements_name)
+
+    # Add these files into the attachments:files:[] on notebook the configuration
+    if 'attachments' not in wanted_notebook:
+        wanted_notebook['attachments'] = {'files': [notebook_name, 'requirements.txt']}
+    else:
+        wanted_notebook['attachments']['files'] = [notebook_name, 'requirements.txt']
+
+    # Create rdf.yaml file on a temporary folder
+    with tempfile.TemporaryDirectory() as tmpdirname:
+        print(f"Temporary directory created on: {tmpdirname}")
+
+        rdf_path = os.path.join(tmpdirname, "rdf.yaml")
+
+        # Write the new collection
+        with open(rdf_path, 'w', encoding='utf8') as f:
+            yaml = YAML()
+            # Choose the settings
+            yaml.preserve_quotes = True
+            yaml.default_flow_style = False
+            yaml.indent(mapping=2, sequence=4, offset=2)
+            yaml.width = 10e10
+            # Save notebook's data
+            yaml.dump(wanted_notebook, f)
+
+        print("The 'rdf.yaml' file was correctly created.")
+
+        os.makedirs(output_path, exist_ok=True)
+        zipfile_path = os.path.join(output_path, f"{notebook_id}.zip")
+
+        # Create the ZIP file with the rdf, notebook and requirements
+        with zipfile.ZipFile(zipfile_path, 'w') as myzip:
+            # Add the rdf.yaml file
+            myzip.write(rdf_path, arcname=os.path.join(notebook_id, "rdf.yaml"))
+            # Add the notebook
+            myzip.write(notebook_local_path, arcname=os.path.join(notebook_id, notebook_name))
+            # Add the requirements
+            myzip.write(requirements_local_path, arcname=os.path.join(notebook_id, "requirements.txt"))
+
+        print(f"ZIP file correctly created on: {zipfile_path}")
+
+def main():
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Export the chosen notebook into a ZIP file that follows the BioImage.IO format.",
+                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument("-n", "--name", help="Name of the notebook you want to export. You need to provide the 'id' from manifest.bioimage.io.yaml.")
+    parser.add_argument("-o", "--output", help="Path to the folder to save the ZIP file.")
+    args = vars(parser.parse_args())
+
+    export_bmz_notebook(notebook_id=args['name'], output_path=args['output'])
+
+if __name__ == "__main__":
+    main()