diff --git a/.github/workflows/export_notebook.yml b/.github/workflows/export_notebook.yml new file mode 100644 index 00000000..60608692 --- /dev/null +++ b/.github/workflows/export_notebook.yml @@ -0,0 +1,30 @@ +name: Update the manifest with DL4MicEverywhere information +on: + # At 05:30 AM of every day + # schedule: + # - cron: "30 5 * * *" + workflow_dispatch: + inputs: + notebook_name: # the variable you can use in place of a matrix + description: 'Name of the notebook you want to export. You need to provide the 'id' from manifest.bioimage.io.yaml.' + type: string + +jobs: + update-manifest: + runs-on: ubuntu-latest + steps: + - name: Clone ZeroCostDL4Mic repository + uses: actions/checkout@v4 + - name: Set up Python 3.9 + uses: actions/setup-python@v5 + with: + python-version: '3.9' + - name: Install requirements + run: pip install -r Tools/CI_requirements.txt + - name: Export the notebook and create a ZIP file with it + run: python3 Tools/export_bmz_notebook.py -n "${{ inputs.notebook_name }}" -o "./tmp" + - name: Upload exported ZIP file + uses: actions/upload-artifact@v4 + with: + name: '${{ inputs.notebook_name }}.zip' + path: './tmp/${{ inputs.notebook_name }}.zip' \ No newline at end of file diff --git a/.github/workflows/mamba.yml b/.github/workflows/mamba.yml index f80aa015..78940336 100644 --- a/.github/workflows/mamba.yml +++ b/.github/workflows/mamba.yml @@ -13,11 +13,10 @@ jobs: - uses: actions/checkout@v2 - uses: conda-incubator/setup-miniconda@v2 with: - # mamba-version: "*" + miniconda-version: "latest" auto-update-conda: true environment-file: .binder/environment.yml python-version: ${{ matrix.python-version }} - # use-mamba: true - name: Conda info shell: bash -el {0} run: conda info diff --git a/.github/workflows/update_manifest_dl4miceverywhere.yml b/.github/workflows/update_manifest_dl4miceverywhere.yml index d38dabc7..3834ead6 100644 --- a/.github/workflows/update_manifest_dl4miceverywhere.yml +++ b/.github/workflows/update_manifest_dl4miceverywhere.yml @@ -18,6 +18,10 @@ jobs: with: repository: HenriquesLab/DL4MicEverywhere path: DL4MicEverywhere + - name: Set up Python 3.9 + uses: actions/setup-python@v5 + with: + python-version: '3.9' - name: Install requirements working-directory: ./ZeroCostDL4Mic run: pip install -r Tools/CI_requirements.txt diff --git a/.github/workflows/update_manifest_versioning.yml b/.github/workflows/update_manifest_versioning.yml index 98ee45d4..e0e3a499 100644 --- a/.github/workflows/update_manifest_versioning.yml +++ b/.github/workflows/update_manifest_versioning.yml @@ -1,4 +1,7 @@ -name: Update versioning on the manifest +# Updates the version from all the notebooks on manifest.bioimage.io.yaml +#using the versions from Latest_Notebook_versions.csv + +name: Update versions on the manifest on: workflow_dispatch: diff --git a/.github/workflows/validate-collection.yml b/.github/workflows/validate-collection.yml index 66a57da7..7fe7ee46 100644 --- a/.github/workflows/validate-collection.yml +++ b/.github/workflows/validate-collection.yml @@ -7,10 +7,16 @@ on: branches: [master] jobs: run: + name: Validate manifest.bioimage.io.yaml runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: install bioimageio.spec + - name: Clone ZeroCostDL4Mic + uses: actions/checkout@v4 + - name: Set up Python 3.9 + uses: actions/setup-python@v5 + with: + python-version: '3.9' + - name: Install bioimageio.spec run: pip install bioimageio.spec lxml - - name: validate collection + - name: Validate collection manifest run: bioimageio validate-partner-collection manifest.bioimage.io.yaml diff --git a/.github/workflows/validate_zero_collection.yaml b/.github/workflows/validate_zero_collection.yaml deleted file mode 100644 index 02a9df9f..00000000 --- a/.github/workflows/validate_zero_collection.yaml +++ /dev/null @@ -1,18 +0,0 @@ -name: validate zero collection - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -jobs: - run: - runs-on: ubuntu-latest - steps: - - name: checkout - uses: actions/checkout@v2 - - name: install bioimageio.spec - run: pip install bioimageio.spec lxml - - name: validate collection - run: bioimageio validate-partner-collection manifest.bioimage.io.yaml diff --git a/Colab_notebooks/Latest_Notebook_versions.csv b/Colab_notebooks/Latest_Notebook_versions.csv index 001f686f..dc225ea1 100644 --- a/Colab_notebooks/Latest_Notebook_versions.csv +++ b/Colab_notebooks/Latest_Notebook_versions.csv @@ -16,7 +16,7 @@ U-Net (2D),2.2.1 U-Net (3D),2.2.1 U-Net (2D) multilabel,2.1.4 Kaibu,1.13.2 -MaskRCNN,1.14.1 +MaskRCNN,1.14.2 Noise2Void (2D),1.16.2 Noise2Void (3D),1.16.2 pix2pix,1.17.3 diff --git a/Tools/check_outdated_notebooks.py b/Tools/check_outdated_notebooks.py new file mode 100644 index 00000000..2b22c2c6 --- /dev/null +++ b/Tools/check_outdated_notebooks.py @@ -0,0 +1,47 @@ +from ruamel.yaml import YAML +import pandas as pd + +from parser_dicts_variables import dict_manifest_to_version + +def check_outdated_notebooks(): + # Read the information from the manifest + with open('manifest.bioimage.io.yaml', 'r', encoding='utf8') as f: + yaml = YAML() + yaml.preserve_quotes = True + manifest_data = yaml.load(f) + + # Read the versions of the notebooks + all_notebook_versions = pd.read_csv('Colab_notebooks/Latest_Notebook_versions.csv', dtype=str) + + list_notebook_need_update = [] + # We are going to check the elements in the collection from the manifest + for element in manifest_data['collection']: + # In case it is an application, we need to update the version + if element['type'] == 'application' and element['id'] != 'notebook_preview': + # Get the different versions of the notebooks (latest_version and manifest) + notebook_version = all_notebook_versions[all_notebook_versions["Notebook"] == dict_manifest_to_version[element['id']]]['Version'].iloc[0] + manifest_version = element['version'] + + # Check if it is the same or different + if notebook_version != manifest_version: + # In case its different, add its id to the list + list_notebook_need_update.append(element['id']) + + return list_notebook_need_update + +def main(): + import argparse + + parser = argparse.ArgumentParser(description="Checks the notebooks that need to be updated and returns a list with its IDs, no arguments required.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + args = vars(parser.parse_args()) + + list_notebook_need_update = check_outdated_notebooks() + + if len(list_notebook_need_update) == 0: + print('') + else: + print(' '.join(list_notebook_need_update)) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/Tools/export_bmz_notebook.py b/Tools/export_bmz_notebook.py new file mode 100644 index 00000000..8ea00607 --- /dev/null +++ b/Tools/export_bmz_notebook.py @@ -0,0 +1,86 @@ +from ruamel.yaml import YAML +import tempfile +import zipfile +import os + +def export_bmz_notebook(notebook_id, output_path): + # Read the information from the manifest + with open("manifest.bioimage.io.yaml", 'r', encoding='utf8') as f: + yaml = YAML() + yaml.preserve_quotes = True + manifest_data = yaml.load(f) + + wanted_notebook = None + # We are going to check the elements in the collection from the manifest + for element in manifest_data['collection']: + # We are only looking for the selected notebook + if element['id'] == notebook_id: + wanted_notebook = element.copy() + break + + # Check if the notebook was found, otherwise raise an Error + if wanted_notebook is None: + raise ValueError(f"Sorry, we were not able to find a notebook with the name: {notebook_id}") + + # Get the path to the notebook + notebook_url = wanted_notebook['config']['dl4miceverywhere']['notebook_url'] + notebook_name = os.path.basename(notebook_url) + notebook_local_path = os.path.join("Colab_notebooks", notebook_name) + + # Get the path to the requirements + requirements_url = wanted_notebook['config']['dl4miceverywhere']['requirements_url'] + requirements_name = os.path.basename(requirements_url) + requirements_local_path = os.path.join("requirements_files", requirements_name) + + # Add these files into the attachments:files:[] on notebook the configuration + if 'attachments' not in wanted_notebook: + wanted_notebook['attachments'] = {'files': [notebook_name, 'requirements.txt']} + else: + wanted_notebook['attachments']['files'] = [notebook_name, 'requirements.txt'] + + # Create rdf.yaml file on a temporary folder + with tempfile.TemporaryDirectory() as tmpdirname: + print(f"Temporary directory created on: {tmpdirname}") + + rdf_path = os.path.join(tmpdirname, "rdf.yaml") + + # Write the new collection + with open(rdf_path, 'w', encoding='utf8') as f: + yaml = YAML() + # Choose the settings + yaml.preserve_quotes = True + yaml.default_flow_style = False + yaml.indent(mapping=2, sequence=4, offset=2) + yaml.width = 10e10 + # Save notebook's data + yaml.dump(wanted_notebook, f) + + print("The 'rdf.yaml' file was correctly created.") + + os.makedirs(output_path, exist_ok=True) + zipfile_path = os.path.join(output_path, f"{notebook_id}.zip") + + # Create the ZIP file with the rdf, notebook and requirements + with zipfile.ZipFile(zipfile_path, 'w') as myzip: + # Add the rdf.yaml file + myzip.write(rdf_path, arcname=os.path.join(notebook_id, "rdf.yaml")) + # Add the notebook + myzip.write(notebook_local_path, arcname=os.path.join(notebook_id, notebook_name)) + # Add the requirements + myzip.write(requirements_local_path, arcname=os.path.join(notebook_id, "requirements.txt")) + + print(f"ZIP file correctly created on: {zipfile_path}") + +def main(): + import argparse + + parser = argparse.ArgumentParser(description="Export the chosen notebook into a ZIP file that follows the BioImage.IO format.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument("-n", "--name", help="Name of the notebook you want to export. You need to provide the 'id' from manifest.bioimage.io.yaml.") + parser.add_argument("-o", "--output", help="Path to the folder to save the ZIP file.") + args = vars(parser.parse_args()) + + export_bmz_notebook(notebook_id=args['name'], output_path=args['output']) + +if __name__ == "__main__": + main() \ No newline at end of file