Merge pull request #595 from bioimage-io/improve_summary

Improve validation summary

README.md

@@ -4,9 +4,9 @@
 
 # Specifications for bioimage.io
 
-This repository contains the specifications of the standard format defined by the bioimage.io community for the content (i.e., models, datasets and applications) in the [bioimage.io website](https://bioimage.io). 
-Each item in the content is always described using a YAML 1.2 file named `rdf.yaml` or `bioimageio.yaml`. 
-This `rdf.yaml` \ `bioimageio.yaml`--- along with the files referenced in it --- can be downloaded from or uploaded to the [bioimage.io website](https://bioimage.io) and may be produced or consumed by bioimage.io-compatible consumers (e.g., image analysis software like ilastik). 
+This repository contains the specifications of the standard format defined by the bioimage.io community for the content (i.e., models, datasets and applications) in the [bioimage.io website](https://bioimage.io).
+Each item in the content is always described using a YAML 1.2 file named `rdf.yaml` or `bioimageio.yaml`.
+This `rdf.yaml` \ `bioimageio.yaml`--- along with the files referenced in it --- can be downloaded from or uploaded to the [bioimage.io website](https://bioimage.io) and may be produced or consumed by bioimage.io-compatible consumers (e.g., image analysis software like ilastik).
 
 [These](https://github.com/bioimage-io/spec-bioimage-io?tab=readme-ov-file#format-version-overview) are the rules and format that bioimage.io-compatible resources must fulfill.
 
@@ -50,7 +50,6 @@ These are primarily intended for syntax highlighting and form generation.
 We provide some [examples for using rdf.yaml files to describe models, applications, notebooks and datasets](https://github.com/bioimage-io/spec-bioimage-io/blob/main/example_descriptions/examples.md),
 and an [example notebook to programmatically access the models, applications, notebooks and datasets descriptions](https://github.com/bioimage-io/spec-bioimage-io/blob/main/example/load_model_and_create_your_own.ipynb).
 
-
 ## 💁 Recommendations
 
 * Due to the limitations of storage services such as Zenodo, which does not support subfolders, it is recommended to place other files in the same directory level of the `rdf.yaml` file and try to avoid using subdirectories.
@@ -63,7 +62,7 @@ The bioimageio CLI has moved entirely to [bioimageio.core](https://github.com/bi
 
 ## 🖥 Installation
 
-bioimageio.spec can be installed with either `conda` or `pip`. 
+bioimageio.spec can be installed with either `conda` or `pip`.
 We recommend installing `bioimageio.core` instead to get access to the Python programmatic features available in the BioImage.IO community:
 
 ```console
@@ -88,7 +87,6 @@ or
 pip install -U bioimageio.spec
 ```
 
-
 ## 🏞 Environment variables
 
 TODO: link to settings in dev docs
@@ -107,6 +105,11 @@ Made with [contrib.rocks](https://contrib.rocks).
 
 ### bioimageio.spec Python package
 
+#### bioimageio.spec 0.5.2post5
+
+* added more information to validation summary
+* deprioritize `Path` objects in the `FileSource` union
+
 #### bioimageio.spec 0.5.2post4
 
 * resolve backup DOIs

bioimageio/spec/VERSION

@@ -1,3 +1,3 @@
 {
-    "version": "0.5.2post4"
+    "version": "0.5.2post5"
 }

bioimageio/spec/_build_description.py

@@ -4,7 +4,12 @@
 from ._internal.io import BioimageioYamlContent
 from ._internal.types import FormatVersionPlaceholder
 from ._internal.validation_context import ValidationContext, validation_context_var
-from .summary import ErrorEntry, ValidationDetail
+from .summary import (
+    WARNING_LEVEL_TO_NAME,
+    ErrorEntry,
+    ValidationContextSummary,
+    ValidationDetail,
+)
 
 ResourceDescrT = TypeVar("ResourceDescrT", bound=ResourceDescrBase)
 
@@ -82,6 +87,12 @@ def build_description_impl(
                 name="extract fields to chose description class",
                 status="failed",
                 errors=errors,
+                context=ValidationContextSummary(
+                    perform_io_checks=context.perform_io_checks,
+                    known_files=context.known_files,
+                    root=str(context.root),
+                    warning_level=WARNING_LEVEL_TO_NAME[context.warning_level],
+                ),
             )
         )
         return ret

bioimageio/spec/_internal/common_nodes.py

@@ -45,6 +45,7 @@
 from ..summary import (
     WARNING_LEVEL_TO_NAME,
     ErrorEntry,
+    ValidationContextSummary,
     ValidationDetail,
     ValidationSummary,
     WarningEntry,
@@ -326,11 +327,14 @@ def _set_init_validation_summary(self):
         self._validation_summary = ValidationSummary(
             name="bioimageio validation",
             source_name=context.source_name,
-            status="passed",
+            type=self.type,
+            format_version=self.format_version,
+            status="failed" if isinstance(self, InvalidDescr) else "passed",
             details=[
                 ValidationDetail(
-                    name=f"initialized {self.type} {self.implemented_format_version}",
+                    name=f"initialized {self.__class__.__name__} to describe {self.type} {self.implemented_format_version}",
                     status="passed",
+                    context=None,  # context for format validation detail is identical
                 )
             ],
         )
@@ -392,6 +396,12 @@ def load(
                 ),
                 status="failed" if errors else "passed",
                 warnings=val_warnings,
+                context=ValidationContextSummary(
+                    perform_io_checks=context.perform_io_checks,
+                    known_files=context.known_files,
+                    root=str(context.root),
+                    warning_level=WARNING_LEVEL_TO_NAME[context.warning_level],
+                ),
             )
         )
 

bioimageio/spec/_internal/io.py

@@ -223,7 +223,7 @@ def get_absolute(
 
 
 FileSource = Annotated[
-    Union[FilePath, HttpUrl, RelativeFilePath, pydantic.HttpUrl],
+    Union[HttpUrl, RelativeFilePath, pydantic.HttpUrl, FilePath],
     Field(union_mode="left_to_right"),
 ]
 PermissiveFileSource = Union[FileSource, str]

bioimageio/spec/summary.py

@@ -1,7 +1,17 @@
 from itertools import chain
 from pathlib import Path
 from types import MappingProxyType
-from typing import Any, Iterable, List, Literal, Mapping, Tuple, Union, no_type_check
+from typing import (
+    Any,
+    Iterable,
+    List,
+    Literal,
+    Mapping,
+    Optional,
+    Tuple,
+    Union,
+    no_type_check,
+)
 
 import rich.console
 import rich.markdown
@@ -97,11 +107,19 @@ class InstalledPackage(TypedDict):
     version: str
 
 
+class ValidationContextSummary(TypedDict):
+    perform_io_checks: bool
+    known_files: Mapping[str, str]
+    root: str
+    warning_level: str
+
+
 class ValidationDetail(BaseModel, extra="allow"):
     name: str
     status: Literal["passed", "failed"]
     errors: List[ErrorEntry] = Field(default_factory=list)
     warnings: List[WarningEntry] = Field(default_factory=list)
+    context: Optional[ValidationContextSummary] = None
 
     def __str__(self):
         return f"{self.__class__.__name__}:\n" + self.format()
@@ -160,6 +178,8 @@ def join_parts(parts: Iterable[Tuple[str, str]]):
 class ValidationSummary(BaseModel, extra="allow"):
     name: str
     source_name: str
+    type: str
+    format_version: str
     status: Literal["passed", "failed"]
     details: List[ValidationDetail]
     env: List[InstalledPackage] = Field(
@@ -204,30 +224,45 @@ def _format_md_table(rows: List[List[str]]) -> str:
         )
         return "\n| " + " |\n| ".join(lines) + " |\n"
 
-    def _format_env(self):
-        if not self.env:
-            return ""
-
-        rows = [["package", "version"]] + [[e["name"], e["version"]] for e in self.env]
-        return self._format_md_table(rows)
-
     def format(
         self,
         hide_tracebacks: bool = False,
         hide_source: bool = False,
         hide_env: bool = False,
         root_loc: Loc = (),
     ) -> str:
-        indent = "   " if root_loc else ""
-        src = "" if hide_source else f"\n{indent}source: {self.source_name}"
-        env = "" if hide_env else self._format_env()
-        details = [["❓", "location", "detail"]]
+        info = self._format_md_table(
+            [[self.status_icon, f"{self.name.strip('.').strip()} {self.status}"]]
+            + ([] if hide_source else [["source", self.source_name]])
+            + [
+                ["format version", f"{self.type} {self.format_version}"],
+            ]
+            + ([] if hide_env else [[e["name"], e["version"]] for e in self.env])
+        )
 
         def format_loc(loc: Loc):
             return "`" + (".".join(map(str, root_loc + loc)) or ".") + "`"
 
-        for d in self.details:
+        details = [["❓", "location", "detail"]]
+        for i, d in enumerate(self.details):
             details.append([d.status_icon, "", d.name])
+            if d.context is not None:
+                details.append(
+                    [
+                        "🔍",
+                        "context.perform_io_checks",
+                        str(d.context["perform_io_checks"]),
+                    ]
+                )
+                if d.context["perform_io_checks"]:
+                    details.append(["🔍", "context.root", d.context["root"]])
+                    for kfn, sha in d.context["known_files"].items():
+                        details.append(["🔍", f"context.known_files.{kfn}", sha])
+
+                details.append(
+                    ["🔍", "context.warning_level", d.context["warning_level"]]
+                )
+
             for entry in d.errors:
                 details.append(["❌", format_loc(entry.loc), entry.msg])
                 if hide_tracebacks:
@@ -261,18 +296,13 @@ def format_loc(loc: Loc):
 
                     details.append(["", "", first_tb_line + tb_rest])
 
-            if d.errors:
-                details.append(["", "", ""])
-
             for entry in d.warnings:
                 details.append(["⚠", format_loc(entry.loc), entry.msg])
-            if d.warnings:
+
+            if i != len(details) - 1:
                 details.append(["", "", ""])
 
-        return (
-            f"{indent}{self.status_icon} {self.name.strip('.')}: {self.status}\n"
-            + f"{src}{env}\n{self._format_md_table(details)}"
-        )
+        return f"{info}{self._format_md_table(details)}"
 
     @no_type_check
     def display(self) -> None: