Metrics redesign (#2326)

* refactor metrics callback * add Evaluator class * remove unused import * add AnomalibMetric base class * cleanup * add create_anomalib_metric function * use new AnomalibMetric class * verify keys when updating metric * use anomalib PR curve for f1max * replace evaluator with metrics arg * revert removing exportable transform method * fix unit tests * add kwargs to lightning models * fix pre-commit issues * remove kwargs key from update config * add Evaluator to metrics init * set default evaluator for image-only models * fix cfa tests * fix model tests * update changelog * remove tests of deprecated callback modules * fix engine tests * fix 201 notebook * fix 600 notebook * add default evaluator to fastflow * revert cfa changes and fix pred score computation * simplify evaluator argument * validate metrics in evaluator * remove unused import * fix pred_label shape validation * fix method name * fix pred_label validator * update validator tests * fix method name in fastflow * add AnomalibMetric functionality to PIMO * add optional default fields and implement for pimo * update aupimo notebooks * use explicit args in lightning model signature * remove kwargs from evaluator * add _resolve_evaluator method * fix config upgrade test * do not force pypi install in mlflow notebook * do not force pypi install in mlflow notebook
openvinotoolkit · Nov 7, 2024 · 6e1d870 · 6e1d870
1 parent 95115f9
commit 6e1d870
Show file tree

Hide file tree

Showing 63 changed files with 737 additions and 1,520 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -29,6 +29,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ### Changed
 
+- Implement new design for metrics by @djdameln https://github.com/openvinotoolkit/anomalib/pull/2326
 - Set permissions for github workflows by @djdameln in https://github.com/openvinotoolkit/anomalib/pull/2127
 - Update timm requirement from <=1.0.3,>=0.5.4 to >=0.5.4,<=1.0.7 by @dependabot in https://github.com/openvinotoolkit/anomalib/pull/2151
 - 🚀 Use gh actions runners for pre-commit checks by @ashwinvaidya17 in https://github.com/openvinotoolkit/anomalib/pull/2160

diff --git a/notebooks/200_models/201_fastflow.ipynb b/notebooks/200_models/201_fastflow.ipynb
@@ -233,7 +233,6 @@
    "source": [
     "engine = Engine(\n",
     "    callbacks=callbacks,\n",
-    "    pixel_metrics=\"AUROC\",\n",
     "    accelerator=\"auto\",  # \\<\"cpu\", \"gpu\", \"tpu\", \"ipu\", \"hpu\", \"auto\">,\n",
     "    devices=1,\n",
     "    logger=False,\n",

diff --git a/notebooks/600_loggers/601_mlflow_logging.ipynb b/notebooks/600_loggers/601_mlflow_logging.ipynb
@@ -38,7 +38,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%pip install -qU anomalib"
+    "%pip install anomalib"
    ]
   },
   {
@@ -56,7 +56,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%pip install -qU anomalib[loggers]"
+    "%pip install anomalib[loggers]"
    ]
   },
   {
@@ -307,7 +307,6 @@
     "\n",
     "engine = Engine(\n",
     "    callbacks=callbacks,\n",
-    "    pixel_metrics=\"AUROC\",\n",
     "    accelerator=\"auto\",\n",
     "    devices=1,\n",
     "    logger=mlflow_logger,  # Logger is set here\n",

diff --git a/notebooks/700_metrics/701a_aupimo.ipynb b/notebooks/700_metrics/701a_aupimo.ipynb
diff --git a/notebooks/700_metrics/701b_aupimo_advanced_i.ipynb b/notebooks/700_metrics/701b_aupimo_advanced_i.ipynb
diff --git a/notebooks/700_metrics/701c_aupimo_advanced_ii.ipynb b/notebooks/700_metrics/701c_aupimo_advanced_ii.ipynb
diff --git a/src/anomalib/callbacks/metrics.py b/src/anomalib/callbacks/metrics.py
diff --git a/src/anomalib/cli/cli.py b/src/anomalib/cli/cli.py
@@ -148,8 +148,6 @@ def add_arguments_to_parser(parser: ArgumentParser) -> None:
             ``Engine`` class should be reflected manually.
         """
         parser.add_argument("--task", type=TaskType | str, default=TaskType.SEGMENTATION)
-        parser.add_argument("--metrics.image", type=list[str] | str | None, default=None)
-        parser.add_argument("--metrics.pixel", type=list[str] | str | None, default=None, required=False)
         parser.add_argument("--logging.log_graph", type=bool, help="Log the model to the logger", default=False)
         if hasattr(parser, "subcommand") and parser.subcommand not in {"export", "predict"}:
             parser.link_arguments("task", "data.init_args.task")
@@ -323,8 +321,6 @@ def instantiate_engine(self) -> None:
 
         engine_args = {
             "task": self._get(self.config_init, "task"),
-            "image_metrics": self._get(self.config_init, "metrics.image"),
-            "pixel_metrics": self._get(self.config_init, "metrics.pixel"),
         }
         trainer_config = {**self._get(self.config_init, "trainer", default={}), **engine_args}
         key = "callbacks"

diff --git a/src/anomalib/data/validators/torch/image.py b/src/anomalib/data/validators/torch/image.py
@@ -585,10 +585,16 @@ def validate_pred_label(pred_label: torch.Tensor | None) -> torch.Tensor | None:
         if pred_label.ndim > 2:
             msg = f"Predicted label must be 1-dimensional or 2-dimensional, got shape {pred_label.shape}."
             raise ValueError(msg)
-        if pred_label.ndim == 2 and pred_label.shape[1] != 1:
-            msg = f"Predicted label with 2 dimensions must have shape [N, 1], got shape {pred_label.shape}."
-            raise ValueError(msg)
-
+        if pred_label.ndim == 2:
+            if pred_label.shape[0] == 1:
+                pred_label = pred_label.squeeze(0)
+            elif pred_label.shape[1] == 1:
+                pred_label = pred_label.squeeze(1)
+            else:
+                msg = (
+                    f"Predicted label with 2 dimensions must have shape [N, 1] or [1, N], got shape {pred_label.shape}."
+                )
+                raise ValueError(msg)
         return pred_label.to(torch.bool)
 
     @staticmethod

diff --git a/src/anomalib/engine/engine.py b/src/anomalib/engine/engine.py
@@ -18,7 +18,6 @@
 
 from anomalib import LearningType, TaskType
 from anomalib.callbacks.checkpoint import ModelCheckpoint
-from anomalib.callbacks.metrics import _MetricsCallback
 from anomalib.callbacks.timer import TimerCallback
 from anomalib.data import AnomalibDataModule, AnomalibDataset, PredictDataset
 from anomalib.deploy import CompressionType, ExportType
@@ -116,8 +115,6 @@ def __init__(
         self,
         callbacks: list[Callback] | None = None,
         task: TaskType | str = TaskType.SEGMENTATION,
-        image_metrics: list[str] | str | dict[str, dict[str, Any]] | None = None,
-        pixel_metrics: list[str] | str | dict[str, dict[str, Any]] | None = None,
         logger: Logger | Iterable[Logger] | bool | None = None,
         default_root_dir: str | Path = "results",
         **kwargs,
@@ -137,12 +134,6 @@ def __init__(
         )
 
         self.task = TaskType(task)
-        self.image_metric_names = image_metrics if image_metrics else ["AUROC", "F1Max"]
-
-        # pixel metrics are only used for segmentation tasks.
-        self.pixel_metric_names = None
-        if self.task == TaskType.SEGMENTATION:
-            self.pixel_metric_names = pixel_metrics if pixel_metrics is not None else ["AUROC", "F1Max"]
 
         self._trainer: Trainer | None = None
 
@@ -375,7 +366,8 @@ def _setup_anomalib_callbacks(self, model: AnomalyModule) -> None:
             _callbacks.append(model.post_processor)
 
         # Add the metrics callback.
-        _callbacks.append(_MetricsCallback(self.task, self.image_metric_names, self.pixel_metric_names))
+        if isinstance(model.evaluator, Callback):
+            _callbacks.append(model.evaluator)
 
         # Add the image visualizer callback if it is passed by the user.
         if not any(isinstance(callback, ImageVisualizer) for callback in self._cache.args["callbacks"]):