Comments

tests/post_training/data/ptq_reference_data.yaml

@@ -26,6 +26,8 @@ torchvision/resnet18_backend_FP32:
   metric_value: 0.6978
 torchvision/resnet18_backend_OV:
   metric_value: 0.6948
+torchvision/resnet18_backend_ONNX:
+  metric_value: 0.6948
 torchvision/resnet18_backend_TORCH:
   metric_value: 0.69152
 torchvision/resnet18_backend_CUDA_TORCH:
@@ -190,16 +192,6 @@ timm/resnest14d_backend_OV:
   metric_value: 0.75
 timm/resnest14d_backend_TORCH:
   metric_value: 0.7485
-timm/resnet18_backend_CUDA_TORCH:
-  metric_value: 0.69748
-timm/resnet18_backend_FP32:
-  metric_value: 0.71502
-timm/resnet18_backend_ONNX:
-  metric_value: 0.71102
-timm/resnet18_backend_OV:
-  metric_value: 0.71116
-timm/resnet18_backend_TORCH:
-  metric_value: 0.70982
 timm/swin_base_patch4_window7_224_backend_FP32:
   metric_value: 0.85274
 timm/swin_base_patch4_window7_224_backend_OV:

tests/post_training/model_scope.py

@@ -66,14 +66,14 @@
         },
         "backends": [BackendType.TORCH, BackendType.OV, BackendType.OPTIMUM],
     },
-    # Torchvision
+    # Torchvision models
     {
         "reported_name": "torchvision/resnet18",
         "model_id": "resnet18",
         "pipeline_cls": ImageClassificationTorchvision,
         "compression_params": {},
-        "backends": [BackendType.FX_TORCH, BackendType.TORCH, BackendType.CUDA_TORCH, BackendType.OV],
-        "batch_size": 1,
+        "backends": [BackendType.FX_TORCH, BackendType.TORCH, BackendType.CUDA_TORCH, BackendType.OV, BackendType.ONNX],
+        "batch_size": 128,
     },
     # Timm models
     {
@@ -256,14 +256,6 @@
         "backends": ALL_PTQ_BACKENDS,
         "batch_size": 128,
     },
-    {
-        "reported_name": "timm/resnet18",
-        "model_id": "resnet18",
-        "pipeline_cls": ImageClassificationTimm,
-        "compression_params": {},
-        "backends": ALL_PTQ_BACKENDS,
-        "batch_size": 128,
-    },
     {
         "reported_name": "timm/swin_base_patch4_window7_224",
         "model_id": "swin_base_patch4_window7_224",

tests/post_training/pipelines/image_classification_base.py

@@ -0,0 +1,80 @@
+# Copyright (c) 2024 Intel Corporation
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import copy
+import os
+
+import numpy as np
+import openvino as ov
+import torch
+from sklearn.metrics import accuracy_score
+from torchvision import datasets
+
+import nncf
+from nncf.common.logging.track_progress import track
+from tests.post_training.pipelines.base import DEFAULT_VAL_THREADS
+from tests.post_training.pipelines.base import PTQTestPipeline
+
+
+class ImageClassificationBase(PTQTestPipeline):
+    """Base pipeline for Image Classification models"""
+
+    def prepare_calibration_dataset(self):
+        dataset = datasets.ImageFolder(root=self.data_dir / "imagenet" / "val", transform=self.transform)
+        loader = torch.utils.data.DataLoader(dataset, batch_size=self.batch_size, num_workers=2, shuffle=False)
+
+        self.calibration_dataset = nncf.Dataset(loader, self.get_transform_calibration_fn())
+
+    def _validate(self):
+        val_dataset = datasets.ImageFolder(root=self.data_dir / "imagenet" / "val", transform=self.transform)
+        val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=1, num_workers=2, shuffle=False)
+
+        dataset_size = len(val_loader)
+
+        # Initialize result tensors for async inference support.
+        predictions = np.zeros((dataset_size))
+        references = -1 * np.ones((dataset_size))
+
+        core = ov.Core()
+
+        if os.environ.get("INFERENCE_NUM_THREADS"):
+            # Set CPU_THREADS_NUM for OpenVINO inference
+            inference_num_threads = os.environ.get("INFERENCE_NUM_THREADS")
+            core.set_property("CPU", properties={"INFERENCE_NUM_THREADS": str(inference_num_threads)})
+
+        ov_model = core.read_model(self.path_compressed_ir)
+        compiled_model = core.compile_model(ov_model, device_name="CPU")
+
+        jobs = int(os.environ.get("NUM_VAL_THREADS", DEFAULT_VAL_THREADS))
+        infer_queue = ov.AsyncInferQueue(compiled_model, jobs)
+
+        with track(total=dataset_size, description="Validation") as pbar:
+
+            def process_result(request, userdata):
+                output_data = request.get_output_tensor().data
+                predicted_label = np.argmax(output_data, axis=1)
+                predictions[userdata] = predicted_label
+                pbar.progress.update(pbar.task, advance=1)
+
+            infer_queue.set_callback(process_result)
+
+            for i, (images, target) in enumerate(val_loader):
+                # W/A for memory leaks when using torch DataLoader and OpenVINO
+                image_copies = copy.deepcopy(images.numpy())
+                infer_queue.start_async(image_copies, userdata=i)
+                references[i] = target
+
+            infer_queue.wait_all()
+
+        acc_top1 = accuracy_score(predictions, references)
+
+        self.run_info.metric_name = "Acc@1"
+        self.run_info.metric_value = acc_top1

tests/post_training/pipelines/image_classification_timm.py

@@ -9,32 +9,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import copy
-import os
-
 import numpy as np
 import onnx
 import openvino as ov
 import timm
 import torch
-from sklearn.metrics import accuracy_score
 from timm.data.transforms_factory import transforms_imagenet_eval
 from timm.layers.config import set_fused_attn
-from torchvision import datasets
 
-import nncf
-from nncf.common.logging.track_progress import track
-from tests.post_training.pipelines.base import DEFAULT_VAL_THREADS
 from tests.post_training.pipelines.base import OV_BACKENDS
 from tests.post_training.pipelines.base import PT_BACKENDS
 from tests.post_training.pipelines.base import BackendType
-from tests.post_training.pipelines.base import PTQTestPipeline
+from tests.post_training.pipelines.image_classification_base import ImageClassificationBase
 
 # Disable using aten::scaled_dot_product_attention
 set_fused_attn(False, False)
 
 
-class ImageClassificationTimm(PTQTestPipeline):
+class ImageClassificationTimm(ImageClassificationBase):
     """Pipeline for Image Classification model from timm repository"""
 
     def prepare_model(self) -> None:
@@ -113,55 +105,3 @@ def transform_fn(data_item):
                 return {self.input_name: np.array(images, dtype=np.float32)}
 
         return transform_fn
-
-    def prepare_calibration_dataset(self):
-        dataset = datasets.ImageFolder(root=self.data_dir / "imagenet" / "val", transform=self.transform)
-        loader = torch.utils.data.DataLoader(dataset, batch_size=self.batch_size, num_workers=2, shuffle=False)
-
-        self.calibration_dataset = nncf.Dataset(loader, self.get_transform_calibration_fn())
-
-    def _validate(self):
-        val_dataset = datasets.ImageFolder(root=self.data_dir / "imagenet" / "val", transform=self.transform)
-        val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=1, num_workers=2, shuffle=False)
-
-        dataset_size = len(val_loader)
-
-        # Initialize result tensors for async inference support.
-        predictions = np.zeros((dataset_size))
-        references = -1 * np.ones((dataset_size))
-
-        core = ov.Core()
-
-        if os.environ.get("INFERENCE_NUM_THREADS"):
-            # Set CPU_THREADS_NUM for OpenVINO inference
-            inference_num_threads = os.environ.get("INFERENCE_NUM_THREADS")
-            core.set_property("CPU", properties={"INFERENCE_NUM_THREADS": str(inference_num_threads)})
-
-        ov_model = core.read_model(self.path_compressed_ir)
-        compiled_model = core.compile_model(ov_model, device_name="CPU")
-
-        jobs = int(os.environ.get("NUM_VAL_THREADS", DEFAULT_VAL_THREADS))
-        infer_queue = ov.AsyncInferQueue(compiled_model, jobs)
-
-        with track(total=dataset_size, description="Validation") as pbar:
-
-            def process_result(request, userdata):
-                output_data = request.get_output_tensor().data
-                predicted_label = np.argmax(output_data, axis=1)
-                predictions[userdata] = predicted_label
-                pbar.progress.update(pbar.task, advance=1)
-
-            infer_queue.set_callback(process_result)
-
-            for i, (images, target) in enumerate(val_loader):
-                # W/A for memory leaks when using torch DataLoader and OpenVINO
-                image_copies = copy.deepcopy(images.numpy())
-                infer_queue.start_async(image_copies, userdata=i)
-                references[i] = target
-
-            infer_queue.wait_all()
-
-        acc_top1 = accuracy_score(predictions, references)
-
-        self.run_info.metric_name = "Acc@1"
-        self.run_info.metric_value = acc_top1

tests/post_training/pipelines/image_classification_torchvision.py

@@ -9,27 +9,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import copy
-import os
-
 import numpy as np
+import onnx
 import openvino as ov
 import torch
-from sklearn.metrics import accuracy_score
 from torch._export import capture_pre_autograd_graph
-from torchvision import datasets
 from torchvision import models
 
-import nncf
-from nncf.common.logging.track_progress import track
 from nncf.torch import disable_patching
-from tests.post_training.pipelines.base import DEFAULT_VAL_THREADS
 from tests.post_training.pipelines.base import PT_BACKENDS
 from tests.post_training.pipelines.base import BackendType
-from tests.post_training.pipelines.base import PTQTestPipeline
+from tests.post_training.pipelines.image_classification_base import ImageClassificationBase
 
 
-class ImageClassificationTorchvision(PTQTestPipeline):
+class ImageClassificationTorchvision(ImageClassificationBase):
     """Pipeline for Image Classification model from torchvision repository"""
 
     models_vs_imagenet_weights = {
@@ -45,27 +38,38 @@ def __init__(self, *args, **kwargs):
         self.input_name: str = None
 
     def prepare_model(self) -> None:
-        if self.backend not in [BackendType.FP32, BackendType.FX_TORCH, BackendType.OV] + PT_BACKENDS:
-            raise RuntimeError(
-                f"Torchvision classification models does not support quantization for {self.backend} backend."
-            )
-
         model_cls = models.__dict__.get(self.model_id)
         self.model_weights = self.models_vs_imagenet_weights[model_cls]
         model = model_cls(weights=self.model_weights)
         model.eval()
 
-        self.input_size = [self.batch_size, 3, 224, 224]
-        self.dummy_tensor = torch.rand(self.input_size)
+        self.static_input_size = [self.batch_size, 3, 224, 224]
+        self.input_size = self.static_input_size.copy()
+        if self.batch_size > 1:  # Dynamic batch_size shape export
+            self.input_size[0] = -1
+
+        self.dummy_tensor = torch.rand(self.static_input_size)
 
         if self.backend == BackendType.FX_TORCH:
             with torch.no_grad():
                 with disable_patching():
-                    self.model = capture_pre_autograd_graph(model, (torch.ones(self.input_size),))
+                    self.model = capture_pre_autograd_graph(model, (self.dummy_tensor,))
 
         elif self.backend in PT_BACKENDS:
             self.model = model
 
+        if self.backend == BackendType.ONNX:
+            onnx_path = self.fp32_model_dir / "model_fp32.onnx"
+            additional_kwargs = {}
+            if self.batch_size > 1:
+                additional_kwargs["input_names"] = ["image"]
+                additional_kwargs["dynamic_axes"] = {"image": {0: "batch"}}
+            torch.onnx.export(
+                model, self.dummy_tensor, onnx_path, export_params=True, opset_version=13, **additional_kwargs
+            )
+            self.model = onnx.load(onnx_path)
+            self.input_name = self.model.graph.input[0].name
+
         elif self.backend in [BackendType.OV, BackendType.FP32]:
             with torch.no_grad():
                 self.model = ov.convert_model(model, example_input=self.dummy_tensor, input=self.input_size)
@@ -115,55 +119,3 @@ def transform_fn(data_item):
                 return {self.input_name: np.array(images, dtype=np.float32)}
 
         return transform_fn
-
-    def prepare_calibration_dataset(self):
-        dataset = datasets.ImageFolder(root=self.data_dir / "imagenet" / "val", transform=self.transform)
-        loader = torch.utils.data.DataLoader(dataset, batch_size=self.batch_size, num_workers=2, shuffle=False)
-
-        self.calibration_dataset = nncf.Dataset(loader, self.get_transform_calibration_fn())
-
-    def _validate(self):
-        val_dataset = datasets.ImageFolder(root=self.data_dir / "imagenet" / "val", transform=self.transform)
-        val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=1, num_workers=2, shuffle=False)
-
-        dataset_size = len(val_loader)
-
-        # Initialize result tensors for async inference support.
-        predictions = np.zeros((dataset_size))
-        references = -1 * np.ones((dataset_size))
-
-        core = ov.Core()
-
-        if os.environ.get("INFERENCE_NUM_THREADS"):
-            # Set CPU_THREADS_NUM for OpenVINO inference
-            inference_num_threads = os.environ.get("INFERENCE_NUM_THREADS")
-            core.set_property("CPU", properties={"INFERENCE_NUM_THREADS": str(inference_num_threads)})
-
-        ov_model = core.read_model(self.path_compressed_ir)
-        compiled_model = core.compile_model(ov_model, device_name="CPU")
-
-        jobs = int(os.environ.get("NUM_VAL_THREADS", DEFAULT_VAL_THREADS))
-        infer_queue = ov.AsyncInferQueue(compiled_model, jobs)
-
-        with track(total=dataset_size, description="Validation") as pbar:
-
-            def process_result(request, userdata):
-                output_data = request.get_output_tensor().data
-                predicted_label = np.argmax(output_data, axis=1)
-                predictions[userdata] = predicted_label
-                pbar.progress.update(pbar.task, advance=1)
-
-            infer_queue.set_callback(process_result)
-
-            for i, (images, target) in enumerate(val_loader):
-                # W/A for memory leaks when using torch DataLoader and OpenVINO
-                image_copies = copy.deepcopy(images.numpy())
-                infer_queue.start_async(image_copies, userdata=i)
-                references[i] = target
-
-            infer_queue.wait_all()
-
-        acc_top1 = accuracy_score(predictions, references)
-
-        self.run_info.metric_name = "Acc@1"
-        self.run_info.metric_value = acc_top1