From 2e7d5e9780d258ce47ea36a538c425d320d9b8fa Mon Sep 17 00:00:00 2001 From: Aleksei Kashapov Date: Fri, 19 Apr 2024 15:36:15 +0200 Subject: [PATCH 1/4] add validation with batch_size --- .../pipelines/image_classification_timm.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/post_training/pipelines/image_classification_timm.py b/tests/post_training/pipelines/image_classification_timm.py index 5cda1d04e6d..6580b3c8822 100644 --- a/tests/post_training/pipelines/image_classification_timm.py +++ b/tests/post_training/pipelines/image_classification_timm.py @@ -126,13 +126,12 @@ def prepare_calibration_dataset(self): def _validate(self): val_dataset = datasets.ImageFolder(root=self.data_dir / "imagenet" / "val", transform=self.transform) - val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=1, num_workers=2, shuffle=False) + val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=self.batch_size, num_workers=2, shuffle=False) dataset_size = len(val_loader) - # Initialize result tensors for async inference support. - predictions = np.zeros((dataset_size)) - references = -1 * np.ones((dataset_size)) + predictions = [[] for _ in range(dataset_size)] + references = [[] for _ in range(dataset_size)] core = ov.Core() @@ -164,8 +163,12 @@ def process_result(request, userdata): references[i] = target infer_queue.wait_all() - - acc_top1 = accuracy_score(predictions, references) + flatten_predictions = [] + flatten_references = [] + for i in range(len(predictions)): + flatten_predictions.extend(predictions[i]) + flatten_references.extend(references[i]) + acc_top1 = accuracy_score(flatten_predictions, flatten_references) self.run_info.metric_name = "Acc@1" self.run_info.metric_value = acc_top1 From e5dd69cb814835a32a19b89d36c4893b901da752 Mon Sep 17 00:00:00 2001 From: Aleksei Kashapov Date: Mon, 29 Apr 2024 15:29:44 +0200 Subject: [PATCH 2/4] set bs=1 for swin --- tests/post_training/model_scope.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/post_training/model_scope.py b/tests/post_training/model_scope.py index 0ad69a54317..76766b25dbb 100644 --- a/tests/post_training/model_scope.py +++ b/tests/post_training/model_scope.py @@ -249,7 +249,7 @@ "model_type": ModelType.TRANSFORMER, }, "backends": [BackendType.OV], - "batch_size": 32, + "batch_size": 1, }, { "reported_name": "timm/swin_base_patch4_window7_224_no_sq", @@ -263,7 +263,7 @@ ), }, "backends": [BackendType.TORCH, BackendType.CUDA_TORCH, BackendType.ONNX], - "batch_size": 128, + "batch_size": 1, }, { "reported_name": "timm/tf_inception_v3", From b975bc459ede440095bb28e9ea541d20d702cd92 Mon Sep 17 00:00:00 2001 From: Aleksei Kashapov Date: Mon, 29 Apr 2024 15:32:16 +0200 Subject: [PATCH 3/4] upd swin metrics for bs=1 --- tests/post_training/data/ptq_reference_data.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/post_training/data/ptq_reference_data.yaml b/tests/post_training/data/ptq_reference_data.yaml index e1a1838336b..499dff19fdb 100644 --- a/tests/post_training/data/ptq_reference_data.yaml +++ b/tests/post_training/data/ptq_reference_data.yaml @@ -185,15 +185,15 @@ timm/resnet18_backend_TORCH: timm/swin_base_patch4_window7_224_backend_FP32: metric_value: 0.85274 timm/swin_base_patch4_window7_224_backend_OV: - metric_value: 0.83566 + metric_value: 0.8366 timm/swin_base_patch4_window7_224_no_sq_backend_FP32: metric_value: 0.85274 timm/swin_base_patch4_window7_224_no_sq_backend_CUDA_TORCH: metric_value: 0.85142 timm/swin_base_patch4_window7_224_no_sq_backend_ONNX: - metric_value: 0.85212 + metric_value: 0.85158 timm/swin_base_patch4_window7_224_no_sq_backend_TORCH: - metric_value: 0.85178 + metric_value: 0.85142 timm/tf_inception_v3_backend_CUDA_TORCH: metric_value: 0.77542 timm/tf_inception_v3_backend_FP32: From 03606b130ffd6da63580b9ab158b252f42ebbeb9 Mon Sep 17 00:00:00 2001 From: Aleksei Kashapov Date: Mon, 29 Apr 2024 15:32:57 +0200 Subject: [PATCH 4/4] swin bs --- tests/post_training/model_scope.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/post_training/model_scope.py b/tests/post_training/model_scope.py index 76766b25dbb..0c4d72af5f2 100644 --- a/tests/post_training/model_scope.py +++ b/tests/post_training/model_scope.py @@ -249,7 +249,6 @@ "model_type": ModelType.TRANSFORMER, }, "backends": [BackendType.OV], - "batch_size": 1, }, { "reported_name": "timm/swin_base_patch4_window7_224_no_sq", @@ -263,7 +262,6 @@ ), }, "backends": [BackendType.TORCH, BackendType.CUDA_TORCH, BackendType.ONNX], - "batch_size": 1, }, { "reported_name": "timm/tf_inception_v3",