WIP tests

nncf/quantization/algorithms/channel_alignment/algorithm.py

@@ -124,9 +124,7 @@ def _apply(
         def filter_func(point: StatisticPoint) -> bool:
             return ChannelAlignment in point.algorithm_to_tensor_collectors and point.target_point == target_point
 
-        for conv_in, add_in, conv_out in tqdm(
-            self._get_node_pairs_by_graph_matcher(nncf_graph), desc="Channel allignment"
-        ):
+        for conv_in, add_in, conv_out in tqdm(self._get_node_pairs(nncf_graph), desc="Channel allignment"):
             target_point, node_in = self._get_target_point_and_node_in(conv_in, add_in)
             tensor_collectors = list(
                 statistic_points.get_algo_statistics_for_node(node_in.node_name, filter_func, ChannelAlignment)
@@ -308,7 +306,7 @@ def get_conv_add_conv_pattern():
         pattern.add_pattern_alternative(get_conv_add_conv_pattern())
         return pattern
 
-    def _get_node_pairs_by_graph_matcher(self, nncf_graph: NNCFGraph):
+    def _get_node_pairs(self, nncf_graph: NNCFGraph):
         pairs = []
         patterns = self._get_target_patterns()
         for subgraph in nncf_graph.find_matching_subgraphs(patterns):
@@ -337,7 +335,7 @@ def get_statistic_points(self, model: TModel) -> StatisticPointsContainer:
         self.nncf_graph = NNCFGraphFactory.create(model)
 
         statistic_container = StatisticPointsContainer()
-        for conv_in, add_in, _ in self._get_node_pairs_by_graph_matcher(self.nncf_graph):
+        for conv_in, add_in, _ in self._get_node_pairs(self.nncf_graph):
             target_point, node_in = self._get_target_point_and_node_in(conv_in, add_in)
             channel_axis = conv_in.metatype.output_channel_axis
             reduction_shape = list(range(len(self.nncf_graph.get_output_edges(node_in)[0].tensor_shape)))

tests/openvino/native/quantization/test_channel_alignment.py

@@ -0,0 +1,53 @@
+# Copyright (c) 2023 Intel Corporation
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Type
+
+import pytest
+
+from nncf.common.graph.graph import NNCFGraph
+from nncf.openvino.graph.layer_attributes import OVConstantLayerAttributesContainer
+from nncf.openvino.graph.metatypes.openvino_metatypes import OVAddMetatype
+from nncf.openvino.graph.metatypes.openvino_metatypes import OVConvolutionMetatype
+from nncf.openvino.graph.model_transformer import OVModelTransformer
+from nncf.quantization.algorithms.channel_alignment.openvino_backend import OVChannelAlignmentAlgoBackend
+from tests.post_training.test_templates.test_channel_alignment import TemplateTestChannelAlignment
+
+
+def create_nncf_graph_for_ca_algo():
+    NNCFGraph()
+
+
+NNCF_GRAPH_FOR_CA = None
+
+
+class TestOVChannelAlignment(TemplateTestChannelAlignment):
+    def get_backend_cls(self) -> Type[OVChannelAlignmentAlgoBackend]:
+        return OVChannelAlignmentAlgoBackend
+
+    def mock_nncf_graph_factory(self, mocker, nncf_graph: NNCFGraph) -> None:
+        mocker.patch("nncf.common.factory.NNCFGraphFactory.create", return_value=nncf_graph)
+
+    def mock_model_transformer_factory(self, mocker) -> None:
+        mocker.patch("nncf.common.factory.ModelTransformerFactory.create", return_value=OVModelTransformer)
+
+    def convert_conv_layer_attrs(self, layer_attributes):
+        return OVConstantLayerAttributesContainer({}, {1: layer_attributes})
+
+    def get_conv_metatype(self):
+        return OVConvolutionMetatype
+
+    def get_add_metatype(self):
+        return OVAddMetatype
+
+    @pytest.fixture(scope="session")
+    def test_params(self):
+        return {"test_get_node_pairs": {"NNCFGraph": {{"bad": None, "good": None}}}}

tests/post_training/test_templates/models.py

@@ -12,6 +12,7 @@
 from nncf.common.graph import NNCFGraph
 from nncf.common.graph.operator_metatypes import InputNoopMetatype
 from nncf.common.graph.operator_metatypes import OutputNoopMetatype
+from tests.common.quantization.metatypes import ConstantTestMetatype
 from tests.common.quantization.mock_graphs import NodeWithType
 from tests.common.quantization.test_filter_constant_nodes import create_mock_graph
 from tests.common.quantization.test_filter_constant_nodes import get_nncf_graph_from_mock_nx_graph
@@ -95,3 +96,67 @@ def __init__(self, matmul_metatype, matmul_layer_attrs=None, nncf_graph_cls=NNCF
         node_edges = [("Input_1", "MatMul_1"), ("MatMul_1", "Output_1")]
         original_mock_graph = create_mock_graph(nodes, node_edges)
         self.nncf_graph = get_nncf_graph_from_mock_nx_graph(original_mock_graph, nncf_graph_cls)
+
+
+class NNCFGraphCA:
+    def __init__(self, conv_metatype, conv_layer_attrs=None, nncf_graph_cls=NNCFGraph):
+        #       Original graph
+        #          Input_1
+        #             |
+        #           Conv_1
+        #             |
+        #           Conv_2
+        #             |
+        #           Output_1
+        nodes = [
+            NodeWithType("Input_1", InputNoopMetatype),
+            NodeWithType("Conv_1_W", ConstantTestMetatype),
+            NodeWithType("Conv_1", conv_metatype, layer_attributes=conv_layer_attrs),
+            NodeWithType("Conv_2_W", ConstantTestMetatype),
+            NodeWithType("Conv_2", conv_metatype, layer_attributes=conv_layer_attrs),
+            NodeWithType("Output_1", OutputNoopMetatype),
+        ]
+        node_edges = [
+            ("Input_1", "Conv_1"),
+            ("Conv_1", "Conv_2"),
+            ("Conv_2", "Output_1"),
+            ("Conv_1_W", "Conv_1"),
+            ("Conv_2_W", "Conv_2"),
+        ]
+        original_mock_graph = create_mock_graph(nodes, node_edges)
+        self.nncf_graph = get_nncf_graph_from_mock_nx_graph(original_mock_graph, nncf_graph_cls)
+
+
+class NNCFGraphCAWithBias:
+    def __init__(self, conv_metatype, add_metatype, conv_layer_attrs=None, nncf_graph_cls=NNCFGraph):
+        #       Original graph
+        #          Input_1
+        #             |
+        #           Conv_1
+        #             |
+        #           Add_1
+        #             |
+        #           Conv_2
+        #             |
+        #           Output_1
+        nodes = [
+            NodeWithType("Input_1", InputNoopMetatype),
+            NodeWithType("Conv_1_W", ConstantTestMetatype),
+            NodeWithType("Conv_1", conv_metatype, layer_attributes=conv_layer_attrs),
+            NodeWithType("Add_1_W", ConstantTestMetatype),
+            NodeWithType("Add_1", add_metatype),
+            NodeWithType("Conv_2_W", ConstantTestMetatype),
+            NodeWithType("Conv_2", conv_metatype, layer_attributes=conv_layer_attrs),
+            NodeWithType("Output_1", OutputNoopMetatype),
+        ]
+        node_edges = [
+            ("Input_1", "Conv_1"),
+            ("Conv_1", "Add_1"),
+            ("Add_1", "Conv_2"),
+            ("Conv_2", "Output_1"),
+            ("Conv_1_W", "Conv_1"),
+            ("Add_1_W", "Add_1"),
+            ("Conv_2_W", "Conv_2"),
+        ]
+        original_mock_graph = create_mock_graph(nodes, node_edges)
+        self.nncf_graph = get_nncf_graph_from_mock_nx_graph(original_mock_graph, nncf_graph_cls)

tests/post_training/test_templates/test_channel_alignment.py

@@ -0,0 +1,234 @@
+# Copyright (c) 2023 Intel Corporation
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#      http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import abstractmethod
+from typing import Type
+
+import numpy as np
+import pytest
+
+from nncf.common.graph.graph import NNCFGraph
+from nncf.common.graph.layer_attributes import ConvolutionLayerAttributes
+from nncf.common.graph.transformations.commands import TargetType
+from nncf.common.tensor_statistics.statistic_point import StatisticPoint
+from nncf.common.tensor_statistics.statistic_point import StatisticPointsContainer
+from nncf.experimental.common.tensor_statistics.collectors import MedianAggregator
+from nncf.experimental.common.tensor_statistics.collectors import QuantileReducer
+from nncf.experimental.common.tensor_statistics.collectors import TensorCollector
+from nncf.quantization.algorithms.channel_alignment.algorithm import ChannelAlignment
+from nncf.quantization.algorithms.channel_alignment.backend import ChannelAlignmentAlgoBackend
+from tests.post_training.test_templates.models import NNCFGraphCA
+from tests.post_training.test_templates.models import NNCFGraphCAWithBias
+
+VALID_CONV_LAYER_ATTRS = [
+    ConvolutionLayerAttributes(
+        weight_requires_grad=False,
+        in_channels=5,
+        out_channels=5,
+        kernel_size=(5, 5),
+        stride=(1, 1),
+        dilations=(1, 1),
+        groups=1,
+        transpose=False,
+        padding_values=(0, 0, 0, 0),
+    )
+]
+
+
+INVALID_CONV_LAYER_ATTRS = [
+    ConvolutionLayerAttributes(
+        weight_requires_grad=False,
+        in_channels=5,
+        out_channels=5,
+        kernel_size=(5, 5),
+        stride=(2, 1),
+        dilations=(1, 1),
+        groups=1,
+        transpose=False,
+        padding_values=(0, 0, 0, 0),
+    ),
+    ConvolutionLayerAttributes(
+        weight_requires_grad=False,
+        in_channels=5,
+        out_channels=5,
+        kernel_size=(5, 5),
+        stride=(1, 1),
+        dilations=(2, 1),
+        groups=1,
+        transpose=False,
+        padding_values=(0, 0, 0, 0),
+    ),
+    ConvolutionLayerAttributes(
+        weight_requires_grad=False,
+        in_channels=5,
+        out_channels=5,
+        kernel_size=(5, 5),
+        stride=(1, 1),
+        dilations=(2, 1),
+        groups=1,
+        transpose=False,
+        padding_values=(0, 0, 0, 0),
+    ),
+    ConvolutionLayerAttributes(
+        weight_requires_grad=False,
+        in_channels=5,
+        out_channels=5,
+        kernel_size=(5, 5),
+        stride=(1, 1),
+        dilations=(1, 1),
+        groups=5,
+        transpose=False,
+        padding_values=(0, 0, 0, 0),
+    ),
+    ConvolutionLayerAttributes(
+        weight_requires_grad=False,
+        in_channels=5,
+        out_channels=5,
+        kernel_size=(5, 5),
+        stride=(1, 1),
+        dilations=(1, 1),
+        groups=1,
+        transpose=False,
+        padding_values=(1, 0, 0, 0),
+    ),
+]
+
+
+class TemplateTestChannelAlignment:
+    @abstractmethod
+    def get_backend_cls(self) -> Type[ChannelAlignmentAlgoBackend]:
+        pass
+
+    @abstractmethod
+    def mock_nncf_graph_factory(self, mocker, nncf_graph: NNCFGraph) -> None:
+        pass
+
+    @abstractmethod
+    def mock_model_transformer_factory(self, mocker, mocked_transformer) -> None:
+        pass
+
+    @abstractmethod
+    def convert_conv_layer_attrs(self, layer_attributes):
+        pass
+
+    @abstractmethod
+    def get_conv_metatype(self):
+        pass
+
+    @abstractmethod
+    def get_add_metatype(self):
+        pass
+
+    @abstractmethod
+    @pytest.fixture(scope="session")
+    def test_params(self):
+        pass
+
+    def test_align_means_scales():
+        pass
+
+    @pytest.mark.parametrize(
+        "layer_attributes,ref_match",
+        [(attr, True) for attr in VALID_CONV_LAYER_ATTRS] + [(attr, False) for attr in INVALID_CONV_LAYER_ATTRS],
+    )
+    def test_get_node_pairs(self, layer_attributes, ref_match):
+        algorithm = ChannelAlignment()
+        algorithm._backend_entity = self.get_backend_cls()
+        conv_layer_attrs = self.convert_conv_layer_attrs(layer_attributes)
+        nncf_graph = NNCFGraphCA(self.get_conv_metatype(), conv_layer_attrs)
+        pairs = algorithm._get_node_pairs(nncf_graph.nncf_graph)
+        if ref_match:
+            assert len(pairs) == 1
+            conv_in, add_in, conv_out = pairs[0]
+            assert conv_in.node_name == "/Conv_1_0"
+            assert add_in is None
+            assert conv_out.node_name == "/Conv_2_0"
+        else:
+            assert len(pairs) == 0
+
+    def _get_nncf_graph(self, with_bias: bool) -> NNCFGraph:
+        cla = self.convert_conv_layer_attrs(VALID_CONV_LAYER_ATTRS[0])
+        if with_bias:
+            return NNCFGraphCAWithBias(self.get_conv_metatype(), self.get_add_metatype(), cla).nncf_graph
+        else:
+            return NNCFGraphCA(self.get_conv_metatype(), cla).nncf_graph
+
+    @pytest.mark.parametrize("with_bias", [True, False])
+    def test_transformation_layout(self, with_bias, mocker):
+        mocked_transformer = mocker.MagicMocker()
+        self.mock_model_transformer_factory(mocker, mocked_transformer)
+        nncf_graph = self._get_nncf_graph(with_bias)
+        self.mock_nncf_graph_factory(mocker, nncf_graph)
+        ChannelAlignment._align_means = mocker.MagicMock(return_value=(None, None))
+        ChannelAlignment._align_scales = mocker.MagicMock(return_value=(None, None, None))
+
+        statistic_points = StatisticPointsContainer()
+        statistic_points.add_statistic_point(StatisticPoint())
+
+    @pytest.mark.parametrize("with_bias", [True, False])
+    def test_get_statistic_points(self, with_bias, mocker):
+        nncf_graph = self._get_nncf_graph(with_bias)
+        self.mock_nncf_graph_factory(mocker, nncf_graph)
+
+        ref_subset_size = "ref_subset_size"
+        ref_inplace = "ref_inplace"
+        algorithm = ChannelAlignment(ref_subset_size, ref_inplace)
+        algorithm._set_backend_entity = mocker.MagicMock()
+        mocked_backed = self.get_backend_cls()
+        ref_stat_collector = "ref_stat_collector"
+        mocked_backed.get_statistic_collector = mocker.MagicMock(return_value=ref_stat_collector)
+        algorithm._backend_entity = mocked_backed
+
+        statistic_container = algorithm.get_statistic_points(None)
+
+        backend_cls = self.get_backend_cls()
+        target_node_name = "/Add_1_0" if with_bias else "/Conv_1_0"
+        target_node = nncf_graph.get_node_by_name(target_node_name)
+        ref_input_port_id, _ = backend_cls.get_activation_port_ids_for_node(target_node)
+
+        assert len(statistic_container) == 1
+        assert target_node_name in statistic_container
+        stat_points = statistic_container[target_node_name]
+        assert len(stat_points) == 1
+
+        assert len(stat_points[0].algorithm_to_tensor_collectors.keys()) == 1
+        assert ChannelAlignment in stat_points[0].algorithm_to_tensor_collectors
+        tensor_collectors = stat_points[0].algorithm_to_tensor_collectors[ChannelAlignment]
+        assert len(tensor_collectors) == 1
+        assert tensor_collectors[0] == ref_stat_collector
+        mocked_backed.get_statistic_collector.assert_called_once_with((0, 2, 3), 1e-4, ref_subset_size, ref_inplace)
+
+        target_point = stat_points[0].target_point
+        assert target_point.target_node_name == target_node_name
+        assert target_point.port_id == ref_input_port_id
+        assert target_point.type == TargetType.POST_LAYER_OPERATION
+
+    @pytest.mark.parametrize("inplace_ref", [False, True])
+    @pytest.mark.parametrize("q_ref", [1e-4, 0.3])
+    def test_statistic_collectors(self, inplace_ref, q_ref):
+        backend_cls = self.get_backend_cls()
+        reduction_shape_ref = (0, 2, 3)
+        num_samples_ref = 123
+        statistic_collector: TensorCollector = backend_cls.get_statistic_collector(
+            reduction_shape=reduction_shape_ref, q=q_ref, num_samples=num_samples_ref, inplace=inplace_ref
+        )
+
+        assert len(statistic_collector.reducers) == 1
+        reducer = statistic_collector.reducers.pop()
+        assert isinstance(reducer, QuantileReducer)
+        assert reducer._reduction_shape == reduction_shape_ref
+        assert np.allclose(reducer._quantile, (q_ref, 1 - q_ref))
+
+        assert len(statistic_collector.aggregators) == 2
+        for aggr in statistic_collector.aggregators.values():
+            assert isinstance(aggr, MedianAggregator)
+            assert aggr.num_samples == num_samples_ref
+            assert not aggr._use_per_sample_stats