From 98e260d6e43b26f209ce19abe82044ba190f3458 Mon Sep 17 00:00:00 2001 From: Simeon Ehrig Date: Thu, 1 Feb 2024 09:36:03 +0100 Subject: [PATCH 1/6] add get_default_filter_chain() - add empty filters --- bashi/filter_backend.py | 22 ++++++ bashi/filter_compiler_name.py | 22 ++++++ bashi/filter_compiler_version.py | 22 ++++++ bashi/filter_software_dependency.py | 22 ++++++ bashi/utils.py | 28 ++++++++ tests/test_filter_adapter.py | 21 ++++++ tests/test_filter_chain.py | 100 ++++++++++++++++++++++++++++ 7 files changed, 237 insertions(+) create mode 100644 bashi/filter_backend.py create mode 100644 bashi/filter_compiler_name.py create mode 100644 bashi/filter_compiler_version.py create mode 100644 bashi/filter_software_dependency.py create mode 100644 tests/test_filter_chain.py diff --git a/bashi/filter_backend.py b/bashi/filter_backend.py new file mode 100644 index 0000000..14e7b8d --- /dev/null +++ b/bashi/filter_backend.py @@ -0,0 +1,22 @@ +"""Filter rules basing on backend names and versions.""" + +from typing import Optional, IO +from bashi.types import ParameterValueTuple + + +def backend_filter( + row: ParameterValueTuple, + output: Optional[IO[str]] = None, +) -> bool: + """Filter rules basing on backend names and versions. + + Args: + row (ParameterValueTuple): parameter-value-tuple to verify. + output (Optional[IO[str]], optional): Writes the reason in the io object why the parameter + value tuple does not pass the filter. If None, no information is provided. The default + value is None. + + Returns: + bool: True, if parameter-value-tuple is valid. + """ + return True diff --git a/bashi/filter_compiler_name.py b/bashi/filter_compiler_name.py new file mode 100644 index 0000000..93945b8 --- /dev/null +++ b/bashi/filter_compiler_name.py @@ -0,0 +1,22 @@ +"""Filter rules basing on host and device compiler names.""" + +from typing import Optional, IO +from bashi.types import ParameterValueTuple + + +def compiler_name_filter( + row: ParameterValueTuple, + output: Optional[IO[str]] = None, +) -> bool: + """Filter rules basing on host and device compiler names. + + Args: + row (ParameterValueTuple): parameter-value-tuple to verify. + output (Optional[IO[str]], optional): Writes the reason in the io object why the parameter + value tuple does not pass the filter. If None, no information is provided. The default + value is None. + + Returns: + bool: True, if parameter-value-tuple is valid. + """ + return True diff --git a/bashi/filter_compiler_version.py b/bashi/filter_compiler_version.py new file mode 100644 index 0000000..56d2cc3 --- /dev/null +++ b/bashi/filter_compiler_version.py @@ -0,0 +1,22 @@ +"""Filter rules basing on host and device compiler names and versions.""" + +from typing import Optional, IO +from bashi.types import ParameterValueTuple + + +def compiler_version_filter( + row: ParameterValueTuple, + output: Optional[IO[str]] = None, +) -> bool: + """Filter rules basing on host and device compiler names and versions. + + Args: + row (ParameterValueTuple): parameter-value-tuple to verify. + output (Optional[IO[str]], optional): Writes the reason in the io object why the parameter + value tuple does not pass the filter. If None, no information is provided. The default + value is None. + + Returns: + bool: True, if parameter-value-tuple is valid. + """ + return True diff --git a/bashi/filter_software_dependency.py b/bashi/filter_software_dependency.py new file mode 100644 index 0000000..c7bc70e --- /dev/null +++ b/bashi/filter_software_dependency.py @@ -0,0 +1,22 @@ +"""Filter rules handling software dependencies and compiler settings.""" + +from typing import Optional, IO +from bashi.types import ParameterValueTuple + + +def software_dependency_filter( + row: ParameterValueTuple, + output: Optional[IO[str]] = None, +) -> bool: + """Filter rules handling software dependencies and compiler settings. + + Args: + row (ParameterValueTuple): parameter-value-tuple to verify. + output (Optional[IO[str]], optional): Writes the reason in the io object why the parameter + value tuple does not pass the filter. If None, no information is provided. The default + value is None. + + Returns: + bool: True, if parameter-value-tuple is valid. + """ + return True diff --git a/bashi/utils.py b/bashi/utils.py index c8c345f..9adb465 100644 --- a/bashi/utils.py +++ b/bashi/utils.py @@ -14,6 +14,10 @@ CombinationList, FilterFunction, ) +from bashi.filter_compiler_name import compiler_name_filter +from bashi.filter_compiler_version import compiler_version_filter +from bashi.filter_backend import backend_filter +from bashi.filter_software_dependency import software_dependency_filter @dataclasses.dataclass @@ -76,6 +80,30 @@ def __call__(self, row: List[ParameterValue]) -> bool: return self.filter_func(ordered_row) +@typechecked +def get_default_filter_chain( + custom_filter_function: FilterFunction = lambda _: True, +) -> FilterFunction: + """Concatenate the bashi filter functions in the default order and return them as one function + with a single entry point. + + Args: + custom_filter_function (FilterFunction): This function is added as the last filter level and + allows the user to add custom filter rules without having to create the entire filter + chain from scratch. Defaults to lambda_:True. + + Returns: + FilterFunction: The filter function chain, which can be directly used in bashi.FilterAdapter + """ + return ( + lambda row: compiler_name_filter(row) + and compiler_version_filter(row) + and backend_filter(row) + and software_dependency_filter(row) + and custom_filter_function(row) + ) + + @typechecked def get_expected_parameter_value_pairs( parameter_matrix: ParameterValueMatrix, diff --git a/tests/test_filter_adapter.py b/tests/test_filter_adapter.py index f91e4c3..43d0082 100644 --- a/tests/test_filter_adapter.py +++ b/tests/test_filter_adapter.py @@ -6,6 +6,10 @@ from typeguard import typechecked from bashi.types import ParameterValueTuple, ParameterValue from bashi.utils import FilterAdapter +from bashi.filter_compiler_name import compiler_name_filter +from bashi.filter_compiler_version import compiler_version_filter +from bashi.filter_backend import backend_filter +from bashi.filter_software_dependency import software_dependency_filter class TestFilterAdapterDataSet1(unittest.TestCase): @@ -97,6 +101,23 @@ def test_lambda(self): filter_adapter = FilterAdapter(self.param_map, lambda row: len(row) == 3) self.assertTrue(filter_adapter(self.test_row), "row has not the length of 3") + # interface test with production filter + def test_compiler_name_filter(self): + error_msg = ( + "the filter should return true every time, " + "because the test data should no trigger any rule" + ) + self.assertTrue( + FilterAdapter(self.param_map, compiler_name_filter)(self.test_row), error_msg + ) + self.assertTrue( + FilterAdapter(self.param_map, compiler_version_filter)(self.test_row), error_msg + ) + self.assertTrue(FilterAdapter(self.param_map, backend_filter)(self.test_row), error_msg) + self.assertTrue( + FilterAdapter(self.param_map, software_dependency_filter)(self.test_row), error_msg + ) + # do a complex test with a different data set class TestFilterAdapterDataSet2(unittest.TestCase): diff --git a/tests/test_filter_chain.py b/tests/test_filter_chain.py new file mode 100644 index 0000000..b61442f --- /dev/null +++ b/tests/test_filter_chain.py @@ -0,0 +1,100 @@ +# pylint: disable=missing-docstring +import unittest +from typing import Dict, List +from collections import OrderedDict +import packaging.version as pkv +from bashi.types import ParameterValue, ParameterValueTuple, FilterFunction +from bashi.utils import get_default_filter_chain, FilterAdapter + + +class TestFilterChain(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.param_val_tuple: ParameterValueTuple = OrderedDict() + cls.param_val_tuple["param1"] = ParameterValue("param-val-name1", pkv.parse("1")) + cls.param_val_tuple["param2"] = ParameterValue("param-val-name2", pkv.parse("2")) + cls.param_val_tuple["param3"] = ParameterValue("param-val-name3", pkv.parse("3")) + + cls.param_map: Dict[int, str] = {} + for index, param_name in enumerate(cls.param_val_tuple.keys()): + cls.param_map[index] = param_name + + cls.test_row: List[ParameterValue] = [] + for param_val in cls.param_val_tuple.values(): + cls.test_row.append(param_val) + + def test_filter_chain_default(self): + filter_chain: FilterFunction = get_default_filter_chain() + self.assertTrue( + filter_chain(self.param_val_tuple), + "The filter should return true every time, " + "because the test data should no trigger any rule", + ) + + def test_filter_chain_custom_filter_pass(self): + def custom_filter(row: ParameterValueTuple): + if "paramNotExist" in row: + return False + return True + + filter_chain: FilterFunction = get_default_filter_chain(custom_filter) + self.assertTrue( + filter_chain(self.param_val_tuple), + "The production filters should return True all the time, because the test data set " + "does not contain any production data. The custom filter should not match the test " + "data.", + ) + + def test_filter_chain_custom_filter_match(self): + def custom_filter(row: ParameterValueTuple): + if "param2" in row: + return False + return True + + filter_chain: FilterFunction = get_default_filter_chain(custom_filter) + self.assertFalse( + filter_chain(self.param_val_tuple), + "The production filters should return True all the time, because the test data set " + "does not contain any production data. The custom filter should match the test data.", + ) + + def test_filter_chain_filter_adapter_default(self): + filter_chain: FilterFunction = get_default_filter_chain() + adapter = FilterAdapter(self.param_map, filter_chain) + + self.assertTrue( + adapter(self.test_row), + "The filter should return true every time, " + "because the test data should no trigger any rule", + ) + + def test_filter_chain_filter_adapter_custom_filter_pass(self): + def custom_filter(row: ParameterValueTuple): + if "paramNotExist2" in row: + return False + return True + + filter_chain: FilterFunction = get_default_filter_chain(custom_filter) + adapter = FilterAdapter(self.param_map, filter_chain) + + self.assertTrue( + adapter(self.test_row), + "The production filters should return True all the time, because the test data set " + "does not contain any production data. The custom filter should not match the test " + "data.", + ) + + def test_filter_chain_filter_adapter_custom_filter_match(self): + def custom_filter(row: ParameterValueTuple): + if "param1" in row: + return False + return True + + filter_chain: FilterFunction = get_default_filter_chain(custom_filter) + adapter = FilterAdapter(self.param_map, filter_chain) + + self.assertFalse( + adapter(self.test_row), + "The production filters should return True all the time, because the test data set " + "does not contain any production data. The custom filter should match the test data.", + ) From 63cade81c0f954b181e87fa8eb911a3b5d8cd7e7 Mon Sep 17 00:00:00 2001 From: Simeon Ehrig Date: Mon, 5 Feb 2024 09:09:55 +0100 Subject: [PATCH 2/6] add helper functions to remove entries from expected parameter-value-pairs --- bashi/utils.py | 50 ++++++++ tests/test_expected_parameter_value_pairs.py | 118 +++++++++++++++++++ 2 files changed, 168 insertions(+) diff --git a/bashi/utils.py b/bashi/utils.py index 9adb465..e5b2ebc 100644 --- a/bashi/utils.py +++ b/bashi/utils.py @@ -162,6 +162,56 @@ def _loop_over_parameter_values( expected_pairs.append(param_val_pair) +@typechecked +def remove_parameter_value_pair( + to_remove: ParameterValuePair, parameter_value_pairs: List[ParameterValuePair] +) -> bool: + """Removes a parameter-value pair with one or two entries from the parameter-value-pair list. If + the parameter-value-pair only has one parameter value, all parameter-value-pairs that contain + the parameter value are removed. + + Args: + to_remove (ParameterValuePair): Parameter-value-pair to remove + param_val_pairs (List[ParameterValuePair]): List of parameter-value-pairs. Will be modified. + + Raises: + RuntimeError: If `to_remove` 0 or more than 2 entries. + + Returns: + bool: True if entry was removed from parameter_value_pairs + """ + if len(to_remove) == 1: + return _remove_single_entry_parameter_value_pair(to_remove, parameter_value_pairs) + + if len(to_remove) == 0 or len(to_remove) > 2: + raise RuntimeError("More than two parameter-values are not allowed") + + try: + parameter_value_pairs.remove(to_remove) + return True + except ValueError: + return False + + +@typechecked +def _remove_single_entry_parameter_value_pair( + to_remove: ParameterValuePair, param_val_pairs: List[ParameterValuePair] +) -> bool: + val_name, val_version = next(iter(to_remove.items())) + + len_before = len(param_val_pairs) + + def filter_function(param_val_pair: ParameterValuePair) -> bool: + if val_name in param_val_pair and param_val_pair[val_name] == val_version: + return False + return True + + param_val_pairs[:] = list(filter(filter_function, param_val_pairs)) + + return len_before != len(param_val_pairs) + + +@typechecked def check_parameter_value_pair_in_combination_list( combination_list: CombinationList, parameter_value_pairs: List[ParameterValuePair], diff --git a/tests/test_expected_parameter_value_pairs.py b/tests/test_expected_parameter_value_pairs.py index c608dc8..faf07a1 100644 --- a/tests/test_expected_parameter_value_pairs.py +++ b/tests/test_expected_parameter_value_pairs.py @@ -20,6 +20,7 @@ from bashi.utils import ( get_expected_parameter_value_pairs, check_parameter_value_pair_in_combination_list, + remove_parameter_value_pair, ) @@ -377,3 +378,120 @@ def test_unrestricted_allpairspy_generator(self): self.assertTrue( check_parameter_value_pair_in_combination_list(comb_list, self.expected_param_val_pairs) ) + + +class TestRemoveExpectedParameterValuePair(unittest.TestCase): + def test_remove_two_entry_parameter_value_pair(self): + OD = OrderedDict + ppv = parse_param_val + + expected_param_value_pairs: List[ParameterValuePair] = parse_expected_val_pairs( + [ + OD({HOST_COMPILER: (GCC, 10), DEVICE_COMPILER: (NVCC, 11.2)}), + OD({HOST_COMPILER: (GCC, 10), DEVICE_COMPILER: (NVCC, 12.0)}), + OD({HOST_COMPILER: (GCC, 9), DEVICE_COMPILER: (NVCC, 12.0)}), + OD({CMAKE: (CMAKE, 3.23), BOOST: (BOOST, 1.83)}), + ] + ) + original_length = len(expected_param_value_pairs) + + # expects one or two entries + self.assertRaises( + RuntimeError, + remove_parameter_value_pair, + OD(), + expected_param_value_pairs, + ) + self.assertRaises( + RuntimeError, + remove_parameter_value_pair, + OD( + { + HOST_COMPILER: ppv((GCC, 9)), + DEVICE_COMPILER: ppv((NVCC, 11.2)), + CMAKE: ppv((CMAKE, 3.23)), + } + ), + expected_param_value_pairs, + ) + + self.assertFalse( + remove_parameter_value_pair( + OD({HOST_COMPILER: ppv((GCC, 9)), DEVICE_COMPILER: ppv((NVCC, 11.2))}), + expected_param_value_pairs, + ) + ) + self.assertEqual(len(expected_param_value_pairs), original_length) + + self.assertTrue( + remove_parameter_value_pair( + OD({HOST_COMPILER: ppv((GCC, 10)), DEVICE_COMPILER: ppv((NVCC, 12.0))}), + expected_param_value_pairs, + ) + ) + self.assertEqual(len(expected_param_value_pairs), original_length - 1) + + self.assertTrue( + remove_parameter_value_pair( + OD({CMAKE: ppv((CMAKE, 3.23)), BOOST: ppv((BOOST, 1.83))}), + expected_param_value_pairs, + ) + ) + self.assertEqual(len(expected_param_value_pairs), original_length - 2) + + def test_remove_single_entry_parameter_value_pair(self): + OD = OrderedDict + ppv = parse_param_val + + expected_param_value_pairs: List[ParameterValuePair] = parse_expected_val_pairs( + [ + OD({HOST_COMPILER: (GCC, 10), DEVICE_COMPILER: (NVCC, 11.2)}), + OD({HOST_COMPILER: (GCC, 10), DEVICE_COMPILER: (NVCC, 12.0)}), + OD({HOST_COMPILER: (GCC, 9), DEVICE_COMPILER: (NVCC, 11.2)}), + OD({HOST_COMPILER: (GCC, 9), DEVICE_COMPILER: (NVCC, 12.0)}), + OD({CMAKE: (CMAKE, 3.23), BOOST: (BOOST, 1.83)}), + ] + ) + original_length = len(expected_param_value_pairs) + + self.assertFalse( + remove_parameter_value_pair( + OD({HOST_COMPILER: ppv((GCC, 12))}), + expected_param_value_pairs, + ) + ) + self.assertEqual(len(expected_param_value_pairs), original_length) + + self.assertFalse( + remove_parameter_value_pair( + OD({HOST_COMPILER: ppv((CLANG, 12))}), + expected_param_value_pairs, + ) + ) + self.assertEqual(len(expected_param_value_pairs), original_length) + + self.assertFalse( + remove_parameter_value_pair( + OD({UBUNTU: ppv((UBUNTU, 20.04))}), + expected_param_value_pairs, + ) + ) + self.assertEqual(len(expected_param_value_pairs), original_length) + + self.assertTrue( + remove_parameter_value_pair( + OD({HOST_COMPILER: ppv((GCC, 9))}), + expected_param_value_pairs, + ) + ) + self.assertEqual(len(expected_param_value_pairs), original_length - 2) + self.assertEqual( + expected_param_value_pairs, + parse_expected_val_pairs( + [ + OD({HOST_COMPILER: (GCC, 10), DEVICE_COMPILER: (NVCC, 11.2)}), + OD({HOST_COMPILER: (GCC, 10), DEVICE_COMPILER: (NVCC, 12.0)}), + OD({CMAKE: (CMAKE, 3.23), BOOST: (BOOST, 1.83)}), + ] + ), + ) From 4b63360c818b5969a4cee325f078baab534a391a Mon Sep 17 00:00:00 2001 From: Simeon Ehrig Date: Mon, 5 Feb 2024 16:29:55 +0100 Subject: [PATCH 3/6] implement generate_combination_list() function --- bashi/generator.py | 53 +++- bashi/globals.py | 20 ++ bashi/versions.py | 88 +++++++ tests/test_expected_parameter_value_pairs.py | 37 +-- tests/test_generate_combination_list.py | 243 +++++++++++++++++++ tests/test_generator.py | 8 - tests/test_params_value_matrix_generator.py | 56 +++++ tests/utils_test.py | 66 +++++ 8 files changed, 524 insertions(+), 47 deletions(-) create mode 100644 bashi/versions.py create mode 100644 tests/test_generate_combination_list.py delete mode 100644 tests/test_generator.py create mode 100644 tests/test_params_value_matrix_generator.py create mode 100644 tests/utils_test.py diff --git a/bashi/generator.py b/bashi/generator.py index 0a94f78..9202ebf 100644 --- a/bashi/generator.py +++ b/bashi/generator.py @@ -1,6 +1,51 @@ -def foo(): - print("Hello bashi.") +"""Functions to generate the combination-list""" +from typing import Dict +from collections import OrderedDict -def add(x: int, y: int): - return x + y +from allpairspy import AllPairs + +from bashi.types import ( + Parameter, + ParameterValueMatrix, + FilterFunction, + Combination, + CombinationList, +) +from bashi.utils import get_default_filter_chain, FilterAdapter + + +def generate_combination_list( + parameter_value_matrix: ParameterValueMatrix, + custom_filter: FilterFunction = lambda _: True, +) -> CombinationList: + """Generate combination-list from the parameter-value-matrix. The combination list contains + all valid parameter-value-pairs at least one time. + + Args: + parameter_value_matrix (ParameterValueMatrix): Input matrix with parameter and + parameter-values. + custom_filter (FilterFunction, optional): Custom filter function to extend bashi + filters. Defaults is lambda _: True. + Returns: + CombinationList: combination-list + """ + filter_chain = get_default_filter_chain(custom_filter) + + param_map: Dict[int, Parameter] = {} + for index, key in enumerate(parameter_value_matrix.keys()): + param_map[index] = key + filter_adapter = FilterAdapter(param_map, filter_chain) + + comb_list: CombinationList = [] + + # convert List[Pair] to CombinationList + for all_pair in AllPairs( # type: ignore + parameters=parameter_value_matrix, n=2, filter_func=filter_adapter + ): + comb: Combination = OrderedDict() + for index, param in enumerate(all_pair._fields): # type: ignore + comb[param] = all_pair[index] # type: ignore + comb_list.append(comb) + + return comb_list diff --git a/bashi/globals.py b/bashi/globals.py index 265352f..be122e6 100644 --- a/bashi/globals.py +++ b/bashi/globals.py @@ -1,5 +1,9 @@ """This module contains constants used in the bashi library.""" +from typing import List +import packaging.version + + # parameter key names, whit special meaning HOST_COMPILER: str = "host_compiler" DEVICE_COMPILER: str = "device_compiler" @@ -12,6 +16,8 @@ HIPCC: str = "hipcc" ICPX: str = "icpx" +COMPILERS: List[str] = [GCC, CLANG, NVCC, CLANG_CUDA, HIPCC, ICPX] + # alpaka backend names ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLE: str = "alpaka_ACC_CPU_B_SEQ_T_SEQ_ENABLE" ALPAKA_ACC_CPU_B_SEQ_T_THREADS_ENABLE: str = "alpaka_ACC_CPU_B_SEQ_T_THREADS_ENABLE" @@ -22,8 +28,22 @@ ALPAKA_ACC_GPU_HIP_ENABLE: str = "alpaka_ACC_GPU_HIP_ENABLE" ALPAKA_ACC_SYCL_ENABLE: str = "alpaka_ACC_SYCL_ENABLE" +BACKENDS: List[str] = [ + ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLE, + ALPAKA_ACC_CPU_B_SEQ_T_THREADS_ENABLE, + ALPAKA_ACC_CPU_B_TBB_T_SEQ_ENABLE, + ALPAKA_ACC_CPU_B_OMP2_T_SEQ_ENABLE, + ALPAKA_ACC_CPU_B_SEQ_T_OMP2_ENABLE, + ALPAKA_ACC_GPU_CUDA_ENABLE, + ALPAKA_ACC_GPU_HIP_ENABLE, + ALPAKA_ACC_SYCL_ENABLE, +] + # software dependencies and compiler configurations UBUNTU: str = "ubuntu" CMAKE: str = "cmake" BOOST: str = "boost" CXX_STANDARD: str = "cxx_standard" + +OFF_VER: packaging.version.Version = packaging.version.parse("0.0.0") +ON_VER: packaging.version.Version = packaging.version.parse("1.0.0") diff --git a/bashi/versions.py b/bashi/versions.py new file mode 100644 index 0000000..cb8a880 --- /dev/null +++ b/bashi/versions.py @@ -0,0 +1,88 @@ +"""Provides all supported software versions""" + +from typing import Dict, List, Union +from collections import OrderedDict +import packaging.version as pkv +from bashi.globals import * # pylint: disable=wildcard-import,unused-wildcard-import +from bashi.types import ParameterValue, ParameterValueMatrix + +VERSIONS: Dict[str, List[Union[str, int, float]]] = { + GCC: [6, 7, 8, 9, 10, 11, 12, 13], + CLANG: [6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17], + NVCC: [ + 10.0, + 10.1, + 10.2, + 11.0, + 11.1, + 11.2, + 11.3, + 11.4, + 11.5, + 11.6, + 11.7, + 11.8, + 12.0, + 12.1, + 12.2, + ], + HIPCC: [5.0, 5.1, 5.2, 5.3, 5.4, 5.5, 5.6, 5.7, 6.0], + ICPX: ["2023.1.0", "2023.2.0"], + UBUNTU: [18.04, 20.04], + CMAKE: [3.18, 3.19, 3.20, 3.21, 3.22, 3.23, 3.24, 3.25, 3.26], + BOOST: [ + "1.74.0", + "1.75.0", + "1.76.0", + "1.77.0", + "1.78.0", + "1.79.0", + "1.80.0", + "1.81.0", + "1.82.0", + ], + CXX_STANDARD: [17, 20], +} + + +def get_parameter_value_matrix() -> ParameterValueMatrix: + """Generates a parameter-value-matrix from all supported compilers, softwares and compilation + configuration. + + Returns: + ParameterValueMatrix: parameter-value-matrix + """ + param_val_matrix: ParameterValueMatrix = OrderedDict() + + extended_version = VERSIONS.copy() + extended_version[CLANG_CUDA] = extended_version[CLANG] + + for compiler_type in [HOST_COMPILER, DEVICE_COMPILER]: + param_val_matrix[compiler_type] = [] + for sw_name, sw_versions in extended_version.items(): + if sw_name in COMPILERS: + for sw_version in sw_versions: + param_val_matrix[compiler_type].append( + ParameterValue(sw_name, pkv.parse(str(sw_version))) + ) + + for backend in BACKENDS: + if backend == ALPAKA_ACC_GPU_CUDA_ENABLE: + param_val_matrix[backend] = [ParameterValue(backend, OFF_VER)] + for cuda_version in extended_version[NVCC]: + param_val_matrix[backend].append( + ParameterValue(backend, pkv.parse(str(cuda_version))) + ) + else: + param_val_matrix[backend] = [ + ParameterValue(backend, OFF_VER), + ParameterValue(backend, ON_VER), + ] + + for other, versions in extended_version.items(): + if not other in COMPILERS + BACKENDS: + param_val_matrix[other] = [] + for version in versions: + param_val_matrix[other].append(ParameterValue(other, pkv.parse(str(version)))) + + return param_val_matrix diff --git a/tests/test_expected_parameter_value_pairs.py b/tests/test_expected_parameter_value_pairs.py index faf07a1..1e2654f 100644 --- a/tests/test_expected_parameter_value_pairs.py +++ b/tests/test_expected_parameter_value_pairs.py @@ -1,16 +1,13 @@ # pylint: disable=missing-docstring import unittest -from typing import List, Union, Tuple +from typing import List from collections import OrderedDict import io -import packaging.version as pkv # allpairspy has no type hints from allpairspy import AllPairs # type: ignore +from utils_test import parse_param_val, parse_param_vals, parse_expected_val_pairs from bashi.types import ( - Parameter, - ParameterValue, - ValueName, ParameterValuePair, ParameterValueMatrix, Combination, @@ -24,36 +21,6 @@ ) -def parse_param_val(param_val: Tuple[ValueName, Union[str, int, float]]) -> ParameterValue: - val_name, val_version = param_val - return ParameterValue(val_name, pkv.parse(str(val_version))) - - -def parse_param_vals( - param_vals: List[Tuple[ValueName, Union[str, int, float]]] -) -> List[ParameterValue]: - parsed_list: List[ParameterValue] = [] - - for param_val in param_vals: - parsed_list.append(parse_param_val(param_val)) - - return parsed_list - - -def parse_expected_val_pairs( - input_list: List[OrderedDict[Parameter, Tuple[ValueName, Union[str, int, float]]]] -) -> List[ParameterValuePair]: - expected_val_pairs: List[ParameterValuePair] = [] - - for param_val_pair in input_list: - tmp_entry: ParameterValuePair = OrderedDict() - for param in param_val_pair: - tmp_entry[param] = parse_param_val(param_val_pair[param]) - expected_val_pairs.append(tmp_entry) - - return expected_val_pairs - - class TestExpectedValuePairs(unittest.TestCase): @classmethod def setUpClass(cls): diff --git a/tests/test_generate_combination_list.py b/tests/test_generate_combination_list.py new file mode 100644 index 0000000..c130095 --- /dev/null +++ b/tests/test_generate_combination_list.py @@ -0,0 +1,243 @@ +# pylint: disable=missing-docstring +import unittest +import os +import io +from collections import OrderedDict +import packaging.version as pkv +from utils_test import parse_param_vals +from bashi.versions import get_parameter_value_matrix +from bashi.generator import generate_combination_list +from bashi.utils import ( + get_expected_parameter_value_pairs, + check_parameter_value_pair_in_combination_list, + remove_parameter_value_pair, +) +from bashi.types import ( + ParameterValue, + ParameterValuePair, + ParameterValueTuple, + ParameterValueMatrix, +) +from bashi.globals import * # pylint: disable=wildcard-import,unused-wildcard-import + + +class TestGeneratorTestData(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.param_matrix: ParameterValueMatrix = OrderedDict() + + cls.param_matrix[HOST_COMPILER] = parse_param_vals( + [(GCC, 10), (GCC, 11), (GCC, 12), (CLANG, 16), (CLANG, 17)] + ) + cls.param_matrix[DEVICE_COMPILER] = parse_param_vals( + [(NVCC, 11.2), (NVCC, 12.0), (GCC, 10), (GCC, 11)] + ) + cls.param_matrix[CMAKE] = parse_param_vals([(CMAKE, 3.22), (CMAKE, 3.23)]) + cls.param_matrix[BOOST] = parse_param_vals([(BOOST, 1.81), (BOOST, 1.82), (BOOST, 1.83)]) + + cls.generated_parameter_value_pairs: List[ + ParameterValuePair + ] = get_expected_parameter_value_pairs(cls.param_matrix) + + def test_generator_without_custom_filter(self): + comb_list = generate_combination_list(self.param_matrix) + + self.assertTrue( + check_parameter_value_pair_in_combination_list( + comb_list, self.generated_parameter_value_pairs + ) + ) + + # TODO(SimeonEhrig): remove expectedFailure, if this PR was merged: + # https://github.com/thombashi/allpairspy/pull/10 + @unittest.expectedFailure + def test_generator_with_custom_filter(self): + def custom_filter(row: ParameterValueTuple) -> bool: + if DEVICE_COMPILER in row and row[DEVICE_COMPILER].name == NVCC: + return False + + if ( + CMAKE in row + and row[CMAKE].version == pkv.parse("3.23") + and BOOST in row + and row[BOOST].version == pkv.parse("1.82") + ): + return False + + return True + + OD = OrderedDict + + for row in [ + OD( + { + HOST_COMPILER: ParameterValue(GCC, pkv.parse("10")), + } + ), + OD( + { + HOST_COMPILER: ParameterValue(GCC, pkv.parse("10")), + DEVICE_COMPILER: ParameterValue(GCC, pkv.parse("10")), + } + ), + OD( + { + HOST_COMPILER: ParameterValue(GCC, pkv.parse("10")), + DEVICE_COMPILER: ParameterValue(GCC, pkv.parse("10")), + CMAKE: ParameterValue(CMAKE, pkv.parse("3.23")), + } + ), + OD( + { + HOST_COMPILER: ParameterValue(GCC, pkv.parse("10")), + DEVICE_COMPILER: ParameterValue(GCC, pkv.parse("10")), + BOOST: ParameterValue(BOOST, pkv.parse("1.82")), + } + ), + OD( + { + HOST_COMPILER: ParameterValue(GCC, pkv.parse("10")), + DEVICE_COMPILER: ParameterValue(GCC, pkv.parse("10")), + CMAKE: ParameterValue(CMAKE, pkv.parse("3.22")), + BOOST: ParameterValue(BOOST, pkv.parse("1.81")), + } + ), + OD( + { + HOST_COMPILER: ParameterValue(GCC, pkv.parse("10")), + DEVICE_COMPILER: ParameterValue(GCC, pkv.parse("10")), + CMAKE: ParameterValue(CMAKE, pkv.parse("3.23")), + BOOST: ParameterValue(BOOST, pkv.parse("1.81")), + } + ), + OD( + { + HOST_COMPILER: ParameterValue(GCC, pkv.parse("10")), + DEVICE_COMPILER: ParameterValue(GCC, pkv.parse("10")), + CMAKE: ParameterValue(CMAKE, pkv.parse("3.22")), + BOOST: ParameterValue(BOOST, pkv.parse("1.82")), + } + ), + ]: + self.assertTrue(custom_filter(row)) + + self.assertFalse( + custom_filter( + OrderedDict( + { + HOST_COMPILER: ParameterValue(GCC, pkv.parse("10")), + DEVICE_COMPILER: ParameterValue(GCC, pkv.parse("10")), + CMAKE: ParameterValue(CMAKE, pkv.parse("3.23")), + BOOST: ParameterValue(BOOST, pkv.parse("1.82")), + } + ) + ) + ) + + comb_list = generate_combination_list( + parameter_value_matrix=self.param_matrix, custom_filter=custom_filter + ) + + reduced_expected_param_val_pairs = self.generated_parameter_value_pairs.copy() + for device_compiler in self.param_matrix[DEVICE_COMPILER]: + if device_compiler.name == NVCC: + self.assertTrue( + remove_parameter_value_pair( + OrderedDict( + {DEVICE_COMPILER: ParameterValue(NVCC, device_compiler.version)} + ), + reduced_expected_param_val_pairs, + ) + ) + + self.assertTrue( + remove_parameter_value_pair( + OrderedDict( + { + CMAKE: ParameterValue(CMAKE, pkv.parse("3.23")), + BOOST: ParameterValue(BOOST, pkv.parse("1.82")), + }, + ), + reduced_expected_param_val_pairs, + ) + ) + + missing_combinations = io.StringIO() + + # because of a bug in the allpairspy library, valid pairs are missing. + try: + self.assertTrue( + check_parameter_value_pair_in_combination_list( + comb_list, reduced_expected_param_val_pairs, missing_combinations + ) + ) + except AssertionError as e: + # remove comment to print missing, valid pairs + # print(f"\n{missing_combinations.getvalue()}") + raise e + + +# set the environment variable NOLONG to skip long running tests +@unittest.skipIf("NOLONG" in os.environ, "Skip long running test") +class TestGeneratorRealData(unittest.TestCase): + def test_generator_without_custom_filter(self): + param_val_matrix = get_parameter_value_matrix() + expected_param_val_pairs = get_expected_parameter_value_pairs(param_val_matrix) + + comb_list = generate_combination_list(param_val_matrix) + + self.assertTrue( + check_parameter_value_pair_in_combination_list(comb_list, expected_param_val_pairs) + ) + + # TODO(SimeonEhrig): remove expectedFailure, if this PR was merged: + # https://github.com/thombashi/allpairspy/pull/10 + @unittest.expectedFailure + def test_generator_with_custom_filter(self): + def custom_filter(row: ParameterValueTuple) -> bool: + if DEVICE_COMPILER in row and row[DEVICE_COMPILER].name == NVCC: + return False + + if ( + CMAKE in row + and row[CMAKE].version == pkv.parse("3.23") + and BOOST in row + and row[BOOST].version == pkv.parse("1.82") + ): + return False + + return True + + param_val_matrix = get_parameter_value_matrix() + reduced_expected_param_val_pairs = get_expected_parameter_value_pairs(param_val_matrix) + + self.assertTrue( + remove_parameter_value_pair( + OrderedDict( + { + CMAKE: ParameterValue(CMAKE, pkv.parse("3.23")), + BOOST: ParameterValue(BOOST, pkv.parse("1.82")), + }, + ), + reduced_expected_param_val_pairs, + ) + ) + + comb_list = generate_combination_list( + parameter_value_matrix=param_val_matrix, custom_filter=custom_filter + ) + + missing_combinations = io.StringIO() + + try: + self.assertTrue( + check_parameter_value_pair_in_combination_list( + comb_list, reduced_expected_param_val_pairs, missing_combinations + ) + ) + except AssertionError as e: + # remove comment to display missing combinations + # missing_combinations_str = missing_combinations.getvalue() + # print(f"\nnumber of missing combinations: {len(missing_combinations_str.split('\n'))}") + # print(f"\n{missing_combinations_str}") + raise e diff --git a/tests/test_generator.py b/tests/test_generator.py deleted file mode 100644 index 2c26122..0000000 --- a/tests/test_generator.py +++ /dev/null @@ -1,8 +0,0 @@ -import unittest - -from bashi.generator import add - - -class TestSum(unittest.TestCase): - def test_sum(self): - self.assertTrue(add(1, 2) == 3) diff --git a/tests/test_params_value_matrix_generator.py b/tests/test_params_value_matrix_generator.py new file mode 100644 index 0000000..a69279a --- /dev/null +++ b/tests/test_params_value_matrix_generator.py @@ -0,0 +1,56 @@ +# pylint: disable=missing-docstring +import unittest +from bashi.versions import VERSIONS, get_parameter_value_matrix +from bashi.globals import * # pylint: disable=wildcard-import,unused-wildcard-import + + +class TestParameterValueGenerator(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.param_val_matrix = get_parameter_value_matrix() + + def test_all_params_in(self): + expected_params = [HOST_COMPILER, DEVICE_COMPILER] + expected_params += BACKENDS + for sw_version in VERSIONS: + if not sw_version in COMPILERS: + expected_params.append(sw_version) + + self.assertEqual( + len(self.param_val_matrix.keys()), + len(expected_params), + f"\nparam_val_matrix: {self.param_val_matrix.keys()}\n" + f"expected_params: {expected_params}]", + ) + + def test_number_host_device_compiler(self): + extended_versions = VERSIONS.copy() + extended_versions[CLANG_CUDA] = extended_versions[CLANG] + number_of_compilers = 0 + for compiler in COMPILERS: + number_of_compilers += len(extended_versions[compiler]) + + self.assertEqual(len(self.param_val_matrix[HOST_COMPILER]), number_of_compilers) + self.assertEqual(len(self.param_val_matrix[DEVICE_COMPILER]), number_of_compilers) + + def test_number_of_backends(self): + for backend in BACKENDS: + if backend == ALPAKA_ACC_GPU_CUDA_ENABLE: + # all nvcc versions are also CUDA SDK versions + # add 1 for disable the backend + number_of_cuda_backends = len(VERSIONS[NVCC]) + 1 + self.assertEqual( + len(self.param_val_matrix[backend]), + number_of_cuda_backends, + f"\nBackend: {backend}", + ) + else: + # normal backends are enabled or not + self.assertEqual(len(self.param_val_matrix[backend]), 2, f"\nBackend: {backend}") + + def test_number_other_software_versions(self): + # all software versions and similar which are not compiler versions adds a parameter-value + # for each version + for sw_name, sw_versions in VERSIONS.items(): + if sw_name not in COMPILERS: + self.assertEqual(len(self.param_val_matrix[sw_name]), len(sw_versions)) diff --git a/tests/utils_test.py b/tests/utils_test.py new file mode 100644 index 0000000..74182fb --- /dev/null +++ b/tests/utils_test.py @@ -0,0 +1,66 @@ +"""Different helper functions for bashi tests.""" + +from typing import List, Union, Tuple +from collections import OrderedDict +import packaging.version as pkv +from bashi.types import ( + Parameter, + ParameterValue, + ParameterValuePair, + ValueName, +) + + +def parse_param_val(param_val: Tuple[ValueName, Union[str, int, float]]) -> ParameterValue: + """Parse a single tuple to a parameter-values. + + Args: + param_val (Tuple[ValueName, Union[str, int, float]]): Tuple to parse + + Returns: + ParameterValue: parsed ParameterValue + """ + val_name, val_version = param_val + return ParameterValue(val_name, pkv.parse(str(val_version))) + + +def parse_param_vals( + param_vals: List[Tuple[ValueName, Union[str, int, float]]] +) -> List[ParameterValue]: + """Parse a list of tuples to a list of parameter-values. + + Args: + param_vals (List[Tuple[ValueName, Union[str, int, float]]]): List to parse + + Returns: + List[ParameterValue]: List of parameter-values + """ + parsed_list: List[ParameterValue] = [] + + for param_val in param_vals: + parsed_list.append(parse_param_val(param_val)) + + return parsed_list + + +def parse_expected_val_pairs( + input_list: List[OrderedDict[Parameter, Tuple[ValueName, Union[str, int, float]]]] +) -> List[ParameterValuePair]: + """Parse list of expected parameter-values to the correct type. + + Args: + input_list (List[OrderedDict[Parameter, Tuple[ValueName, Union[str, int, float]]]]): + List to parse + + Returns: + List[ParameterValuePair]: Parsed parameter-type list + """ + expected_val_pairs: List[ParameterValuePair] = [] + + for param_val_pair in input_list: + tmp_entry: ParameterValuePair = OrderedDict() + for param in param_val_pair: + tmp_entry[param] = parse_param_val(param_val_pair[param]) + expected_val_pairs.append(tmp_entry) + + return expected_val_pairs From 8d9764c3840e606094d46286662c5ed652161509 Mon Sep 17 00:00:00 2001 From: Simeon Ehrig Date: Wed, 7 Feb 2024 13:00:43 +0100 Subject: [PATCH 4/6] change type of ParameterValuePair to named tuple - introduce new type ParameterValueSingle - add option to remove_parameter_value_pair() remove all pairs only by given name independent of the version number --- bashi/types.py | 8 +- bashi/utils.py | 109 +++++- docs/naming.md | 5 +- tests/test_expected_parameter_value_pairs.py | 354 +++++++++++++++++-- tests/test_generate_combination_list.py | 31 +- tests/utils_test.py | 24 +- 6 files changed, 456 insertions(+), 75 deletions(-) diff --git a/bashi/types.py b/bashi/types.py index 71ef54b..8f349ce 100644 --- a/bashi/types.py +++ b/bashi/types.py @@ -10,7 +10,13 @@ ParameterValue = NamedTuple("ParameterValue", [("name", ValueName), ("version", ValueVersion)]) ParameterValueList: TypeAlias = List[ParameterValue] ParameterValueMatrix: TypeAlias = OrderedDict[Parameter, ParameterValueList] -ParameterValuePair: TypeAlias = OrderedDict[Parameter, ParameterValue] +ParameterValueSingle = NamedTuple( + "ParameterValueSingle", [("parameter", Parameter), ("parameterValue", ParameterValue)] +) +ParameterValuePair = NamedTuple( + "ParameterValuePair", + [("first", ParameterValueSingle), ("second", ParameterValueSingle)], +) ParameterValueTuple: TypeAlias = OrderedDict[Parameter, ParameterValue] Combination: TypeAlias = OrderedDict[Parameter, ParameterValue] CombinationList: TypeAlias = List[Combination] diff --git a/bashi/utils.py b/bashi/utils.py index e5b2ebc..de8033d 100644 --- a/bashi/utils.py +++ b/bashi/utils.py @@ -1,14 +1,16 @@ """Different helper functions for bashi""" -from typing import Dict, List, IO +from typing import Dict, List, IO, Union from collections import OrderedDict import dataclasses import sys from typeguard import typechecked +import packaging.version from bashi.types import ( Parameter, ParameterValue, ParameterValueTuple, + ParameterValueSingle, ParameterValuePair, ParameterValueMatrix, CombinationList, @@ -104,6 +106,50 @@ def get_default_filter_chain( ) +@typechecked +def create_parameter_value_pair( # pylint: disable=too-many-arguments + parameter1: str, + value_name1: str, + value_version1: Union[int, float, str, packaging.version.Version], + parameter2: str, + value_name2: str, + value_version2: Union[int, float, str, packaging.version.Version], +) -> ParameterValuePair: + """Create parameter-value-pair from the given arguments. Parse parameter-versions if required. + + Args: + parameter1 (str): name of the first parameter + value_name1 (str): name of the first value-name + value_version1 (Union[int, float, str, packaging.version.Version]): version of first + value-version + parameter2 (str): name of the second parameter + value_name2 (str): name of the second value-name + value_version2 (Union[int, float, str, packaging.version.Version]): version of the second + value-version + + Returns: + ParameterValuePair: parameter-value-pair + """ + if isinstance(value_version1, packaging.version.Version): + parsed_value_version1: packaging.version.Version = value_version1 + else: + parsed_value_version1: packaging.version.Version = packaging.version.parse( + str(value_version1) + ) + + if isinstance(value_version2, packaging.version.Version): + parsed_value_version2: packaging.version.Version = value_version2 + else: + parsed_value_version2: packaging.version.Version = packaging.version.parse( + str(value_version2) + ) + + return ParameterValuePair( + ParameterValueSingle(parameter1, ParameterValue(value_name1, parsed_value_version1)), + ParameterValueSingle(parameter2, ParameterValue(value_name2, parsed_value_version2)), + ) + + @typechecked def get_expected_parameter_value_pairs( parameter_matrix: ParameterValueMatrix, @@ -156,35 +202,44 @@ def _loop_over_parameter_values( """ for v1_name, v1_version in parameters[v1_parameter]: for v2_name, v2_version in parameters[v2_parameter]: - param_val_pair: ParameterValuePair = OrderedDict() - param_val_pair[v1_parameter] = ParameterValue(v1_name, v1_version) - param_val_pair[v2_parameter] = ParameterValue(v2_name, v2_version) - expected_pairs.append(param_val_pair) + expected_pairs.append( + create_parameter_value_pair( + v1_parameter, v1_name, v1_version, v2_parameter, v2_name, v2_version + ) + ) @typechecked def remove_parameter_value_pair( - to_remove: ParameterValuePair, parameter_value_pairs: List[ParameterValuePair] + to_remove: Union[ParameterValueSingle, ParameterValuePair], + parameter_value_pairs: List[ParameterValuePair], + all_versions: bool = False, ) -> bool: """Removes a parameter-value pair with one or two entries from the parameter-value-pair list. If the parameter-value-pair only has one parameter value, all parameter-value-pairs that contain the parameter value are removed. Args: - to_remove (ParameterValuePair): Parameter-value-pair to remove + to_remove (Union[ParameterValueSingle, ParameterValuePair]): Parameter-value-single or + parameter-value-pair to remove param_val_pairs (List[ParameterValuePair]): List of parameter-value-pairs. Will be modified. - + all_versions (bool): If it is `True` and `to_remove` has type of `ParameterValuePair`, + removes all parameter-value-pairs witch matches the value-names independent of the + value-version. Defaults to False. Raises: - RuntimeError: If `to_remove` 0 or more than 2 entries. + RuntimeError: If `all_versions=True` and `to_remove` is a `ParameterValueSingle` Returns: bool: True if entry was removed from parameter_value_pairs """ - if len(to_remove) == 1: + if isinstance(to_remove, ParameterValueSingle): + if all_versions: + raise RuntimeError("all_versions=True is not support for ParameterValueSingle") + return _remove_single_entry_parameter_value_pair(to_remove, parameter_value_pairs) - if len(to_remove) == 0 or len(to_remove) > 2: - raise RuntimeError("More than two parameter-values are not allowed") + if all_versions: + return _remove_parameter_value_pair_all_versions(to_remove, parameter_value_pairs) try: parameter_value_pairs.remove(to_remove) @@ -195,14 +250,34 @@ def remove_parameter_value_pair( @typechecked def _remove_single_entry_parameter_value_pair( - to_remove: ParameterValuePair, param_val_pairs: List[ParameterValuePair] + to_remove: ParameterValueSingle, param_val_pairs: List[ParameterValuePair] ) -> bool: - val_name, val_version = next(iter(to_remove.items())) + len_before = len(param_val_pairs) + + def filter_function(param_val_pair: ParameterValuePair) -> bool: + for param_val_entry in param_val_pair: + if param_val_entry == to_remove: + return False + return True + + param_val_pairs[:] = list(filter(filter_function, param_val_pairs)) + + return len_before != len(param_val_pairs) + +@typechecked +def _remove_parameter_value_pair_all_versions( + to_remove: ParameterValuePair, param_val_pairs: List[ParameterValuePair] +) -> bool: len_before = len(param_val_pairs) def filter_function(param_val_pair: ParameterValuePair) -> bool: - if val_name in param_val_pair and param_val_pair[val_name] == val_version: + if ( + param_val_pair.first.parameter == to_remove.first.parameter + and param_val_pair.second.parameter == to_remove.second.parameter + and param_val_pair.first.parameterValue.name == to_remove.first.parameterValue.name + and param_val_pair.second.parameterValue.name == to_remove.second.parameterValue.name + ): return False return True @@ -232,8 +307,8 @@ def check_parameter_value_pair_in_combination_list( missing_expected_param = False for ex_param_val_pair in parameter_value_pairs: - param1, param_val1 = list(ex_param_val_pair.items())[0] - param2, param_val2 = list(ex_param_val_pair.items())[1] + param1, param_val1 = ex_param_val_pair[0] + param2, param_val2 = ex_param_val_pair[1] found = False for comb in combination_list: # comb contains all parameters, therefore a check is not required diff --git a/docs/naming.md b/docs/naming.md index 64f2ac4..999127e 100644 --- a/docs/naming.md +++ b/docs/naming.md @@ -5,7 +5,7 @@ The [pair-wise testing](https://en.wikipedia.org/wiki/All-pairs_testing) takes a The real Python types are implemented in [types.py](../bashi/types.py) - **parameter** (`str`): A `parameter` represents a software component like the host compiler or a specific software like `CMake` or `Boost`. A `parameter` names a list of `parameter-values` and expresses how a `parameter-value` is used. -- **parameter-value** (`Tuple[value-name: str, value-version: packaging.version.Version]`): A `parameter-value` represents of a specific version of a `parameter`, for example `GCC 10`, `nvcc 12.2` or `CMake 3.28`. The pair wise generator takes on `parameter-value` of each `parameter` for is combinatorics. +- **parameter-value** (`NamedTuple[value-name: str, value-version: packaging.version.Version]`): A `parameter-value` represents of a specific version of a `parameter`, for example `GCC 10`, `nvcc 12.2` or `CMake 3.28`. The pair wise generator takes on `parameter-value` of each `parameter` for is combinatorics. - **value-name** (`str`): The `value-name` is the first part of the `parameter-value`. It names a specific software component, like `GCC` or `CMake`. If the `parameter` names a specific software, `parameter` and `value-name` are equal. - **value-version** (`packaging.version.Version`): The `value-version` is the second part of the `parameter-value`. It defines a specific version of a software component such like `12.2` or `3.12`. - **parameter-value-list** (`parameter: str = List[parameter-value: Tuple[value-name: str, value-version: packaging.version.Version]]`): A `parameter-value-list` is a list of `parameter-values` assigned to a `parameter`. For example: @@ -16,4 +16,5 @@ The real Python types are implemented in [types.py](../bashi/types.py) - **parameter-value-tuple** (`OrderedList[parameter: str, parameter-value: Tuple[value-name: str, value-version: packaging.version.Version]]`): A `parameter-value-tuple` is a list of one ore more `parameter-value`s. The `parameter-value-tuple` is created from a `parameter-value-matrix` and each `parameter-value` is assigned to a different `parameter`. This means, each `parameter-value` is from a different `parameter-value-list` in a `parameter-value-matrix`. The `parameter-value-tuple` has the same or a smaller number of entries as the number of `parameters` in a `parameter-value-matrix`. - **combination** (`OrderedList[parameter: str, parameter-value: Tuple[value-name: str, value-version: packaging.version.Version]]`): A `combination` is a `parameter-value-tuple` with the same number of `parameter-value`s as the number of input `parameters`. - **combination-list** (`List[OrderedList[parameter : str, parameter-value: Tuple[value-name: str, value-version: packaging.version.Version]]]`): A `combination-list` is a list of `combination`s an the result of the pair-wise generator. -- **parameter-value-pair** (`OrderedList[parameter: str, parameter-value: Tuple[value-name: str, value-version: packaging.version.Version]]`): A `parameter-value-pair` is a `parameter-value-tuple` with exact two `parameter-values`. The pair-wise generator guaranties that each `parameter-value-pair`, which can be created by the given `parameter-value-matrix` exists at least in one `combination` of the `combination-list`. The only exception is, if a `parameter-value-pair` is forbidden by a filter rule. +- **parameter-value-single** (`NamedTuple[parameter: str, parameter-value: NamedTuple[value-name: str, value-version: packaging.version.Version]]`): A `parameter-value-single` connects a `parameter` with a single `parameter-value`. +- **parameter-value-pair** (`NamedTuple[first: NamedTuple[parameter: str, parameter-value: NamedTuple[value-name: str, value-version: packaging.version.Version]], second: NamedTuple[parameter: str, parameter-value: NamedTuple[value-name: str, value-version: packaging.version.Version]]]`): A `parameter-value-pair` is a `parameter-value-tuple` with exact two `parameter-values`. The pair-wise generator guaranties that each `parameter-value-pair`, which can be created by the given `parameter-value-matrix` exists at least in one `combination` of the `combination-list`. The only exception is, if a `parameter-value-pair` is forbidden by a filter rule. diff --git a/tests/test_expected_parameter_value_pairs.py b/tests/test_expected_parameter_value_pairs.py index 1e2654f..3b1b6d3 100644 --- a/tests/test_expected_parameter_value_pairs.py +++ b/tests/test_expected_parameter_value_pairs.py @@ -1,16 +1,19 @@ # pylint: disable=missing-docstring import unittest -from typing import List +from typing import List, Dict from collections import OrderedDict import io +import packaging.version as pkv # allpairspy has no type hints from allpairspy import AllPairs # type: ignore from utils_test import parse_param_val, parse_param_vals, parse_expected_val_pairs from bashi.types import ( + Parameter, + ParameterValue, + ParameterValueSingle, ParameterValuePair, ParameterValueMatrix, - Combination, CombinationList, ) from bashi.globals import * # pylint: disable=wildcard-import,unused-wildcard-import @@ -18,9 +21,162 @@ get_expected_parameter_value_pairs, check_parameter_value_pair_in_combination_list, remove_parameter_value_pair, + create_parameter_value_pair, ) +class TestCreateParameterValuePair(unittest.TestCase): + def test_create_parameter_value_pair_type_str(self): + param1 = "param1" + param2 = "param2" + name1 = "name1" + name2 = "name2" + ver1 = "1.0" + ver2 = "2.0" + + created_param_val_pair = create_parameter_value_pair( + param1, name1, ver1, param2, name2, ver2 + ) + + expected_param_val_pair = ParameterValuePair( + ParameterValueSingle( + param1, + ParameterValue(name1, pkv.parse(ver1)), + ), + ParameterValueSingle( + param2, + ParameterValue(name2, pkv.parse(ver2)), + ), + ) + + self.assertEqual(type(created_param_val_pair), type(expected_param_val_pair)) + + self.assertEqual( + created_param_val_pair, + expected_param_val_pair, + ) + + def test_create_parameter_value_pair_type_float(self): + param1 = "param1" + param2 = "param2" + name1 = "name1" + name2 = "name2" + ver1 = 1.0 + ver2 = 2.0 + + created_param_val_pair = create_parameter_value_pair( + param1, name1, ver1, param2, name2, ver2 + ) + + expected_param_val_pair = ParameterValuePair( + ParameterValueSingle( + param1, + ParameterValue(name1, pkv.parse(str(ver1))), + ), + ParameterValueSingle( + param2, + ParameterValue(name2, pkv.parse(str(ver2))), + ), + ) + + self.assertEqual(type(created_param_val_pair), type(expected_param_val_pair)) + + self.assertEqual( + created_param_val_pair, + expected_param_val_pair, + ) + + def test_create_parameter_value_pair_type_int(self): + param1 = "param1" + param2 = "param2" + name1 = "name1" + name2 = "name2" + ver1 = 1 + ver2 = 2 + + created_param_val_pair = create_parameter_value_pair( + param1, name1, ver1, param2, name2, ver2 + ) + + expected_param_val_pair = ParameterValuePair( + ParameterValueSingle( + param1, + ParameterValue(name1, pkv.parse(str(ver1))), + ), + ParameterValueSingle( + param2, + ParameterValue(name2, pkv.parse(str(ver2))), + ), + ) + + self.assertEqual(type(created_param_val_pair), type(expected_param_val_pair)) + + self.assertEqual( + created_param_val_pair, + expected_param_val_pair, + ) + + def test_create_parameter_value_pair_type_version(self): + param1 = "param1" + param2 = "param2" + name1 = "name1" + name2 = "name2" + ver1 = pkv.parse("1.0") + ver2 = pkv.parse("2.0") + + created_param_val_pair = create_parameter_value_pair( + param1, name1, ver1, param2, name2, ver2 + ) + + expected_param_val_pair = ParameterValuePair( + ParameterValueSingle( + param1, + ParameterValue(name1, ver1), + ), + ParameterValueSingle( + param2, + ParameterValue(name2, ver2), + ), + ) + + self.assertEqual(type(created_param_val_pair), type(expected_param_val_pair)) + + self.assertEqual( + created_param_val_pair, + expected_param_val_pair, + ) + + def test_create_parameter_value_pair_type_mixed(self): + param1 = "param1" + param2 = "param2" + name1 = "name1" + name2 = "name2" + ver1 = pkv.parse("1.0") + ver2 = "2.0" + + created_param_val_pair = create_parameter_value_pair( + param1, name1, ver1, param2, name2, ver2 + ) + + expected_param_val_pair = ParameterValuePair( + ParameterValueSingle( + param1, + ParameterValue(name1, ver1), + ), + ParameterValueSingle( + param2, + ParameterValue(name2, pkv.parse(ver2)), + ), + ) + + self.assertEqual(type(created_param_val_pair), type(expected_param_val_pair)) + + self.assertEqual( + created_param_val_pair, + expected_param_val_pair, + ) + + class TestExpectedValuePairs(unittest.TestCase): @classmethod def setUpClass(cls): @@ -35,9 +191,9 @@ def setUpClass(cls): cls.param_matrix[CMAKE] = parse_param_vals([(CMAKE, 3.22), (CMAKE, 3.23)]) cls.param_matrix[BOOST] = parse_param_vals([(BOOST, 1.81), (BOOST, 1.82), (BOOST, 1.83)]) - cls.generated_parameter_value_pairs: List[ - ParameterValuePair - ] = get_expected_parameter_value_pairs(cls.param_matrix) + cls.generated_parameter_value_pairs: List[ParameterValuePair] = ( + get_expected_parameter_value_pairs(cls.param_matrix) + ) OD = OrderedDict @@ -348,9 +504,8 @@ def test_unrestricted_allpairspy_generator(self): class TestRemoveExpectedParameterValuePair(unittest.TestCase): - def test_remove_two_entry_parameter_value_pair(self): + def test_remove_parameter_value_pair(self): OD = OrderedDict - ppv = parse_param_val expected_param_value_pairs: List[ParameterValuePair] = parse_expected_val_pairs( [ @@ -362,29 +517,9 @@ def test_remove_two_entry_parameter_value_pair(self): ) original_length = len(expected_param_value_pairs) - # expects one or two entries - self.assertRaises( - RuntimeError, - remove_parameter_value_pair, - OD(), - expected_param_value_pairs, - ) - self.assertRaises( - RuntimeError, - remove_parameter_value_pair, - OD( - { - HOST_COMPILER: ppv((GCC, 9)), - DEVICE_COMPILER: ppv((NVCC, 11.2)), - CMAKE: ppv((CMAKE, 3.23)), - } - ), - expected_param_value_pairs, - ) - self.assertFalse( remove_parameter_value_pair( - OD({HOST_COMPILER: ppv((GCC, 9)), DEVICE_COMPILER: ppv((NVCC, 11.2))}), + create_parameter_value_pair(HOST_COMPILER, GCC, 9, DEVICE_COMPILER, NVCC, 11.2), expected_param_value_pairs, ) ) @@ -392,7 +527,7 @@ def test_remove_two_entry_parameter_value_pair(self): self.assertTrue( remove_parameter_value_pair( - OD({HOST_COMPILER: ppv((GCC, 10)), DEVICE_COMPILER: ppv((NVCC, 12.0))}), + create_parameter_value_pair(HOST_COMPILER, GCC, 10, DEVICE_COMPILER, NVCC, 12.0), expected_param_value_pairs, ) ) @@ -400,13 +535,13 @@ def test_remove_two_entry_parameter_value_pair(self): self.assertTrue( remove_parameter_value_pair( - OD({CMAKE: ppv((CMAKE, 3.23)), BOOST: ppv((BOOST, 1.83))}), + create_parameter_value_pair(CMAKE, CMAKE, 3.23, BOOST, BOOST, 1.83), expected_param_value_pairs, ) ) self.assertEqual(len(expected_param_value_pairs), original_length - 2) - def test_remove_single_entry_parameter_value_pair(self): + def test_remove_parameter_value_single(self): OD = OrderedDict ppv = parse_param_val @@ -421,9 +556,18 @@ def test_remove_single_entry_parameter_value_pair(self): ) original_length = len(expected_param_value_pairs) + # all_versions=True is not support for ParameterValueSingle + self.assertRaises( + RuntimeError, + remove_parameter_value_pair, + ParameterValueSingle(HOST_COMPILER, ppv((GCC, 12))), + expected_param_value_pairs, + True, + ) + self.assertFalse( remove_parameter_value_pair( - OD({HOST_COMPILER: ppv((GCC, 12))}), + ParameterValueSingle(HOST_COMPILER, ppv((GCC, 12))), expected_param_value_pairs, ) ) @@ -431,7 +575,7 @@ def test_remove_single_entry_parameter_value_pair(self): self.assertFalse( remove_parameter_value_pair( - OD({HOST_COMPILER: ppv((CLANG, 12))}), + ParameterValueSingle(HOST_COMPILER, ppv((CLANG, 12))), expected_param_value_pairs, ) ) @@ -439,7 +583,7 @@ def test_remove_single_entry_parameter_value_pair(self): self.assertFalse( remove_parameter_value_pair( - OD({UBUNTU: ppv((UBUNTU, 20.04))}), + ParameterValueSingle(UBUNTU, ppv((UBUNTU, 20.04))), expected_param_value_pairs, ) ) @@ -447,7 +591,7 @@ def test_remove_single_entry_parameter_value_pair(self): self.assertTrue( remove_parameter_value_pair( - OD({HOST_COMPILER: ppv((GCC, 9))}), + ParameterValueSingle(HOST_COMPILER, ppv((GCC, 9))), expected_param_value_pairs, ) ) @@ -462,3 +606,143 @@ def test_remove_single_entry_parameter_value_pair(self): ] ), ) + + def test_remove_parameter_value_pair_all_versions(self): + versions = { + GCC: [9, 10, 11, 12, 13], + CLANG: [13, 14, 15, 16, 17], + NVCC: [11.0, 11.1, 11.2, 11.3, 11.4], + HIPCC: [5.0, 5.1, 5.2, 5.3], + CMAKE: [3.22, 3.23, 3.24], + BOOST: [1.80, 1.81, 1.82], + } + + param_val_matrix: ParameterValueMatrix = OrderedDict() + for compiler in [HOST_COMPILER, DEVICE_COMPILER]: + param_val_matrix[compiler] = [] + for compiler_name in [GCC, CLANG, NVCC, HIPCC]: + for compiler_version in versions[compiler_name]: + param_val_matrix[compiler].append( + ParameterValue(compiler_name, pkv.parse(str(compiler_version))) + ) + + for sw in [CMAKE, BOOST]: + param_val_matrix[sw] = [] + for version in versions[sw]: + param_val_matrix[sw].append(ParameterValue(sw, pkv.parse(str(version)))) + + reduced_param_value_pairs = get_expected_parameter_value_pairs(param_val_matrix) + + expected_number_of_reduced_pairs = len(reduced_param_value_pairs) + + expected_reduced_param_value_pairs = reduced_param_value_pairs.copy() + + # remove single value to verify that default flag is working + example_single_pair = create_parameter_value_pair( + HOST_COMPILER, + NVCC, + 11.0, + DEVICE_COMPILER, + NVCC, + 11.3, + ) + + expected_reduced_param_value_pairs.remove(example_single_pair) + + self.assertTrue( + remove_parameter_value_pair( + to_remove=example_single_pair, + parameter_value_pairs=reduced_param_value_pairs, + ) + ) + + # remove single entry + expected_number_of_reduced_pairs -= 1 + self.assertEqual(len(reduced_param_value_pairs), expected_number_of_reduced_pairs) + + reduced_param_value_pairs.sort() + expected_reduced_param_value_pairs.sort() + self.assertEqual(reduced_param_value_pairs, expected_reduced_param_value_pairs) + + # remove all expected tuples, where host and device compiler is nvcc + def filter_function1(param_val_pair: ParameterValuePair) -> bool: + if ( + param_val_pair.first.parameter == HOST_COMPILER + and param_val_pair.second.parameter == DEVICE_COMPILER + ): + if ( + param_val_pair.first.parameterValue.name == NVCC + and param_val_pair.second.parameterValue.name == NVCC + ): + return False + + return True + + expected_reduced_param_value_pairs[:] = list( + filter(filter_function1, expected_reduced_param_value_pairs) + ) + + self.assertTrue( + remove_parameter_value_pair( + to_remove=create_parameter_value_pair( + HOST_COMPILER, + NVCC, + 0, + DEVICE_COMPILER, + NVCC, + 0, + ), + parameter_value_pairs=reduced_param_value_pairs, + all_versions=True, + ) + ) + + # remove number of pairs, where host and device compiler is nvcc + # -1 because we removed already a combination manually before + expected_number_of_reduced_pairs -= len(versions[NVCC]) * len(versions[NVCC]) - 1 + self.assertEqual(len(reduced_param_value_pairs), expected_number_of_reduced_pairs) + + reduced_param_value_pairs.sort() + expected_reduced_param_value_pairs.sort() + self.assertEqual(reduced_param_value_pairs, expected_reduced_param_value_pairs) + + # remove all combinations where HIPCC is the host compiler and nvcc the device compiler + def filter_function2(param_val_pair: ParameterValuePair) -> bool: + if ( + param_val_pair.first.parameter == HOST_COMPILER + and param_val_pair.second.parameter == DEVICE_COMPILER + ): + if ( + param_val_pair.first.parameterValue.name == HIPCC + and param_val_pair.second.parameterValue.name == NVCC + ): + return False + + return True + + expected_reduced_param_value_pairs[:] = list( + filter(filter_function2, expected_reduced_param_value_pairs) + ) + + self.assertTrue( + remove_parameter_value_pair( + to_remove=create_parameter_value_pair( + HOST_COMPILER, + HIPCC, + 0, + DEVICE_COMPILER, + NVCC, + 0, + ), + parameter_value_pairs=reduced_param_value_pairs, + all_versions=True, + ) + ) + + # remove number pairs, where host compiler is HIPCC and device compiler is nvcc + expected_number_of_reduced_pairs -= len(versions[HIPCC]) * len(versions[NVCC]) + self.assertEqual(len(reduced_param_value_pairs), expected_number_of_reduced_pairs) + + reduced_param_value_pairs.sort() + expected_reduced_param_value_pairs.sort() + self.assertEqual(reduced_param_value_pairs, expected_reduced_param_value_pairs) diff --git a/tests/test_generate_combination_list.py b/tests/test_generate_combination_list.py index c130095..988f7b5 100644 --- a/tests/test_generate_combination_list.py +++ b/tests/test_generate_combination_list.py @@ -11,9 +11,11 @@ get_expected_parameter_value_pairs, check_parameter_value_pair_in_combination_list, remove_parameter_value_pair, + create_parameter_value_pair, ) from bashi.types import ( ParameterValue, + ParameterValueSingle, ParameterValuePair, ParameterValueTuple, ParameterValueMatrix, @@ -35,9 +37,9 @@ def setUpClass(cls): cls.param_matrix[CMAKE] = parse_param_vals([(CMAKE, 3.22), (CMAKE, 3.23)]) cls.param_matrix[BOOST] = parse_param_vals([(BOOST, 1.81), (BOOST, 1.82), (BOOST, 1.83)]) - cls.generated_parameter_value_pairs: List[ - ParameterValuePair - ] = get_expected_parameter_value_pairs(cls.param_matrix) + cls.generated_parameter_value_pairs: List[ParameterValuePair] = ( + get_expected_parameter_value_pairs(cls.param_matrix) + ) def test_generator_without_custom_filter(self): comb_list = generate_combination_list(self.param_matrix) @@ -143,8 +145,8 @@ def custom_filter(row: ParameterValueTuple) -> bool: if device_compiler.name == NVCC: self.assertTrue( remove_parameter_value_pair( - OrderedDict( - {DEVICE_COMPILER: ParameterValue(NVCC, device_compiler.version)} + ParameterValueSingle( + DEVICE_COMPILER, ParameterValue(NVCC, device_compiler.version) ), reduced_expected_param_val_pairs, ) @@ -152,11 +154,13 @@ def custom_filter(row: ParameterValueTuple) -> bool: self.assertTrue( remove_parameter_value_pair( - OrderedDict( - { - CMAKE: ParameterValue(CMAKE, pkv.parse("3.23")), - BOOST: ParameterValue(BOOST, pkv.parse("1.82")), - }, + create_parameter_value_pair( + CMAKE, + CMAKE, + "3.23", + BOOST, + BOOST, + "1.82", ), reduced_expected_param_val_pairs, ) @@ -213,12 +217,7 @@ def custom_filter(row: ParameterValueTuple) -> bool: self.assertTrue( remove_parameter_value_pair( - OrderedDict( - { - CMAKE: ParameterValue(CMAKE, pkv.parse("3.23")), - BOOST: ParameterValue(BOOST, pkv.parse("1.82")), - }, - ), + create_parameter_value_pair(CMAKE, CMAKE, "3.23", BOOST, BOOST, "1.82"), reduced_expected_param_val_pairs, ) ) diff --git a/tests/utils_test.py b/tests/utils_test.py index 74182fb..f6753f4 100644 --- a/tests/utils_test.py +++ b/tests/utils_test.py @@ -9,6 +9,7 @@ ParameterValuePair, ValueName, ) +from bashi.utils import create_parameter_value_pair def parse_param_val(param_val: Tuple[ValueName, Union[str, int, float]]) -> ParameterValue: @@ -58,9 +59,24 @@ def parse_expected_val_pairs( expected_val_pairs: List[ParameterValuePair] = [] for param_val_pair in input_list: - tmp_entry: ParameterValuePair = OrderedDict() - for param in param_val_pair: - tmp_entry[param] = parse_param_val(param_val_pair[param]) - expected_val_pairs.append(tmp_entry) + if len(param_val_pair) != 2: + raise RuntimeError("input_list needs to have two entries") + + it = iter(param_val_pair.items()) + param1, param_val1 = next(it) + val_name1, val_version1 = param_val1 + param2, param_val2 = next(it) + val_name2, val_version2 = param_val2 + + expected_val_pairs.append( + create_parameter_value_pair( + param1, + val_name1, + val_version1, + param2, + val_name2, + val_version2, + ) + ) return expected_val_pairs From 5bb3749de1a4298d6bdc4af2266b5e9c702322b5 Mon Sep 17 00:00:00 2001 From: Simeon Ehrig Date: Wed, 7 Feb 2024 13:17:57 +0100 Subject: [PATCH 5/6] update black CI job --- .github/workflows/testDeploy.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testDeploy.yml b/.github/workflows/testDeploy.yml index 3988491..26f3e0d 100644 --- a/.github/workflows/testDeploy.yml +++ b/.github/workflows/testDeploy.yml @@ -6,9 +6,11 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: rickstaa/action-black@v1 + - uses: psf/black@stable with: - black_args: ". --check" + options: "--check --verbose" + src: "./" + version: "~= 24.0" unit-tests: runs-on: ubuntu-latest From b323da5d4cb8269b92907abb6fc715f58367b69f Mon Sep 17 00:00:00 2001 From: Simeon Ehrig Date: Wed, 7 Feb 2024 13:57:51 +0100 Subject: [PATCH 6/6] replace allpairspy with covertable library - Both library provides an pair-wise generator algorithm. - There is a know bug in the allpairspy library, which causes that not all valid parameter-value-pairs are generated, if filter rules are used. The bug exists since two years. --- bashi/generator.py | 34 +++++++++++--------- pyproject.toml | 2 +- tests/test_expected_parameter_value_pairs.py | 16 ++++----- tests/test_generate_combination_list.py | 19 +++-------- 4 files changed, 32 insertions(+), 39 deletions(-) diff --git a/bashi/generator.py b/bashi/generator.py index 9202ebf..0f3b55f 100644 --- a/bashi/generator.py +++ b/bashi/generator.py @@ -1,18 +1,19 @@ """Functions to generate the combination-list""" -from typing import Dict +from typing import Dict, List from collections import OrderedDict -from allpairspy import AllPairs +from covertable import make from bashi.types import ( Parameter, + ParameterValue, ParameterValueMatrix, FilterFunction, Combination, CombinationList, ) -from bashi.utils import get_default_filter_chain, FilterAdapter +from bashi.utils import get_default_filter_chain def generate_combination_list( @@ -32,20 +33,21 @@ def generate_combination_list( """ filter_chain = get_default_filter_chain(custom_filter) - param_map: Dict[int, Parameter] = {} - for index, key in enumerate(parameter_value_matrix.keys()): - param_map[index] = key - filter_adapter = FilterAdapter(param_map, filter_chain) - comb_list: CombinationList = [] - # convert List[Pair] to CombinationList - for all_pair in AllPairs( # type: ignore - parameters=parameter_value_matrix, n=2, filter_func=filter_adapter - ): - comb: Combination = OrderedDict() - for index, param in enumerate(all_pair._fields): # type: ignore - comb[param] = all_pair[index] # type: ignore - comb_list.append(comb) + all_pairs: List[Dict[Parameter, ParameterValue]] = make( + factors=parameter_value_matrix, + length=2, + pre_filter=filter_chain, + ) # type: ignore + + # convert List[Dict[Parameter, ParameterValue]] to CombinationList + for all_pair in all_pairs: + tmp_comb: Combination = OrderedDict() + # covertable does not keep the ordering of the parameters + # therefore we sort it + for param in parameter_value_matrix.keys(): + tmp_comb[param] = all_pair[param] + comb_list.append(tmp_comb) return comb_list diff --git a/pyproject.toml b/pyproject.toml index 5f61a27..22cf465 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ classifiers= [ "Operating System :: OS Independent", ] dependencies = [ - "allpairspy == 2.5.1", + "covertable == 2.1.0", "typeguard", "packaging" ] diff --git a/tests/test_expected_parameter_value_pairs.py b/tests/test_expected_parameter_value_pairs.py index 3b1b6d3..ec3cd29 100644 --- a/tests/test_expected_parameter_value_pairs.py +++ b/tests/test_expected_parameter_value_pairs.py @@ -5,9 +5,8 @@ import io import packaging.version as pkv -# allpairspy has no type hints -from allpairspy import AllPairs # type: ignore from utils_test import parse_param_val, parse_param_vals, parse_expected_val_pairs +from covertable import make from bashi.types import ( Parameter, ParameterValue, @@ -488,15 +487,16 @@ def test_check_parameter_value_pair_in_combination_list_complete_list_plus_wrong self.assertEqual(output_wrong_many_pairs_list, expected_output_many_wrong_pairs_list) - def test_unrestricted_allpairspy_generator(self): + def test_unrestricted_covertable_generator(self): comb_list: CombinationList = [] # pylance shows a warning, because it cannot determine the concrete type of a namedtuple, # which is returned by AllPairs - for all_pair in AllPairs(parameters=self.param_matrix): # type: ignore - comb: Combination = OrderedDict() - for index, param in enumerate(all_pair._fields): # type: ignore - comb[param] = all_pair[index] # type: ignore - comb_list.append(comb) + all_pairs: List[Dict[Parameter, ParameterValue]] = make( + factors=self.param_matrix + ) # type: ignore + + for all_pair in all_pairs: + comb_list.append(OrderedDict(all_pair)) self.assertTrue( check_parameter_value_pair_in_combination_list(comb_list, self.expected_param_val_pairs) diff --git a/tests/test_generate_combination_list.py b/tests/test_generate_combination_list.py index 988f7b5..29b3b66 100644 --- a/tests/test_generate_combination_list.py +++ b/tests/test_generate_combination_list.py @@ -50,9 +50,6 @@ def test_generator_without_custom_filter(self): ) ) - # TODO(SimeonEhrig): remove expectedFailure, if this PR was merged: - # https://github.com/thombashi/allpairspy/pull/10 - @unittest.expectedFailure def test_generator_with_custom_filter(self): def custom_filter(row: ParameterValueTuple) -> bool: if DEVICE_COMPILER in row and row[DEVICE_COMPILER].name == NVCC: @@ -168,7 +165,6 @@ def custom_filter(row: ParameterValueTuple) -> bool: missing_combinations = io.StringIO() - # because of a bug in the allpairspy library, valid pairs are missing. try: self.assertTrue( check_parameter_value_pair_in_combination_list( @@ -177,7 +173,7 @@ def custom_filter(row: ParameterValueTuple) -> bool: ) except AssertionError as e: # remove comment to print missing, valid pairs - # print(f"\n{missing_combinations.getvalue()}") + print(f"\n{missing_combinations.getvalue()}") raise e @@ -194,14 +190,8 @@ def test_generator_without_custom_filter(self): check_parameter_value_pair_in_combination_list(comb_list, expected_param_val_pairs) ) - # TODO(SimeonEhrig): remove expectedFailure, if this PR was merged: - # https://github.com/thombashi/allpairspy/pull/10 - @unittest.expectedFailure def test_generator_with_custom_filter(self): def custom_filter(row: ParameterValueTuple) -> bool: - if DEVICE_COMPILER in row and row[DEVICE_COMPILER].name == NVCC: - return False - if ( CMAKE in row and row[CMAKE].version == pkv.parse("3.23") @@ -236,7 +226,8 @@ def custom_filter(row: ParameterValueTuple) -> bool: ) except AssertionError as e: # remove comment to display missing combinations - # missing_combinations_str = missing_combinations.getvalue() - # print(f"\nnumber of missing combinations: {len(missing_combinations_str.split('\n'))}") - # print(f"\n{missing_combinations_str}") + missing_combinations_str = missing_combinations.getvalue() + print(f"\n{missing_combinations_str}") + number_of_combs = len(missing_combinations_str.split("\n")) + print(f"\nnumber of missing combinations: {number_of_combs}") raise e