diff --git a/docs/vspec2x.md b/docs/vspec2x.md index 1f6cb91d..c97511a3 100644 --- a/docs/vspec2x.md +++ b/docs/vspec2x.md @@ -12,7 +12,7 @@ The supported arguments might look like this ``` usage: vspec2x.py [-h] [-I dir] [-e EXTENDED_ATTRIBUTES] [-s] [--abort-on-unknown-attribute] [--abort-on-name-style] - [--format format] [--uuid] [--no_expand] [-o overlays] [-u unit_file] + [--format format] [--uuid] [--no_expand] [-o overlays] [-u unit_file] [-q quantity_file] [-vt vspec_types_file] [-ot ] [--json-all-extended-attributes] [--json-pretty] [--yaml-all-extended-attributes] [-v version] [--all-idl-features] [--gqlfield GQLFIELD GQLFIELD] @@ -35,7 +35,7 @@ All done. This assumes you checked out the [COVESA Vehicle Signal Specification](https://github.com/covesa/vehicle_signal_specification) which contains vss-tools including vspec2x as a submodule. -The `-I` parameter adds a directory to search for includes referenced in you `.vspec` files. `-I` can be used multiple times to specify more include directories. The `-u` parameter specifies the unit file to use. +The `-I` parameter adds a directory to search for includes referenced in you `.vspec` files. `-I` can be used multiple times to specify more include directories. The `-u` parameter specifies the unit file(s) to use. The `-q` parameter specifies quantity file(s) to use. The first positional argument - `../spec/VehicleSignalSpecification.vspec` in the example - gives the (root) `.vspec` file to be converted. The second positional argument - `vss.json` in the example - is the output file. @@ -201,6 +201,18 @@ When deciding which units to use the tooling use the following logic: See the [FAQ](../FAQ.md) for more information on how to define own units. +### Handling of quantities + +For units it is required to define `quantity`, previously called `domain`. +COVESA maintains a [quantity file](https://github.com/COVESA/vehicle_signal_specification/blob/master/spec/quantities.yaml) for the standard VSS catalog. + +When deciding which quantities to use the tooling use the following logic: + +* If `-q ` is used then the specified quantity files will be used. Default quantities will not be considered. +* If `-q` is not used the tool will check for a file called `quantities.yaml` in the same directory as the root `*.vspec` file. + +As of today use of quantity files is optional, and tooling will only give a warning if a unit use a quantity not specified in a quantity file. + ## Handling of overlays and extensions `vspec2x` allows composition of several overlays on top of a base vspec, to extend the model or overwrite certain metadata. Check [VSS documentation](https://covesa.github.io/vehicle_signal_specification/introduction/) on the concept of overlays. diff --git a/tests/model/test_contants.py b/tests/model/test_contants.py index 218764d0..ebe10ce4 100644 --- a/tests/model/test_contants.py +++ b/tests/model/test_contants.py @@ -9,7 +9,8 @@ import pytest import os -from vspec.model.constants import VSSType, VSSDataType, VSSUnitCollection, StringStyle, VSSTreeType, VSSUnit +from vspec.model.constants import VSSType, VSSDataType, VSSUnitCollection, StringStyle, VSSTreeType +from vspec.model.constants import VSSUnit, VSSQuantity @pytest.mark.parametrize("style_enum, style_str", @@ -132,3 +133,14 @@ def test_unit(): assert item.quantity == "myquantity" # String subclass so just comparing shall get "myid" assert item == "myid" + + +def test_quantity(): + """ Test Quantity class """ + item = VSSQuantity("myid", "mydefinition", "myremark", "mycomment") + assert item.value == "myid" + assert item.definition == "mydefinition" + assert item.remark == "myremark" + assert item.comment == "mycomment" + # String subclass so just comparing shall get "myid" + assert item == "myid" diff --git a/tests/vspec/test_units/quantities.yaml b/tests/vspec/test_units/quantities.yaml new file mode 100644 index 00000000..0ea9d0b6 --- /dev/null +++ b/tests/vspec/test_units/quantities.yaml @@ -0,0 +1,3 @@ +# Default file for testing +volume: + definition: Extent of a three‑dimensional geometrical shape (ISO 80000-3:2019) diff --git a/tests/vspec/test_units/quantity_volym.yaml b/tests/vspec/test_units/quantity_volym.yaml new file mode 100644 index 00000000..37885223 --- /dev/null +++ b/tests/vspec/test_units/quantity_volym.yaml @@ -0,0 +1,2 @@ +volym: + definition: Volume in swedish diff --git a/tests/vspec/test_units/test_units.py b/tests/vspec/test_units/test_units.py index 70da2af9..1e226880 100644 --- a/tests/vspec/test_units/test_units.py +++ b/tests/vspec/test_units/test_units.py @@ -10,6 +10,7 @@ import pytest import os +from typing import Optional # #################### Helper methods ############################# @@ -20,19 +21,33 @@ def change_test_dir(request, monkeypatch): monkeypatch.chdir(request.fspath.dirname) -def run_unit(vspec_file, unit_argument, expected_file): +def run_unit(vspec_file, unit_argument, expected_file, quantity_argument="", + grep_present: bool = True, grep_string: Optional[str] = None): test_str = "../../../vspec2json.py --json-pretty " + \ - vspec_file + " " + unit_argument + " out.json > out.txt 2>&1" + vspec_file + " " + unit_argument + " " + quantity_argument + " out.json > out.txt 2>&1" result = os.system(test_str) assert os.WIFEXITED(result) assert os.WEXITSTATUS(result) == 0 test_str = "diff out.json " + expected_file result = os.system(test_str) - os.system("rm -f out.json out.txt") + os.system("rm -f out.json") assert os.WIFEXITED(result) assert os.WEXITSTATUS(result) == 0 + # Verify expected quntity + + if grep_string is not None: + test_str = 'grep \"' + grep_string + '\" out.txt > /dev/null' + result = os.system(test_str) + assert os.WIFEXITED(result) + if grep_present: + assert os.WEXITSTATUS(result) == 0 + else: + assert os.WEXITSTATUS(result) == 1 + + os.system("rm -f out.txt") + def run_unit_error(vspec_file, unit_argument, grep_error): test_str = "../../../vspec2json.py --json-pretty " + \ @@ -106,3 +121,47 @@ def test_unit_error_missing_file(change_test_dir): def test_unit_on_branch(change_test_dir): run_unit_error("unit_on_branch.vspec", "-u units_all.yaml", "cannot have unit") + + +# Quantity tests +def test_implicit_quantity(change_test_dir): + run_unit( + "signals_with_special_units.vspec", + "--unit-file units_all.yaml", + "expected_special.json", + "", False, "has not been defined") + + +def test_explicit_quantity(change_test_dir): + run_unit( + "signals_with_special_units.vspec", + "--unit-file units_all.yaml", + "expected_special.json", + "-q quantities.yaml", False, "has not been defined") + + +def test_explicit_quantity_2(change_test_dir): + run_unit( + "signals_with_special_units.vspec", + "--unit-file units_all.yaml", + "expected_special.json", + "--quantity-file quantities.yaml", False, "has not been defined") + + +def test_explicit_quantity_warning(change_test_dir): + """ + We should get two warnings as the quantity file contain "volym", not "volume" + """ + run_unit( + "signals_with_special_units.vspec", + "--unit-file units_all.yaml", + "expected_special.json", + "-q quantity_volym.yaml", True, "Quantity volume used by unit puncheon has not been defined") + + +def test_quantity_redefinition(change_test_dir): + run_unit( + "signals_with_special_units.vspec", + "--unit-file units_all.yaml", + "expected_special.json", + "-q quantity_volym.yaml -q quantity_volym.yaml", True, "Redefinition of quantity volym") diff --git a/vspec/__init__.py b/vspec/__init__.py index 26ba1ec6..f90b838a 100755 --- a/vspec/__init__.py +++ b/vspec/__init__.py @@ -23,7 +23,7 @@ from .model.vsstree import VSSNode from .model.exceptions import ImpossibleMergeException, IncompleteElementException -from .model.constants import VSSTreeType, VSSUnitCollection +from .model.constants import VSSTreeType, VSSUnitCollection, VSSQuantityCollection nestable_types = set(["branch", "struct"]) @@ -863,6 +863,29 @@ def create_tree_uuids(root: VSSNode): namespace_uuid, vss_element.qualified_name()).hex +def load_quantities(vspec_file: str, quantity_files: List[str]): + + total_nbr_quantities = 0 + if not quantity_files: + # Search for a file quantities.yaml in same directory as vspec file + vspec_dir = os.path.dirname(os.path.realpath(vspec_file)) + default_vss_quantity_file = vspec_dir + os.path.sep + 'quantities.yaml' + if os.path.exists(default_vss_quantity_file): + total_nbr_quantities = VSSQuantityCollection.load_config_file(default_vss_quantity_file) + logging.info(f"Added {total_nbr_quantities} quantities from {default_vss_quantity_file}") + else: + for quantity_file in quantity_files: + nbr_quantities = VSSQuantityCollection.load_config_file(quantity_file) + if (nbr_quantities == 0): + logging.warning(f"Warning: No quantities found in {quantity_file}") + else: + logging.info(f"Added {nbr_quantities} quantities from {quantity_file}") + total_nbr_quantities += nbr_quantities + + if (total_nbr_quantities == 0): + logging.info("No quantities defined!") + + def load_units(vspec_file: str, unit_files: List[str]): total_nbr_units = 0 diff --git a/vspec/model/constants.py b/vspec/model/constants.py index 89e6cbf0..ac588fc7 100644 --- a/vspec/model/constants.py +++ b/vspec/model/constants.py @@ -12,6 +12,7 @@ # Constant Types and Mappings # # noinspection PyPackageRequirements +from __future__ import annotations import re import logging import sys @@ -37,7 +38,7 @@ class VSSUnit(str): quantity: Optional[str] = None # Typically quantity, like "Voltage" def __new__(cls, id: str, unit: Optional[str] = None, definition: Optional[str] = None, - quantity: Optional[str] = None) -> 'VSSUnit': + quantity: Optional[str] = None) -> VSSUnit: self = super().__new__(cls, id) self.id = id self.unit = unit @@ -50,6 +51,28 @@ def value(self): return self +class VSSQuantity(str): + """String subclass for storing quantity information. + """ + id: str # Identifier preferably taken from a standard, like ISO 80000 + definition: str # Explanation of quantity, for example reference to standard + remark: Optional[str] = None # remark as defined in for example ISO 80000 + comment: Optional[str] = None + + def __new__(cls, id: str, definition: str, remark: Optional[str] = None, + comment: Optional[str] = None) -> VSSQuantity: + self = super().__new__(cls, id) + self.id = id + self.definition = definition + self.remark = remark + self.comment = comment + return self + + @property + def value(self): + return self + + class EnumMetaWithReverseLookup(EnumMeta): """This class extends EnumMeta and adds: - from_str(str): reverse lookup @@ -169,7 +192,12 @@ def load_config_file(cls, config_file: str) -> int: logging.error("No quantity (domain) found for unit %s", k) sys.exit(-1) + if VSSQuantityCollection.get_quantity(quantity) is None: + logging.info("Quantity %s used by unit %s has not been defined", quantity, k) + unit_node = VSSUnit(k, unit, definition, quantity) + if k in cls.units: + logging.warning("Redefinition of unit %s", k) cls.units[k] = unit_node return added_configs @@ -181,6 +209,43 @@ def get_unit(cls, id: str) -> Optional[VSSUnit]: return None +class VSSQuantityCollection(): + + quantities: Dict[str, VSSQuantity] = dict() + + @classmethod + def load_config_file(cls, config_file: str) -> int: + added_quantities = 0 + with open(config_file) as my_yaml_file: + my_quantities = yaml.safe_load(my_yaml_file) + added_quantities = len(my_quantities) + for k, v in my_quantities.items(): + if "definition" in v: + definition = v["definition"] + else: + logging.error("No definition found for quantity %s", k) + sys.exit(-1) + remark = None + if "remark" in v: + remark = v["remark"] + comment = None + if "comment" in v: + comment = v["comment"] + + quantity_node = VSSQuantity(k, definition, remark, comment) + if k in cls.quantities: + logging.warning("Redefinition of quantity %s", k) + cls.quantities[k] = quantity_node + return added_quantities + + @classmethod + def get_quantity(cls, id: str) -> Optional[VSSQuantity]: + if id in cls.quantities: + return cls.quantities[id] + else: + return None + + class VSSTreeType(Enum, metaclass=EnumMetaWithReverseLookup): SIGNAL_TREE = "signal_tree" DATA_TYPE_TREE = "data_type_tree" diff --git a/vspec2x.py b/vspec2x.py index 07e19a85..64fa10a3 100755 --- a/vspec2x.py +++ b/vspec2x.py @@ -87,6 +87,8 @@ def main(arguments): help='Add overlay that will be layered on top of the VSS file in the order they appear.') parser.add_argument('-u', '--unit-file', action='append', metavar='unit_file', type=str, default=[], help='Unit file to be used for generation. Argument -u may be used multiple times.') + parser.add_argument('-q', '--quantity-file', action='append', metavar='quantity_file', type=str, default=[], + help='Quantity file to be used for generation. Argument -uqmay be used multiple times.') parser.add_argument('vspec_file', metavar='', help='The vehicle specification file to convert.') parser.add_argument('output_file', metavar='', @@ -152,6 +154,7 @@ def main(arguments): if args.uuid: print_uuid = True + vspec.load_quantities(args.vspec_file, args.quantity_file) vspec.load_units(args.vspec_file, args.unit_file) # Warn if unsupported feature is used