Adding tool support for quantities

docs/vspec2x.md

@@ -12,7 +12,7 @@ The supported arguments might look like this
 
  ```
 usage: vspec2x.py [-h] [-I dir] [-e EXTENDED_ATTRIBUTES] [-s] [--abort-on-unknown-attribute] [--abort-on-name-style]
-                  [--format format] [--uuid] [--no_expand] [-o overlays] [-u unit_file]
+                  [--format format] [--uuid] [--no_expand] [-o overlays] [-u unit_file] [-q quantity_file]
                   [-vt vspec_types_file] [-ot <types_output_file>]
                   [--json-all-extended-attributes] [--json-pretty]
                   [--yaml-all-extended-attributes] [-v version] [--all-idl-features] [--gqlfield GQLFIELD GQLFIELD]
@@ -35,7 +35,7 @@ All done.
 
 This assumes you checked out the [COVESA Vehicle Signal Specification](https://github.com/covesa/vehicle_signal_specification) which contains vss-tools including vspec2x as a submodule.
 
-The `-I` parameter adds a directory to search for includes referenced in you `.vspec` files. `-I` can be used multiple times to specify more include directories. The `-u` parameter specifies the unit file to use.
+The `-I` parameter adds a directory to search for includes referenced in you `.vspec` files. `-I` can be used multiple times to specify more include directories. The `-u` parameter specifies the unit file(s) to use. The `-q` parameter specifies quantity file(s) to use.
 
 The first positional argument - `../spec/VehicleSignalSpecification.vspec` in the example  - gives the (root) `.vspec` file to be converted. The second positional argument  - `vss.json` in the example - is the output file.
 
@@ -201,6 +201,18 @@ When deciding which units to use the tooling use the following logic:
 
 See the [FAQ](../FAQ.md) for more information on how to define own units.
 
+### Handling of quantities
+
+For units it is required to define `quantity`, previously called `domain`.
+COVESA maintains a [quantity file](https://github.com/COVESA/vehicle_signal_specification/blob/master/spec/quantities.yaml) for the standard VSS catalog.
+
+When deciding which quantities to use the tooling use the following logic:
+
+* If `-q <file>` is used then the specified quantity files will be used. Default quantities will not be considered.
+* If `-q` is not used the tool will check for a file called `quantities.yaml` in the same directory as the root `*.vspec` file.
+
+As of today use of quantity files is optional, and tooling will only give a warning if a unit use a quantity not specified in a quantity file.
+
 ## Handling of overlays and extensions
 `vspec2x` allows composition of several overlays on top of a base vspec, to extend the model or overwrite certain metadata. Check [VSS documentation](https://covesa.github.io/vehicle_signal_specification/introduction/) on the concept of overlays.
 

tests/model/test_contants.py

@@ -9,7 +9,8 @@
 import pytest
 import os
 
-from vspec.model.constants import VSSType, VSSDataType, VSSUnitCollection, StringStyle, VSSTreeType, VSSUnit
+from vspec.model.constants import VSSType, VSSDataType, VSSUnitCollection, StringStyle, VSSTreeType
+from vspec.model.constants import VSSUnit, VSSQuantity
 
 
 @pytest.mark.parametrize("style_enum, style_str",
@@ -132,3 +133,14 @@ def test_unit():
     assert item.quantity == "myquantity"
     # String subclass so just comparing shall get "myid"
     assert item == "myid"
+
+
+def test_quantity():
+    """ Test Quantity class """
+    item = VSSQuantity("myid", "mydefinition", "myremark", "mycomment")
+    assert item.value == "myid"
+    assert item.definition == "mydefinition"
+    assert item.remark == "myremark"
+    assert item.comment == "mycomment"
+    # String subclass so just comparing shall get "myid"
+    assert item == "myid"

tests/vspec/test_units/quantities.yaml

@@ -0,0 +1,3 @@
+# Default file for testing
+volume:
+  definition: Extent of a three‑dimensional geometrical shape (ISO 80000-3:2019)

tests/vspec/test_units/quantity_volym.yaml

@@ -0,0 +1,2 @@
+volym:
+  definition: Volume in swedish

tests/vspec/test_units/test_units.py

@@ -10,6 +10,7 @@
 
 import pytest
 import os
+from typing import Optional
 
 
 # #################### Helper methods #############################
@@ -20,19 +21,33 @@ def change_test_dir(request, monkeypatch):
     monkeypatch.chdir(request.fspath.dirname)
 
 
-def run_unit(vspec_file, unit_argument, expected_file):
+def run_unit(vspec_file, unit_argument, expected_file, quantity_argument="",
+             grep_present: bool = True, grep_string: Optional[str] = None):
     test_str = "../../../vspec2json.py --json-pretty " + \
-        vspec_file + " " + unit_argument + " out.json > out.txt 2>&1"
+        vspec_file + " " + unit_argument + " " + quantity_argument + " out.json > out.txt 2>&1"
     result = os.system(test_str)
     assert os.WIFEXITED(result)
     assert os.WEXITSTATUS(result) == 0
 
     test_str = "diff out.json " + expected_file
     result = os.system(test_str)
-    os.system("rm -f out.json out.txt")
+    os.system("rm -f out.json")
     assert os.WIFEXITED(result)
     assert os.WEXITSTATUS(result) == 0
 
+    # Verify expected quntity
+
+    if grep_string is not None:
+        test_str = 'grep \"' + grep_string + '\" out.txt > /dev/null'
+        result = os.system(test_str)
+        assert os.WIFEXITED(result)
+        if grep_present:
+            assert os.WEXITSTATUS(result) == 0
+        else:
+            assert os.WEXITSTATUS(result) == 1
+
+    os.system("rm -f out.txt")
+
 
 def run_unit_error(vspec_file, unit_argument, grep_error):
     test_str = "../../../vspec2json.py --json-pretty " + \
@@ -106,3 +121,47 @@ def test_unit_error_missing_file(change_test_dir):
 
 def test_unit_on_branch(change_test_dir):
     run_unit_error("unit_on_branch.vspec", "-u units_all.yaml", "cannot have unit")
+
+
+# Quantity tests
+def test_implicit_quantity(change_test_dir):
+    run_unit(
+        "signals_with_special_units.vspec",
+        "--unit-file units_all.yaml",
+        "expected_special.json",
+        "", False, "has not been defined")
+
+
+def test_explicit_quantity(change_test_dir):
+    run_unit(
+        "signals_with_special_units.vspec",
+        "--unit-file units_all.yaml",
+        "expected_special.json",
+        "-q quantities.yaml", False, "has not been defined")
+
+
+def test_explicit_quantity_2(change_test_dir):
+    run_unit(
+        "signals_with_special_units.vspec",
+        "--unit-file units_all.yaml",
+        "expected_special.json",
+        "--quantity-file quantities.yaml", False, "has not been defined")
+
+
+def test_explicit_quantity_warning(change_test_dir):
+    """
+    We should get two warnings as the quantity file contain "volym", not "volume"
+    """
+    run_unit(
+        "signals_with_special_units.vspec",
+        "--unit-file units_all.yaml",
+        "expected_special.json",
+        "-q quantity_volym.yaml", True, "Quantity volume used by unit puncheon has not been defined")
+
+
+def test_quantity_redefinition(change_test_dir):
+    run_unit(
+        "signals_with_special_units.vspec",
+        "--unit-file units_all.yaml",
+        "expected_special.json",
+        "-q quantity_volym.yaml -q quantity_volym.yaml", True, "Redefinition of quantity volym")

vspec/__init__.py

@@ -23,7 +23,7 @@
 
 from .model.vsstree import VSSNode
 from .model.exceptions import ImpossibleMergeException, IncompleteElementException
-from .model.constants import VSSTreeType, VSSUnitCollection
+from .model.constants import VSSTreeType, VSSUnitCollection, VSSQuantityCollection
 
 nestable_types = set(["branch", "struct"])
 
@@ -863,6 +863,29 @@ def create_tree_uuids(root: VSSNode):
             namespace_uuid, vss_element.qualified_name()).hex
 
 
+def load_quantities(vspec_file: str, quantity_files: List[str]):
+
+    total_nbr_quantities = 0
+    if not quantity_files:
+        # Search for a file quantities.yaml in same directory as vspec file
+        vspec_dir = os.path.dirname(os.path.realpath(vspec_file))
+        default_vss_quantity_file = vspec_dir + os.path.sep + 'quantities.yaml'
+        if os.path.exists(default_vss_quantity_file):
+            total_nbr_quantities = VSSQuantityCollection.load_config_file(default_vss_quantity_file)
+            logging.info(f"Added {total_nbr_quantities} quantities from {default_vss_quantity_file}")
+    else:
+        for quantity_file in quantity_files:
+            nbr_quantities = VSSQuantityCollection.load_config_file(quantity_file)
+            if (nbr_quantities == 0):
+                logging.warning(f"Warning: No quantities found in {quantity_file}")
+            else:
+                logging.info(f"Added {nbr_quantities} quantities from {quantity_file}")
+                total_nbr_quantities += nbr_quantities
+
+    if (total_nbr_quantities == 0):
+        logging.info("No quantities defined!")
+
+
 def load_units(vspec_file: str, unit_files: List[str]):
 
     total_nbr_units = 0

vspec/model/constants.py

@@ -12,6 +12,7 @@
 # Constant Types and Mappings
 #
 # noinspection PyPackageRequirements
+from __future__ import annotations
 import re
 import logging
 import sys
@@ -37,7 +38,7 @@ class VSSUnit(str):
     quantity: Optional[str] = None  # Typically quantity, like "Voltage"
 
     def __new__(cls, id: str, unit: Optional[str] = None, definition: Optional[str] = None,
-                quantity: Optional[str] = None) -> 'VSSUnit':
+                quantity: Optional[str] = None) -> VSSUnit:
         self = super().__new__(cls, id)
         self.id = id
         self.unit = unit
@@ -50,6 +51,28 @@ def value(self):
         return self
 
 
+class VSSQuantity(str):
+    """String subclass for storing quantity information.
+    """
+    id: str  # Identifier preferably taken from a standard, like ISO 80000
+    definition: str  # Explanation of quantity, for example reference to standard
+    remark: Optional[str] = None  # remark as defined in for example ISO 80000
+    comment: Optional[str] = None
+
+    def __new__(cls, id: str, definition: str, remark: Optional[str] = None,
+                comment: Optional[str] = None) -> VSSQuantity:
+        self = super().__new__(cls, id)
+        self.id = id
+        self.definition = definition
+        self.remark = remark
+        self.comment = comment
+        return self
+
+    @property
+    def value(self):
+        return self
+
+
 class EnumMetaWithReverseLookup(EnumMeta):
     """This class extends EnumMeta and adds:
      - from_str(str): reverse lookup
@@ -169,7 +192,12 @@ def load_config_file(cls, config_file: str) -> int:
                     logging.error("No quantity (domain) found for unit %s", k)
                     sys.exit(-1)
 
+                if VSSQuantityCollection.get_quantity(quantity) is None:
+                    logging.info("Quantity %s used by unit %s has not been defined", quantity, k)
+
                 unit_node = VSSUnit(k, unit, definition, quantity)
+                if k in cls.units:
+                    logging.warning("Redefinition of unit %s", k)
                 cls.units[k] = unit_node
         return added_configs
 
@@ -181,6 +209,43 @@ def get_unit(cls, id: str) -> Optional[VSSUnit]:
             return None
 
 
+class VSSQuantityCollection():
+
+    quantities: Dict[str, VSSQuantity] = dict()
+
+    @classmethod
+    def load_config_file(cls, config_file: str) -> int:
+        added_quantities = 0
+        with open(config_file) as my_yaml_file:
+            my_quantities = yaml.safe_load(my_yaml_file)
+            added_quantities = len(my_quantities)
+            for k, v in my_quantities.items():
+                if "definition" in v:
+                    definition = v["definition"]
+                else:
+                    logging.error("No definition found for quantity %s", k)
+                    sys.exit(-1)
+                remark = None
+                if "remark" in v:
+                    remark = v["remark"]
+                comment = None
+                if "comment" in v:
+                    comment = v["comment"]
+
+                quantity_node = VSSQuantity(k, definition, remark, comment)
+                if k in cls.quantities:
+                    logging.warning("Redefinition of quantity %s", k)
+                cls.quantities[k] = quantity_node
+        return added_quantities
+
+    @classmethod
+    def get_quantity(cls, id: str) -> Optional[VSSQuantity]:
+        if id in cls.quantities:
+            return cls.quantities[id]
+        else:
+            return None
+
+
 class VSSTreeType(Enum, metaclass=EnumMetaWithReverseLookup):
     SIGNAL_TREE = "signal_tree"
     DATA_TYPE_TREE = "data_type_tree"

vspec2x.py

@@ -87,6 +87,8 @@ def main(arguments):
                         help='Add overlay that will be layered on top of the VSS file in the order they appear.')
     parser.add_argument('-u', '--unit-file', action='append', metavar='unit_file', type=str, default=[],
                         help='Unit file to be used for generation. Argument -u may be used multiple times.')
+    parser.add_argument('-q', '--quantity-file', action='append', metavar='quantity_file', type=str, default=[],
+                        help='Quantity file to be used for generation. Argument -uqmay be used multiple times.')
     parser.add_argument('vspec_file', metavar='<vspec_file>',
                         help='The vehicle specification file to convert.')
     parser.add_argument('output_file', metavar='<output_file>',
@@ -152,6 +154,7 @@ def main(arguments):
     if args.uuid:
         print_uuid = True
 
+    vspec.load_quantities(args.vspec_file, args.quantity_file)
     vspec.load_units(args.vspec_file, args.unit_file)
 
     # Warn if unsupported feature is used