cim2pp: extracted getting default classes, added generic setting data…

…types from CGMES XMI schema (#2172)

* cim2pp: extracted getting default classes, added generic setting datatypes from CGMES XMI schema

* cim2pp: extracted getting default classes, added generic setting datatypes from CGMES XMI schema

CHANGELOG.rst

@@ -34,6 +34,7 @@ Change Log
 - [ADDED] option to use a second tap changer for the trafo element
 - [CHANGED] parameters of function merge_internal_net_and_equivalent_external_net()
 - [FIXED] :code:`convert_format.py`: update the attributes of the characteristic objects to match the new characteristic
+- [CHANGED] cim2pp: extracted getting default classes, added generic setting datatypes from CGMES XMI schema
 
 
 [2.13.1] - 2023-05-12

pandapower/converter/cim/cim2pp/build_pp_net.py

@@ -92,34 +92,51 @@ def convert_to_pp(self, convert_line_to_switch: bool = False, line_r_limit: floa
                       sort=False, ignore_index=True)[['rdfId', 'nominalVoltage']]
 
         # --------- convert busses ---------
-        self.classes_dict['connectivityNodesCim16'].ConnectivityNodesCim16(cimConverter=self).convert_connectivity_nodes_cim16()
+        self.classes_dict['connectivityNodesCim16'].ConnectivityNodesCim16(
+            cimConverter=self).convert_connectivity_nodes_cim16()
         # --------- convert external networks ---------
-        self.classes_dict['externalNetworkInjectionsCim16'].ExternalNetworkInjectionsCim16(cimConverter=self).convert_external_network_injections_cim16()
+        self.classes_dict['externalNetworkInjectionsCim16'].ExternalNetworkInjectionsCim16(
+            cimConverter=self).convert_external_network_injections_cim16()
         # --------- convert lines ---------
-        self.classes_dict['acLineSegmentsCim16'].AcLineSegmentsCim16(cimConverter=self).convert_ac_line_segments_cim16(convert_line_to_switch, line_r_limit, line_x_limit)
-        self.classes_dict['dcLineSegmentsCim16'].DcLineSegmentsCim16(cimConverter=self).convert_dc_line_segments_cim16()
+        self.classes_dict['acLineSegmentsCim16'].AcLineSegmentsCim16(
+            cimConverter=self).convert_ac_line_segments_cim16(convert_line_to_switch, line_r_limit, line_x_limit)
+        self.classes_dict['dcLineSegmentsCim16'].DcLineSegmentsCim16(
+            cimConverter=self).convert_dc_line_segments_cim16()
         # --------- convert switches ---------
         self.classes_dict['switchesCim16'].SwitchesCim16(cimConverter=self).convert_switches_cim16()
         # --------- convert loads ---------
-        self.classes_dict['energyConcumersCim16'].EnergyConsumersCim16(cimConverter=self).convert_energy_consumers_cim16()
+        self.classes_dict['energyConcumersCim16'].EnergyConsumersCim16(
+            cimConverter=self).convert_energy_consumers_cim16()
         self.classes_dict['conformLoadsCim16'].ConformLoadsCim16(cimConverter=self).convert_conform_loads_cim16()
-        self.classes_dict['nonConformLoadsCim16'].NonConformLoadsCim16(cimConverter=self).convert_non_conform_loads_cim16()
-        self.classes_dict['stationSuppliesCim16'].StationSuppliesCim16(cimConverter=self).convert_station_supplies_cim16()
+        self.classes_dict['nonConformLoadsCim16'].NonConformLoadsCim16(
+            cimConverter=self).convert_non_conform_loads_cim16()
+        self.classes_dict['stationSuppliesCim16'].StationSuppliesCim16(
+            cimConverter=self).convert_station_supplies_cim16()
         # --------- convert generators ---------
-        self.classes_dict['synchronousMachinesCim16'].SynchronousMachinesCim16(cimConverter=self).convert_synchronous_machines_cim16()
-        self.classes_dict['asynchronousMachinesCim16'].AsynchronousMachinesCim16(cimConverter=self).convert_asynchronous_machines_cim16()
-        self.classes_dict['energySourcesCim16'].EnergySourceCim16(cimConverter=self).convert_energy_sources_cim16()
+        self.classes_dict['synchronousMachinesCim16'].SynchronousMachinesCim16(
+            cimConverter=self).convert_synchronous_machines_cim16()
+        self.classes_dict['asynchronousMachinesCim16'].AsynchronousMachinesCim16(
+            cimConverter=self).convert_asynchronous_machines_cim16()
+        self.classes_dict['energySourcesCim16'].EnergySourceCim16(
+            cimConverter=self).convert_energy_sources_cim16()
         # --------- convert shunt elements ---------
-        self.classes_dict['linearShuntCompensatorCim16'].LinearShuntCompensatorCim16(cimConverter=self).convert_linear_shunt_compensator_cim16()
-        self.classes_dict['nonLinearShuntCompensatorCim16'].NonLinearShuntCompensatorCim16(cimConverter=self).convert_nonlinear_shunt_compensator_cim16()
-        self.classes_dict['staticVarCompensatorCim16'].StaticVarCompensatorCim16(cimConverter=self).convert_static_var_compensator_cim16()
+        self.classes_dict['linearShuntCompensatorCim16'].LinearShuntCompensatorCim16(
+            cimConverter=self).convert_linear_shunt_compensator_cim16()
+        self.classes_dict['nonLinearShuntCompensatorCim16'].NonLinearShuntCompensatorCim16(
+            cimConverter=self).convert_nonlinear_shunt_compensator_cim16()
+        self.classes_dict['staticVarCompensatorCim16'].StaticVarCompensatorCim16(
+            cimConverter=self).convert_static_var_compensator_cim16()
         # --------- convert impedance elements ---------
-        self.classes_dict['equivalentBranchesCim16'].EquivalentBranchesCim16(cimConverter=self).convert_equivalent_branches_cim16()
-        self.classes_dict['seriesCompensatorsCim16'].SeriesCompensatorsCim16(cimConverter=self).convert_series_compensators_cim16()
+        self.classes_dict['equivalentBranchesCim16'].EquivalentBranchesCim16(
+            cimConverter=self).convert_equivalent_branches_cim16()
+        self.classes_dict['seriesCompensatorsCim16'].SeriesCompensatorsCim16(
+            cimConverter=self).convert_series_compensators_cim16()
         # --------- convert extended ward and ward elements ---------
-        self.classes_dict['equivalentInjectionsCim16'].EquivalentInjectionsCim16(cimConverter=self).convert_equivalent_injections_cim16()
+        self.classes_dict['equivalentInjectionsCim16'].EquivalentInjectionsCim16(
+            cimConverter=self).convert_equivalent_injections_cim16()
         # --------- convert transformers ---------
-        self.classes_dict['powerTransformersCim16'].PowerTransformersCim16(cimConverter=self).convert_power_transformers_cim16()
+        self.classes_dict['powerTransformersCim16'].PowerTransformersCim16(
+            cimConverter=self).convert_power_transformers_cim16()
 
         # create the geo coordinates
         gl_or_dl = str(self.kwargs.get('use_GL_or_DL_profile', 'both')).lower()
@@ -135,7 +152,8 @@ def convert_to_pp(self, convert_line_to_switch: bool = False, line_r_limit: floa
         if self.cim['gl']['Location'].index.size > 0 and self.cim['gl']['PositionPoint'].index.size > 0 and \
                 use_gl_profile:
             try:
-                self.classes_dict['geoCoordinatesFromGLCim16'].GeoCoordinatesFromGLCim16(cimConverter=self).add_geo_coordinates_from_gl_cim16()
+                self.classes_dict['geoCoordinatesFromGLCim16'].GeoCoordinatesFromGLCim16(
+                    cimConverter=self).add_geo_coordinates_from_gl_cim16()
             except Exception as e:
                 self.logger.warning("Creating the geo coordinates failed, returning the net without geo coordinates!")
                 self.logger.exception(e)
@@ -151,7 +169,8 @@ def convert_to_pp(self, convert_line_to_switch: bool = False, line_r_limit: floa
                 self.cim['dl']['DiagramObjectPoint'].index.size > 0 and self.net.bus_geodata.index.size == 0 and \
                 use_dl_profile:
             try:
-                self.classes_dict['coordinatesFromDLCim16'].CoordinatesFromDLCim16(cimConverter=self).add_coordinates_from_dl_cim16(diagram_name=kwargs.get('diagram_name', None))
+                self.classes_dict['coordinatesFromDLCim16'].CoordinatesFromDLCim16(
+                    cimConverter=self).add_coordinates_from_dl_cim16(diagram_name=kwargs.get('diagram_name', None))
             except Exception as e:
                 self.logger.warning("Creating the coordinates failed, returning the net without coordinates!")
                 self.logger.exception(e)
@@ -165,7 +184,8 @@ def convert_to_pp(self, convert_line_to_switch: bool = False, line_r_limit: floa
         self.net = pp_tools.set_pp_col_types(net=self.net)
 
         # create transformer tap controller
-        self.classes_dict['tapController'].TapController(cimConverter=self).create_tap_controller_for_power_transformers()
+        self.classes_dict['tapController'].TapController(
+            cimConverter=self).create_tap_controller_for_power_transformers()
 
         self.logger.info("Running a power flow.")
         self.report_container.add_log(Report(
@@ -211,21 +231,6 @@ def convert_to_pp(self, convert_line_to_switch: bool = False, line_r_limit: floa
             self.net.measurement = self.net.measurement[0:0]
             if not kwargs.get('ignore_errors', True):
                 raise e
-        try:
-            # TODO: think on whether to remove whole function
-            if kwargs.get('update_assets_from_sv', False):
-                CreateMeasurements(self.net, self.cim).update_assets_from_sv()
-        except Exception as e:
-            self.logger.warning("Updating the assets failed!")
-            self.logger.exception(e)
-            self.report_container.add_log(Report(
-                level=LogLevel.ERROR, code=ReportCode.ERROR_CONVERTING,
-                message="Updating the assets failed!"))
-            self.report_container.add_log(Report(
-                level=LogLevel.EXCEPTION, code=ReportCode.EXCEPTION_CONVERTING,
-                message=traceback.format_exc()))
-            if not kwargs.get('ignore_errors', True):
-                raise e
         # a special fix for BB and NB mixed networks:
         # fuse boundary ConnectivityNodes with their TopologicalNodes
         bus_t = self.net.bus.reset_index(level=0, drop=False)

pandapower/converter/cim/cim2pp/convert_measurements.py

@@ -34,46 +34,6 @@ def _copy_to_measurement(self, input_df: pd.DataFrame):
             if one_attr in input_df.columns:
                 self.net[pp_type][one_attr][start_index_pp_net:] = input_df[one_attr][:]
 
-    def update_assets_from_sv(self):
-        self.logger.info("--------------------------- Updating assets from SV ---------------------------")
-        time_start = time.time()
-        sc = cim_tools.get_pp_net_special_columns_dict()
-        # get the measurements from the sv profile and set the Terminal as index
-        sv_powerflow = self.cim['sv']['SvPowerFlow'][['Terminal', 'p', 'q']]
-
-        # update sgen
-        temp = self.net.sgen[[sc['o_id'], sc['t'], 'p_mw', 'q_mvar', 'sn_mva']]
-        temp = pd.merge(temp, sv_powerflow, how='left', left_on=sc['t'], right_on='Terminal')
-        # fix load sign
-        temp['p'] = -temp['p']
-        temp['q'] = -temp['q']
-        temp['p'].fillna(temp['p_mw'], inplace=True)
-        temp['q'].fillna(temp['q_mvar'], inplace=True)
-        temp['new_sn_mva'] = (temp['p'] ** 2 + temp['q'] ** 2) ** .5
-        self.net.sgen.p_mw = temp['p'][:]
-        self.net.sgen.q_mvar = temp['q'][:]
-        self.net.sgen.sn_mva = temp[["new_sn_mva", "sn_mva"]].max(axis=1)
-
-        # update load
-        temp = self.net.load[[sc['o_id'], sc['t'], 'p_mw', 'q_mvar']]
-        temp = pd.merge(temp, sv_powerflow, how='left', left_on=sc['t'], right_on='Terminal')
-        temp['p'].fillna(temp['p_mw'], inplace=True)
-        temp['q'].fillna(temp['q_mvar'], inplace=True)
-        self.net.load.p_mw = temp['p'][:]
-        self.net.load.q_mvar = temp['q'][:]
-
-        # update ward
-        temp = self.net.ward[[sc['o_id'], sc['t'], 'ps_mw', 'qs_mvar', 'pz_mw', 'qz_mvar']]
-        temp = pd.merge(temp, sv_powerflow, how='left', left_on=sc['t'], right_on='Terminal')
-        temp['p'].fillna(temp['ps_mw'], inplace=True)
-        temp['q'].fillna(temp['qs_mvar'], inplace=True)
-        self.net.ward.ps_mw = temp['p'][:]
-        self.net.ward.qs_mvar = temp['q'][:]
-        self.net.ward.pz_mw = temp['p'][:]
-        self.net.ward.qz_mvar = temp['q'][:]
-
-        self.logger.info("Needed time for updating the assets: %ss" % (time.time() - time_start))
-
     def create_measurements_from_analog(self):
         self.logger.info("------------------------- Creating measurements from Analog -------------------------")
         time_start = time.time()

pandapower/converter/cim/cim2pp/converter_classes/coordinates/coordinatesFromDLCim16.py

@@ -38,6 +38,9 @@ def add_coordinates_from_dl_cim16(self, diagram_name: str = None):
             dl_do.rename(columns={'rdfId': 'DiagramObject'}, inplace=True)
         dl_data = pd.merge(dl_do, self.cimConverter.cim['dl']['DiagramObjectPoint'], how='left', on='DiagramObject')
         dl_data.drop(columns=['rdfId', 'Diagram', 'DiagramObject'], inplace=True)
+        # make sure that the columns 'xPosition' and 'yPosition' are floats
+        dl_data['xPosition'] = dl_data['xPosition'].astype(float)
+        dl_data['yPosition'] = dl_data['yPosition'].astype(float)
         # the coordinates for the buses
         buses = self.cimConverter.net.bus.reset_index()
         buses = buses[['index', sc['o_id']]]

pandapower/converter/cim/cim2pp/converter_classes/coordinates/geoCoordinatesFromGLCim16.py

@@ -26,6 +26,9 @@ def add_geo_coordinates_from_gl_cim16(self):
             self.cimConverter.cim['gl']['Location'][['rdfId', 'PowerSystemResources']], how='left',
             left_on='Location', right_on='rdfId')
         gl_data.drop(columns=['Location', 'rdfId'], inplace=True)
+        # make sure that the columns 'xPosition' and 'yPosition' are floats
+        gl_data['xPosition'] = gl_data['xPosition'].astype(float)
+        gl_data['yPosition'] = gl_data['yPosition'].astype(float)
         bus_geo = gl_data.rename(columns={'PowerSystemResources': 'Substation'})
         cn = self.cimConverter.cim['eq']['ConnectivityNode'][['rdfId', 'ConnectivityNodeContainer']]
         cn = pd.concat([cn, self.cimConverter.cim['eq_bd']['ConnectivityNode'][['rdfId', 'ConnectivityNodeContainer']]])

pandapower/converter/cim/cim2pp/from_cim.py

@@ -9,6 +9,7 @@
 from . import build_pp_net
 from .. import cim_classes
 from .. import interfaces
+from . import converter_classes as std_converter_classes
 
 logger = logging.getLogger('cim.cim2pp.from_cim')
 
@@ -38,8 +39,30 @@ def from_cim_dict(cim_parser: cim_classes.CimParser, log_debug=False, convert_li
     :param custom_converter_classes: Dict to inject classes for different functionality. Optional, default: None
     :return: The pandapower net.
     """
-    from . import converter_classes as std_converter_classes
-    converter_classes: dict = {
+    converter_classes = get_converter_classes()
+    if custom_converter_classes is not None:
+        for key in custom_converter_classes:
+            converter_classes[key] = custom_converter_classes.get(key)
+
+    # repair the input CIM data
+    if repair_cim is not None and repair_cim_class is not None:
+        repair_cim = repair_cim_class().deserialize(repair_cim, report_container=cim_parser.get_report_container())
+        repair_cim.repair(cim_parser.get_cim_dict(), report_container=cim_parser.get_report_container())
+
+    cim_converter = build_pp_net.CimConverter(cim_parser=cim_parser, converter_classes=converter_classes, **kwargs)
+    pp_net = cim_converter.convert_to_pp(convert_line_to_switch=convert_line_to_switch, line_r_limit=line_r_limit,
+                                         line_x_limit=line_x_limit, log_debug=log_debug, **kwargs)
+
+    # repair the output pandapower network
+    if repair_pp is not None and repair_pp_class is not None:
+        repair_pp = repair_pp_class().deserialize(repair_pp, report_container=cim_parser.get_report_container())
+        repair_pp.repair(pp_net, report_container=cim_parser.get_report_container())
+
+    return pp_net
+
+
+def get_converter_classes():
+    converter_classes: Dict[str,classmethod] = {
         'connectivityNodesCim16': std_converter_classes.connectivitynodes.connectivityNodesCim16,
         'externalNetworkInjectionsCim16': std_converter_classes.externalnetworks.externalNetworkInjectionsCim16,
         'acLineSegmentsCim16': std_converter_classes.lines.acLineSegmentsCim16,
@@ -63,25 +86,7 @@ def from_cim_dict(cim_parser: cim_classes.CimParser, log_debug=False, convert_li
         'geoCoordinatesFromGLCim16': std_converter_classes.coordinates.geoCoordinatesFromGLCim16,
         'coordinatesFromDLCim16': std_converter_classes.coordinates.coordinatesFromDLCim16,
     }
-    if custom_converter_classes is not None:
-        for key in custom_converter_classes:
-            converter_classes[key] = custom_converter_classes.get(key)
-
-    # repair the input CIM data
-    if repair_cim is not None and repair_cim_class is not None:
-        repair_cim = repair_cim_class().deserialize(repair_cim, report_container=cim_parser.get_report_container())
-        repair_cim.repair(cim_parser.get_cim_dict(), report_container=cim_parser.get_report_container())
-
-    cim_converter = build_pp_net.CimConverter(cim_parser=cim_parser, converter_classes=converter_classes, **kwargs)
-    pp_net = cim_converter.convert_to_pp(convert_line_to_switch=convert_line_to_switch, line_r_limit=line_r_limit,
-                                         line_x_limit=line_x_limit, log_debug=log_debug, **kwargs)
-
-    # repair the output pandapower network
-    if repair_pp is not None and repair_pp_class is not None:
-        repair_pp = repair_pp_class().deserialize(repair_pp, report_container=cim_parser.get_report_container())
-        repair_pp.repair(pp_net, report_container=cim_parser.get_report_container())
-
-    return pp_net
+    return converter_classes
 
 
 def from_cim(file_list: List[str] = None, encoding: str = 'utf-8', convert_line_to_switch: bool = False,
@@ -98,8 +103,6 @@ def from_cim(file_list: List[str] = None, encoding: str = 'utf-8', convert_line_
     - create_measurements (str): Set this parameter to 'SV' to create measurements for the pandapower net from the SV
     profile. Set it to 'Analog' to create measurements from Analogs. If the parameter is not set or is set to None, no
     measurements will be created.
-    - update_assets_from_sv (bool): Set this parameter to True to update the assets (sgens, loads, wards, ...) with
-    values from the SV profile. Default: False.
     - use_GL_or_DL_profile (str): Choose the profile to use for converting coordinates. Set it to 'GL' to use the GL
     profile (Usually lat and long coordinates). Set it to 'DL' to use the DL profile (Usually x, y coordinates for
     displaying control room schema). Set it to 'both' to let the converter choose the profile. The converter will