diff --git a/cellpy/__init__.py b/cellpy/__init__.py index 05e9ace4..e28c67e7 100644 --- a/cellpy/__init__.py +++ b/cellpy/__init__.py @@ -21,23 +21,29 @@ import warnings import cellpy._version + + from cellpy.parameters import prms # TODO: this might give circular ref from cellpy.parameters import prmreader + +__version__ = cellpy._version.__version__ + from cellpy.readers import cellreader, dbreader, filefinder, do from cellpy.readers.core import Q, ureg -__version__ = cellpy._version.__version__ +# Q, ureg = cellpy.readers.core.get_pint_unit_registry() + logging.getLogger(__name__).addHandler(logging.NullHandler()) # TODO: (v2.0) remove this and enforce using for example `import cellpy.session as clp` and then # run `prmreader.initialize` in that `__init__` instead: -init = prmreader.initialize +init = parameters.prmreader.initialize init() # TODO: (v2.0) remove this and enforce using `cellpy.get` (or `cellpy.cellreader.get`) instead: get = cellreader.get -print_instruments = cellreader.print_instruments +print_instruments = readers.cellreader.print_instruments __all__ = [ "cellreader", diff --git a/cellpy/libs/__init__.py b/cellpy/libs/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cellpy/libs/apipkg/__init__.py b/cellpy/libs/apipkg/__init__.py new file mode 100644 index 00000000..efd65767 --- /dev/null +++ b/cellpy/libs/apipkg/__init__.py @@ -0,0 +1,39 @@ +""" +apipkg: control the exported namespace of a Python package. + +see https://pypi.python.org/pypi/apipkg + +(c) holger krekel, 2009 - MIT license +""" +from __future__ import annotations + +__all__ = ["initpkg", "ApiModule", "AliasModule", "__version__", "distribution_version"] +import sys +from typing import Any + +from ._alias_module import AliasModule +from ._importing import distribution_version as distribution_version +from ._module import _initpkg +from ._module import ApiModule +from ._version import version as __version__ + + +def initpkg( + pkgname: str, + exportdefs: dict[str, Any], + attr: dict[str, object] | None = None, + eager: bool = False, +) -> ApiModule: + """initialize given package from the export definitions.""" + attr = attr or {} + mod = sys.modules.get(pkgname) + + mod = _initpkg(mod, pkgname, exportdefs, attr=attr) + + # eagerload in bypthon to avoid their monkeypatching breaking packages + if "bpython" in sys.modules or eager: + for module in list(sys.modules.values()): + if isinstance(module, ApiModule): + getattr(module, "__dict__") + + return mod diff --git a/cellpy/libs/apipkg/_alias_module.py b/cellpy/libs/apipkg/_alias_module.py new file mode 100644 index 00000000..91efda2e --- /dev/null +++ b/cellpy/libs/apipkg/_alias_module.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from types import ModuleType + +from ._importing import importobj + + +def AliasModule(modname: str, modpath: str, attrname: str | None = None) -> ModuleType: + cached_obj: object | None = None + + def getmod() -> object: + nonlocal cached_obj + if cached_obj is None: + cached_obj = importobj(modpath, attrname) + return cached_obj + + x = modpath + ("." + attrname if attrname else "") + repr_result = f"" + + class AliasModule(ModuleType): + def __repr__(self) -> str: + return repr_result + + def __getattribute__(self, name: str) -> object: + try: + return getattr(getmod(), name) + except ImportError: + if modpath == "pytest" and attrname is None: + # hack for pylibs py.test + return None + else: + raise + + def __setattr__(self, name: str, value: object) -> None: + setattr(getmod(), name, value) + + def __delattr__(self, name: str) -> None: + delattr(getmod(), name) + + return AliasModule(str(modname)) diff --git a/cellpy/libs/apipkg/_importing.py b/cellpy/libs/apipkg/_importing.py new file mode 100644 index 00000000..ea458705 --- /dev/null +++ b/cellpy/libs/apipkg/_importing.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +import os +import sys + + +def _py_abspath(path: str) -> str: + """ + special version of abspath + that will leave paths from jython jars alone + """ + if path.startswith("__pyclasspath__"): + return path + else: + return os.path.abspath(path) + + +def distribution_version(name: str) -> str | None: + """try to get the version of the named distribution, + returns None on failure""" + if sys.version_info >= (3, 8): + from importlib.metadata import PackageNotFoundError, version + else: + from importlib_metadata import PackageNotFoundError, version + try: + return version(name) + except PackageNotFoundError: + return None + + +def importobj(modpath: str, attrname: str | None) -> object: + """imports a module, then resolves the attrname on it""" + module = __import__(modpath, None, None, ["__doc__"]) + if not attrname: + return module + + retval = module + names = attrname.split(".") + for x in names: + retval = getattr(retval, x) + return retval diff --git a/cellpy/libs/apipkg/_module.py b/cellpy/libs/apipkg/_module.py new file mode 100644 index 00000000..d828959a --- /dev/null +++ b/cellpy/libs/apipkg/_module.py @@ -0,0 +1,184 @@ +from __future__ import annotations + +import sys +import threading +from types import ModuleType +from typing import Any +from typing import Callable +from typing import cast +from typing import Iterable + +from ._alias_module import AliasModule +from ._importing import _py_abspath +from ._importing import importobj +from ._syncronized import _synchronized + + +class ApiModule(ModuleType): + """the magical lazy-loading module standing""" + + def __docget(self) -> str | None: + try: + return self.__doc + except AttributeError: + if "__doc__" in self.__map__: + return cast(str, self.__makeattr("__doc__")) + else: + return None + + def __docset(self, value: str) -> None: + self.__doc = value + + __doc__ = property(__docget, __docset) # type: ignore + __map__: dict[str, tuple[str, str]] + + def __init__( + self, + name: str, + importspec: dict[str, Any], + implprefix: str | None = None, + attr: dict[str, Any] | None = None, + ) -> None: + super().__init__(name) + self.__name__ = name + self.__all__ = [x for x in importspec if x != "__onfirstaccess__"] + self.__map__ = {} + self.__implprefix__ = implprefix or name + if attr: + for name, val in attr.items(): + setattr(self, name, val) + for name, importspec in importspec.items(): + if isinstance(importspec, dict): + subname = f"{self.__name__}.{name}" + apimod = ApiModule(subname, importspec, implprefix) + sys.modules[subname] = apimod + setattr(self, name, apimod) + else: + parts = importspec.split(":") + modpath = parts.pop(0) + attrname = parts and parts[0] or "" + if modpath[0] == ".": + modpath = implprefix + modpath + + if not attrname: + subname = f"{self.__name__}.{name}" + apimod = AliasModule(subname, modpath) + sys.modules[subname] = apimod + if "." not in name: + setattr(self, name, apimod) + else: + self.__map__[name] = (modpath, attrname) + + def __repr__(self): + repr_list = [f"") + return "".join(repr_list) + + @_synchronized + def __makeattr(self, name, isgetattr=False): + """lazily compute value for name or raise AttributeError if unknown.""" + target = None + if "__onfirstaccess__" in self.__map__: + target = self.__map__.pop("__onfirstaccess__") + fn = cast(Callable[[], None], importobj(*target)) + fn() + try: + modpath, attrname = self.__map__[name] + except KeyError: + # __getattr__ is called when the attribute does not exist, but it may have + # been set by the onfirstaccess call above. Infinite recursion is not + # possible as __onfirstaccess__ is removed before the call (unless the call + # adds __onfirstaccess__ to __map__ explicitly, which is not our problem) + if target is not None and name != "__onfirstaccess__": + return getattr(self, name) + # Attribute may also have been set during a concurrent call to __getattr__ + # which executed after this call was already waiting on the lock. Check + # for a recently set attribute while avoiding infinite recursion: + # * Don't call __getattribute__ if __makeattr was called from a data + # descriptor such as the __doc__ or __dict__ properties, since data + # descriptors are called as part of object.__getattribute__ + # * Only call __getattribute__ if there is a possibility something has set + # the attribute we're looking for since __getattr__ was called + if threading is not None and isgetattr: + return super().__getattribute__(name) + raise AttributeError(name) + else: + result = importobj(modpath, attrname) + setattr(self, name, result) + # in a recursive-import situation a double-del can happen + self.__map__.pop(name, None) + return result + + def __getattr__(self, name): + return self.__makeattr(name, isgetattr=True) + + def __dir__(self) -> Iterable[str]: + yield from super().__dir__() + yield from self.__map__ + + @property + def __dict__(self) -> dict[str, Any]: # type: ignore + # force all the content of the module + # to be loaded when __dict__ is read + dictdescr = ModuleType.__dict__["__dict__"] # type: ignore + ns: dict[str, Any] = dictdescr.__get__(self) + if ns is not None: + hasattr(self, "some") + for name in self.__all__: + try: + self.__makeattr(name) + except AttributeError: + pass + return ns + + +_PRESERVED_MODULE_ATTRS = { + "__file__", + "__version__", + "__loader__", + "__path__", + "__package__", + "__doc__", + "__spec__", + "__dict__", +} + + +def _initpkg(mod: ModuleType | None, pkgname, exportdefs, attr=None) -> ApiModule: + """Helper for initpkg. + + Python 3.3+ uses finer grained locking for imports, and checks sys.modules before + acquiring the lock to avoid the overhead of the fine-grained locking. This + introduces a race condition when a module is imported by multiple threads + concurrently - some threads will see the initial module and some the replacement + ApiModule. We avoid this by updating the existing module in-place. + + """ + if mod is None: + d = {"__file__": None, "__spec__": None} + d.update(attr) + mod = ApiModule(pkgname, exportdefs, implprefix=pkgname, attr=d) + sys.modules[pkgname] = mod + return mod + else: + f = getattr(mod, "__file__", None) + if f: + f = _py_abspath(f) + mod.__file__ = f + if hasattr(mod, "__path__"): + mod.__path__ = [_py_abspath(p) for p in mod.__path__] + if "__doc__" in exportdefs and hasattr(mod, "__doc__"): + del mod.__doc__ + for name in dir(mod): + if name not in _PRESERVED_MODULE_ATTRS: + delattr(mod, name) + + # Updating class of existing module as per importlib.util.LazyLoader + mod.__class__ = ApiModule + apimod = cast(ApiModule, mod) + ApiModule.__init__(apimod, pkgname, exportdefs, implprefix=pkgname, attr=attr) + return apimod diff --git a/cellpy/libs/apipkg/_syncronized.py b/cellpy/libs/apipkg/_syncronized.py new file mode 100644 index 00000000..71d3c2cb --- /dev/null +++ b/cellpy/libs/apipkg/_syncronized.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +import functools +import threading + + +def _synchronized(wrapped_function): + """Decorator to synchronise __getattr__ calls.""" + + # Lock shared between all instances of ApiModule to avoid possible deadlocks + lock = threading.RLock() + + @functools.wraps(wrapped_function) + def synchronized_wrapper_function(*args, **kwargs): + with lock: + return wrapped_function(*args, **kwargs) + + return synchronized_wrapper_function diff --git a/cellpy/libs/apipkg/_version.py b/cellpy/libs/apipkg/_version.py new file mode 100644 index 00000000..ddb90c2d --- /dev/null +++ b/cellpy/libs/apipkg/_version.py @@ -0,0 +1 @@ +version = "3.0.2" diff --git a/cellpy/libs/apipkg/py.typed b/cellpy/libs/apipkg/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/cellpy/readers/__init__.py b/cellpy/readers/__init__.py index 8b137891..3bc529e3 100644 --- a/cellpy/readers/__init__.py +++ b/cellpy/readers/__init__.py @@ -1 +1,29 @@ +from cellpy.libs.apipkg import initpkg +initpkg( + __name__, + { + "externals": { + "numpy": "numpy", + "openpyxl": "openpyxl", + "pandas": "pandas", + "pint": "pint", + }, + # "core": { + # "Data": "cellpy.readers.core:Data", + # "BaseDbReader": "cellpy.readers.core:BaseDbReader", + # "FileID": "cellpy.readers.core:FileID", + # "Q": "cellpy.readers.core:Q", + # "convert_from_simple_unit_label_to_string_unit_label": "cellpy.readers.core:convert_from_simple_unit_label_to_string_unit_label", + # "generate_default_factory": "cellpy.readers.core:generate_default_factory", + # "identify_last_data_point": "cellpy.readers.core:identify_last_data_point", + # "instrument_configurations": "cellpy.readers.core:instrument_configurations", + # "interpolate_y_on_x": "cellpy.readers.core:interpolate_y_on_x", + # "pickle_protocol": "cellpy.readers.core:pickle_protocol", + # "xldate_as_datetime": "cellpy.readers.core:xldate_as_datetime", + # }, + # "internals": { + # "OtherPath": "cellpy.internals.core:OtherPath", + # }, + }, +) diff --git a/cellpy/readers/cellreader.py b/cellpy/readers/cellreader.py index 4b377956..4312e1be 100644 --- a/cellpy/readers/cellreader.py +++ b/cellpy/readers/cellreader.py @@ -7,8 +7,7 @@ Example: >>> c = cellpy.get(["super_battery_run_01.res", "super_battery_run_02.res"]) # loads and merges the runs >>> voltage_curves = c.get_cap() - >>> c.save("super_battery_run.hdf") - + >>> c.save("super_battery_run.h5") """ import collections @@ -25,15 +24,12 @@ import warnings from pathlib import Path from typing import Union, Sequence, List, Optional, Iterable +from typing import TYPE_CHECKING from dataclasses import asdict -import numpy as np -import openpyxl -import pandas as pd -from pandas.errors import PerformanceWarning -from pint.errors import DimensionalityError -from pint import Quantity -from scipy import interpolate +from . import externals +from cellpy.readers import core +import cellpy.internals.core as internals from cellpy.exceptions import ( DeprecatedFeature, @@ -67,20 +63,6 @@ CellpyMetaIndividualTest, ) -from cellpy.readers.core import ( - Data, - FileID, - Q, - convert_from_simple_unit_label_to_string_unit_label, - generate_default_factory, - identify_last_data_point, - instrument_configurations, - interpolate_y_on_x, - pickle_protocol, - xldate_as_datetime, -) -from cellpy.internals.core import OtherPath - DIGITS_C_RATE = 5 HEADERS_NORMAL = get_headers_normal() # TODO @jepe refactor this (not needed) @@ -95,12 +77,18 @@ # TODO: @jepe - performance warnings - mixed types within cols (pytables) -warnings.filterwarnings("ignore", category=pd.io.pytables.PerformanceWarning) -pd.set_option("mode.chained_assignment", None) # "raise", "warn", None +# warnings.filterwarnings("ignore", category=externals.pandas.io.pytables.PerformanceWarning) +# externals.pandas.set_option("mode.chained_assignment", None) # "raise", "warn", None _module_logger = logging.getLogger(__name__) +# def core.Q(value, *args, **kwargs): +# """Convert value to pint quantity.""" +# ureg, Q = core.get_pint_unit_registry() +# return Q(value, *args, **kwargs) + + class CellpyCell: """Main class for working and storing data. @@ -230,7 +218,6 @@ def __init__( output_units (dict): sent to cellpy.parameters.internal_settings.get_default_output_units debug (bool): set to True if you want to see debug messages. """ - # TODO v 1.1: move to data (allow for multiple testers for same cell) if tester is None: self.tester = prms.Instruments.tester @@ -294,8 +281,8 @@ def __init__( self.limit_data_points = None self.ensure_step_table = prms.Reader.ensure_step_table self.ensure_summary_table = prms.Reader.ensure_summary_table - self.raw_datadir = OtherPath(prms.Paths.rawdatadir) - self.cellpy_datadir = OtherPath(prms.Paths.cellpydatadir) + self.raw_datadir = internals.OtherPath(prms.Paths.rawdatadir) + self.cellpy_datadir = internals.OtherPath(prms.Paths.cellpydatadir) self.auto_dirs = prms.Reader.auto_dirs # v2.0 # - headers and instruments @@ -316,7 +303,7 @@ def initialize(self): """Initialize the CellpyCell object with empty Data instance.""" logging.debug("Initializing...") - self._data = Data() + self._data = core.Data() # the batch utility might be using session name # the cycle and ica collector are using session name @@ -419,7 +406,7 @@ def _dump_cellpy_unit(self, value, parameter): logging.critical(f"Parsing {parameter} ({value})") try: - c = Q(value) + c = core.Q(value) c_unit = c.units self.cellpy_units[parameter] = f"{c_unit}" logging.critical(f"Updated your cellpy_units['{parameter}'] to '{c_unit}'") @@ -593,7 +580,7 @@ def split_many(self, base_cycles=None): if base_cycles is None: all_cycles = self.get_cycle_numbers() - base_cycles = int(np.median(all_cycles)) + base_cycles = int(externals.numpy.median(all_cycles)) cells = list() if not isinstance(base_cycles, (list, tuple)): @@ -630,12 +617,12 @@ def split_many(self, base_cycles=None): new_cell.data.steps = steptable0 new_cell.data.raw = data0 new_cell.data.summary = summary0 - new_cell.data = identify_last_data_point(new_cell.data) + new_cell.data = core.identify_last_data_point(new_cell.data) old_cell.data.steps = steptable old_cell.data.raw = data old_cell.data.summary = summary - old_cell.data = identify_last_data_point(old_cell.data) + old_cell.data = core.identify_last_data_point(old_cell.data) cells.append(new_cell) @@ -650,7 +637,7 @@ def __register_external_readers(self): def register_instrument_readers(self): """Register instrument readers.""" - self.instrument_factory = generate_default_factory() + self.instrument_factory = core.generate_default_factory() # instruments = find_all_instruments() # for instrument_id, instrument in instruments.items(): # self.instrument_factory.register_builder(instrument_id, instrument) @@ -912,7 +899,7 @@ def _check_raw(self, file_names, abort_on_missing=False): ids = dict() for f in file_names: logging.debug(f"checking raw file {f}") - fid = FileID(f) + fid = core.FileID(f) # logging.debug(fid) if fid.name is None: warnings.warn(f"file does not exist: {f}") @@ -931,12 +918,12 @@ def _check_raw(self, file_names, abort_on_missing=False): ids[name] = int(fid.last_modified) return ids - def _check_cellpy_file(self, filename: OtherPath): + def _check_cellpy_file(self, filename: "OtherPath"): """Get the file-ids for the cellpy_file.""" - if not isinstance(filename, OtherPath): + if not isinstance(filename, internals.OtherPath): logging.debug("filename must be an OtherPath object") - filename = OtherPath(filename) + filename = internals.OtherPath(filename) use_full_filename_path = False parent_level = prms._cellpyfile_root # noqa @@ -954,7 +941,7 @@ def _check_cellpy_file(self, filename: OtherPath): # copy the file to temporary directory (this will take some time, and therefore it is # probably best not to put your cellpy files in a remote directory yet): filename = filename.copy() - store = pd.HDFStore(filename) + store = externals.pandas.HDFStore(filename) except Exception as e: logging.debug(f"could not open cellpy-file ({e})") return None @@ -1245,7 +1232,7 @@ def from_raw( logging.debug("loading raw file:") logging.debug(f"{file_name}") if is_a_file: - file_name = OtherPath(file_name) + file_name = internals.OtherPath(file_name) if not file_name.is_file(): raise NoDataFound(f"Could not find the file {file_name}") @@ -1365,8 +1352,8 @@ def load( logging.debug("loading cellpy-file (hdf5):") logging.debug(cellpy_file) logging.debug(f"{type(cellpy_file)=}") - cellpy_file = OtherPath(cellpy_file) - with pickle_protocol(PICKLE_PROTOCOL): + cellpy_file = internals.OtherPath(cellpy_file) + with core.pickle_protocol(PICKLE_PROTOCOL): logging.debug(f"using pickle protocol {PICKLE_PROTOCOL}") data = self._load_hdf5(cellpy_file, parent_level, accept_old, selector=selector) logging.debug("cellpy-file loaded") @@ -1400,7 +1387,7 @@ def _get_cellpy_file_version(self, filename, meta_dir=None, parent_level=None): if parent_level is None: parent_level = prms._cellpyfile_root - with pd.HDFStore(filename) as store: + with externals.pandas.HDFStore(filename) as store: try: meta_table = store.select(parent_level + meta_dir) except KeyError: @@ -1496,7 +1483,7 @@ def _load_hdf5_current_version(self, filename, parent_level=None, selector=None) logging.debug(f"filename: {filename}") logging.debug(f"selector: {selector}") - with pd.HDFStore(filename) as store: + with externals.pandas.HDFStore(filename) as store: ( data, meta_table, @@ -1538,7 +1525,7 @@ def _load_hdf5_v7(self, filename, selector=None, **kwargs): logging.debug(f"filename: {filename}") logging.debug(f"selector: {selector}") - with pd.HDFStore(filename) as store: + with externals.pandas.HDFStore(filename) as store: data, meta_table = self._create_initial_data_set_from_cellpy_file(meta_dir, parent_level, store) self._check_keys_in_cellpy_file(meta_dir, parent_level, raw_dir, store, summary_dir) self._extract_summary_from_cellpy_file(data, parent_level, store, summary_dir, selector=selector) @@ -1570,7 +1557,7 @@ def _load_hdf5_v6(self, filename, selector=None): fid_dir = "/fid" meta_dir = "/info" - with pd.HDFStore(filename) as store: + with externals.pandas.HDFStore(filename) as store: data, meta_table = self._create_initial_data_set_from_cellpy_file( meta_dir, parent_level, @@ -1628,7 +1615,7 @@ def _load_hdf5_v5(self, filename, selector=None): fid_dir = "/fid" meta_dir = "/info" - with pd.HDFStore(filename) as store: + with externals.pandas.HDFStore(filename) as store: data, meta_table = self._create_initial_data_set_from_cellpy_file(meta_dir, parent_level, store) self._check_keys_in_cellpy_file(meta_dir, parent_level, raw_dir, store, summary_dir) self._extract_summary_from_cellpy_file( @@ -1695,7 +1682,7 @@ def _load_old_hdf5_v3_to_v4(self, filename): _summary_dir = "/dfsummary" _fid_dir = "/fidtable" - with pd.HDFStore(filename) as store: + with externals.pandas.HDFStore(filename) as store: data, meta_table = self._create_initial_data_set_from_cellpy_file(meta_dir, parent_level, store) self._check_keys_in_cellpy_file(meta_dir, parent_level, _raw_dir, store, _summary_dir) @@ -1745,11 +1732,11 @@ def _create_initial_data_set_from_cellpy_file(self, meta_dir, parent_level, stor if test_dependent_meta_dir is not None: common_meta_table = store.select(parent_level + meta_dir) test_dependent_meta = store.select(parent_level + test_dependent_meta_dir) - data = Data() + data = core.Data() # data.cellpy_file_version = CELLPY_FILE_VERSION return data, common_meta_table, test_dependent_meta - data = Data() + data = core.Data() meta_table = None try: @@ -1830,9 +1817,9 @@ def _unpack_selector(self, selector): # TODO @jepe: move this to its own module (e.g. as a cellpy-loader in instruments?): def _extract_summary_from_cellpy_file( self, - data: Data, + data: "Data", parent_level: str, - store: pd.HDFStore, + store: "externals.pandas.HDFStore", summary_dir: str, selector: Union[None, str] = None, upgrade_from_to: tuple = None, @@ -1905,7 +1892,7 @@ def _extract_steps_from_cellpy_file( except Exception as e: print(e) logging.debug("could not get steps from cellpy-file") - data.steps = pd.DataFrame() + data.steps = externals.pandas.DataFrame() warnings.warn(f"Unhandled exception raised: {e}") # TODO @jepe: move this to its own module (e.g. as a cellpy-loader in instruments?): @@ -1930,9 +1917,9 @@ def _extract_fids_from_cellpy_file(self, fid_dir, parent_level, store, upgrade_f # TODO @jepe: move this to its own module (e.g. as a cellpy-loader in instruments?): def _extract_meta_from_cellpy_file( self, - data: Data, - meta_table: pd.DataFrame, - test_dependent_meta_table: pd.DataFrame, + data: "Data", + meta_table: "externals.pandas.DataFrame", + test_dependent_meta_table: "externals.pandas.DataFrame", filename: Union[Path, str], upgrade_from_to: tuple = None, ) -> None: @@ -1974,8 +1961,8 @@ def _extract_meta_from_cellpy_file( # TODO @jepe: move this to its own module (e.g. as a cellpy-loader in instruments?): def _extract_meta_from_old_cellpy_file_max_v7( self, - data: Data, - meta_table: pd.DataFrame, + data: "Data", + meta_table: "externals.pandas.DataFrame", filename: Union[Path, str], upgrade_from_to: tuple, ) -> None: @@ -2012,7 +1999,7 @@ def _extract_meta_from_old_cellpy_file_max_v7( v = v[0] if not isinstance(v, str): logging.debug(f"{v} is not of type string") - v = convert_from_simple_unit_label_to_string_unit_label(key, v) + v = core.convert_from_simple_unit_label_to_string_unit_label(key, v) data.raw_units[key] = v except KeyError: logging.critical(f"missing key in meta_table: {h5_key}") @@ -2055,11 +2042,11 @@ def _create_infotable(self): raise IOError(f"raw unit for {key} ({value}) must be of type string, not {type(value)}") new_info_table[h5_key] = value - new_info_table = pd.DataFrame.from_records([new_info_table]) - new_info_table_test_dependent = pd.DataFrame.from_records([new_info_table_test_dependent]) + new_info_table = externals.pandas.DataFrame.from_records([new_info_table]) + new_info_table_test_dependent = externals.pandas.DataFrame.from_records([new_info_table_test_dependent]) fidtable = self._convert2fid_table(cell) - fidtable = pd.DataFrame(fidtable) + fidtable = externals.pandas.DataFrame(fidtable) # TODO: test_dependent with several tests (and possibly merge with FID) # TODO: save # TODO: load old @@ -2119,9 +2106,9 @@ def _convert2fid_list(tbl): lengths = [] min_amount = 0 for counter, item in enumerate(tbl["raw_data_name"]): - fid = FileID() + fid = core.FileID() try: - fid.name = OtherPath(item).name + fid.name = internals.OtherPath(item).name except NotImplementedError: fid.name = item fid.full_name = tbl["raw_data_full_name"][counter] @@ -2183,7 +2170,7 @@ def _append(self, t1, t2, merge_summary=False, merge_step_table=False, recalc=Tr start_time_2 = t2.meta_common.start_datetime if self.tester in ["arbin_res"]: - diff_time = xldate_as_datetime(start_time_2) - xldate_as_datetime(start_time_1) + diff_time = core.xldate_as_datetime(start_time_2) - core.xldate_as_datetime(start_time_1) else: diff_time = start_time_2 - start_time_1 diff_time = diff_time.total_seconds() @@ -2218,7 +2205,7 @@ def _append(self, t1, t2, merge_summary=False, merge_step_table=False, recalc=Tr logging.debug("not doing recalc") # merging logging.debug("performing concat") - raw = pd.concat([t1.raw, t2.raw], ignore_index=True) + raw = externals.pandas.concat([t1.raw, t2.raw], ignore_index=True) data.raw = raw data.loaded_from.append(t2.loaded_from) step_table_made = False @@ -2259,7 +2246,7 @@ def _append(self, t1, t2, merge_summary=False, merge_step_table=False, recalc=Tr t2.summary[data_point_header] = t2.summary[data_point_header] + last_data_point - summary2 = pd.concat([t1.summary, t2.summary], ignore_index=True) + summary2 = externals.pandas.concat([t1.summary, t2.summary], ignore_index=True) data.summary = summary2 else: @@ -2270,7 +2257,7 @@ def _append(self, t1, t2, merge_summary=False, merge_step_table=False, recalc=Tr cycle_index_header = self.headers_normal.cycle_index_txt t2.steps[self.headers_step_table.cycle] = t2.raw[self.headers_step_table.cycle] + last_cycle - steps2 = pd.concat([t1.steps, t2.steps], ignore_index=True) + steps2 = externals.pandas.concat([t1.steps, t2.steps], ignore_index=True) data.steps = steps2 else: logging.debug("could not merge step tables " "(non-existing) -" "create them first!") @@ -2289,9 +2276,9 @@ def _validate_step_table(self, simple=False): if not self.data.has_steps: return False - no_cycles_raw = np.amax(d[self.headers_normal.cycle_index_txt]) + no_cycles_raw = externals.numpy.amax(d[self.headers_normal.cycle_index_txt]) headers_step_table = self.headers_step_table - no_cycles_step_table = np.amax(s[headers_step_table.cycle]) + no_cycles_step_table = externals.numpy.amax(s[headers_step_table.cycle]) if simple: logging.debug(" (simple)") @@ -2309,7 +2296,7 @@ def _validate_step_table(self, simple=False): for j in range(1, no_cycles_raw + 1): cycle_number = j no_steps_raw = len( - np.unique( + externals.numpy.unique( d.loc[ d[self.headers_normal.cycle_index_txt] == cycle_number, self.headers_normal.step_index_txt, @@ -2458,7 +2445,7 @@ def get_step_numbers( if trim_taper_steps: logging.info( "Trimming taper steps is currently not" - "possible when returning pd.DataFrame. " + "possible when returning externals.pandas.DataFrame. " "Do it manually instead." ) out = st[st[shdr.type].isin(steptypes) & st[shdr.cycle].isin(cycle_numbers)] @@ -2517,7 +2504,7 @@ def load_step_specifications(self, file_name, short=False): # # for arbin at least). # raise NotImplementedError - step_specs = pd.read_csv(file_name, sep=prms.Reader.sep) + step_specs = externals.pandas.read_csv(file_name, sep=prms.Reader.sep) if "step" not in step_specs.columns: logging.info("Missing column: step") raise IOError @@ -2926,7 +2913,7 @@ def _select_step(self, cycle, step): y_txt = self.headers_normal.voltage_txt x_txt = self.headers_normal.discharge_capacity_txt # jepe fix - # no_cycles=np.amax(test.raw[c_txt]) + # no_cycles=externals.numpy.amax(test.raw[c_txt]) # print d.columns if not any(test.raw.columns == c_txt): @@ -3189,12 +3176,12 @@ def to_excel( if get_cap_kwargs is not None: get_cap_method_kwargs.update(get_cap_kwargs) - border = openpyxl.styles.Border() + border = externals.openpyxl.styles.Border() face_color = "00EEEEEE" - meta_alignment_left = openpyxl.styles.Alignment(horizontal="left", vertical="bottom") + meta_alignment_left = externals.openpyxl.styles.Alignment(horizontal="left", vertical="bottom") meta_width = 34 - meta_alignment_right = openpyxl.styles.Alignment(horizontal="right", vertical="bottom") - fill = openpyxl.styles.PatternFill(start_color=face_color, end_color=face_color, fill_type="solid") + meta_alignment_right = externals.openpyxl.styles.Alignment(horizontal="right", vertical="bottom") + fill = externals.openpyxl.styles.PatternFill(start_color=face_color, end_color=face_color, fill_type="solid") if filename is None: pre = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") @@ -3210,9 +3197,9 @@ def to_excel( raw_units = self.raw_units.to_frame() raw_units.index = "raw_units_" + raw_units.index - meta_common_frame = pd.concat([meta_common_frame, cellpy_units, raw_units]) + meta_common_frame = externals.pandas.concat([meta_common_frame, cellpy_units, raw_units]) - with pd.ExcelWriter(filename, engine="openpyxl") as writer: + with externals.pandas.ExcelWriter(filename, engine="externals.openpyxl") as writer: meta_common_frame.to_excel(writer, sheet_name="meta_common", **to_excel_method_kwargs) meta_test_dependent_frame.to_excel(writer, sheet_name="meta_test_dependent", **to_excel_method_kwargs) summary_frame.to_excel(writer, sheet_name="summary", **to_excel_method_kwargs) @@ -3417,7 +3404,7 @@ def save( logging.info("If you really want to do it, use save with force=True") return - outfile_all = OtherPath(filename) + outfile_all = internals.OtherPath(filename) if not outfile_all.suffix: logging.debug("No suffix given - adding one") outfile_all = outfile_all.with_suffix(f".{extension}") @@ -3466,9 +3453,9 @@ def save( common_meta_dir = prms._cellpyfile_common_meta # noqa fid_dir = prms._cellpyfile_fid # noqa test_dependent_meta_dir = prms._cellpyfile_test_dependent_meta # noqa - warnings.simplefilter("ignore", PerformanceWarning) + warnings.simplefilter("ignore", externals.pandas.errors.PerformanceWarning) try: - with pickle_protocol(PICKLE_PROTOCOL): + with core.pickle_protocol(PICKLE_PROTOCOL): store = self._save_to_hdf5( fid_dir, fid_table, @@ -3486,7 +3473,7 @@ def save( finally: store.close() logging.debug(" all -> hdf5 OK") - warnings.simplefilter("default", PerformanceWarning) + warnings.simplefilter("default", externals.pandas.errors.PerformanceWarning) # del store # --- finished saving to hdf5 ------------------------------- @@ -3505,7 +3492,7 @@ def _save_to_hdf5( step_dir, summary_dir, ): - store = pd.HDFStore( + store = externals.pandas.HDFStore( outfile_all, complib=prms._cellpyfile_complib, complevel=prms._cellpyfile_complevel, @@ -3573,7 +3560,7 @@ def _fix_dtype_step_table(self, dataset): return for col in cols: if col not in [hst.cycle, hst.sub_step, hst.info]: - dataset.steps[col] = dataset.steps[col].apply(pd.to_numeric) + dataset.steps[col] = dataset.steps[col].apply(externals.pandas.to_numeric) else: dataset.steps[col] = dataset.steps[col].astype("str") return dataset @@ -3588,7 +3575,7 @@ def _cap_mod_summary(self, summary, capacity_modifier="reset"): chargecap = 0.0 dischargecap = 0.0 - # TODO: @jepe - use pd.loc[row,column] + # TODO: @jepe - use externals.pandas.loc[row,column] if capacity_modifier == "reset": for index, row in summary.iterrows(): @@ -3625,7 +3612,7 @@ def _cap_mod_normal(self, capacity_modifier="reset", allctypes=True): if capacity_modifier == "reset": # discharge cycles - no_cycles = np.amax(raw[cycle_index_header]) + no_cycles = externals.numpy.amax(raw[cycle_index_header]) for j in range(1, no_cycles + 1): cap_type = "discharge" e_header = discharge_energy_index_header @@ -3635,8 +3622,8 @@ def _cap_mod_normal(self, capacity_modifier="reset", allctypes=True): steps = discharge_cycles[j] txt = "Cycle %i (discharge): " % j logging.debug(txt) - # TODO: @jepe - use pd.loc[row,column] e.g. pd.loc[:,"charge_cap"] - # for col or pd.loc[(pd.["step"]==1),"x"] + # TODO: @jepe - use externals.pandas.loc[row,column] e.g. externals.pandas.loc[:,"charge_cap"] + # for col or externals.pandas.loc[(externals.pandas.["step"]==1),"x"] selection = (raw[cycle_index_header] == j) & (raw[step_index_header].isin(steps)) c0 = raw[selection].iloc[0][cap_header] e0 = raw[selection].iloc[0][e_header] @@ -3712,7 +3699,7 @@ def get_raw( additional_headers: Optional[list] = None, as_frame: bool = True, scaler: Optional[float] = None, - ) -> Union[pd.DataFrame, List[np.array]]: + ) -> Union["externals.pandas.DataFrame", List["externals.numpy.array"]]: """Returns the values for column with given header (in raw units). Args: @@ -3990,7 +3977,7 @@ def get_dcap( the factor is obtained from the self.get_converter_to_specific() method. mode (string): 'gravimetric', 'areal' or 'absolute'. Defaults to 'gravimetric'. Used if converter is not provided (or None). - as_frame (bool): if True: returns pd.DataFrame instead of capacity, voltage series. + as_frame (bool): if True: returns externals.pandas.DataFrame instead of capacity, voltage series. **kwargs (dict): additional keyword arguments sent to the internal _get_cap method. Returns: @@ -4003,7 +3990,7 @@ def get_dcap( dc, v = self._get_cap(cycle, "discharge", converter=converter, **kwargs) if as_frame: - cycle_df = pd.concat([v, dc], axis=1) + cycle_df = externals.pandas.concat([v, dc], axis=1) return cycle_df else: return dc, v @@ -4025,7 +4012,7 @@ def get_ccap( the factor is obtained from the self.get_converter_to_specific() method. mode (string): 'gravimetric', 'areal' or 'absolute'. Defaults to 'gravimetric'. Used if converter is not provided (or None). - as_frame (bool): if True: returns pd.DataFrame instead of capacity, voltage series. + as_frame (bool): if True: returns externals.pandas.DataFrame instead of capacity, voltage series. **kwargs (dict): additional keyword arguments sent to the internal _get_cap method. Returns: @@ -4038,7 +4025,7 @@ def get_ccap( cc, v = self._get_cap(cycle, "charge", converter=converter, **kwargs) if as_frame: - cycle_df = pd.concat([v, cc], axis=1) + cycle_df = externals.pandas.concat([v, cc], axis=1) return cycle_df else: return cc, v @@ -4081,7 +4068,7 @@ def get_cap( - "forth-and-forth" - discharge (or charge) also starts at 0 (or shift if not shift=0.0) - insert_nan (bool): insert a np.nan between the charge and discharge curves. + insert_nan (bool): insert a externals.numpy.nan between the charge and discharge curves. Defaults to True for "forth-and-forth", else False shift: start-value for charge (or discharge) (typically used when plotting shifted-capacity). @@ -4175,7 +4162,7 @@ def first(df): insert_nan = False if insert_nan: - _nan = pd.DataFrame({"capacity": [np.nan], "voltage": [np.nan]}) + _nan = externals.pandas.DataFrame({"capacity": [externals.numpy.nan], "voltage": [externals.numpy.nan]}) converter_kwargs = dict() if mass is not None: @@ -4199,14 +4186,14 @@ def first(df): capacity = None voltage = None specific_converter = self.get_converter_to_specific(mode=mode, **converter_kwargs) - cycle_df = pd.DataFrame() + cycle_df = externals.pandas.DataFrame() initial = True for current_cycle in cycle: - cc = pd.DataFrame() - cv = pd.DataFrame() - dc = pd.DataFrame() - dv = pd.DataFrame() + cc = externals.pandas.DataFrame() + cv = externals.pandas.DataFrame() + dc = externals.pandas.DataFrame() + dv = externals.pandas.DataFrame() error = False try: cc, cv = self.get_ccap( @@ -4322,14 +4309,14 @@ def first(df): try: # processing first if not _first_step_c.empty: - _first_df = pd.DataFrame( + _first_df = externals.pandas.DataFrame( { "voltage": _first_step_v, "capacity": _first_step_c, } ) if interpolated: - _first_df = interpolate_y_on_x( + _first_df = core.interpolate_y_on_x( _first_df, y=y_col, x=x_col, @@ -4338,22 +4325,22 @@ def first(df): direction=first_interpolation_direction, ) if insert_nan: - _first_df = pd.concat([_first_df, _nan]) + _first_df = externals.pandas.concat([_first_df, _nan]) if categorical_column: _first_df["direction"] = -1 else: - _first_df = pd.DataFrame() + _first_df = externals.pandas.DataFrame() # processing last if not _last_step_c.empty: - _last_df = pd.DataFrame( + _last_df = externals.pandas.DataFrame( { "voltage": _last_step_v.values, "capacity": _last_step_c.values, } ) if interpolated: - _last_df = interpolate_y_on_x( + _last_df = core.interpolate_y_on_x( _last_df, y=y_col, x=x_col, @@ -4362,11 +4349,11 @@ def first(df): direction=last_interpolation_direction, ) if insert_nan: - _last_df = pd.concat([_last_df, _nan]) + _last_df = externals.pandas.concat([_last_df, _nan]) if categorical_column: _last_df["direction"] = 1 else: - _last_df = pd.DataFrame() + _last_df = externals.pandas.DataFrame() if interpolate_along_cap: if method == "forth": @@ -4378,7 +4365,7 @@ def first(df): except AttributeError: logging.info(f"Could not extract cycle {current_cycle}") else: - c = pd.concat([_first_df, _last_df], axis=0) + c = externals.pandas.concat([_first_df, _last_df], axis=0) if label_cycle_number: c.insert(0, "cycle", current_cycle) # c["cycle"] = current_cycle @@ -4386,7 +4373,7 @@ def first(df): if cycle_df.empty: cycle_df = c else: - cycle_df = pd.concat([cycle_df, c], axis=0) + cycle_df = externals.pandas.concat([cycle_df, c], axis=0) if capacity_then_voltage: cols = cycle_df.columns.to_list() new_cols = [ @@ -4406,12 +4393,12 @@ def first(df): _non_empty_c.append(_last_step_c) _non_empty_v.append(_last_step_v) - c = pd.concat(_non_empty_c, axis=0) - v = pd.concat(_non_empty_v, axis=0) + c = externals.pandas.concat(_non_empty_c, axis=0) + v = externals.pandas.concat(_non_empty_v, axis=0) if not c.empty: - capacity = pd.concat([capacity, c], axis=0) - voltage = pd.concat([voltage, v], axis=0) + capacity = externals.pandas.concat([capacity, c], axis=0) + voltage = externals.pandas.concat([voltage, v], axis=0) if return_dataframe: return cycle_df @@ -4465,8 +4452,8 @@ def _get_cap( _v.append(selected_step[self.headers_normal.voltage_txt]) _c.append(selected_step[column_txt] * converter) try: - voltage = pd.concat(_v, axis=0) - cap = pd.concat(_c, axis=0) + voltage = externals.pandas.concat(_v, axis=0) + cap = externals.pandas.concat(_c, axis=0) except Exception: logging.debug("could not find any steps for this cycle") raise NullData(f"no steps found (c:{cycle} s:{step} type:{cap_type})") @@ -4486,7 +4473,7 @@ def get_ocv( interpolated=False, dx=None, number_of_points=None, - ) -> pd.DataFrame: + ) -> externals.pandas.DataFrame: """Get the open circuit voltage relaxation curves. Args: @@ -4512,7 +4499,7 @@ def get_ocv( if cycles is None: cycles = self.get_cycle_numbers() else: - if not isinstance(cycles, (list, tuple, np.ndarray)): + if not isinstance(cycles, (list, tuple, externals.numpy.ndarray)): cycles = [cycles] else: remove_first = False @@ -4550,7 +4537,7 @@ def get_ocv( groupby_list = [cycle_label, step_label] for name, group in selected_df.groupby(groupby_list): - new_group = interpolate_y_on_x( + new_group = core.interpolate_y_on_x( group, x=step_time_label, y=voltage_label, @@ -4562,7 +4549,7 @@ def get_ocv( new_group[i] = j new_dfs.append(new_group) - selected_df = pd.concat(new_dfs) + selected_df = externals.pandas.concat(new_dfs) return selected_df @@ -4570,9 +4557,9 @@ def get_number_of_cycles(self, steptable=None): """Get the number of cycles in the test.""" if steptable is None: d = self.data.raw - number_of_cycles = np.amax(d[self.headers_normal.cycle_index_txt]) + number_of_cycles = externals.numpy.amax(d[self.headers_normal.cycle_index_txt]) else: - number_of_cycles = np.amax(steptable[self.headers_step_table.cycle]) + number_of_cycles = externals.numpy.amax(steptable[self.headers_step_table.cycle]) return number_of_cycles def get_rates(self, steptable=None, agg="first", direction=None): @@ -4755,7 +4742,7 @@ def total_time_at_voltage_level( if not col_has_date_time_dtype: logging.debug("converting date_time to datetime64[ns]") - v[date_time_hdr] = pd.to_datetime(v[date_time_hdr], format=date_time_format) + v[date_time_hdr] = externals.pandas.to_datetime(v[date_time_hdr], format=date_time_format) if duplicated: logging.debug("removing duplicated date_time values") @@ -4815,12 +4802,12 @@ def nominal_capacity_as_absolute( if value is None: value = self.data.nom_cap - value = Q(value, self.cellpy_units["nominal_capacity"]) + value = core.Q(value, self.cellpy_units["nominal_capacity"]) if nom_cap_specifics == "gravimetric": - specific = Q(specific, self.cellpy_units["mass"]) + specific = core.Q(specific, self.cellpy_units["mass"]) elif nom_cap_specifics == "areal": - specific = Q(specific, self.cellpy_units["area"]) + specific = core.Q(specific, self.cellpy_units["area"]) elif nom_cap_specifics == "absolute": specific = 1 @@ -4829,13 +4816,13 @@ def nominal_capacity_as_absolute( raise NotImplementedError("volumetric not implemented yet") if convert_charge_units: - conversion_factor_charge = Q(1, self.cellpy_units["charge"]) / Q(1, self.data.raw_units["charge"]) + conversion_factor_charge = core.Q(1, self.cellpy_units["charge"]) / core.Q(1, self.data.raw_units["charge"]) else: conversion_factor_charge = 1.0 try: absolute_value = (value * conversion_factor_charge * specific).to_reduced_units().to("Ah") - except DimensionalityError as e: + except externals.pint.errors.PerformanceWarning as e: print(" DimensionalityError ".center(80, "=")) print("Could not convert nominal capacity to absolute value!") print( @@ -4895,7 +4882,7 @@ def with_cellpy_unit(self, parameter, as_str=False): if as_str: return f"{_value} {_unit}" - return Q(_value, _unit) + return core.Q(_value, _unit) def to_cellpy_unit(self, value, physical_property): """Convert value to cellpy units. @@ -4909,12 +4896,12 @@ def to_cellpy_unit(self, value, physical_property): the value in cellpy units """ logging.debug(f"value {value} is numeric? {isinstance(value, numbers.Number)}") - logging.debug(f"value {value} is a pint quantity? {isinstance(value, Quantity)}") + logging.debug(f"value {value} is a pint quantity? {isinstance(value, externals.pint.Quantity)}") - if not isinstance(value, Quantity): + if not isinstance(value, externals.pint.Quantity): if isinstance(value, numbers.Number): try: - value = Q(value, self.data.raw_units[physical_property]) + value = core.Q(value, self.data.raw_units[physical_property]) logging.debug(f"With unit from raw-units: {value}") except NoDataFound: raise NoDataFound( @@ -4925,9 +4912,9 @@ def to_cellpy_unit(self, value, physical_property): except KeyError as e: raise KeyError("You have to provide a valid physical_property") from e elif isinstance(value, tuple): - value = Q(*value) + value = core.Q(*value) else: - value = Q(value) + value = core.Q(value) value = value.to(self.cellpy_units[physical_property]) @@ -4944,16 +4931,16 @@ def unit_scaler_from_raw(self, unit, physical_property): Returns (numeric): conversion factor (scaler) """ - logging.debug(f"value {unit} is a pint quantity? {isinstance(unit, Quantity)}") + logging.debug(f"value {unit} is a pint quantity? {isinstance(unit, externals.pint.Quantity)}") old_unit = self.data.raw_units[physical_property] - value = Q(1, old_unit) + value = core.Q(1, old_unit) value = value.to(unit) return value.m def get_converter_to_specific( self, - dataset: Data = None, + dataset: "Data" = None, value: float = None, from_units: CellpyUnits = None, to_units: CellpyUnits = None, @@ -4985,22 +4972,22 @@ def get_converter_to_specific( if mode == "gravimetric": value = value or dataset.mass - value = Q(value, new_units["mass"]) - to_unit_specific = Q(1.0, new_units["specific_gravimetric"]) + value = core.Q(value, new_units["mass"]) + to_unit_specific = core.Q(1.0, new_units["specific_gravimetric"]) elif mode == "areal": value = value or dataset.active_electrode_area - value = Q(value, new_units["area"]) - to_unit_specific = Q(1.0, new_units["specific_areal"]) + value = core.Q(value, new_units["area"]) + to_unit_specific = core.Q(1.0, new_units["specific_areal"]) elif mode == "volumetric": value = value or dataset.volume - value = Q(value, new_units["volume"]) - to_unit_specific = Q(1.0, new_units["specific_volumetric"]) + value = core.Q(value, new_units["volume"]) + to_unit_specific = core.Q(1.0, new_units["specific_volumetric"]) elif mode == "absolute": - value = Q(1.0, None) - to_unit_specific = Q(1.0, None) + value = core.Q(1.0, None) + to_unit_specific = core.Q(1.0, None) elif mode is None: return 1.0 @@ -5009,8 +4996,8 @@ def get_converter_to_specific( logging.debug(f"mode={mode} not supported!") return 1.0 - from_unit_cap = Q(1.0, old_units["charge"]) - to_unit_cap = Q(1.0, new_units["charge"]) + from_unit_cap = core.Q(1.0, old_units["charge"]) + to_unit_cap = core.Q(1.0, new_units["charge"]) # from unit is always in absolute values: from_unit = from_unit_cap @@ -5202,12 +5189,12 @@ def _is_listtype(x): @staticmethod def _bounds(x): - return np.amin(x), np.amax(x) + return externals.numpy.amin(x), externals.numpy.amax(x) @staticmethod def _roundup(x): n = 1000.0 - x = np.ceil(x * n) + x = externals.numpy.ceil(x * n) x /= n return x @@ -5224,6 +5211,8 @@ def _reverse(x): def _select_y(self, x, y, points): # uses interpolation to select y = f(x) + from scipy import interpolate + min_x, max_x = self._bounds(x) if x[0] > x[-1]: # need to reverse @@ -5882,7 +5871,7 @@ def _make_summar_legacy( logging.debug(f"(dt: {(time.time() - time_00):4.2f}s)") - def _generate_absolute_summary_columns(self, data, _first_step_txt, _second_step_txt) -> Data: + def _generate_absolute_summary_columns(self, data, _first_step_txt, _second_step_txt) -> "Data": summary = data.summary summary[self.headers_summary.coulombic_efficiency] = 100 * summary[_second_step_txt] / summary[_first_step_txt] summary[self.headers_summary.cumulated_coulombic_efficiency] = summary[ @@ -5955,7 +5944,7 @@ def _generate_absolute_summary_columns(self, data, _first_step_txt, _second_step return data - def _generate_specific_summary_columns(self, data: Data, mode: str, specific_columns: Sequence) -> Data: + def _generate_specific_summary_columns(self, data: "Data", mode: str, specific_columns: Sequence) -> "Data": specific_converter = self.get_converter_to_specific(dataset=data, mode=mode) summary = data.summary for col in specific_columns: @@ -5964,11 +5953,11 @@ def _generate_specific_summary_columns(self, data: Data, mode: str, specific_col data.summary = summary return data - def _c_rates_to_summary(self, data: Data) -> Data: + def _c_rates_to_summary(self, data: "Data") -> "Data": logging.debug("Extracting C-rates") def rate_to_cellpy_units(rate): - conversion_factor = Q(1.0, self.data.raw_units["current"]) / Q(1.0, self.cellpy_units["current"]) + conversion_factor = core.Q(1.0, self.data.raw_units["current"]) / core.Q(1.0, self.cellpy_units["current"]) conversion_factor = conversion_factor.to_reduced_units().magnitude return rate * conversion_factor @@ -6012,12 +6001,12 @@ def rate_to_cellpy_units(rate): def _equivalent_cycles_to_summary( self, - data: Data, + data: "Data", _first_step_txt: str, _second_step_txt: str, nom_cap: float, normalization_cycles: Union[Sequence, int, None], - ) -> Data: + ) -> "Data": # The method currently uses the charge capacity for calculating equivalent cycles. This # can be easily extended to also allow for choosing the discharge capacity later on if # it turns out that to be needed. @@ -6321,7 +6310,7 @@ def _dev_update_merge(self, t1, t2): if not t1.raw.empty: t1.raw = t1.raw.iloc[:-1] - raw2 = pd.concat([t1.raw, t2.raw], ignore_index=True) + raw2 = externals.pandas.concat([t1.raw, t2.raw], ignore_index=True) test.raw = raw2 else: test = t2 @@ -6334,7 +6323,7 @@ def _dev_update_make_steps(self, **kwargs): # Note! hard-coding header name (might fail if changing default headers) from_data_point = self.data.steps.iloc[-1].point_first new_steps = self.make_step_table(from_data_point=from_data_point, **kwargs) - merged_steps = pd.concat([old_steps, new_steps]).reset_index(drop=True) + merged_steps = externals.pandas.concat([old_steps, new_steps]).reset_index(drop=True) self.data.steps = merged_steps def _dev_update_make_summary(self, **kwargs): @@ -6348,7 +6337,7 @@ def _dev_update_make_summary(self, **kwargs): # For later: # (Remark! need to solve how to merge cumulated columns) # new_summary = self.make_summary(from_cycle=from_cycle) - # merged_summary = pd.concat([old_summary, new_summary]).reset_index(drop=True) + # merged_summary = externals.pandas.concat([old_summary, new_summary]).reset_index(drop=True) # self.data.summary = merged_summary def _dev_update_from_raw(self, file_names=None, data_points=None, **kwargs): @@ -6547,7 +6536,7 @@ def get( else: load_cellpy_file = True - filename = OtherPath(cellpy_file) + filename = internals.OtherPath(cellpy_file) if isinstance(filename, (list, tuple)): logging.debug("got a list or tuple of names") @@ -6555,7 +6544,7 @@ def get( else: logging.debug("got a single name") logging.debug(f"{filename=}") - filename = OtherPath(filename) + filename = internals.OtherPath(filename) if ( auto_pick_cellpy_format and instrument not in instruments_with_colliding_file_suffix @@ -6569,7 +6558,7 @@ def get( logging.debug(f"checked if the files were similar") if similar: load_cellpy_file = True - filename = OtherPath(cellpy_file) + filename = internals.OtherPath(cellpy_file) except Exception as e: logging.debug(f"Error during checking if similar: {e}") logging.debug("Setting load_cellpy_file to False") @@ -6700,7 +6689,7 @@ def print_instruments(): print(80 * "=") print(f"Implemented instrument loaders") print(80 * "=") - for name, value in instrument_configurations().items(): + for name, value in core.instrument_configurations().items(): print(name) instrument_models = value["__all__"].copy() instrument_models.remove("default") @@ -6859,8 +6848,8 @@ def _load_and_save_to_excel(): def _check_excel(): - import openpyxl - from openpyxl.styles import Border, Side + import externals.openpyxl + from externals.openpyxl.styles import Border, Side import pandas as pd from pathlib import Path @@ -6869,14 +6858,14 @@ def _check_excel(): excel_file = Path("../../tmp/nothing.xlsx") to_excel_method_kwargs = {"index": True, "header": True} - df = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}) + df = externals.pandas.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}) n_rows, n_cols = df.shape - with pd.ExcelWriter(excel_file, engine="openpyxl") as writer: + with externals.pandas.ExcelWriter(excel_file, engine="externals.openpyxl") as writer: df.to_excel(writer, sheet_name="first", **to_excel_method_kwargs) ws = writer.sheets["first"] border = Border() face_color = "00EEEEEE" - fill = openpyxl.styles.PatternFill(start_color=face_color, end_color=face_color, fill_type="solid") + fill = externals.openpyxl.styles.PatternFill(start_color=face_color, end_color=face_color, fill_type="solid") for cell in ws["A"]: print(cell) @@ -6905,7 +6894,7 @@ def _check_new_dot_get_methods(): ) # pprint(c.headers_normal) cycles_a = [1, 2, 3] - cycles_b = np.array([1, 2, 3]) + cycles_b = externals.numpy.array([1, 2, 3]) cycles_c = [1, 2, 3] a = c.get_timestamp(cycles_a, with_index=True, units="raw") print(f"{cycles_a=} raw".center(80, "-")) diff --git a/cellpy/readers/core.py b/cellpy/readers/core.py index 286bb553..193b20a2 100644 --- a/cellpy/readers/core.py +++ b/cellpy/readers/core.py @@ -17,8 +17,10 @@ import warnings from typing import Any, Tuple, Dict, List, Union, TypeVar, Optional -import numpy as np -import pandas as pd +from . import externals + +# import numpy as np +# import pandas as pd import pint from scipy import interpolate @@ -41,14 +43,13 @@ LOADERS_NOT_READY_FOR_PROD = ["ext_nda_reader"] # used by the instruments_configurations helper function (move?) # pint (https://pint.readthedocs.io/en/stable/) -ureg = pint.UnitRegistry() +ureg = externals.pint.UnitRegistry() try: ureg.formatter.default_format = "~P" except AttributeError: ureg.default_format = "~P" Q = ureg.Quantity - # TODO: in future versions (maybe 1.1.0) we should "copy-paste" the whole pathlib module # from CPython and add the functionality we need to it. This will make # it easier to keep up with changes in the pathlib module. @@ -395,9 +396,9 @@ def __init__(self, **kwargs): self.raw_units = get_default_raw_units() self.raw_limits = get_default_raw_limits() - self.raw = pd.DataFrame() - self.summary = pd.DataFrame() - self.steps = pd.DataFrame() + self.raw = externals.pandas.DataFrame() + self.summary = externals.pandas.DataFrame() + self.steps = externals.pandas.DataFrame() self.meta_common = CellpyMetaCommon() # TODO: v2.0 consider making this a list of several CellpyMetaIndividualTest @@ -590,7 +591,7 @@ def populate_defaults(self): @property def empty(self): """Check if the data object is empty.""" - if isinstance(self, pd.DataFrame): + if isinstance(self, externals.pandas.DataFrame): raise TypeError( "Data is a DataFrame (should be a Data object). " "You probably have a bug in your code. " @@ -1021,8 +1022,8 @@ def collect_capacity_curves( else: logging.warning(f"collect_capacity_curve received unknown key-word argument: {arg=}") - minimum_v_value = np.inf - maximum_v_value = -np.inf + minimum_v_value = externals.numpy.inf + maximum_v_value = -externals.numpy.inf charge_list = [] cycles = kwargs.pop("cycle", None) @@ -1055,11 +1056,11 @@ def collect_capacity_curves( except NullData as e: logging.warning(e) - d = pd.DataFrame() + d = externals.pandas.DataFrame() d.name = cycle charge_list.append(d) else: - d = pd.DataFrame({"q": q, "v": v}) + d = externals.pandas.DataFrame({"q": q, "v": v}) # d.name = f"{cycle}" d.name = cycle charge_list.append(d) @@ -1126,9 +1127,9 @@ def interpolate_y_on_x( f = interpolate.interp1d(xs, ys, bounds_error=bounds_error, **kwargs) if new_x is None: if number_of_points: - new_x = np.linspace(x_min, x_max, number_of_points) + new_x = externals.numpy.linspace(x_min, x_max, number_of_points) else: - new_x = np.arange(x_min, x_max, dx) + new_x = externals.numpy.arange(x_min, x_max, dx) else: # TODO: @jepe - make this better (and safer) @@ -1136,11 +1137,11 @@ def interpolate_y_on_x( logging.critical("EXPERIMENTAL FEATURE - USE WITH CAUTION") logging.critical(f"start, end, number_of_points = {new_x}") _x_min, _x_max, _number_of_points = new_x - new_x = np.linspace(_x_min, _x_max, _number_of_points, dtype=float) + new_x = externals.numpy.linspace(_x_min, _x_max, _number_of_points, dtype=float) new_y = f(new_x) - new_df = pd.DataFrame({x: new_x, y: new_y}) + new_df = externals.pandas.DataFrame({x: new_x, y: new_y}) return new_df @@ -1224,9 +1225,9 @@ def group_by_interpolate( x_max = df[x].max() x_min = df[x].min() if number_of_points: - new_x = np.linspace(x_max, x_min, number_of_points) + new_x = externals.numpy.linspace(x_max, x_min, number_of_points) else: - new_x = np.arange(x_max, x_min, dx) + new_x = externals.numpy.arange(x_max, x_min, dx) new_dfs = [] keys = [] @@ -1244,14 +1245,14 @@ def group_by_interpolate( new_dfs.append(new_group) if tidy: - new_df = pd.concat(new_dfs) + new_df = externals.pandas.concat(new_dfs) else: if individual_x_cols: - new_df = pd.concat(new_dfs, axis=1, keys=keys) + new_df = externals.pandas.concat(new_dfs, axis=1, keys=keys) group_by.append(header_name) new_df.columns.names = group_by else: - new_df = pd.concat(new_dfs) + new_df = externals.pandas.concat(new_dfs) new_df = new_df.pivot(index=x, columns=group_by[0], values=y) time_01 = time.time() - time_00 diff --git a/cellpy/readers/dbreader.py b/cellpy/readers/dbreader.py index 620c1413..152725c3 100644 --- a/cellpy/readers/dbreader.py +++ b/cellpy/readers/dbreader.py @@ -15,7 +15,8 @@ from typing import Optional from cellpy.parameters import prms -from cellpy.readers.core import BaseDbReader + +from cellpy.readers import core # logger = logging.getLogger(__name__) @@ -38,7 +39,7 @@ def __repr__(self): return f"" -class Reader(BaseDbReader): +class Reader(core.BaseDbReader): def __init__( self, db_file=None, diff --git a/cellpy/readers/instruments/neware_xlsx.py b/cellpy/readers/instruments/neware_xlsx.py index dacdf11d..88d8a6e8 100644 --- a/cellpy/readers/instruments/neware_xlsx.py +++ b/cellpy/readers/instruments/neware_xlsx.py @@ -1,4 +1,5 @@ """neware xlsx exported data""" + from dataclasses import dataclass import datetime import logging @@ -9,6 +10,7 @@ import pandas as pd from dateutil.parser import parse +import cellpy.readers.core from cellpy import prms from cellpy.exceptions import WrongFileVersion from cellpy.parameters.internal_settings import HeaderDict, get_headers_normal @@ -47,9 +49,7 @@ class DataLoader(BaseLoader): def __init__(self, *args, **kwargs): """initiates the neware xlsx reader class""" self.raw_headers_normal = None # the column headers defined by Neware - self.cellpy_headers_normal = ( - get_headers_normal() - ) # the column headers defined by cellpy + self.cellpy_headers_normal = get_headers_normal() # the column headers defined by cellpy self.normal_headers_renaming_dict = None # renaming dict for the headers self.config_params = None # configuration parameters @@ -190,9 +190,7 @@ def loader(self, name, **kwargs): data.raw = data_df data.raw_data_files_length.append(len(data_df)) - data.summary = ( - pd.DataFrame() - ) # creating an empty frame - loading summary is not implemented yet + data.summary = pd.DataFrame() # creating an empty frame - loading summary is not implemented yet data = self._post_process(data) data = self.identify_last_data_point(data) @@ -229,13 +227,9 @@ def _post_process(self, data): hdr_step_time = self.cellpy_headers_normal.step_time_txt hdr_test_time = self.cellpy_headers_normal.test_time_txt if hdr_step_time in data.raw: - data.raw[hdr_step_time] = pd.to_timedelta( - data.raw[hdr_step_time] - ).dt.total_seconds() + data.raw[hdr_step_time] = pd.to_timedelta(data.raw[hdr_step_time]).dt.total_seconds() if hdr_test_time in data.raw: - data.raw[hdr_test_time] = pd.to_timedelta( - data.raw[hdr_test_time] - ).dt.total_seconds() + data.raw[hdr_test_time] = pd.to_timedelta(data.raw[hdr_test_time]).dt.total_seconds() if split_the_capacity: logging.debug("splitting capacity") @@ -256,18 +250,14 @@ def _post_process(self, data): try: # data.raw[test_time_txt] = pd.to_timedelta(data.raw[test_time_txt]) # cellpy is not ready for this # data.raw[step_time_txt] = pd.to_timedelta(data.raw[step_time_txt]) # cellpy is not ready for this - data.raw[date_time_txt] = pd.to_datetime( - data.raw[date_time_txt], format=DATE_TIME_FORMAT - ) + data.raw[date_time_txt] = pd.to_datetime(data.raw[date_time_txt], format=DATE_TIME_FORMAT) except ValueError: logging.debug("could not convert to datetime format") if set_index: hdr_data_point = self.cellpy_headers_normal.data_point_txt if data.raw.index.name != hdr_data_point: - data.raw = data.raw.set_index( - hdr_data_point, drop=False - ) # TODO: check if this is standard + data.raw = data.raw.set_index(hdr_data_point, drop=False) # TODO: check if this is standard if forward_fill_ir: logging.debug("forward filling ir") @@ -318,27 +308,19 @@ def _query(self): if file_format == "xls": engine = "xlrd" - logging.debug( - f"parsing with pandas.read_excel using {engine} (old format): {self.name}" - ) + logging.debug(f"parsing with pandas.read_excel using {engine} (old format): {self.name}") raise WrongFileVersion("reading old xls not implemented yet") elif file_format == "xlsx": engine = "openpyxl" - logging.critical( - f"parsing with pandas.read_excel using {engine}: {self.name}" - ) + logging.critical(f"parsing with pandas.read_excel using {engine}: {self.name}") else: - raise IOError( - f"Could not read {file_name}, {file_format} not supported yet" - ) + raise IOError(f"Could not read {file_name}, {file_format} not supported yet") # -------------- meta data -------------- try: - unit_frame = pd.read_excel( - file_name, engine=engine, sheet_name=unit_sheet, header=None - ) + unit_frame = pd.read_excel(file_name, engine=engine, sheet_name=unit_sheet, header=None) except ValueError as e: print(f"could not parse {unit_sheet} in file: {e}") print(f"most likely this file is not appropriate for cellpy") @@ -360,9 +342,7 @@ def _query(self): print(f"could not parse unit sheet: {e}") try: - test_frame = pd.read_excel( - file_name, engine=engine, sheet_name=test_sheet, header=None - ) + test_frame = pd.read_excel(file_name, engine=engine, sheet_name=test_sheet, header=None) except ValueError as e: print(f"could not parse {test_sheet} in file: {e}") print(f"It is very likely that this file is not appropriate for cellpy!") @@ -404,9 +384,7 @@ def _query(self): # combining the step and data frames data_frame[[hdr_date]] = data_frame[[hdr_date]].apply(pd.to_datetime) - step_frame[[hdr_start, hdr_end]] = step_frame[[hdr_start, hdr_end]].apply( - pd.to_datetime - ) + step_frame[[hdr_start, hdr_end]] = step_frame[[hdr_start, hdr_end]].apply(pd.to_datetime) data_frame[hdr_cycle] = 0 data_frame[hdr_step_index] = 0 @@ -419,10 +397,7 @@ def _query(self): mask = ( (data_frame[hdr_date] > start_date) - | ( - (data_frame[hdr_date] == start_date) - & (data_frame[hdr_step] == step) - ) + | ((data_frame[hdr_date] == start_date) & (data_frame[hdr_step] == step)) ) & ( (data_frame[hdr_date] < end_date) | ((data_frame[hdr_date] == end_date) & (data_frame[hdr_step] == step))