adding pandas as lazy import to parameters, and fabric to internals

cellpy/internals/__init__.py

@@ -0,0 +1,10 @@
+from cellpy.libs.apipkg import initpkg
+
+initpkg(
+    __name__,
+    {
+        "externals": {
+            "fabric": "fabric",
+        },
+    },
+)

cellpy/internals/core.py

@@ -26,8 +26,8 @@
     cast,
 )
 
-import fabric
-
+# import fabric
+from . import externals as externals
 from cellpy.exceptions import UnderDefined
 from cellpy.internals import otherpath
 from cellpy.internals.otherpath import (
@@ -115,7 +115,7 @@ def check_connection(
     connect_kwargs, host = p.connection_info()
 
     path_separator = "/"  # only supports unix-like systems
-    with fabric.Connection(host, connect_kwargs=connect_kwargs) as conn:
+    with externals.fabric.Connection(host, connect_kwargs=connect_kwargs) as conn:
         try:
             t1 = time.perf_counter()
             try:
@@ -147,8 +147,7 @@ def check_connection(
 
         except FileNotFoundError as e:
             print(
-                f"   - FileNotFoundError: Could not perform directory listing in {p.raw_path} on {host}."
-                f"\n     {e}"
+                f"   - FileNotFoundError: Could not perform directory listing in {p.raw_path} on {host}." f"\n     {e}"
             )
 
     return info

cellpy/internals/otherpath.py

@@ -24,8 +24,8 @@
     cast,
 )
 
-import fabric
-
+# import fabric
+from . import externals as externals
 from cellpy.exceptions import UnderDefined
 
 NEW_OTHER_PATH_VERSION = (3, 12)
@@ -490,7 +490,7 @@ def _get_connection_info(self, testing: bool = False) -> Tuple[Dict, str]:
         return connect_kwargs, host
 
     def _copy_with_fabric(self, host: str, connect_kwargs: dict, destination: Union[str, S, pathlib.Path]):
-        with fabric.Connection(host, connect_kwargs=connect_kwargs) as conn:
+        with externals.fabric.Connection(host, connect_kwargs=connect_kwargs) as conn:
             try:
                 t1 = time.time()
                 conn.get(self.raw_path, str(destination / self.name))
@@ -499,7 +499,7 @@ def _copy_with_fabric(self, host: str, connect_kwargs: dict, destination: Union[
                 raise FileNotFoundError(f"Could not find file {self.raw_path} on {host}") from e
 
     def _stat_with_fabric(self, host: str, connect_kwargs: dict) -> ExternalStatResult:
-        with fabric.Connection(host, connect_kwargs=connect_kwargs) as conn:
+        with externals.fabric.Connection(host, connect_kwargs=connect_kwargs) as conn:
             try:
                 t1 = time.time()
                 sftp_conn = conn.sftp()
@@ -523,7 +523,7 @@ def _listdir_with_fabric(
 
         path_separator = "/"  # only supports unix-like systems
         t1 = time.time()
-        with fabric.Connection(host, connect_kwargs=connect_kwargs) as conn:
+        with externals.fabric.Connection(host, connect_kwargs=connect_kwargs) as conn:
             try:
                 t1 = time.time()
                 sftp_conn = conn.sftp()
@@ -584,7 +584,7 @@ def _glob_with_fabric(
         path_separator = "/"
         logging.info(f"glob_str: {glob_str}")
         logging.info("using fabric to glob")
-        with fabric.Connection(host, connect_kwargs=connect_kwargs) as conn:
+        with externals.fabric.Connection(host, connect_kwargs=connect_kwargs) as conn:
             try:
                 t1 = time.time()
                 sftp_conn = conn.sftp()
@@ -1063,7 +1063,7 @@ def _get_connection_info(self, testing: bool = False) -> Tuple[Dict, str]:
         return connect_kwargs, host
 
     def _copy_with_fabric(self, host: str, connect_kwargs: dict, destination: Union[str, S, pathlib.Path]):
-        with fabric.Connection(host, connect_kwargs=connect_kwargs) as conn:
+        with externals.fabric.Connection(host, connect_kwargs=connect_kwargs) as conn:
             try:
                 t1 = time.time()
                 conn.get(self.raw_path, str(destination / self.name))
@@ -1072,7 +1072,7 @@ def _copy_with_fabric(self, host: str, connect_kwargs: dict, destination: Union[
                 raise FileNotFoundError(f"Could not find file {self.raw_path} on {host}") from e
 
     def _stat_with_fabric(self, host: str, connect_kwargs: dict) -> ExternalStatResult:
-        with fabric.Connection(host, connect_kwargs=connect_kwargs) as conn:
+        with externals.fabric.Connection(host, connect_kwargs=connect_kwargs) as conn:
             try:
                 t1 = time.time()
                 sftp_conn = conn.sftp()
@@ -1096,7 +1096,7 @@ def _listdir_with_fabric(
 
         path_separator = "/"  # only supports unix-like systems
         t1 = time.time()
-        with fabric.Connection(host, connect_kwargs=connect_kwargs) as conn:
+        with externals.fabric.Connection(host, connect_kwargs=connect_kwargs) as conn:
             try:
                 t1 = time.time()
                 sftp_conn = conn.sftp()
@@ -1157,7 +1157,7 @@ def _glob_with_fabric(
         path_separator = "/"
         logging.info(f"glob_str: {glob_str}")
         logging.info("using fabric to glob")
-        with fabric.Connection(host, connect_kwargs=connect_kwargs) as conn:
+        with externals.fabric.Connection(host, connect_kwargs=connect_kwargs) as conn:
             try:
                 t1 = time.time()
                 sftp_conn = conn.sftp()

cellpy/parameters/__init__.py

@@ -1 +1,11 @@
+from cellpy.libs.apipkg import initpkg
 
+initpkg(
+    __name__,
+    {
+        "externals": {
+            "pandas": "pandas",
+            "box": "box",
+        },
+    },
+)

cellpy/parameters/internal_settings.py

@@ -7,8 +7,8 @@
 from typing import List, Optional
 
 # TODO: crashed for python 3.10 using pandas<=2.1
-import pandas as pd
-
+# import pandas as pd
+from . import externals as externals
 from cellpy import prms
 
 CELLPY_FILE_VERSION = 8
@@ -123,7 +123,7 @@ def digest(self, as_list: bool = False, **kwargs):
 
     def to_frame(self):
         """Converts to pandas dataframe"""
-        df = pd.DataFrame.from_dict(asdict(self), orient="index")
+        df = externals.pandas.DataFrame.from_dict(asdict(self), orient="index")
         df.index.name = "key"
         n_rows, n_cols = df.shape
         if n_cols == 1:
@@ -286,7 +286,7 @@ def get(self, key):
 
     def to_frame(self):
         """Converts to pandas dataframe"""
-        df = pd.DataFrame.from_dict(asdict(self), orient="index")
+        df = externals.pandas.DataFrame.from_dict(asdict(self), orient="index")
         df.index.name = "key"
         n_rows, n_cols = df.shape
         if n_cols == 1:

cellpy/parameters/prmreader.py

@@ -10,13 +10,14 @@
 from dataclasses import asdict, dataclass
 from pprint import pprint
 
-import box
+# import box
 import dotenv
 from rich import print
 import ruamel
 from ruamel.yaml import YAML
 from ruamel.yaml.error import YAMLError
 
+from . import externals as externals
 from cellpy.exceptions import ConfigFileNotRead, ConfigFileNotWritten
 from cellpy.parameters import prms
 from cellpy.parameters.internal_settings import OTHERPATHS
@@ -187,9 +188,9 @@ def _update_prms(config_dict, resolve_paths=True):
                 z = config_dict[key][k]
                 if isinstance(z, dict):
                     y = getattr(_config_attr, k)
-                    z = box.Box({**y, **z})
+                    z = externals.box.Box({**y, **z})
                 if isinstance(z, ruamel.yaml.comments.CommentedMap):
-                    z = box.Box(z)
+                    z = externals.box.Box(z)
                 setattr(_config_attr, k, z)
         else:
             logging.info("\n  not-supported prm: %s" % key)
@@ -413,7 +414,7 @@ def info():
         if key.startswith("_") and not key.startswith("__") and prms._debug:  # NOQA
             print(f"Internal: {key} (type={type(current_object)}): {current_object}")
 
-        elif isinstance(current_object, box.Box):
+        elif isinstance(current_object, externals.box.Box):
             print()
             print(f" {key} [OLD-TYPE PRM] ".center(80, "-"))
             for subkey in current_object:
@@ -422,18 +423,14 @@ def info():
 
         elif key == "Paths":
             print(" Paths ".center(80, "-"))
-            attributes = {
-                k: v for k, v in vars(current_object).items() if not k.startswith("_")
-            }
+            attributes = {k: v for k, v in vars(current_object).items() if not k.startswith("_")}
             for attr in OTHERPATHS:
                 attributes[attr] = getattr(current_object, attr)
             print(attributes)
 
         elif isinstance(current_object, (prms.CellPyConfig, prms.CellPyDataConfig)):
             # print(" NEW-TYPE PRM ".center(80, "="))
-            attributes = {
-                k: v for k, v in vars(current_object).items() if not k.startswith("_")
-            }
+            attributes = {k: v for k, v in vars(current_object).items() if not k.startswith("_")}
             print(f" {key} ".center(80, "-"))
             print(attributes)
             print()

cellpy/parameters/prms.py

@@ -9,9 +9,9 @@
 
 # Using TYPE_CHECKING to avoid circular imports
 # (this will only work without from __future__ import annotations for python 3.11 and above)
-from cellpy.internals.core import OtherPath
 
-import box
+from . import externals as externals
+from cellpy.internals.core import OtherPath
 
 # When adding prms, please
 #   1) check / update the internal_settings.py file as well to
@@ -310,9 +310,9 @@ class InstrumentsClass(CellPyConfig):
 
     tester: Union[str, None]
     custom_instrument_definitions_file: Union[str, None]
-    Arbin: box.Box
-    Maccor: box.Box
-    Neware: box.Box
+    Arbin: externals.box.Box
+    Maccor: externals.box.Box
+    Neware: externals.box.Box
 
 
 # Pre-defined instruments:
@@ -331,13 +331,13 @@ class InstrumentsClass(CellPyConfig):
     "SQL_Driver": "SQL Server",
 }
 
-Arbin = box.Box(_Arbin)
+Arbin = externals.box.Box(_Arbin)
 
 _Maccor = {"default_model": "one"}
-Maccor = box.Box(_Maccor)
+Maccor = externals.box.Box(_Maccor)
 
 _Neware = {"default_model": "one"}
-Neware = box.Box(_Neware)
+Neware = externals.box.Box(_Neware)
 
 Instruments = InstrumentsClass(
     tester=None,  # TODO: moving this to DataSetClass (deprecate)

cellpy/readers/__init__.py

@@ -8,22 +8,11 @@
             "openpyxl": "openpyxl",
             "pandas": "pandas",
             "pint": "pint",
+            # "pyodbc": "pyodbc",
+            # "sqlalchemy": "sqlalchemy",
+            # "tqdm": "tqdm",
+            # "xlrd": "xlrd",
+            "fabric": "fabric",
         },
-        # "core": {
-        #     "Data": "cellpy.readers.core:Data",
-        #     "BaseDbReader": "cellpy.readers.core:BaseDbReader",
-        #     "FileID": "cellpy.readers.core:FileID",
-        #     "Q": "cellpy.readers.core:Q",
-        #     "convert_from_simple_unit_label_to_string_unit_label": "cellpy.readers.core:convert_from_simple_unit_label_to_string_unit_label",
-        #     "generate_default_factory": "cellpy.readers.core:generate_default_factory",
-        #     "identify_last_data_point": "cellpy.readers.core:identify_last_data_point",
-        #     "instrument_configurations": "cellpy.readers.core:instrument_configurations",
-        #     "interpolate_y_on_x": "cellpy.readers.core:interpolate_y_on_x",
-        #     "pickle_protocol": "cellpy.readers.core:pickle_protocol",
-        #     "xldate_as_datetime": "cellpy.readers.core:xldate_as_datetime",
-        # },
-        # "internals": {
-        #     "OtherPath": "cellpy.internals.core:OtherPath",
-        # },
     },
 )

cellpy/readers/cellreader.py

@@ -27,7 +27,7 @@
 from typing import TYPE_CHECKING
 from dataclasses import asdict
 
-from . import externals
+from . import externals as externals
 from cellpy.readers import core
 import cellpy.internals.core as internals
 

cellpy/readers/core.py

@@ -17,12 +17,7 @@
 import warnings
 from typing import Any, Tuple, Dict, List, Union, TypeVar, Optional
 
-from . import externals
-
-# import numpy as np
-# import pandas as pd
-import pint
-from scipy import interpolate
+from . import externals as externals
 
 from cellpy.exceptions import NullData
 from cellpy.internals.core import OtherPath
@@ -1106,6 +1101,7 @@ def interpolate_y_on_x(
 
     # TODO: allow for giving a fixed interpolation range (x-values).
     #  Remember to treat extrapolation properly (e.g. replace with NaN?).
+    from scipy import interpolate
 
     if x is None:
         x = df.columns[0]

cellpy/readers/dbreader.py

@@ -8,14 +8,10 @@
 import warnings
 from dataclasses import asdict
 from datetime import datetime
+from typing import List, Optional
 
-import numpy as np
-import pandas as pd
-from typing import List
-from typing import Optional
-
+from . import externals as externals
 from cellpy.parameters import prms
-
 from cellpy.readers import core
 
 # logger = logging.getLogger(__name__)
@@ -269,8 +265,8 @@ def _find_out_what_rows_to_skip(self):
 
     def _lookup_unit(self, label):
         units = {
-            "int": np.int32,
-            "float": np.float64,
+            "int": externals.numpy.int32,
+            "float": externals.numpy.float64,
             "str": str,
             "bol": bool,
             "cat": str,
@@ -305,7 +301,7 @@ def _open_sheet(self):
         logging.debug(f"Trying to open the file {self.db_file}")
         logging.debug(f"Number of rows (no means all): {nrows}")
         logging.debug(f"Skipping the following rows: {rows_to_skip}")
-        work_book = pd.ExcelFile(self.db_file, engine="openpyxl")
+        work_book = externals.pandas.ExcelFile(self.db_file, engine="openpyxl")
         try:
             sheet = work_book.parse(
                 table_name,

cellpy/readers/filefinder.py

@@ -10,8 +10,7 @@
 from typing import Optional, Union, List, Tuple
 import warnings
 
-import fabric
-
+from . import externals as externals
 import cellpy.exceptions
 from cellpy.parameters import prms
 from cellpy.internals.core import OtherPath
@@ -108,7 +107,7 @@ def find_in_raw_file_directory(
             f = _file_list[0]
         else:
             # TODO: make a better error-message if the d.raw_path does not exist:
-            with fabric.Connection(host, connect_kwargs=connect_kwargs) as conn:
+            with externals.fabric.Connection(host, connect_kwargs=connect_kwargs) as conn:
                 find_command = f'find -L {d.raw_path} -name "{glob_txt}"'
                 out = conn.run(f"{find_command}", hide="both", warn=True)
             if out.return_code != 0: