adding pandas, scipy interpolate, openpyxl and pint to lazy import in…

… readers (though pint is used in core, so it will be loaded on initial cellpy import)

cellpy/__init__.py

@@ -21,23 +21,29 @@
 import warnings
 
 import cellpy._version
+
+
 from cellpy.parameters import prms  # TODO: this might give circular ref
 from cellpy.parameters import prmreader
+
+__version__ = cellpy._version.__version__
+
 from cellpy.readers import cellreader, dbreader, filefinder, do
 from cellpy.readers.core import Q, ureg
 
-__version__ = cellpy._version.__version__
+# Q, ureg = cellpy.readers.core.get_pint_unit_registry()
+
 
 logging.getLogger(__name__).addHandler(logging.NullHandler())
 
 # TODO: (v2.0) remove this and enforce using for example `import cellpy.session as clp` and then
 #  run `prmreader.initialize` in that `__init__` instead:
-init = prmreader.initialize
+init = parameters.prmreader.initialize
 init()
 
 # TODO: (v2.0) remove this and enforce using `cellpy.get` (or `cellpy.cellreader.get`) instead:
 get = cellreader.get
-print_instruments = cellreader.print_instruments
+print_instruments = readers.cellreader.print_instruments
 
 __all__ = [
     "cellreader",

cellpy/libs/apipkg/__init__.py

@@ -0,0 +1,39 @@
+"""
+apipkg: control the exported namespace of a Python package.
+
+see https://pypi.python.org/pypi/apipkg
+
+(c) holger krekel, 2009 - MIT license
+"""
+from __future__ import annotations
+
+__all__ = ["initpkg", "ApiModule", "AliasModule", "__version__", "distribution_version"]
+import sys
+from typing import Any
+
+from ._alias_module import AliasModule
+from ._importing import distribution_version as distribution_version
+from ._module import _initpkg
+from ._module import ApiModule
+from ._version import version as __version__
+
+
+def initpkg(
+    pkgname: str,
+    exportdefs: dict[str, Any],
+    attr: dict[str, object] | None = None,
+    eager: bool = False,
+) -> ApiModule:
+    """initialize given package from the export definitions."""
+    attr = attr or {}
+    mod = sys.modules.get(pkgname)
+
+    mod = _initpkg(mod, pkgname, exportdefs, attr=attr)
+
+    # eagerload in bypthon to avoid their monkeypatching breaking packages
+    if "bpython" in sys.modules or eager:
+        for module in list(sys.modules.values()):
+            if isinstance(module, ApiModule):
+                getattr(module, "__dict__")
+
+    return mod

cellpy/libs/apipkg/_alias_module.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from types import ModuleType
+
+from ._importing import importobj
+
+
+def AliasModule(modname: str, modpath: str, attrname: str | None = None) -> ModuleType:
+    cached_obj: object | None = None
+
+    def getmod() -> object:
+        nonlocal cached_obj
+        if cached_obj is None:
+            cached_obj = importobj(modpath, attrname)
+        return cached_obj
+
+    x = modpath + ("." + attrname if attrname else "")
+    repr_result = f"<AliasModule {modname!r} for {x!r}>"
+
+    class AliasModule(ModuleType):
+        def __repr__(self) -> str:
+            return repr_result
+
+        def __getattribute__(self, name: str) -> object:
+            try:
+                return getattr(getmod(), name)
+            except ImportError:
+                if modpath == "pytest" and attrname is None:
+                    # hack for pylibs py.test
+                    return None
+                else:
+                    raise
+
+        def __setattr__(self, name: str, value: object) -> None:
+            setattr(getmod(), name, value)
+
+        def __delattr__(self, name: str) -> None:
+            delattr(getmod(), name)
+
+    return AliasModule(str(modname))

cellpy/libs/apipkg/_importing.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+import os
+import sys
+
+
+def _py_abspath(path: str) -> str:
+    """
+    special version of abspath
+    that will leave paths from jython jars alone
+    """
+    if path.startswith("__pyclasspath__"):
+        return path
+    else:
+        return os.path.abspath(path)
+
+
+def distribution_version(name: str) -> str | None:
+    """try to get the version of the named distribution,
+    returns None on failure"""
+    if sys.version_info >= (3, 8):
+        from importlib.metadata import PackageNotFoundError, version
+    else:
+        from importlib_metadata import PackageNotFoundError, version
+    try:
+        return version(name)
+    except PackageNotFoundError:
+        return None
+
+
+def importobj(modpath: str, attrname: str | None) -> object:
+    """imports a module, then resolves the attrname on it"""
+    module = __import__(modpath, None, None, ["__doc__"])
+    if not attrname:
+        return module
+
+    retval = module
+    names = attrname.split(".")
+    for x in names:
+        retval = getattr(retval, x)
+    return retval

cellpy/libs/apipkg/_module.py

@@ -0,0 +1,184 @@
+from __future__ import annotations
+
+import sys
+import threading
+from types import ModuleType
+from typing import Any
+from typing import Callable
+from typing import cast
+from typing import Iterable
+
+from ._alias_module import AliasModule
+from ._importing import _py_abspath
+from ._importing import importobj
+from ._syncronized import _synchronized
+
+
+class ApiModule(ModuleType):
+    """the magical lazy-loading module standing"""
+
+    def __docget(self) -> str | None:
+        try:
+            return self.__doc
+        except AttributeError:
+            if "__doc__" in self.__map__:
+                return cast(str, self.__makeattr("__doc__"))
+            else:
+                return None
+
+    def __docset(self, value: str) -> None:
+        self.__doc = value
+
+    __doc__ = property(__docget, __docset)  # type: ignore
+    __map__: dict[str, tuple[str, str]]
+
+    def __init__(
+        self,
+        name: str,
+        importspec: dict[str, Any],
+        implprefix: str | None = None,
+        attr: dict[str, Any] | None = None,
+    ) -> None:
+        super().__init__(name)
+        self.__name__ = name
+        self.__all__ = [x for x in importspec if x != "__onfirstaccess__"]
+        self.__map__ = {}
+        self.__implprefix__ = implprefix or name
+        if attr:
+            for name, val in attr.items():
+                setattr(self, name, val)
+        for name, importspec in importspec.items():
+            if isinstance(importspec, dict):
+                subname = f"{self.__name__}.{name}"
+                apimod = ApiModule(subname, importspec, implprefix)
+                sys.modules[subname] = apimod
+                setattr(self, name, apimod)
+            else:
+                parts = importspec.split(":")
+                modpath = parts.pop(0)
+                attrname = parts and parts[0] or ""
+                if modpath[0] == ".":
+                    modpath = implprefix + modpath
+
+                if not attrname:
+                    subname = f"{self.__name__}.{name}"
+                    apimod = AliasModule(subname, modpath)
+                    sys.modules[subname] = apimod
+                    if "." not in name:
+                        setattr(self, name, apimod)
+                else:
+                    self.__map__[name] = (modpath, attrname)
+
+    def __repr__(self):
+        repr_list = [f"<ApiModule {self.__name__!r}"]
+        if hasattr(self, "__version__"):
+            repr_list.append(f" version={self.__version__!r}")
+        if hasattr(self, "__file__"):
+            repr_list.append(f" from {self.__file__!r}")
+        repr_list.append(">")
+        return "".join(repr_list)
+
+    @_synchronized
+    def __makeattr(self, name, isgetattr=False):
+        """lazily compute value for name or raise AttributeError if unknown."""
+        target = None
+        if "__onfirstaccess__" in self.__map__:
+            target = self.__map__.pop("__onfirstaccess__")
+            fn = cast(Callable[[], None], importobj(*target))
+            fn()
+        try:
+            modpath, attrname = self.__map__[name]
+        except KeyError:
+            # __getattr__ is called when the attribute does not exist, but it may have
+            # been set by the onfirstaccess call above. Infinite recursion is not
+            # possible as __onfirstaccess__ is removed before the call (unless the call
+            # adds __onfirstaccess__ to __map__ explicitly, which is not our problem)
+            if target is not None and name != "__onfirstaccess__":
+                return getattr(self, name)
+            # Attribute may also have been set during a concurrent call to __getattr__
+            # which executed after this call was already waiting on the lock. Check
+            # for a recently set attribute while avoiding infinite recursion:
+            # * Don't call __getattribute__ if __makeattr was called from a data
+            #   descriptor such as the __doc__ or __dict__ properties, since data
+            #   descriptors are called as part of object.__getattribute__
+            # * Only call __getattribute__ if there is a possibility something has set
+            #   the attribute we're looking for since __getattr__ was called
+            if threading is not None and isgetattr:
+                return super().__getattribute__(name)
+            raise AttributeError(name)
+        else:
+            result = importobj(modpath, attrname)
+            setattr(self, name, result)
+            # in a recursive-import situation a double-del can happen
+            self.__map__.pop(name, None)
+            return result
+
+    def __getattr__(self, name):
+        return self.__makeattr(name, isgetattr=True)
+
+    def __dir__(self) -> Iterable[str]:
+        yield from super().__dir__()
+        yield from self.__map__
+
+    @property
+    def __dict__(self) -> dict[str, Any]:  # type: ignore
+        # force all the content of the module
+        # to be loaded when __dict__ is read
+        dictdescr = ModuleType.__dict__["__dict__"]  # type: ignore
+        ns: dict[str, Any] = dictdescr.__get__(self)
+        if ns is not None:
+            hasattr(self, "some")
+            for name in self.__all__:
+                try:
+                    self.__makeattr(name)
+                except AttributeError:
+                    pass
+        return ns
+
+
+_PRESERVED_MODULE_ATTRS = {
+    "__file__",
+    "__version__",
+    "__loader__",
+    "__path__",
+    "__package__",
+    "__doc__",
+    "__spec__",
+    "__dict__",
+}
+
+
+def _initpkg(mod: ModuleType | None, pkgname, exportdefs, attr=None) -> ApiModule:
+    """Helper for initpkg.
+
+    Python 3.3+ uses finer grained locking for imports, and checks sys.modules before
+    acquiring the lock to avoid the overhead of the fine-grained locking. This
+    introduces a race condition when a module is imported by multiple threads
+    concurrently - some threads will see the initial module and some the replacement
+    ApiModule. We avoid this by updating the existing module in-place.
+
+    """
+    if mod is None:
+        d = {"__file__": None, "__spec__": None}
+        d.update(attr)
+        mod = ApiModule(pkgname, exportdefs, implprefix=pkgname, attr=d)
+        sys.modules[pkgname] = mod
+        return mod
+    else:
+        f = getattr(mod, "__file__", None)
+        if f:
+            f = _py_abspath(f)
+        mod.__file__ = f
+        if hasattr(mod, "__path__"):
+            mod.__path__ = [_py_abspath(p) for p in mod.__path__]
+        if "__doc__" in exportdefs and hasattr(mod, "__doc__"):
+            del mod.__doc__
+        for name in dir(mod):
+            if name not in _PRESERVED_MODULE_ATTRS:
+                delattr(mod, name)
+
+        # Updating class of existing module as per importlib.util.LazyLoader
+        mod.__class__ = ApiModule
+        apimod = cast(ApiModule, mod)
+        ApiModule.__init__(apimod, pkgname, exportdefs, implprefix=pkgname, attr=attr)
+        return apimod

cellpy/libs/apipkg/_syncronized.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+import functools
+import threading
+
+
+def _synchronized(wrapped_function):
+    """Decorator to synchronise __getattr__ calls."""
+
+    # Lock shared between all instances of ApiModule to avoid possible deadlocks
+    lock = threading.RLock()
+
+    @functools.wraps(wrapped_function)
+    def synchronized_wrapper_function(*args, **kwargs):
+        with lock:
+            return wrapped_function(*args, **kwargs)
+
+    return synchronized_wrapper_function

cellpy/libs/apipkg/_version.py

@@ -0,0 +1 @@
+version = "3.0.2"

cellpy/readers/__init__.py

@@ -1 +1,29 @@
+from cellpy.libs.apipkg import initpkg
 
+initpkg(
+    __name__,
+    {
+        "externals": {
+            "numpy": "numpy",
+            "openpyxl": "openpyxl",
+            "pandas": "pandas",
+            "pint": "pint",
+        },
+        # "core": {
+        #     "Data": "cellpy.readers.core:Data",
+        #     "BaseDbReader": "cellpy.readers.core:BaseDbReader",
+        #     "FileID": "cellpy.readers.core:FileID",
+        #     "Q": "cellpy.readers.core:Q",
+        #     "convert_from_simple_unit_label_to_string_unit_label": "cellpy.readers.core:convert_from_simple_unit_label_to_string_unit_label",
+        #     "generate_default_factory": "cellpy.readers.core:generate_default_factory",
+        #     "identify_last_data_point": "cellpy.readers.core:identify_last_data_point",
+        #     "instrument_configurations": "cellpy.readers.core:instrument_configurations",
+        #     "interpolate_y_on_x": "cellpy.readers.core:interpolate_y_on_x",
+        #     "pickle_protocol": "cellpy.readers.core:pickle_protocol",
+        #     "xldate_as_datetime": "cellpy.readers.core:xldate_as_datetime",
+        # },
+        # "internals": {
+        #     "OtherPath": "cellpy.internals.core:OtherPath",
+        # },
+    },
+)