Merge main

.ci_support/environment-docs.yml

@@ -9,17 +9,17 @@ dependencies:
 - conda_subprocess =0.0.4
 - cloudpickle =3.0.0
 - gitpython =3.1.43
-- h5io_browser =0.0.15
+- h5io_browser =0.0.16
 - h5py =3.11.0
 - jinja2 =3.1.4
-- monty =2024.5.24
+- monty =2024.7.12
 - numpy =1.26.4
 - pandas =2.2.2
-- pint =0.24.1
+- pint =0.24.3
 - psutil =6.0.0
 - pyfileindex =0.0.25
-- pyiron_snippets =0.1.2
-- pympipool =0.8.5
+- pyiron_snippets =0.1.3
+- executorlib =0.0.1
 - pysqa =0.1.21
 - pytables =3.9.2
 - sqlalchemy =2.0.31

.ci_support/environment-mini.yml

@@ -0,0 +1,18 @@
+channels:
+- conda-forge
+dependencies:
+- cloudpickle =3.0.0
+- h5io_browser =0.0.16
+- h5py =3.11.0
+- monty =2024.7.12
+- numpy =1.26.4
+- pandas =2.2.2
+- psutil =6.0.0
+- pyfileindex =0.0.25
+- pyiron_snippets =0.1.3
+- executorlib =0.0.1
+- pysqa =0.1.21
+- pytables =3.9.2
+- sqlalchemy =2.0.31
+- tqdm =4.66.4
+- traitlets =5.14.3

.ci_support/environment-old.yml

@@ -13,7 +13,7 @@ dependencies:
 - psutil =5.8.0
 - pyfileindex =0.0.16
 - pyiron_snippets =0.1.1
-- pympipool =0.8.0
+- executorlib =0.0.1
 - pysqa =0.1.12
 - pytables =3.6.1
 - sqlalchemy =2.0.22

.ci_support/environment.yml

@@ -7,17 +7,17 @@ dependencies:
 - conda_subprocess =0.0.4
 - cloudpickle =3.0.0
 - gitpython =3.1.43
-- h5io_browser =0.0.15
+- h5io_browser =0.0.16
 - h5py =3.11.0
 - jinja2 =3.1.4
-- monty =2024.5.24
+- monty =2024.7.12
 - numpy =1.26.4
 - pandas =2.2.2
-- pint =0.24.1
+- pint =0.24.3
 - psutil =6.0.0
 - pyfileindex =0.0.25
-- pyiron_snippets =0.1.2
-- pympipool =0.8.5
+- pyiron_snippets =0.1.3
+- executorlib =0.0.1
 - pysqa =0.1.21
 - pytables =3.9.2
 - sqlalchemy =2.0.31

.github/workflows/dependabot.yml

@@ -23,6 +23,7 @@ jobs:
           sed -i "/${package}/s/${from}/${to}/g" binder/environment.yml
           sed -i "/${package}/s/${from}/${to}/g" .ci_support/environment.yml
           sed -i "/${package}/s/${from}/${to}/g" .ci_support/environment-docs.yml
+          sed -i "/${package}/s/${from}/${to}/g" .ci_support/environment-mini.yml
       - name: UpdateDependabotPR commit
         run: |
           git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"

.github/workflows/minimal.yml

@@ -0,0 +1,31 @@
+# This workflow is used to run the unittest of pyiron
+
+name: Minimal
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Setup Mambaforge
+      uses: conda-incubator/setup-miniconda@v3
+      with:
+        python-version: '3.12'
+        miniforge-variant: Mambaforge
+        channels: conda-forge
+        channel-priority: strict
+        activate-environment: my-env
+        environment-file: .ci_support/environment-mini.yml
+        use-mamba: true
+    - name: Test
+      shell: bash -l {0}
+      timeout-minutes: 10
+      run: |
+        python .ci_support/pyironconfig.py
+        bash .ci_support/pip_install.sh
+        python -m unittest discover tests/unit

.github/workflows/unittests.yml

@@ -62,9 +62,3 @@ jobs:
     - name: Coveralls
       if: matrix.label == 'linux-64-py-3-11'
       uses: coverallsapp/github-action@v2
-    - name: Codacy
-      if:  matrix.label == 'linux-64-py-3-11' && github.event_name != 'push'
-      continue-on-error: True
-      shell: bash -l {0}
-      run: |
-        python-codacy-coverage -r coverage.xml

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.5.1
+    rev: v0.5.2
     hooks:
       - id: ruff
         name: ruff lint

README.md

@@ -18,7 +18,7 @@ simulation in `pyiron_atomistics`.
   can be wrapped in `pyiron_base` to enable parameter studies with thousands or millions of calculation.
 * The calculation can either be executed locally on the same computer or on high performance computing (HPC) resources.
   The python simple queuing system adapter [pysqa](https://pysqa.readthedocs.io) is used to interface with the HPC 
-  queuing systems directly from python and the [pympipool](https://pympipool.readthedocs.io) package is employed to 
+  queuing systems directly from python and the [pympipool](https://executorlib.readthedocs.io) package is employed to 
   assign dedicated resources like multiple CPU cores and GPUs to individual python functions.
 * Scientific data is efficiently stored using the [hierarchical data format (HDF)](https://www.hdfgroup.org) via the 
   [h5py](https://www.h5py.org) python library and more specifically the [h5io](https://github.com/h5io) packages to 

binder/environment.yml

@@ -6,17 +6,17 @@ dependencies:
 - conda_subprocess =0.0.4
 - cloudpickle =3.0.0
 - gitpython =3.1.43
-- h5io_browser =0.0.15
+- h5io_browser =0.0.16
 - h5py =3.11.0
 - jinja2 =3.1.4
-- monty =2024.5.24
+- monty =2024.7.12
 - numpy =1.26.4
 - pandas =2.2.2
-- pint =0.24.1
+- pint =0.24.3
 - psutil =6.0.0
 - pyfileindex =0.0.25
-- pyiron_snippets =0.1.2
-- pympipool =0.8.5
+- pyiron_snippets =0.1.3
+- executorlib =0.0.1
 - pysqa =0.1.21
 - pytables =3.9.2
 - sqlalchemy =2.0.31

docs/tutorial.md

@@ -131,7 +131,7 @@ selected. Still it is important to mention, that assigning 120 CPU cores does no
 function. Only by implementing internal parallelization inside the python functions with solutions like 
 [concurrent.futures.ProcessPoolExecutor](https://docs.python.org/3/library/concurrent.futures.html#processpoolexecutor)
 it is possible to parallelize the execution of python functions on a single compute node. Finally, the pyiron developers
-released the [pympipool](https://pympipool.readthedocs.io) to enable parallelization of python functions as well as the
+released the [pympipool](https://executorlib.readthedocs.io) to enable parallelization of python functions as well as the
 direct assignment of GPU resources inside a given queuing system allocation over multiple compute nodes using the 
 hierarchical queuing system [flux](https://flux-framework.org). 
 

pyiron_base/_tests.py

@@ -45,7 +45,7 @@ def setUp(self):
 
     def _assert_equal_numpy(self, a, b, msg=None):
         try:
-            np.testing.assert_array_equal(a, b, err_msg=msg)
+            np.testing.assert_array_equal(a, b, err_msg=msg if msg is not None else "")
         except AssertionError as e:
             raise self.failureException(*e.args) from None
 

pyiron_base/jobs/datamining.py

@@ -255,7 +255,7 @@ def create_table(self, file, job_status_list, executor=None, enforce_update=Fals
             The executor, if given, must not naively pickle the mapped functions or
             arguments, as PyironTable relies on lambda functions internally.  Use
             with executors that rely on dill or cloudpickle instead.  Pyiron
-            provides such executors in the `pympipool` sub packages.
+            provides such executors in the `executorlib` sub packages.
 
         Args:
             file (FileHDFio): HDF were the previous state of the table is stored
@@ -785,7 +785,7 @@ def update_table(self, job_status_list=None):
             self.project.db.item_update({"timestart": datetime.now()}, self.job_id)
         with self.project_hdf5.open("input") as hdf5_input:
             if self._executor_type is None and self.server.cores > 1:
-                self._executor_type = "pympipool.Executor"
+                self._executor_type = "executorlib.Executor"
             if self._executor_type is not None:
                 with self._get_executor(max_workers=self.server.cores) as exe:
                     self._pyiron_table.create_table(

pyiron_base/jobs/job/generic.py

@@ -1620,15 +1620,15 @@ def _get_executor(self, max_workers=None):
                 "No executor type defined - Please set self.executor_type."
             )
         elif (
-            self._executor_type == "pympipool.Executor"
+            self._executor_type == "executorlib.Executor"
             and platform.system() == "Darwin"
         ):
             # The Mac firewall might prevent connections based on the network address - especially Github CI
             return import_class(self._executor_type)(
                 max_cores=max_workers, hostname_localhost=True
             )
-        elif self._executor_type == "pympipool.Executor":
-            # The pympipool Executor defines max_cores rather than max_workers
+        elif self._executor_type == "executorlib.Executor":
+            # The executorlib Executor defines max_cores rather than max_workers
             return import_class(self._executor_type)(max_cores=max_workers)
         elif isinstance(self._executor_type, str):
             return import_class(self._executor_type)(max_workers=max_workers)

pyiron_base/jobs/job/interactive.py

@@ -268,16 +268,21 @@ def interactive_flush(self, path="interactive", include_last_step=False):
                     step=self.interactive_write_frequency,
                     include_last=include_last_step,
                 )
-                if (
-                    len(data) > 0
-                    and isinstance(data[0], list)
-                    and len(np.shape(data)) == 1
-                ):
-                    self._extend_hdf(h5=h5, path=path, key=key, data=data)
-                elif np.array(data).dtype == np.dtype("O"):
-                    self._extend_hdf(h5=h5, path=path, key=key, data=data)
-                else:
-                    self._extend_hdf(h5=h5, path=path, key=key, data=np.array(data))
+                try:
+                    if (
+                        len(data) > 0
+                        and isinstance(data[0], list)
+                        and len(np.shape(data)) == 1
+                    ):
+                        self._extend_hdf(h5=h5, path=path, key=key, data=data)
+                    elif np.array(data).dtype == np.dtype("O"):
+                        self._extend_hdf(h5=h5, path=path, key=key, data=data)
+                    else:
+                        self._extend_hdf(h5=h5, path=path, key=key, data=np.array(data))
+                except ValueError:
+                    self._extend_hdf(
+                        h5=h5, path=path, key=key, data=np.array(data, dtype="object")
+                    )
                 self.interactive_cache[key] = []
 
     def interactive_open(self):

pyiron_base/jobs/job/runfunction.py

@@ -10,7 +10,6 @@
 from datetime import datetime
 from typing import List, Optional, Tuple
 
-from jinja2 import Template
 from pyiron_snippets.deprecate import deprecate
 
 from pyiron_base.jobs.job.wrapper import JobWrapper
@@ -1008,6 +1007,8 @@ def execute_job_with_calculate_function(job):
 
 
 def _generate_flux_execute_string(job, database_is_disabled):
+    from jinja2 import Template
+
     if not database_is_disabled:
         executable_template = Template(
             "#!/bin/bash\n"