Merge branch 'dev' into ci-linux-arm64

.clang-tidy

@@ -2,5 +2,5 @@
 # FIXME: all performance-* reports
 # FIXME: all cert-* reports
 # FIXME: all bugprone-* reports
-Checks: -*,bugprone-*,-bugprone-unhandled-self-assignment,-bugprone-parent-virtual-call,-bugprone-narrowing-conversions,-bugprone-exception-escape,-bugprone-string-literal-with-embedded-nul,cppcoreguidelines-slicing,mpi-*,readability-non-const-parameter,performance-for-range-copy,modernize-*,-modernize-use-trailing-return-type,-modernize-use-bool-literals,-modernize-avoid-c-arrays,-modernize-use-auto,-modernize-return-braced-init-list
+Checks: -*,bugprone-*,-bugprone-unhandled-self-assignment,-bugprone-parent-virtual-call,-bugprone-narrowing-conversions,-bugprone-exception-escape,-bugprone-string-literal-with-embedded-nul,cppcoreguidelines-slicing,mpi-*,readability-non-const-parameter,performance-*,modernize-*,-modernize-use-trailing-return-type,-modernize-use-bool-literals,-modernize-avoid-c-arrays,-modernize-use-auto,-modernize-return-braced-init-list
 HeaderFilterRegex: '((^(?!\/share\/openPMD\/).*)*include\/openPMD\/.+\.hpp|src\/^(?!binding).+\.cpp$)'

.github/workflows/macos.yml

@@ -40,7 +40,8 @@ jobs:
           -DopenPMD_USE_MPI=ON    \
           -DopenPMD_USE_HDF5=ON   \
           -DopenPMD_USE_ADIOS2=ON \
-          -DopenPMD_USE_INVASIVE_TESTS=ON
+          -DopenPMD_USE_INVASIVE_TESTS=ON \
+          -DMPIEXEC_EXECUTABLE=".github/workflows/mpirun_workaround.sh"
         cmake --build build --parallel 3
         ctest --test-dir build --verbose
 

.github/workflows/mpirun_workaround.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+# mpiexec currently seems to have a bug where it tries to parse parameters
+# of the launched application when they start with a dash, e.g.
+# `mpiexec python ./openpmd-pipe --infile in.bp --outfile out.bp`
+# leads to:
+# >  An unrecognized option was included on the mpiexec command line:
+# >
+# >    Option: --infile
+# >
+# >  Please use the "mpiexec --help" command to obtain a list of all
+# >  supported options.
+#
+# This script provides a workaround by putting the called sub-command into
+# a script in a temporary file.
+
+mpiexec -n 1 ls --all \
+    && echo "MPIRUN WORKING AGAIN, PLEASE REMOVE WORKAROUND" >&2 \
+    && exit 1 \
+    || true
+
+mpirun_args=()
+
+script_file="$(mktemp)"
+
+cleanup() {
+    rm "$script_file"
+}
+trap cleanup EXIT
+
+while true; do
+    case "$1" in
+        -c | -np | --np | -n | --n )
+            mpirun_args+=("$1" "$2")
+            shift
+            shift
+            ;;
+        *)
+            break
+            ;;
+    esac
+done
+
+echo -e '#!/usr/bin/env bash\n' > "$script_file"
+for item in "$@"; do
+    echo -n "'$item' " >> "$script_file"
+done
+
+chmod +x "$script_file"
+
+mpirun "${mpirun_args[@]}" "$script_file"

.pre-commit-config.yaml

@@ -66,7 +66,7 @@ repos:
 # clang-format v13
 #   to run manually, use .github/workflows/clang-format/clang-format.sh
 - repo: https://github.com/pre-commit/mirrors-clang-format
-  rev: v17.0.4
+  rev: v17.0.6
   hooks:
   - id: clang-format
     # By default, the clang-format hook configures:
@@ -80,7 +80,7 @@ repos:
 
 # Autoremoves unused Python imports
 - repo: https://github.com/hadialqattan/pycln
-  rev: v2.3.0
+  rev: v2.4.0
   hooks:
   - id: pycln
     name: pycln (python)

Dockerfile

@@ -5,8 +5,8 @@ FROM       quay.io/pypa/manylinux2010_x86_64 as build-env
 # FROM       quay.io/pypa/manylinux1_x86_64 as build-env
 ENV        DEBIAN_FRONTEND noninteractive
 
-# Python 3.8-3.11 via "38 39 311"
-ARG        PY_VERSIONS="38 39 310 311"
+# Python 3.8-3.12 via "38 39 311 312"
+ARG        PY_VERSIONS="38 39 310 311 312"
 
 # static libs need relocatable symbols for linking to shared python lib
 ENV        CFLAGS="-fPIC ${CFLAGS}"
@@ -162,7 +162,7 @@ FROM       debian:bullseye
 ENV        DEBIAN_FRONTEND noninteractive
 COPY --from=build-env /wheelhouse/openPMD_api-*-cp311-cp311-manylinux2010_x86_64.whl .
 RUN        apt-get update \
-           && apt-get install -y --no-install-recommends python3.10 python3-distutils ca-certificates curl \
+           && apt-get install -y --no-install-recommends python3.11 python3-distutils ca-certificates curl \
            && rm -rf /var/lib/apt/lists/*
 RUN        python3.11 --version \
            && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \
@@ -172,6 +172,21 @@ RUN        python3.11 -c "import openpmd_api as io; print(io.__version__); print
 RUN        python3.11 -m openpmd_api.ls --help
 RUN        openpmd-ls --help
 
+# test in fresh env: Debian:Bullseye + Python 3.12
+FROM       debian:bullseye
+ENV        DEBIAN_FRONTEND noninteractive
+COPY --from=build-env /wheelhouse/openPMD_api-*-cp312-cp312-manylinux2010_x86_64.whl .
+RUN        apt-get update \
+           && apt-get install -y --no-install-recommends python3.12 python3-distutils ca-certificates curl \
+           && rm -rf /var/lib/apt/lists/*
+RUN        python3.12 --version \
+           && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \
+           && python3.12 get-pip.py \
+           && python3.12 -m pip install openPMD_api-*-cp312-cp312-manylinux2010_x86_64.whl
+RUN        python3.12 -c "import openpmd_api as io; print(io.__version__); print(io.variants)"
+RUN        python3.12 -m openpmd_api.ls --help
+RUN        openpmd-ls --help
+
 # copy binary artifacts (wheels)
 FROM       quay.io/pypa/manylinux2010_x86_64
 MAINTAINER Axel Huebl <[email protected]>

README.md

@@ -115,7 +115,7 @@ while those can be built either with or without:
 
 Optional language bindings:
 * Python:
-  * Python 3.8 - 3.11
+  * Python 3.8 - 3.12
   * pybind11 2.11.1+
   * numpy 1.15+
   * mpi4py 2.1+ (optional, for MPI)
@@ -428,6 +428,7 @@ Previously supported by the Consortium for Advanced Modeling of Particles Accele
 Supported by the Exascale Computing Project (17-SC-20-SC), a collaborative effort of two U.S. Department of Energy organizations (Office of Science and the National Nuclear Security Administration).
 This project has received funding from the European Unions Horizon 2020 research and innovation programme under grant agreement No 654220.
 This work was partially funded by the Center of Advanced Systems Understanding (CASUS), which is financed by Germany's Federal Ministry of Education and Research (BMBF) and by the Saxon Ministry for Science, Culture and Tourism (SMWK) with tax funds on the basis of the budget approved by the Saxon State Parliament.
+Supported by the HElmholtz Laser Plasma Metadata Initiative (HELPMI) project (ZT-I-PF-3-066), funded by the "Initiative and Networking Fund" of the Helmholtz Association in the framework of the "Helmholtz Metadata Collaboration" project call 2022.
 
 ### Transitive Contributions
 

Singularity

@@ -7,7 +7,7 @@ From: debian:unstable
 Welcome to the openPMD-api container.
 This container contains a pre-installed openPMD-api library.
 This container provides serial I/O.
-Supported backends are HDF5 and ADIOS1.
+Supported backends are HDF5 and ADIOS2.
 Supported frontends are C++11 and Python3.
 
 %setup
@@ -25,24 +25,19 @@ Supported frontends are C++11 and Python3.
         ipython3 \
         python3-dev \
         pybind11-dev \
-        libadios-bin libadios-dev \
         libglib2.0-dev libbz2-dev libibverbs-dev libnetcdf-dev \
         libhdf5-dev && \
     rm -rf /var/lib/apt/lists/*
 
     # python3-numpy
 
     # missing: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=900804
-    # libadios-openmpi-dev
     # libopenmpi-dev libhdf5-openmpi-dev
 
-    # libadios2-dev
-
     cd $(mktemp -d)
     cmake /opt/openpmd-api       \
         -DopenPMD_USE_MPI=OFF    \
         -DopenPMD_USE_HDF5=ON    \
-        -DopenPMD_USE_ADIOS1=ON  \
         -DopenPMD_USE_ADIOS2=OFF \
         -DopenPMD_USE_PYTHON=ON  \
         -DopenPMD_BUILD_TESTING=OFF       \
@@ -61,6 +56,5 @@ Supported frontends are C++11 and Python3.
 %labels
     openPMD_HAVE_MPI OFF
     openPMD_HAVE_HDF5 ON
-    openPMD_HAVE_ADIOS1 ON
     openPMD_HAVE_ADIOS2 OFF
     openPMD_HAVE_PYTHON ON

docs/Doxyfile

@@ -1,7 +1,7 @@
 PROJECT_NAME      = "openPMD-api"
 XML_OUTPUT        = xml
 INPUT             = ../src ../include ../README.md
-EXCLUDE_PATTERNS  = *CommonADIOS1IOHandler.cpp
+#EXCLUDE_PATTERNS  = *CommonADIOS1IOHandler.cpp
 
 # TAGFILES += "cppreference-doxygen-web.tag.xml=http://en.cppreference.com/w/"
 BUILTIN_STL_SUPPORT = YES
@@ -24,7 +24,6 @@ MACRO_EXPANSION   = YES
 PREDEFINED        = DOXYGEN_SHOULD_SKIP_THIS \
                     openPMD_HAVE_MPI=1 \
                     openPMD_HAVE_HDF5=1 \
-                    openPMD_HAVE_ADIOS1=1 \
                     openPMD_HAVE_ADIOS2=1 \
                     openPMD_HAVE_PYTHON=1 \
                     OPENPMD_private:=private \

docs/source/analysis/paraview.rst

@@ -22,7 +22,7 @@ openPMD
 -------
 
 openPMD files can be visualized with ParaView 5.9+, using 5.11+ is recommended.
-ParaView supports ADIOS1, ADIOS2 and HDF5 files, as it implements against the Python bindings of openPMD-api.
+ParaView supports ADIOS2 and HDF5 files, as it implements against the Python bindings of openPMD-api.
 
 For openPMD output to be recognized, create a small textfile with ``.pmd`` ending per data series, which can be opened with ParaView:
 

docs/source/backends/adios2.rst

@@ -105,6 +105,11 @@ The default behavior may be restored by setting the :ref:`JSON parameter <backen
 Best Practice at Large Scale
 ----------------------------
 
+ADIOS2 distinguishes between "heavy" data of arbitrary size (i.e. the "actual" data) and lightweight metadata.
+
+Heavy I/O
+.........
+
 A benefitial configuration depends heavily on:
 
 1. Hardware: filesystem type, specific file striping, network infrastructure and available RAM on the aggregator nodes.
@@ -135,6 +140,24 @@ The preferred backend usually depends on the system's native software stack.
 For fine-tuning at extreme scale or for exotic systems, please refer to the ADIOS2 manual and talk to your filesystem admins and the ADIOS2 authors.
 Be aware that extreme-scale I/O is a research topic after all.
 
+Metadata
+........
+
+ADIOS2 will implicitly aggregate metadata specified from parallel MPI processes.
+Duplicate specification of metadata is eliminated in this process.
+Unlike in HDF5, specifying metadata collectively is not required and is even detrimental to performance.
+The :ref:`JSON/TOML key <backendconfig>` ``adios2.attribute_writing_ranks`` can be used to restrict attribute writing to only a select handful of ranks (most typically a single one).
+The ADIOS2 backend of the openPMD-api will then ignore attributes from all other MPI ranks.
+
+.. tip::
+
+  Treat metadata specification as a collective operation in order to retain compatibility with HDF5, and then specify ``adios2.attribute_writing_ranks = 0`` in order to achieve best performance in ADIOS2.
+
+.. warning::
+
+  The ADIOS2 backend may also use attributes to encode openPMD groups (ref. "group table").
+  The ``adios2.attribute_writing_ranks`` key also applies to those attributes, i.e. also group creation must be treated as collective then (at least on the specified ranks).
+
 Experimental group table feature
 --------------------------------
 

docs/source/details/adios2.json

@@ -2,6 +2,7 @@
   "adios2": {
     "engine": {
       "type": "sst",
+      "preferred_flush_target": "disk",
       "parameters": {
         "BufferGrowthFactor": "2.0",
         "QueueLimit": "2"
@@ -17,6 +18,7 @@
           }
         }
       ]
-    }
+    },
+    "attribute_writing_ranks": 0
   }
 }

docs/source/details/adios2.toml

@@ -1,5 +1,12 @@
+[adios2]
+# ignore all attribute writes not issued on these ranks
+# can also be a list if multiple ranks need to be given
+# however rank 0 should be the most common option here
+attribute_writing_ranks = 0
+
 [adios2.engine]
 type = "sst"
+preferred_flush_target = "disk"
 
 [adios2.engine.parameters]
 BufferGrowthFactor = "2.0"

docs/source/details/backendconfig.rst

@@ -113,7 +113,7 @@ A full configuration of the ADIOS2 backend:
 .. literalinclude:: adios2.toml
    :language: toml
 
-All keys found under ``adios2.dataset`` are applicable globally as well as per dataset, keys found under ``adios2.engine`` only globally.
+All keys found under ``adios2.dataset`` are applicable globally as well as per dataset, any other keys such as those found under ``adios2.engine`` only globally.
 Explanation of the single keys:
 
 * ``adios2.engine.type``: A string that is passed directly to ``adios2::IO:::SetEngine`` for choosing the ADIOS2 engine to be used.
@@ -142,6 +142,11 @@ Explanation of the single keys:
   The openPMD-api will automatically use a fallback implementation for the span-based Put() API if any operator is added to a dataset.
   This workaround is enabled on a per-dataset level.
   The workaround can be completely deactivated by specifying ``{"adios2": {"use_span_based_put": true}}`` or it can alternatively be activated indiscriminately for all datasets by specifying ``{"adios2": {"use_span_based_put": false}}``.
+* ``adios2.attribute_writing_ranks``: A list of MPI ranks that define metadata. ADIOS2 attributes will be written only from those ranks, any other ranks will be ignored. Can be either a list of integers or a single integer.
+
+.. hint::
+
+  Specifying ``adios2.attribute_writing_ranks`` can lead to serious serialization performance improvements at large scale.
 
 Operations specified inside ``adios2.dataset.operators`` will be applied to ADIOS2 datasets in writing as well as in reading.
 Beginning with ADIOS2 2.8.0, this can be used to specify decompressor settings:

docs/source/details/mpi.rst

@@ -42,6 +42,8 @@ Functionality                Behavior           Description
        If you want to support all backends equally, treat as a collective operation.
        Note that openPMD represents constant record components with attributes, thus inheriting this for ``::makeConstant``.
 
+       When treating attribute definitions as collective, we advise specifying the ADIOS2 :ref:`JSON/TOML key <backendconfig>` ``adios2.attribute_writing_ranks`` for metadata aggregation scalabilty, typically as ``adios2.attribute_writing_ranks = 0``.
+
 .. [4] We usually open iterations delayed on first access. This first access is usually the ``flush()`` call after a ``storeChunk``/``loadChunk`` operation. If the first access is non-collective, an explicit, collective ``Iteration::open()`` can be used to have the files already open.
        Alternatively, iterations might be accessed for the first time by immediate operations such as ``::availableChunks()``.
 

docs/source/dev/dependencies.rst

@@ -27,7 +27,6 @@ Optional: I/O backends
 
 * `JSON <https://en.wikipedia.org/wiki/JSON>`_
 * `HDF5 <https://support.hdfgroup.org/HDF5>`_ 1.8.13+
-* `ADIOS1 <https://www.olcf.ornl.gov/center-projects/adios>`_ 1.13.1+ (deprecated)
 * `ADIOS2 <https://github.com/ornladios/ADIOS2>`_ 2.7.0+
 
 while those can be build either with or without:
@@ -39,7 +38,7 @@ Optional: language bindings
 
 * Python:
 
-  * Python 3.8 - 3.11
+  * Python 3.8 - 3.12
   * pybind11 2.11.1+
   * numpy 1.15+
   * mpi4py 2.1+ (optional, for MPI)

docs/source/index.rst

@@ -58,6 +58,7 @@ Previously supported by the Consortium for Advanced Modeling of Particles Accele
 Supported by the Exascale Computing Project (17-SC-20-SC), a collaborative effort of two U.S. Department of Energy organizations (Office of Science and the National Nuclear Security Administration).
 This project has received funding from the European Unions Horizon 2020 research and innovation programme under grant agreement No 654220.
 This work was partially funded by the Center of Advanced Systems Understanding (CASUS), which is financed by Germany's Federal Ministry of Education and Research (BMBF) and by the Saxon Ministry for Science, Culture and Tourism (SMWK) with tax funds on the basis of the budget approved by the Saxon State Parliament.
+Supported by the HElmholtz Laser Plasma Metadata Initiative (HELPMI) project (ZT-I-PF-3-066), funded by the "Initiative and Networking Fund" of the Helmholtz Association in the framework of the "Helmholtz Metadata Collaboration" project call 2022.
 
 
 .. toctree::

examples/10_streaming_read.cpp

@@ -46,7 +46,7 @@ int main()
 
         for (size_t i = 0; i < 3; ++i)
         {
-            std::string dim = dimensions[i];
+            std::string const &dim = dimensions[i];
             RecordComponent rc = electronPositions[dim];
             loadedChunks[i] = rc.loadChunk<position_t>(
                 Offset(rc.getDimensionality(), 0), rc.getExtent());
@@ -60,7 +60,7 @@ int main()
 
         for (size_t i = 0; i < 3; ++i)
         {
-            std::string dim = dimensions[i];
+            std::string const &dim = dimensions[i];
             Extent const &extent = extents[i];
             std::cout << "\ndim: " << dim << "\n" << std::endl;
             auto chunk = loadedChunks[i];

examples/8_benchmark_parallel.cpp

@@ -12,7 +12,7 @@
 #include <vector>
 
 #if openPMD_HAVE_MPI
-inline void print_help(std::string const program_name)
+inline void print_help(std::string const &program_name)
 {
     std::cout << "Usage: " << program_name << "\n";
     std::cout << "Run a simple parallel write and read benchmark.\n\n";
@@ -27,7 +27,7 @@ inline void print_help(std::string const program_name)
     std::cout << "    " << program_name << "  # for a strong scaling\n";
 }
 
-inline void print_version(std::string const program_name)
+inline void print_version(std::string const &program_name)
 {
     std::cout << program_name << " (openPMD-api) " << openPMD::getVersion()
               << "\n";
@@ -46,6 +46,7 @@ int main(int argc, char *argv[])
 
     // CLI parsing
     std::vector<std::string> str_argv;
+    str_argv.reserve(argc);
     for (int i = 0; i < argc; ++i)
         str_argv.emplace_back(argv[i]);
     bool weak_scaling = false;

examples/8a_benchmark_write_parallel.cpp

@@ -240,8 +240,7 @@ std::vector<std::string> getBackends()
 {
     std::vector<std::string> res;
 #if openPMD_HAVE_ADIOS2
-    if (auxiliary::getEnvString("OPENPMD_BP_BACKEND", "NOT_SET") != "ADIOS1")
-        res.emplace_back(".bp");
+    res.emplace_back(".bp");
 #endif
 
 #if openPMD_HAVE_HDF5

examples/8b_benchmark_read_parallel.cpp

@@ -198,8 +198,7 @@ std::vector<std::string> getBackends()
 {
     std::vector<std::string> res;
 #if openPMD_HAVE_ADIOS2
-    if (auxiliary::getEnvString("OPENPMD_BP_BACKEND", "NOT_SET") != "ADIOS1")
-        res.emplace_back(".bp");
+    res.emplace_back(".bp");
     if (auxiliary::getEnvString("OPENPMD_BENCHMARK_USE_BACKEND", "NOT_SET") ==
         "ADIOS")
         return res;