From 3a60e7714f6143c8fc7bf89809f2167d058359ee Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 15 Jan 2025 21:04:09 -0800 Subject: [PATCH] Release: 0.16.1 (#1676) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Release: 0.16.1 Signed-off-by: Axel Huebl * Fix new_version.py * Update version in windows.yml * Reformat with yapf3 * Fix greedy Regex parsing * Changelog: Add PRs #1681 #1679 #1678 * Fix: CMake Internal Control (#1678) The `openPMD_USE_INTERNAL_*` options are for general internal/external control. They were accidentally overwritten by new tar flags and sometimes incomplete. Signed-off-by: Axel Huebl * Revert "CMake Tar: TOML11 3.7.1 (#1673)" (#1679) This reverts commit aea4b72388351e89214c0c3c3246acde2375e3ec. * Replace deprecated makeSuite with loadTestsFromTestCase (#1681) Removed with Python 3.13 * Doc: Fix Shipped Internally (#1682) Downloaded now. Signed-off-by: Axel Huebl * `PYBIND11_FINDPYTHON=ON` (#1684) Reuse our `find_package(Python ...)` call and use new CMake logic in pybind11. https://pybind11.readthedocs.io/en/stable/compiling.html#modules-with-cmake https://cmake.org/cmake/help/latest/command/find_package.html#config-mode-version-selection Signed-off-by: Axel Huebl * Fix ICX build (#1690) Error was: /home/runner/work/openPMD-api/openPMD-api/include/openPMD/backend/Container.hpp:262:32: error: no member named 'm_container' in 'Container' 262 | container().swap(other.m_container); * Fix Leftover ADIOS1 Mentions (#1683) * Fix Leftover ADIOS1 Mentions Signed-off-by: Axel Huebl * Remove ADIOS1 from examples --------- Signed-off-by: Axel Huebl Co-authored-by: Franz Pöschel * Explicitly specify ADIOS2 components in openPMDConfig.cmake (#1693) * Explicitly specify ADIOS2 components in openPMDConfig.cmake When including openPMD in downstream CMake and that CMake script contains Fortran, ADIOS2 will automatically detect that. But since we do not reexport the ADIOS2 headers, we only need the components that openPMD explicitly uses: CXX and MPI. * Same in CMakeLists.txt * Remove mpirun_workaround.sh (#1698) Seems to no longer be necessary in the affected CI runs * Properly check for empty HDF5_VERSION in CMake (#1702) The old test apparently invoked some weird CMake legacy behavior. This should make the pinned HDF5 Conda version unnecessary (#1701) since the version with missing HDF5_VERSION is correctly dealt with. * HDF5: Delete and re-create attribute when overwriting with diff. type (#1697) * HDF5: Delete and re-create attribute when overwriting with diff. type * WIP: Change datatype in test Somehow truncation test is broken by this * Add forgotten H5Aclose() call * TOUCH IOTask: Avoid setting files as dirty in non-write modes (#1704) * Bugfix: Consider dirty upon touch() only in write modes * Add error check to avoid writing in read-only mode * Changelog * Use ULONG for particle patches * Upgrade appleclang14 build to MacOS 13 > The macOS-12 environment is deprecated, consider switching to macOS-13, macOS-14 (macos-latest) or macOS-15. For more details, see https://github.com/actions/runner-images/issues/10721 * Update Changelog * Release Date --------- Signed-off-by: Axel Huebl Co-authored-by: Franz Pöschel --- .github/workflows/linux.yml | 3 +- .github/workflows/macos.yml | 5 +- .github/workflows/mpirun_workaround.sh | 52 ------------ .github/workflows/windows.yml | 2 +- CHANGELOG.rst | 35 ++++++++ CITATION.cff | 2 +- CMakeLists.txt | 14 ++-- README.md | 6 +- cmake/dependencies/catch.cmake | 74 ++++++++--------- cmake/dependencies/json.cmake | 74 ++++++++--------- cmake/dependencies/pybind11.cmake | 79 +++++++++--------- cmake/dependencies/toml11.cmake | 78 +++++++++--------- docs/Doxyfile | 1 - docs/source/conf.py | 4 +- docs/source/dev/dependencies.rst | 2 +- docs/source/dev/linking.rst | 4 +- docs/source/index.rst | 2 +- docs/source/maintenance/release_channels.rst | 2 +- examples/12_span_write.cpp | 8 +- examples/13_write_dynamic_configuration.cpp | 3 - examples/13_write_dynamic_configuration.py | 3 - examples/7_extended_write_serial.cpp | 5 -- examples/7_extended_write_serial.py | 5 -- include/openPMD/backend/Container.hpp | 2 +- include/openPMD/version.hpp | 2 +- new_version.py | 86 ++++++++++++++++---- openPMDConfig.cmake.in | 7 +- setup.py | 11 ++- src/IO/ADIOS/ADIOS2IOHandler.cpp | 12 ++- src/IO/HDF5/HDF5IOHandler.cpp | 44 ++++++++-- src/IO/JSON/JSONIOHandlerImpl.cpp | 18 +++- test/SerialIOTest.cpp | 12 ++- test/python/unittest/Test.py | 4 +- 33 files changed, 384 insertions(+), 277 deletions(-) delete mode 100755 .github/workflows/mpirun_workaround.sh diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 9592a71223..faad7cd9e0 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -328,8 +328,7 @@ jobs: -DopenPMD_USE_MPI=ON \ -DopenPMD_USE_HDF5=ON \ -DopenPMD_USE_ADIOS2=ON \ - -DopenPMD_USE_INVASIVE_TESTS=ON \ - -DMPIEXEC_EXECUTABLE=".github/workflows/mpirun_workaround.sh" + -DopenPMD_USE_INVASIVE_TESTS=ON cmake --build build --parallel 2 cd build ctest --output-on-failure diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 782acf0bc0..db003484a0 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -40,13 +40,12 @@ jobs: -DopenPMD_USE_MPI=ON \ -DopenPMD_USE_HDF5=ON \ -DopenPMD_USE_ADIOS2=ON \ - -DopenPMD_USE_INVASIVE_TESTS=ON \ - -DMPIEXEC_EXECUTABLE=".github/workflows/mpirun_workaround.sh" + -DopenPMD_USE_INVASIVE_TESTS=ON cmake --build build --parallel 3 ctest --test-dir build --verbose appleclang14_py: - runs-on: macos-12 + runs-on: macos-13 if: github.event.pull_request.draft == false steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/mpirun_workaround.sh b/.github/workflows/mpirun_workaround.sh deleted file mode 100755 index e953b34647..0000000000 --- a/.github/workflows/mpirun_workaround.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash - -# mpiexec currently seems to have a bug where it tries to parse parameters -# of the launched application when they start with a dash, e.g. -# `mpiexec python ./openpmd-pipe --infile in.bp --outfile out.bp` -# leads to: -# > An unrecognized option was included on the mpiexec command line: -# > -# > Option: --infile -# > -# > Please use the "mpiexec --help" command to obtain a list of all -# > supported options. -# -# This script provides a workaround by putting the called sub-command into -# a script in a temporary file. - -ls="$(which ls)" -mpiexec "$ls" -m \ - && echo "MPIRUN WORKING AGAIN, PLEASE REMOVE WORKAROUND" >&2 \ - && exit 1 \ - || true - -mpirun_args=() - -script_file="$(mktemp)" - -cleanup() { - rm "$script_file" -} -trap cleanup EXIT - -while true; do - case "$1" in - -c | -np | --np | -n | --n ) - mpirun_args+=("$1" "$2") - shift - shift - ;; - *) - break - ;; - esac -done - -echo -e '#!/usr/bin/env bash\n' > "$script_file" -for item in "$@"; do - echo -n "'$item' " >> "$script_file" -done - -chmod +x "$script_file" - -mpirun "${mpirun_args[@]}" "$script_file" diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index efb6d6d762..183bcc4f7e 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -42,7 +42,7 @@ jobs: python3.exe -m pip wheel . if(!$?) { Exit $LASTEXITCODE } - python3.exe -m pip install openPMD_api-0.16.0.dev0-cp39-cp39-win_amd64.whl + python3.exe -m pip install openPMD_api-0.16.1-cp39-cp39-win_amd64.whl if(!$?) { Exit $LASTEXITCODE } python3.exe -c "import openpmd_api as api; print(api.variants)" diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 409997c5b8..f37d7f0cb1 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -3,6 +3,41 @@ Changelog ========= +0.16.1 +------ +**Date:** 2025-01-15 + +New Backends, Extension, Perf. & Memory + +This is the 0.16.0 release but with internal version bumps where they were missing. + +Changes to "0.16.0" +^^^^^^^^^^^^^^^^^^^ + +Bug Fixes +""""""""" + +- HDF5: Delete and re-create attribute when overwriting with diff. type #1697 +- TOUCH IOTask: Avoid setting files as dirty in non-write modes #1704 +- Fix CMake variables for controlling internal dependencies #1678 +- Bump toml11 dependency to ``v4.2.0`` by default #1679 +- Set ``PYBIND11_FINDPYTHON=ON`` #1684 +- Properly check for empty ``HDF5_VERSION`` in CMake #1702 +- Fix ICX build #1690 +- Explicitly specify ADIOS2 components in openPMDConfig.cmake #1693 +- Example 12: Use ``ULONG`` for particle patches #1710 + +Other +""""" + +- Replace deprecated Python unittest API call #1681 +- Remove ``mpirun_workaround.sh`` #1698 +- CI: Upgrade appleclang14 build to MacOS 13 #1703 +- Docs: + + - Fix Shipped Internally #1682 + - Fix leftover ADIOS2 mentions #1683 + 0.16.0 ------ **Date:** 2024-10-07 diff --git a/CITATION.cff b/CITATION.cff index a810842b0c..a0f24d52ea 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -37,7 +37,7 @@ contact: orcid: https://orcid.org/0000-0003-1943-7141 email: axelhuebl@lbl.gov title: "openPMD-api: C++ & Python API for Scientific I/O with openPMD" -version: 0.16.0-dev +version: 0.16.1 repository-code: https://github.com/openPMD/openPMD-api doi: 10.14278/rodare.27 license: LGPL-3.0-or-later diff --git a/CMakeLists.txt b/CMakeLists.txt index a0cc161f4a..9adaeb94ec 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,7 +2,7 @@ # cmake_minimum_required(VERSION 3.22.0) -project(openPMD VERSION 0.16.0) # LANGUAGES CXX +project(openPMD VERSION 0.16.1) # LANGUAGES CXX # the openPMD "markup"/"schema" standard version set(openPMD_STANDARD_VERSION 1.1.0) @@ -314,7 +314,7 @@ endif() string(CONCAT openPMD_HDF5_STATUS "") # version: lower limit if(openPMD_HAVE_HDF5) - if(HDF5_VERSION STREQUAL "") + if("${HDF5_VERSION}" STREQUAL "") message(WARNING "HDF5_VERSION is empty. Now assuming it is 1.8.13 or newer.") else() if(HDF5_VERSION VERSION_LESS 1.8.13) @@ -360,20 +360,24 @@ if(openPMD_HDF5_STATUS) endif() # external library: ADIOS2 (optional) +set(openPMD_REQUIRED_ADIOS2_COMPONENTS CXX) +if(openPMD_HAVE_MPI) + list(APPEND openPMD_REQUIRED_ADIOS2_COMPONENTS MPI) +endif() if(openPMD_USE_ADIOS2 STREQUAL AUTO) - find_package(ADIOS2 2.7.0 CONFIG) + find_package(ADIOS2 2.7.0 CONFIG COMPONENTS ${openPMD_REQUIRED_ADIOS2_COMPONENTS}) if(ADIOS2_FOUND) set(openPMD_HAVE_ADIOS2 TRUE) else() set(openPMD_HAVE_ADIOS2 FALSE) endif() elseif(openPMD_USE_ADIOS2) - find_package(ADIOS2 2.7.0 REQUIRED CONFIG) + find_package(ADIOS2 2.7.0 REQUIRED CONFIG COMPONENTS ${openPMD_REQUIRED_ADIOS2_COMPONENTS}) set(openPMD_HAVE_ADIOS2 TRUE) else() set(openPMD_HAVE_ADIOS2 FALSE) endif() -# TODO: Check if ADIOS2 is parallel when openPMD_HAVE_MPI is ON +unset(openPMD_REQUIRED_ADIOS2_COMPONENTS) # external library: pybind11 (optional) include(${openPMD_SOURCE_DIR}/cmake/dependencies/pybind11.cmake) diff --git a/README.md b/README.md index 2fc4c7be82..d63cb54530 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ Required: * CMake 3.22.0+ * C++17 capable compiler, e.g., g++ 7+, clang 7+, MSVC 19.15+, icpc 19+, icpx -Shipped internally in `share/openPMD/thirdParty/`: +Shipped internally (downloaded by CMake unless `openPMD_SUPERBUILD=OFF` is set): * [Catch2](https://github.com/catchorg/Catch2) 2.13.10+ ([BSL-1.0](https://github.com/catchorg/Catch2/blob/master/LICENSE.txt)) * [pybind11](https://github.com/pybind/pybind11) 2.13.0+ ([new BSD](https://github.com/pybind/pybind11/blob/master/LICENSE)) * [NLohmann-JSON](https://github.com/nlohmann/json) 3.9.1+ ([MIT](https://github.com/nlohmann/json/blob/develop/LICENSE.MIT)) @@ -304,7 +304,7 @@ export CMAKE_PREFIX_PATH=$HOME/somepath:$CMAKE_PREFIX_PATH Use the following lines in your project's `CMakeLists.txt`: ```cmake # supports: COMPONENTS MPI NOMPI HDF5 ADIOS2 -find_package(openPMD 0.16.0 CONFIG) +find_package(openPMD 0.16.1 CONFIG) if(openPMD_FOUND) target_link_libraries(YourTarget PRIVATE openPMD::openPMD) @@ -332,7 +332,7 @@ set(openPMD_INSTALL OFF) # or instead use: set(openPMD_USE_PYTHON OFF) FetchContent_Declare(openPMD GIT_REPOSITORY "https://github.com/openPMD/openPMD-api.git" - GIT_TAG "0.16.0") + GIT_TAG "0.16.1") FetchContent_MakeAvailable(openPMD) ``` diff --git a/cmake/dependencies/catch.cmake b/cmake/dependencies/catch.cmake index 956d66493e..bc7a3dcc89 100644 --- a/cmake/dependencies/catch.cmake +++ b/cmake/dependencies/catch.cmake @@ -1,49 +1,49 @@ function(find_catch2) if(TARGET Catch2::Catch2) message(STATUS "Catch2::Catch2 target already imported") - elseif(openPMD_catch_src) - message(STATUS "Compiling local Catch2 ...") - message(STATUS "Catch2 source path: ${openPMD_catch_src}") - if(NOT IS_DIRECTORY ${openPMD_catch_src}) - message(FATAL_ERROR "Specified directory openPMD_catch_src='${openPMD_catch_src}' does not exist!") - endif() - elseif(openPMD_catch_tar) - message(STATUS "Downloading Catch2 ...") - message(STATUS "Catch2 source: ${openPMD_catch_tar}") elseif(openPMD_USE_INTERNAL_CATCH) - message(STATUS "Downloading Catch2 ...") - message(STATUS "Catch2 repository: ${openPMD_catch_repo} (${openPMD_catch_branch})") + if(openPMD_catch_src) + message(STATUS "Compiling local Catch2 ...") + message(STATUS "Catch2 source path: ${openPMD_catch_src}") + if(NOT IS_DIRECTORY ${openPMD_catch_src}) + message(FATAL_ERROR "Specified directory openPMD_catch_src='${openPMD_catch_src}' does not exist!") + endif() + elseif(openPMD_catch_tar) + message(STATUS "Downloading Catch2 ...") + message(STATUS "Catch2 source: ${openPMD_catch_tar}") + elseif(openPMD_catch_branch) + message(STATUS "Downloading Catch2 ...") + message(STATUS "Catch2 repository: ${openPMD_catch_repo} (${openPMD_catch_branch})") + endif() endif() if(TARGET Catch2::Catch2) # nothing to do, target already exists in the superbuild - elseif(openPMD_USE_INTERNAL_CATCH OR openPMD_catch_src OR openPMD_catch_tar) - if(openPMD_catch_src) - add_subdirectory(${openPMD_catch_src} _deps/localCatch2-build/) + elseif(openPMD_USE_INTERNAL_CATCH AND openPMD_catch_src) + add_subdirectory(${openPMD_catch_src} _deps/localCatch2-build/) + elseif(openPMD_USE_INTERNAL_CATCH AND (openPMD_catch_tar OR openPMD_catch_branch)) + include(FetchContent) + if(openPMD_catch_tar) + FetchContent_Declare(fetchedCatch2 + URL ${openPMD_catch_tar} + URL_HASH ${openPMD_catch_tar_hash} + BUILD_IN_SOURCE OFF + ) else() - include(FetchContent) - if(openPMD_catch_tar) - FetchContent_Declare(fetchedCatch2 - URL ${openPMD_catch_tar} - URL_HASH ${openPMD_catch_tar_hash} - BUILD_IN_SOURCE OFF - ) - else() - FetchContent_Declare(fetchedCatch2 - GIT_REPOSITORY ${openPMD_catch_repo} - GIT_TAG ${openPMD_catch_branch} - BUILD_IN_SOURCE OFF - ) - endif() - FetchContent_MakeAvailable(fetchedCatch2) - - # advanced fetch options - mark_as_advanced(FETCHCONTENT_BASE_DIR) - mark_as_advanced(FETCHCONTENT_FULLY_DISCONNECTED) - mark_as_advanced(FETCHCONTENT_QUIET) - #mark_as_advanced(FETCHCONTENT_SOURCE_DIR_FETCHEDCatch2) - mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED) - #mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED_FETCHEDCatch2) + FetchContent_Declare(fetchedCatch2 + GIT_REPOSITORY ${openPMD_catch_repo} + GIT_TAG ${openPMD_catch_branch} + BUILD_IN_SOURCE OFF + ) endif() + FetchContent_MakeAvailable(fetchedCatch2) + + # advanced fetch options + mark_as_advanced(FETCHCONTENT_BASE_DIR) + mark_as_advanced(FETCHCONTENT_FULLY_DISCONNECTED) + mark_as_advanced(FETCHCONTENT_QUIET) + #mark_as_advanced(FETCHCONTENT_SOURCE_DIR_FETCHEDCatch2) + mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED) + #mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED_FETCHEDCatch2) elseif(NOT openPMD_USE_INTERNAL_CATCH) find_package(Catch2 2.13.10 CONFIG REQUIRED) message(STATUS "Catch2: Found version '${Catch2_VERSION}'") diff --git a/cmake/dependencies/json.cmake b/cmake/dependencies/json.cmake index 8b22eb80f8..751ffc87f5 100644 --- a/cmake/dependencies/json.cmake +++ b/cmake/dependencies/json.cmake @@ -1,49 +1,49 @@ function(find_json) if(TARGET nlohmann_json::nlohmann_json) message(STATUS "nlohmann_json::nlohmann_json target already imported") - elseif(openPMD_json_src) - message(STATUS "Compiling local nlohmann_json ...") - message(STATUS "nlohmann_json source path: ${openPMD_json_src}") - if(NOT IS_DIRECTORY ${openPMD_json_src}) - message(FATAL_ERROR "Specified directory openPMD_json_src='${openPMD_json_src}' does not exist!") - endif() - elseif(openPMD_json_tar) - message(STATUS "Downloading nlohmann_json ...") - message(STATUS "nlohmann_json source: ${openPMD_json_tar}") elseif(openPMD_USE_INTERNAL_JSON) - message(STATUS "Downloading nlohmann_json ...") - message(STATUS "nlohmann_json repository: ${openPMD_json_repo} (${openPMD_json_branch})") + if(openPMD_json_src) + message(STATUS "Compiling local nlohmann_json ...") + message(STATUS "nlohmann_json source path: ${openPMD_json_src}") + if(NOT IS_DIRECTORY ${openPMD_json_src}) + message(FATAL_ERROR "Specified directory openPMD_json_src='${openPMD_json_src}' does not exist!") + endif() + elseif(openPMD_json_tar) + message(STATUS "Downloading nlohmann_json ...") + message(STATUS "nlohmann_json source: ${openPMD_json_tar}") + elseif(openPMD_json_branch) + message(STATUS "Downloading nlohmann_json ...") + message(STATUS "nlohmann_json repository: ${openPMD_json_repo} (${openPMD_json_branch})") + endif() endif() if(TARGET nlohmann_json::nlohmann_json) # nothing to do, target already exists in the superbuild - elseif(openPMD_USE_INTERNAL_JSON OR openPMD_json_src OR openPMD_json_tar) - if(openPMD_json_src) - add_subdirectory(${openPMD_json_src} _deps/localnlohmann_json-build/) + elseif(openPMD_USE_INTERNAL_JSON AND openPMD_json_src) + add_subdirectory(${openPMD_json_src} _deps/localnlohmann_json-build/) + elseif(openPMD_USE_INTERNAL_JSON AND (openPMD_json_tar OR openPMD_json_branch)) + include(FetchContent) + if(openPMD_json_tar) + FetchContent_Declare(fetchednlohmann_json + URL ${openPMD_json_tar} + URL_HASH ${openPMD_json_tar_hash} + BUILD_IN_SOURCE OFF + ) else() - include(FetchContent) - if(openPMD_json_tar) - FetchContent_Declare(fetchednlohmann_json - URL ${openPMD_json_tar} - URL_HASH ${openPMD_json_tar_hash} - BUILD_IN_SOURCE OFF - ) - else() - FetchContent_Declare(fetchednlohmann_json - GIT_REPOSITORY ${openPMD_json_repo} - GIT_TAG ${openPMD_json_branch} - BUILD_IN_SOURCE OFF - ) - endif() - FetchContent_MakeAvailable(fetchednlohmann_json) - - # advanced fetch options - mark_as_advanced(FETCHCONTENT_BASE_DIR) - mark_as_advanced(FETCHCONTENT_FULLY_DISCONNECTED) - mark_as_advanced(FETCHCONTENT_QUIET) - #mark_as_advanced(FETCHCONTENT_SOURCE_DIR_FETCHEDnlohmann_json) - mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED) - #mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED_FETCHEDnlohmann_json) + FetchContent_Declare(fetchednlohmann_json + GIT_REPOSITORY ${openPMD_json_repo} + GIT_TAG ${openPMD_json_branch} + BUILD_IN_SOURCE OFF + ) endif() + FetchContent_MakeAvailable(fetchednlohmann_json) + + # advanced fetch options + mark_as_advanced(FETCHCONTENT_BASE_DIR) + mark_as_advanced(FETCHCONTENT_FULLY_DISCONNECTED) + mark_as_advanced(FETCHCONTENT_QUIET) + #mark_as_advanced(FETCHCONTENT_SOURCE_DIR_FETCHEDnlohmann_json) + mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED) + #mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED_FETCHEDnlohmann_json) elseif(NOT openPMD_USE_INTERNAL_JSON) find_package(nlohmann_json 3.9.1 CONFIG REQUIRED) message(STATUS "nlohmann_json: Found version '${nlohmann_json_VERSION}'") diff --git a/cmake/dependencies/pybind11.cmake b/cmake/dependencies/pybind11.cmake index 742aafa7dc..70899de816 100644 --- a/cmake/dependencies/pybind11.cmake +++ b/cmake/dependencies/pybind11.cmake @@ -1,49 +1,54 @@ function(find_pybind11) if(TARGET pybind11::module) message(STATUS "pybind11::module target already imported") - elseif(openPMD_pybind11_src) - message(STATUS "Compiling local pybind11 ...") - message(STATUS "pybind11 source path: ${openPMD_pybind11_src}") - if(NOT IS_DIRECTORY ${openPMD_pybind11_src}) - message(FATAL_ERROR "Specified directory openPMD_pybind11_src='${openPMD_pybind11_src}' does not exist!") - endif() - elseif(openPMD_pybind11_tar) - message(STATUS "Downloading pybind11 ...") - message(STATUS "pybind11 source: ${openPMD_pybind11_tar}") elseif(openPMD_USE_INTERNAL_PYBIND11) - message(STATUS "Downloading pybind11 ...") - message(STATUS "pybind11 repository: ${openPMD_pybind11_repo} (${openPMD_pybind11_branch})") + if(openPMD_pybind11_src) + message(STATUS "Compiling local pybind11 ...") + message(STATUS "pybind11 source path: ${openPMD_pybind11_src}") + if(NOT IS_DIRECTORY ${openPMD_pybind11_src}) + message(FATAL_ERROR "Specified directory openPMD_pybind11_src='${openPMD_pybind11_src}' does not exist!") + endif() + elseif(openPMD_pybind11_tar) + message(STATUS "Downloading pybind11 ...") + message(STATUS "pybind11 source: ${openPMD_pybind11_tar}") + elseif(openPMD_pybind11_branch) + message(STATUS "Downloading pybind11 ...") + message(STATUS "pybind11 repository: ${openPMD_pybind11_repo} (${openPMD_pybind11_branch})") + endif() endif() + + # rely on our find_package(Python ...) call + # https://pybind11.readthedocs.io/en/stable/compiling.html#modules-with-cmake + set(PYBIND11_FINDPYTHON ON) + if(TARGET pybind11::module) # nothing to do, target already exists in the superbuild - elseif(openPMD_USE_INTERNAL_PYBIND11 OR openPMD_pybind11_src) - if(openPMD_pybind11_src) - add_subdirectory(${openPMD_pybind11_src} _deps/localpybind11-build/) - else() - include(FetchContent) - if(openPMD_pybind11_tar) - FetchContent_Declare(fetchedpybind11 - URL ${openPMD_pybind11_tar} - URL_HASH ${openPMD_pybind11_tar_hash} - BUILD_IN_SOURCE OFF - ) - else() - FetchContent_Declare(fetchedpybind11 - GIT_REPOSITORY ${openPMD_pybind11_repo} - GIT_TAG ${openPMD_pybind11_branch} + elseif(openPMD_USE_INTERNAL_PYBIND11 AND openPMD_pybind11_src) + add_subdirectory(${openPMD_pybind11_src} _deps/localpybind11-build/) + elseif(openPMD_USE_INTERNAL_PYBIND11 AND (openPMD_pybind11_tar OR openPMD_pybind11_branch)) + include(FetchContent) + if(openPMD_pybind11_tar) + FetchContent_Declare(fetchedpybind11 + URL ${openPMD_pybind11_tar} + URL_HASH ${openPMD_pybind11_tar_hash} BUILD_IN_SOURCE OFF - ) - endif() - FetchContent_MakeAvailable(fetchedpybind11) - - # advanced fetch options - mark_as_advanced(FETCHCONTENT_BASE_DIR) - mark_as_advanced(FETCHCONTENT_FULLY_DISCONNECTED) - mark_as_advanced(FETCHCONTENT_QUIET) - mark_as_advanced(FETCHCONTENT_SOURCE_DIR_FETCHEDpybind11) - mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED) - mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED_FETCHEDpybind11) + ) + else() + FetchContent_Declare(fetchedpybind11 + GIT_REPOSITORY ${openPMD_pybind11_repo} + GIT_TAG ${openPMD_pybind11_branch} + BUILD_IN_SOURCE OFF + ) endif() + FetchContent_MakeAvailable(fetchedpybind11) + + # advanced fetch options + mark_as_advanced(FETCHCONTENT_BASE_DIR) + mark_as_advanced(FETCHCONTENT_FULLY_DISCONNECTED) + mark_as_advanced(FETCHCONTENT_QUIET) + mark_as_advanced(FETCHCONTENT_SOURCE_DIR_FETCHEDpybind11) + mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED) + mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED_FETCHEDpybind11) elseif(NOT openPMD_USE_INTERNAL_PYBIND11) if(openPMD_USE_PYTHON STREQUAL AUTO) find_package(pybind11 2.13.0 CONFIG) diff --git a/cmake/dependencies/toml11.cmake b/cmake/dependencies/toml11.cmake index 6ad9990db2..c1f8832a81 100644 --- a/cmake/dependencies/toml11.cmake +++ b/cmake/dependencies/toml11.cmake @@ -1,49 +1,49 @@ function(find_toml11) if(TARGET toml11::toml11) message(STATUS "toml11::toml11 target already imported") - elseif(openPMD_toml11_src) - message(STATUS "Compiling local toml11 ...") - message(STATUS "toml11 source path: ${openPMD_toml11_src}") - if(NOT IS_DIRECTORY ${openPMD_toml11_src}) - message(FATAL_ERROR "Specified directory openPMD_toml11_src='${openPMD_toml11_src}' does not exist!") - endif() - elseif(openPMD_toml11_tar) - message(STATUS "Downloading toml11 ...") - message(STATUS "toml11 source: ${openPMD_toml11_tar}") elseif(openPMD_USE_INTERNAL_TOML11) - message(STATUS "Downloading toml11 ...") - message(STATUS "toml11 repository: ${openPMD_toml11_repo} (${openPMD_toml11_branch})") + if(openPMD_toml11_src) + message(STATUS "Compiling local toml11 ...") + message(STATUS "toml11 source path: ${openPMD_toml11_src}") + if(NOT IS_DIRECTORY ${openPMD_toml11_src}) + message(FATAL_ERROR "Specified directory openPMD_toml11_src='${openPMD_toml11_src}' does not exist!") + endif() + elseif(openPMD_toml11_tar) + message(STATUS "Downloading toml11 ...") + message(STATUS "toml11 source: ${openPMD_toml11_tar}") + elseif(openPMD_toml11_branch) + message(STATUS "Downloading toml11 ...") + message(STATUS "toml11 repository: ${openPMD_toml11_repo} (${openPMD_toml11_branch})") + endif() endif() if(TARGET toml11::toml11) # nothing to do, target already exists in the superbuild - elseif(openPMD_USE_INTERNAL_TOML11 OR openPMD_toml11_src) - if(openPMD_toml11_src) - add_subdirectory(${openPMD_toml11_src} _deps/localtoml11-build/) - else() - include(FetchContent) - if(openPMD_toml11_tar) - FetchContent_Declare(fetchedtoml11 - URL ${openPMD_toml11_tar} - URL_HASH ${openPMD_toml11_tar_hash} - BUILD_IN_SOURCE OFF - ) - else() - FetchContent_Declare(fetchedtoml11 - GIT_REPOSITORY ${openPMD_toml11_repo} - GIT_TAG ${openPMD_toml11_branch} + elseif(openPMD_USE_INTERNAL_TOML11 AND openPMD_toml11_src) + add_subdirectory(${openPMD_toml11_src} _deps/localtoml11-build/) + elseif(openPMD_USE_INTERNAL_TOML11 AND (openPMD_toml11_tar OR openPMD_toml11_branch)) + include(FetchContent) + if(openPMD_toml11_tar) + FetchContent_Declare(fetchedtoml11 + URL ${openPMD_toml11_tar} + URL_HASH ${openPMD_toml11_tar_hash} BUILD_IN_SOURCE OFF - ) - endif() - FetchContent_MakeAvailable(fetchedtoml11) - - # advanced fetch options - mark_as_advanced(FETCHCONTENT_BASE_DIR) - mark_as_advanced(FETCHCONTENT_FULLY_DISCONNECTED) - mark_as_advanced(FETCHCONTENT_QUIET) - #mark_as_advanced(FETCHCONTENT_SOURCE_DIR_FETCHEDtoml11) - mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED) - #mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED_FETCHEDtoml11) + ) + else() + FetchContent_Declare(fetchedtoml11 + GIT_REPOSITORY ${openPMD_toml11_repo} + GIT_TAG ${openPMD_toml11_branch} + BUILD_IN_SOURCE OFF + ) endif() + FetchContent_MakeAvailable(fetchedtoml11) + + # advanced fetch options + mark_as_advanced(FETCHCONTENT_BASE_DIR) + mark_as_advanced(FETCHCONTENT_FULLY_DISCONNECTED) + mark_as_advanced(FETCHCONTENT_QUIET) + #mark_as_advanced(FETCHCONTENT_SOURCE_DIR_FETCHEDtoml11) + mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED) + #mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED_FETCHEDtoml11) elseif(NOT openPMD_USE_INTERNAL_TOML11) # toml11 4.0 was a breaking change. This is reflected in the library's CMake # logic: version 4.0 is not accepted by a call to find_package(toml11 3.7). @@ -64,10 +64,10 @@ set(openPMD_toml11_src "" "Local path to toml11 source directory (preferred if set)") # tarball fetcher -set(openPMD_toml11_tar "https://github.com/ToruNiina/toml11/archive/refs/tags/v3.7.1.tar.gz" +set(openPMD_toml11_tar "https://github.com/ToruNiina/toml11/archive/refs/tags/v4.2.0.tar.gz" CACHE STRING "Remote tarball link to pull and build toml11 from if(openPMD_USE_INTERNAL_TOML11)") -set(openPMD_toml11_tar_hash "SHA256=afeaa9aa0416d4b6b2cd3897ca55d9317084103077b32a852247d8efd4cf6068" +set(openPMD_toml11_tar_hash "SHA256=9287971cd4a1a3992ef37e7b95a3972d1ae56410e7f8e3f300727ab1d6c79c2c" CACHE STRING "Hash checksum of the tarball of toml11 if(openPMD_USE_INTERNAL_TOML11)") diff --git a/docs/Doxyfile b/docs/Doxyfile index 471c3cef28..16f734aa4f 100644 --- a/docs/Doxyfile +++ b/docs/Doxyfile @@ -1,7 +1,6 @@ PROJECT_NAME = "openPMD-api" XML_OUTPUT = xml INPUT = ../src ../include ../README.md -#EXCLUDE_PATTERNS = *CommonADIOS1IOHandler.cpp # TAGFILES += "cppreference-doxygen-web.tag.xml=http://en.cppreference.com/w/" BUILTIN_STL_SUPPORT = YES diff --git a/docs/source/conf.py b/docs/source/conf.py index 668ef4c8ae..f93e0a2763 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -86,9 +86,9 @@ # built documents. # # The short X.Y version. -version = u'0.16.0' +version = u'0.16.1' # The full version, including alpha/beta/rc tags. -release = u'0.16.0-dev' +release = u'0.16.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/source/dev/dependencies.rst b/docs/source/dev/dependencies.rst index c0318aa683..87667f0efb 100644 --- a/docs/source/dev/dependencies.rst +++ b/docs/source/dev/dependencies.rst @@ -15,7 +15,7 @@ Required Shipped internally ------------------ -The following libraries are shipped internally in ``share/openPMD/thirdParty/`` for convenience: +The following libraries are downloaded by CMake for convenience, unless ``openPMD_SUPERBUILD=OFF`` is set: * `Catch2 `_ 2.13.10+ (`BSL-1.0 `__) * `pybind11 `_ 2.13.0+ (`new BSD `_) diff --git a/docs/source/dev/linking.rst b/docs/source/dev/linking.rst index 191612f541..5048ddbf1d 100644 --- a/docs/source/dev/linking.rst +++ b/docs/source/dev/linking.rst @@ -23,7 +23,7 @@ Use the following lines in your project's ``CMakeLists.txt``: .. code-block:: cmake # supports: COMPONENTS MPI NOMPI HDF5 ADIOS2 - find_package(openPMD 0.16.0 CONFIG) + find_package(openPMD 0.16.1 CONFIG) if(openPMD_FOUND) target_link_libraries(YourTarget PRIVATE openPMD::openPMD) @@ -53,7 +53,7 @@ Just replace the ``add_subdirectory`` call with: set(openPMD_USE_PYTHON OFF) FetchContent_Declare(openPMD GIT_REPOSITORY "https://github.com/openPMD/openPMD-api.git" - GIT_TAG "0.16.0") + GIT_TAG "0.16.1") FetchContent_MakeAvailable(openPMD) diff --git a/docs/source/index.rst b/docs/source/index.rst index 5fc4475364..e38b87d486 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -44,7 +44,7 @@ openPMD-api version supported openPMD standard versions ======================== =================================== ``2.0.0+`` ``2.0.0+`` (not released yet) ``1.0.0+`` ``1.0.1-1.1.0`` (not released yet) -``0.13.1-0.15.1`` (beta) ``1.0.0-1.1.0`` +``0.13.1-0.16.1`` (beta) ``1.0.0-1.1.0`` ``0.1.0-0.12.0`` (alpha) ``1.0.0-1.1.0`` ======================== =================================== diff --git a/docs/source/maintenance/release_channels.rst b/docs/source/maintenance/release_channels.rst index bd40f7f784..9c43711013 100644 --- a/docs/source/maintenance/release_channels.rst +++ b/docs/source/maintenance/release_channels.rst @@ -39,7 +39,7 @@ Brew We maintain a `homebrew tap `_ for `openPMD `_. Provides the C++ and Python API for users. Supports OSX and Linux. -Its source-only Formula for the latest release includes (Open)MPI support and lacks the ADIOS1 backend. +Its source-only Formula for the latest release includes (Open)MPI support. Example workflow for a new release: diff --git a/examples/12_span_write.cpp b/examples/12_span_write.cpp index e2efcffdaa..201e062ff9 100644 --- a/examples/12_span_write.cpp +++ b/examples/12_span_write.cpp @@ -44,9 +44,11 @@ void span_write(std::string const &filename) } for (auto record : {"numParticlesOffset", "numParticles"}) { - patches[record].resetDataset({Datatype::INT, {1}}); - *patches[record].storeChunk({0}, {1}).currentBuffer().data() = - 42; + patches[record].resetDataset({Datatype::ULONG, {1}}); + *patches[record] + .storeChunk({0}, {1}) + .currentBuffer() + .data() = 42; } Record electronPositions = iteration.particles["e"]["position"]; diff --git a/examples/13_write_dynamic_configuration.cpp b/examples/13_write_dynamic_configuration.cpp index 73ce0be095..b480cb2f00 100644 --- a/examples/13_write_dynamic_configuration.cpp +++ b/examples/13_write_dynamic_configuration.cpp @@ -39,9 +39,6 @@ iteration_encoding = "group_based" # The following is only relevant in read mode defer_iteration_parsing = true -[adios1.dataset] -transform = "blosc:compressor=zlib,shuffle=bit,lvl=5;nometa" - [adios2.engine] type = "bp4" diff --git a/examples/13_write_dynamic_configuration.py b/examples/13_write_dynamic_configuration.py index f3e1e63bd5..a0f7b0c213 100644 --- a/examples/13_write_dynamic_configuration.py +++ b/examples/13_write_dynamic_configuration.py @@ -25,9 +25,6 @@ # The following is only relevant in read mode defer_iteration_parsing = true -[adios1.dataset] -transform = "blosc:compressor=zlib,shuffle=bit,lvl=5;nometa" - [adios2.engine] type = "bp4" diff --git a/examples/7_extended_write_serial.cpp b/examples/7_extended_write_serial.cpp index 6b69635fc6..63d9cb38ff 100644 --- a/examples/7_extended_write_serial.cpp +++ b/examples/7_extended_write_serial.cpp @@ -107,11 +107,6 @@ int main() auto d = io::Dataset(dtype, io::Extent{2, 5}); std::string datasetConfig = R"END( { - "adios1": { - "dataset": { - "transform": "blosc:compressor=zlib,shuffle=bit,lvl=1;nometa" - } - }, "adios2": { "dataset": { "operators": [ diff --git a/examples/7_extended_write_serial.py b/examples/7_extended_write_serial.py index d9b50693cb..f63a5c8371 100755 --- a/examples/7_extended_write_serial.py +++ b/examples/7_extended_write_serial.py @@ -106,11 +106,6 @@ # written to disk d = Dataset(partial_mesh.dtype, extent=[2, 5]) dataset_config = { - "adios1": { - "dataset": { - "transform": "blosc:compressor=zlib,shuffle=bit,lvl=1;nometa" - } - }, "adios2": { "dataset": { "operators": [{ diff --git a/include/openPMD/backend/Container.hpp b/include/openPMD/backend/Container.hpp index 58b07bd48a..cee50f9baf 100644 --- a/include/openPMD/backend/Container.hpp +++ b/include/openPMD/backend/Container.hpp @@ -259,7 +259,7 @@ class Container : virtual public Attributable void swap(Container &other) { - container().swap(other.m_container); + container().swap(other.container()); } mapped_type &at(key_type const &key) diff --git a/include/openPMD/version.hpp b/include/openPMD/version.hpp index 9f32154d92..ead912a23c 100644 --- a/include/openPMD/version.hpp +++ b/include/openPMD/version.hpp @@ -29,7 +29,7 @@ */ #define OPENPMDAPI_VERSION_MAJOR 0 #define OPENPMDAPI_VERSION_MINOR 16 -#define OPENPMDAPI_VERSION_PATCH 0 +#define OPENPMDAPI_VERSION_PATCH 1 #define OPENPMDAPI_VERSION_LABEL "" /** @} */ diff --git a/new_version.py b/new_version.py index 59f2e4e49d..a261c99add 100755 --- a/new_version.py +++ b/new_version.py @@ -14,13 +14,11 @@ # Maintainer Inputs ########################################################### -print( - """Hi there, this is an openPMD maintainer tool to update the source +print("""Hi there, this is an openPMD maintainer tool to update the source code of openPMD-api to a new version. For it to work, you need write access on the source directory and you should be working in a clean git branch without ongoing -rebase/merge/conflict resolves and without unstaged changes.""" -) +rebase/merge/conflict resolves and without unstaged changes.""") # check source dir # REPO_DIR = Path(__file__).parent.parent.parent.absolute() @@ -40,6 +38,7 @@ VERSION_STR = f"{MAJOR}.{MINOR}.{PATCH}" VERSION_STR_SUFFIX = VERSION_STR + (f"-{SUFFIX}" if SUFFIX else "") +VERSION_STR_SUFFIX_WITH_DOT = VERSION_STR + (f".{SUFFIX}" if SUFFIX else "") print() print(f"Your new version is: {VERSION_STR_SUFFIX}") @@ -51,20 +50,41 @@ for line in f: match = re.search(r"find_package.*openPMD *([^ ]*) *CONFIG\).*", line) if match: - OLD_VERSION_STR = match.group(1) + OLD_VERSION_STR_README = match.group(1) + break + +with open(str(REPO_DIR.joinpath("CMakeLists.txt")), encoding="utf-8") as f: + for line in f: + match = re.search(r"project\(openPMD *VERSION *(.*)\)", line) + if match: + OLD_VERSION_STR_CMAKE = match.group(1) break +OLD_VERSION_TAG = "" +with open(str(REPO_DIR.joinpath("include/openPMD/version.hpp")), + encoding="utf-8") as f: + for line in f: + match = re.search(r'#define OPENPMDAPI_VERSION_LABEL "([^"]+)"', line) + if match: + OLD_VERSION_TAG = match.group(1) + break + +OLD_VERSION_SUFFIX = f"(-{OLD_VERSION_TAG})?" if OLD_VERSION_TAG else "" +# The order of the alternatives is important, since the Regex parser +# should greedily include the old version suffix +OLD_VERSION_STR = \ + f"({re.escape(OLD_VERSION_STR_CMAKE)}{OLD_VERSION_SUFFIX})" + \ + f"|({re.escape(OLD_VERSION_STR_README)})" + print(f"The old version is: {OLD_VERSION_STR}") print() - REPLY = input("Is this information correct? Will now start updating! [y/N] ") print() if REPLY not in ["Y", "y", ""]: print("You did not confirm with 'y', aborting.") sys.exit(1) - # Ask for new ################################################################# print("""We will now run a few sed commands on your source directory.\n""") @@ -86,18 +106,56 @@ f.write(cmakelists_content) -def generic_replace(filename): +def generic_replace(filename, previous, after): filename = str(REPO_DIR.joinpath(filename)) with open(filename, encoding="utf-8") as f: content = f.read() - content = re.sub(re.escape(OLD_VERSION_STR), VERSION_STR, content) + content = re.sub(previous, after, content, flags=re.MULTILINE) with open(filename, "w", encoding="utf-8") as f: f.write(content) -for file in ["docs/source/dev/linking.rst", "README.md"]: - generic_replace(file) +for file in [ + "docs/source/dev/linking.rst", + "README.md", +]: + generic_replace(file, previous=OLD_VERSION_STR, after=VERSION_STR) + +for file in ["CITATION.cff", "test/SerialIOTest.cpp"]: + generic_replace(file, previous=OLD_VERSION_STR, after=VERSION_STR_SUFFIX) + +generic_replace( + "docs/source/index.rst", + previous=r"``0.13.1-[^`]*``", + after=f"``0.13.1-{VERSION_STR}``", +) +setup_py_path = str(REPO_DIR.joinpath("setup.py")) +with open(setup_py_path, encoding="utf-8") as f: + for line in f: + match = re.search(r"version='([^']+)',", line) + if match: + PREVIOUS_PIP_VERSION = match.group(1) + break +generic_replace("setup.py", + previous=PREVIOUS_PIP_VERSION, + after=VERSION_STR_SUFFIX_WITH_DOT) +generic_replace( + ".github/workflows/windows.yml", + previous=f"{PREVIOUS_PIP_VERSION}0?", + after=(f"{VERSION_STR_SUFFIX_WITH_DOT}0" + if SUFFIX else VERSION_STR_SUFFIX_WITH_DOT), +) +generic_replace( + "docs/source/conf.py", + previous=r"^version.*=.*", + after=f"version = u'{VERSION_STR}'", +) +generic_replace( + "docs/source/conf.py", + previous=r"^release.*=.*", + after=f"release = u'{VERSION_STR_SUFFIX}'", +) version_hpp_path = str(REPO_DIR.joinpath("include/openPMD/version.hpp")) with open(version_hpp_path, encoding="utf-8") as f: @@ -122,8 +180,6 @@ def replace(key, value): # Epilogue #################################################################### -print( - """Done. Please check your source, e.g. via +print("""Done. Please check your source, e.g. via git diff -now and commit the changes if no errors occurred.""" -) +now and commit the changes if no errors occurred.""") diff --git a/openPMDConfig.cmake.in b/openPMDConfig.cmake.in index 3f6902e503..4cf4ff1732 100644 --- a/openPMDConfig.cmake.in +++ b/openPMDConfig.cmake.in @@ -31,7 +31,12 @@ set(openPMD_HDF5_FOUND ${openPMD_HAVE_HDF5}) set(openPMD_HAVE_ADIOS2 @openPMD_HAVE_ADIOS2@) if(openPMD_HAVE_ADIOS2) - find_dependency(ADIOS2) + set(openPMD_REQUIRED_ADIOS2_COMPONENTS CXX) + if(openPMD_HAVE_MPI) + LIST(APPEND openPMD_REQUIRED_ADIOS2_COMPONENTS MPI) + endif() + find_dependency(ADIOS2 COMPONENTS ${openPMD_REQUIRED_ADIOS2_COMPONENTS}) + unset(openPMD_REQUIRED_ADIOS2_COMPONENTS) endif() set(openPMD_ADIOS2_FOUND ${openPMD_HAVE_ADIOS2}) diff --git a/setup.py b/setup.py index 1c4e7476d3..b368be19a4 100644 --- a/setup.py +++ b/setup.py @@ -44,12 +44,19 @@ def build_extension(self, ext): if not extdir.endswith(os.path.sep): extdir += os.path.sep + pyv = sys.version_info cmake_args = [ + # Python: use the calling interpreter in CMake + # https://cmake.org/cmake/help/latest/module/FindPython.html#hints + # https://cmake.org/cmake/help/latest/command/find_package.html#config-mode-version-selection + '-DPython_ROOT_DIR=' + sys.prefix, + f'-DPython_FIND_VERSION={pyv.major}.{pyv.minor}.{pyv.micro}', + '-DPython_FIND_VERSION_EXACT=TRUE', + '-DPython_FIND_STRATEGY=LOCATION', '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + os.path.join(extdir, "openpmd_api"), # '-DCMAKE_RUNTIME_OUTPUT_DIRECTORY=' + extdir, '-DopenPMD_PYTHON_OUTPUT_DIRECTORY=' + extdir, - '-DPython_EXECUTABLE=' + sys.executable, '-DopenPMD_USE_PYTHON:BOOL=ON', # variants '-DopenPMD_USE_MPI:BOOL=' + openPMD_USE_MPI, @@ -174,7 +181,7 @@ def build_extension(self, ext): setup( name='openPMD-api', # note PEP-440 syntax: x.y.zaN but x.y.z.devN - version='0.16.0.dev', + version='0.16.1', author='Axel Huebl, Franz Poeschel, Fabian Koller, Junmin Gu', author_email='axelhuebl@lbl.gov, f.poeschel@hzdr.de', maintainer='Axel Huebl', diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 2946b047f0..05cf7cd689 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -772,7 +772,7 @@ void ADIOS2IOHandlerImpl::createDataset( if (parameters.joinedDimension.has_value()) { error::throwOperationUnsupportedInBackend( - "ADIOS1", "Joined Arrays require ADIOS2 >= v2.9"); + "ADIOS2", "Joined Arrays require ADIOS2 >= v2.9"); } #endif if (!writable->written) @@ -1520,7 +1520,15 @@ void ADIOS2IOHandlerImpl::touch( Writable *writable, Parameter const &) { auto file = refreshFileFromParent(writable, /* preferParentFile = */ false); - m_dirty.emplace(std::move(file)); + if (access::write(m_handler->m_backendAccess)) + { + m_dirty.emplace(std::move(file)); + } + else if (m_fileData.find(file) == m_fileData.end()) + { + throw error::Internal( + "ADIOS2: Tried activating a file that is not open."); + } } adios2::Mode ADIOS2IOHandlerImpl::adios2AccessMode(std::string const &fullPath) diff --git a/src/IO/HDF5/HDF5IOHandler.cpp b/src/IO/HDF5/HDF5IOHandler.cpp index 32a9e80d74..874768adbe 100644 --- a/src/IO/HDF5/HDF5IOHandler.cpp +++ b/src/IO/HDF5/HDF5IOHandler.cpp @@ -471,7 +471,7 @@ void HDF5IOHandlerImpl::createDataset( if (parameters.joinedDimension.has_value()) { error::throwOperationUnsupportedInBackend( - "ADIOS1", "Joined Arrays currently only supported in ADIOS2"); + "HDF5", "Joined Arrays currently only supported in ADIOS2"); } if (!writable->written) @@ -1694,8 +1694,7 @@ void HDF5IOHandlerImpl::writeAttribute( "[HDF5] Internal error: Failed to get HDF5 datatype during attribute " "write"); std::string name = parameters.name; - if (H5Aexists(node_id, name.c_str()) == 0) - { + auto create_attribute_anew = [&]() { hid_t dataspace = getH5DataSpace(att); VERIFY( dataspace >= 0, @@ -1717,14 +1716,47 @@ void HDF5IOHandlerImpl::writeAttribute( status == 0, "[HDF5] Internal error: Failed to close HDF5 dataspace during " "attribute write"); - } - else + }; + if (H5Aexists(node_id, name.c_str()) != 0) { attribute_id = H5Aopen(node_id, name.c_str(), H5P_DEFAULT); VERIFY( - node_id >= 0, + attribute_id >= 0, "[HDF5] Internal error: Failed to open HDF5 attribute during " "attribute write"); + /* + * Only reuse the old attribute if it had the same type. + */ + hid_t type_id = H5Aget_type(attribute_id); + VERIFY( + type_id >= 0, + "[HDF5] Internal error: Failed to inquire HDF5 attribute type " + "during attribute write"); + auto equal = H5Tequal(type_id, dataType); + VERIFY( + equal >= 0, + "[HDF5] Internal error: Failed to compare HDF5 attribute types " + "during attribute write"); + if (equal == 0) // unequal + { + status = H5Aclose(attribute_id); + VERIFY( + status == 0, + "[HDF5] Internal error: Failed to close previous HDF5 " + "attribute " + "during attribute write"); + status = H5Adelete(node_id, name.c_str()); + VERIFY( + status == 0, + "[HDF5] Internal error: Failed to delete previous HDF5 " + "attribute " + "during attribute write"); + create_attribute_anew(); + } + } + else + { + create_attribute_anew(); } using DT = Datatype; diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index e06aa36ed8..b1046c4602 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -25,6 +25,7 @@ #include "openPMD/Error.hpp" #include "openPMD/IO/AbstractIOHandler.hpp" #include "openPMD/IO/AbstractIOHandlerImpl.hpp" +#include "openPMD/IO/Access.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/JSON_internal.hpp" #include "openPMD/auxiliary/Memory.hpp" @@ -158,6 +159,11 @@ JSONIOHandlerImpl::~JSONIOHandlerImpl() = default; std::future JSONIOHandlerImpl::flush() { AbstractIOHandlerImpl::flush(); + if (access::readOnly(m_handler->m_backendAccess) && !m_dirty.empty()) + { + throw error::Internal( + "JSON backend: Cannot have dirty files in read-only modes."); + } for (auto const &file : m_dirty) { putJsonContents(file, false); @@ -283,7 +289,7 @@ void JSONIOHandlerImpl::createDataset( if (parameter.joinedDimension.has_value()) { error::throwOperationUnsupportedInBackend( - "ADIOS1", "Joined Arrays currently only supported in ADIOS2"); + "JSON", "Joined Arrays currently only supported in ADIOS2"); } if (!writable->written) @@ -1044,7 +1050,15 @@ void JSONIOHandlerImpl::touch( Writable *writable, Parameter const &) { auto file = refreshFileFromParent(writable); - m_dirty.emplace(std::move(file)); + if (access::write(m_handler->m_backendAccess)) + { + m_dirty.emplace(std::move(file)); + } + else if (m_jsonVals.find(file) == m_jsonVals.end()) + { + throw error::Internal( + "ADIOS2: Tried activating a file that is not open."); + } } auto JSONIOHandlerImpl::getFilehandle(File const &fileName, Access access) diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 7323a32582..834b35ca02 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -6363,7 +6363,7 @@ void adios2_bp5_no_steps(bool usesteps) IO.DefineAttribute("/openPMD", std::string("1.1.0")); IO.DefineAttribute("/openPMDextension", uint32_t(0)); IO.DefineAttribute("/software", std::string("openPMD-api")); - IO.DefineAttribute("/softwareVersion", std::string("0.16.0-dev")); + IO.DefineAttribute("/softwareVersion", std::string("0.16.1")); IO.DefineAttribute("/data/0/dt", double(1)); IO.DefineAttribute( @@ -7477,6 +7477,8 @@ void groupbased_read_write(std::string const &ext) E_x.setAttribute("updated_in_run", 0); E_y.setAttribute("updated_in_run", 0); + E_y.setAttribute("changed_datatype_in_run", 0); + write.close(); } { @@ -7495,6 +7497,8 @@ void groupbased_read_write(std::string const &ext) E_x.setAttribute("updated_in_run", 1); E_y.setAttribute("updated_in_run", 1); + E_y.setAttribute("changed_datatype_in_run", "one"); + write.close(); } { @@ -7506,6 +7510,9 @@ void groupbased_read_write(std::string const &ext) REQUIRE(E_x_0_fromRun0.getAttribute("updated_in_run").get() == 0); REQUIRE(E_x_1_fromRun1.getAttribute("updated_in_run").get() == 1); REQUIRE(E_y_0_fromRun1.getAttribute("updated_in_run").get() == 1); + REQUIRE( + E_y_0_fromRun1.getAttribute("changed_datatype_in_run") + .get() == "one"); auto chunk_E_x_0_fromRun0 = E_x_0_fromRun0.loadChunk({0}, {1}); auto chunk_E_x_1_fromRun1 = E_x_1_fromRun1.loadChunk({0}, {1}); @@ -7516,6 +7523,7 @@ void groupbased_read_write(std::string const &ext) REQUIRE(*chunk_E_x_0_fromRun0 == 0); REQUIRE(*chunk_E_x_1_fromRun1 == 1); REQUIRE(*chunk_E_y_0_fromRun1 == 1); + read.close(); } // check that truncation works correctly @@ -7529,12 +7537,14 @@ void groupbased_read_write(std::string const &ext) E_x.storeChunkRaw(&data, {0}, {1}); E_x.setAttribute("updated_in_run", 2); + write.close(); } { Series read(filename, Access::READ_ONLY); REQUIRE(read.iterations.size() == 1); REQUIRE(read.iterations.count(2) == 1); + read.close(); } } diff --git a/test/python/unittest/Test.py b/test/python/unittest/Test.py index f094b46a2b..081d9eabf7 100644 --- a/test/python/unittest/Test.py +++ b/test/python/unittest/Test.py @@ -16,8 +16,8 @@ # Define the test suite. def suite(): suites = [ - unittest.makeSuite(APITest), - ] + unittest.TestLoader().loadTestsFromTestCase(testCaseClass=APITest), + ] return unittest.TestSuite(suites)