diff --git a/CMakeLists.txt b/CMakeLists.txt index 95d5c76b50..888095e113 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -157,6 +157,7 @@ option(BUILD_TESTING "Build testing" OFF) include(CTest) mark_as_advanced(CLEAR BUILD_TESTING) +adios_option(BigWhoop "Enable support for BigWhoop transforms" AUTO) adios_option(Blosc2 "Enable support for c-blosc-2 transforms" AUTO) adios_option(BZip2 "Enable support for BZip2 transforms" AUTO) adios_option(ZFP "Enable support for ZFP transforms" AUTO) @@ -263,7 +264,7 @@ endif() set(ADIOS2_CONFIG_OPTS - DataMan DataSpaces HDF5 HDF5_VOL MHS SST Fortran MPI Python PIP Blosc2 BZip2 + DataMan DataSpaces HDF5 HDF5_VOL MHS SST Fortran MPI Python PIP BigWhoop Blosc2 BZip2 LIBPRESSIO MGARD MGARD_MDR PNG SZ ZFP DAOS IME O_DIRECT Sodium Catalyst SysVShMem UCX ZeroMQ Profiling Endian_Reverse Derived_Variable AWSSDK XRootD GPU_Support CUDA Kokkos Kokkos_CUDA Kokkos_HIP Kokkos_SYCL Campaign KVCACHE diff --git a/cmake/DetectOptions.cmake b/cmake/DetectOptions.cmake index 436587056b..763e16a58f 100644 --- a/cmake/DetectOptions.cmake +++ b/cmake/DetectOptions.cmake @@ -70,6 +70,16 @@ endfunction() # Multithreading find_package(Threads REQUIRED) +# BigWhoop +if(ADIOS2_USE_BigWhoop STREQUAL AUTO) + find_package(BWC CONFIG) +elseif(ADIOS2_USE_BigWhoop) + find_package(BWC REQUIRED CONFIG) +endif() +if(BWC_FOUND) + set(ADIOS2_HAVE_BigWhoop TRUE) +endif() + # Blosc2 if(ADIOS2_USE_Blosc2 STREQUAL AUTO) find_package(Blosc2 2.10.1 QUIET) diff --git a/cmake/adios2-config-common.cmake.in b/cmake/adios2-config-common.cmake.in index f13fb2c05b..69d71b2974 100644 --- a/cmake/adios2-config-common.cmake.in +++ b/cmake/adios2-config-common.cmake.in @@ -86,6 +86,11 @@ if(NOT @BUILD_SHARED_LIBS@) find_dependency(Blosc2 2.10.1) endif() + set(ADIOS2_HAVE_BigWhoop @ADIOS2_HAVE_BigWhoop@) + if(ADIOS2_HAVE_BigWhoop) + find_dependency(BigWhoop) + endif() + set(ADIOS2_HAVE_BZip2 @ADIOS2_HAVE_BZip2@) if(ADIOS2_HAVE_BZip2) find_dependency(BZip2) diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt index 1925e9c936..1664558549 100644 --- a/source/adios2/CMakeLists.txt +++ b/source/adios2/CMakeLists.txt @@ -358,6 +358,11 @@ if(ADIOS2_HAVE_DataSpaces) target_link_libraries(adios2_core_mpi PRIVATE DataSpaces::DataSpaces) endif() +if(ADIOS2_HAVE_BigWhoop) + target_sources(adios2_core PRIVATE operator/compress/CompressBigWhoop.cpp) + target_link_libraries(adios2_core PRIVATE bwc::bwclib) +endif() + set(maybe_adios2_blosc2) if(ADIOS2_HAVE_Blosc2) target_sources(adios2_core PRIVATE operator/compress/CompressBlosc.cpp) diff --git a/source/adios2/common/ADIOSTypes.h b/source/adios2/common/ADIOSTypes.h index acf9a8f663..331b2cedca 100644 --- a/source/adios2/common/ADIOSTypes.h +++ b/source/adios2/common/ADIOSTypes.h @@ -438,6 +438,18 @@ std::ostream &operator<<(std::ostream &os, const T &value); namespace ops { +// BWC PARAMETERS +#ifdef ADIOS2_HAVE_BIGWHOOP +constexpr char LossyBWC[] = "bigwhoop"; +namespace bigwhoop +{ +namespace key +{ +constexpr char rate[] = "rate"; +} +} +#endif + // SZ PARAMETERS #ifdef ADIOS2_HAVE_SZ diff --git a/source/adios2/core/Info.cpp b/source/adios2/core/Info.cpp index ae5dca63b3..aa9fd106f4 100644 --- a/source/adios2/core/Info.cpp +++ b/source/adios2/core/Info.cpp @@ -62,6 +62,9 @@ void adios2_available_engines(size_t *nentries, const char *const **list) } static const char *const operators[] = { +#ifdef ADIOS2_HAVE_BIGWHOOP + "BigWhoop", +#endif #ifdef ADIOS2_HAVE_BZIP2 "BZip2", #endif diff --git a/source/adios2/core/Operator.h b/source/adios2/core/Operator.h index d87e6788b4..52a34a762d 100644 --- a/source/adios2/core/Operator.h +++ b/source/adios2/core/Operator.h @@ -37,6 +37,7 @@ class Operator COMPRESS_SZ = 6, COMPRESS_ZFP = 7, COMPRESS_MGARDPLUS = 8, + COMPRESS_BIGWHOOP = 9, REFACTOR_MDR = 41, CALLBACK_SIGNATURE1 = 51, CALLBACK_SIGNATURE2 = 52, diff --git a/source/adios2/operator/OperatorFactory.cpp b/source/adios2/operator/OperatorFactory.cpp index 87139a92ff..77d8fc6b2f 100644 --- a/source/adios2/operator/OperatorFactory.cpp +++ b/source/adios2/operator/OperatorFactory.cpp @@ -14,6 +14,10 @@ #include "adios2/operator/plugin/PluginOperator.h" #include +#ifdef ADIOS2_HAVE_BIGWHOOP +#include "adios2/operator/compress/CompressBigWhoop.h" +#endif + #ifdef ADIOS2_HAVE_BLOSC2 #include "adios2/operator/compress/CompressBlosc.h" #endif @@ -60,6 +64,8 @@ std::string OperatorTypeToString(const Operator::OperatorType type) { switch (type) { + case Operator::COMPRESS_BIGWHOOP: + return "bigwhoop"; case Operator::COMPRESS_BLOSC: return "blosc"; case Operator::COMPRESS_BZIP2: @@ -93,7 +99,13 @@ std::shared_ptr MakeOperator(const std::string &type, const Params &pa const std::string typeLowerCase = helper::LowerCase(type); - if (typeLowerCase == "blosc") + if (typeLowerCase == "bigwhoop") + { +#ifdef ADIOS2_HAVE_BIGWHOOP + ret = std::make_shared(parameters); +#endif + } + else if (typeLowerCase == "blosc") { #ifdef ADIOS2_HAVE_BLOSC2 ret = std::make_shared(parameters); diff --git a/source/adios2/operator/compress/CompressBigWhoop.cpp b/source/adios2/operator/compress/CompressBigWhoop.cpp new file mode 100644 index 0000000000..69e2aaa296 --- /dev/null +++ b/source/adios2/operator/compress/CompressBigWhoop.cpp @@ -0,0 +1,280 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * CompressBigWhoop.cpp + * + * Created on: Jan 26, 2024 + * Author: Gregor Weiss gregor.weiss@hlrs.de + */ +#include "CompressBigWhoop.h" +#include "adios2/helper/adiosFunctions.h" +#include +#include + +namespace adios2 +{ +namespace core +{ +namespace compress +{ + +/** + * Returns a adios2::dims object that contains targetDims entries. If the input + * dimensionality is smaller the additional dimensions are set to defaultDimSize. + * Else the entries at the front are multiplied and collapsed. + * @param dimension + * @param type + * @param targetDims + * @param enforceDims + * @return refined dimensions + */ +Dims ConvertBwcDims(const Dims &dimensions, const DataType type, const size_t targetDims, + const bool enforceDims, const size_t defaultDimSize = 1); + +/** + * Returns BWC supported bwc_precision based on adios string type + * @param type adios type as string, see GetDataType in helper/adiosType.inl + * @return bwc_precision + */ +bwc_precision GetBWCType(DataType type); + +CompressBigWhoop::CompressBigWhoop(const Params ¶meters) +: Operator("bigwhoop", COMPRESS_BIGWHOOP, "compress", parameters) +{ +} + +size_t CompressBigWhoop::Operate(const char *dataIn, const Dims &blockStart, const Dims &blockCount, + const DataType type, char *bufferOut) +{ + const uint8_t bufferVersion = 1; + size_t bufferOutOffset = 0; + + MakeCommonHeader(bufferOut, bufferOutOffset, bufferVersion); + + const size_t ndims = blockCount.size(); + + // bwc V1 metadata + PutParameter(bufferOut, bufferOutOffset, ndims); + for (const auto &d : blockCount) + { + PutParameter(bufferOut, bufferOutOffset, d); + } + PutParameter(bufferOut, bufferOutOffset, type); + // bwc V1 metadata end + + Dims convertedDims = ConvertBwcDims(blockCount, type, 5, true, 1); + + // Allocate BigWhoop coder + bwc_precision bwcType = GetBWCType(type); + bwc_codec *coder = bwc_alloc_coder(convertedDims[0], convertedDims[1], convertedDims[2], + convertedDims[3], convertedDims[4], bwcType); + if (coder == nullptr) + { + helper::Throw("Operator", "CompressBigWhoop", "Operate", + "BigWhoop failed to make coder codec"); + } + + // Rate control + std::string rate = "32"; + auto itRate = m_Parameters.find("rate"); + const bool hasRate = itRate != m_Parameters.end(); + rate = hasRate ? itRate->second : rate; + + // Quantization setting + auto itQM = m_Parameters.find("qm"); + const bool hasQM = itQM != m_Parameters.end(); + if (hasQM) + { + const int QM = + helper::StringTo(itQM->second, "setting 'qm' in call to CompressBigWhoop\n"); + bwc_set_qm(coder, QM); + } + + // Decomposition setting + auto itTile = m_Parameters.find("tile"); + const bool hasTile = itTile != m_Parameters.end(); + if (hasTile) + { + const int tile = + helper::StringTo(itTile->second, "setting 'tile' in call to CompressBigWhoop\n"); + bwc_set_tiles(coder, tile, tile, tile, tile, bwc_tile_sizeof); + } + auto itPrecincts = m_Parameters.find("precincts"); + const bool hasPrecincts = itPrecincts != m_Parameters.end(); + if (hasPrecincts) + { + const int precincts = helper::StringTo( + itPrecincts->second, "setting 'precincts' in call to CompressBigWhoop\n"); + bwc_set_precincts(coder, precincts, precincts, precincts, precincts); + } + auto itCB = m_Parameters.find("codeblocks"); + const bool hasCB = itCB != m_Parameters.end(); + if (hasCB) + { + const int cb = helper::StringTo(itCB->second, + "setting 'codeblocks' in call to CompressBigWhoop\n"); + bwc_set_codeblocks(coder, cb, cb, cb, cb); + } + auto itDecomp = m_Parameters.find("decomposition"); + const bool hasDecomp = itDecomp != m_Parameters.end(); + if (hasDecomp) + { + const int decomp = helper::StringTo( + itDecomp->second, "setting 'decomposition' in call to CompressBigWhoop\n"); + bwc_set_decomp(coder, decomp, decomp, decomp, decomp); + } + + // Compression + bwc_stream *stream = + bwc_init_stream(const_cast(dataIn), bufferOut + bufferOutOffset, comp); + bwc_create_compression(coder, stream, const_cast(rate.data())); + size_t sizeOut = (size_t)bwc_compress(coder, stream); + + if (sizeOut == 0) + { + helper::Throw("Operator", "CompressBigWhoop", "Operate(Compress)", + "BigWhoop failed, compressed buffer size is 0"); + } + + bufferOutOffset += sizeOut; + + free(stream); + bwc_free_codec(coder); + + return bufferOutOffset; +} + +size_t CompressBigWhoop::InverseOperate(const char *bufferIn, const size_t sizeIn, char *dataOut) +{ + size_t bufferInOffset = 1; // skip operator type + const uint8_t bufferVersion = GetParameter(bufferIn, bufferInOffset); + bufferInOffset += 2; // skip two reserved bytes + + if (bufferVersion == 1) + { + return DecompressV1(bufferIn + bufferInOffset, sizeIn - bufferInOffset, dataOut); + } + else if (bufferVersion == 2) + { + // TODO: if a Version 2 buffer is being implemented, put it here + // and keep the DecompressV1 routine for backward compatibility + } + else + { + helper::Throw("Operator", "CompressBigWhoop", "InverseOperate", + "invalid BigWhoop buffer version" + + std::to_string(bufferVersion)); + } + + return 0; +} + +bool CompressBigWhoop::IsDataTypeValid(const DataType type) const +{ + if (type == DataType::Float || type == DataType::Double || type == DataType::FloatComplex || + type == DataType::DoubleComplex) + { + return true; + } + return false; +} + +// PRIVATE + +size_t CompressBigWhoop::DecompressV1(const char *bufferIn, const size_t sizeIn, char *dataOut) +{ + // Do NOT remove even if the buffer version is updated. Data might be still + // in legacy formats. This function must be kept for backward compatibility. + // If a newer buffer format is implemented, create another function, e.g. + // DecompressV2 and keep this function for decompressing legacy data. + + size_t bufferInOffset = 0; + + const size_t ndims = GetParameter(bufferIn, bufferInOffset); + Dims blockCount(ndims); + for (size_t i = 0; i < ndims; ++i) + { + blockCount[i] = GetParameter(bufferIn, bufferInOffset); + } + const DataType type = GetParameter(bufferIn, bufferInOffset); + + Dims convertedDims = ConvertBwcDims(blockCount, type, 5, true, 1); + + // Allocate BigWhoop decoder + bwc_codec *decoder = bwc_alloc_decoder(); + + // Decompression + bwc_stream *stream = + bwc_init_stream(const_cast(bufferIn) + bufferInOffset, dataOut, decomp); + bwc_create_decompression(decoder, stream, 0); + bwc_decompress(decoder, stream); + + free(stream); + bwc_free_codec(decoder); + + return helper::GetTotalSize(convertedDims, helper::GetDataTypeSize(type)); +} + +Dims ConvertBwcDims(const Dims &dimensions, const DataType type, const size_t targetDims, + const bool enforceDims, const size_t defaultDimSize) +{ + if (targetDims < 1) + { + helper::Throw("Core", "Operator", "ConvertBwcDims", + "only accepts targetDims > 0"); + } + + Dims ret = dimensions; + + while (ret.size() > targetDims) + { + ret[1] *= ret[0]; + ret.erase(ret.begin()); + } + + while (enforceDims && ret.size() < targetDims) + { + ret.push_back(defaultDimSize); + } + + if (type == helper::GetDataType>() || + type == helper::GetDataType>()) + { + ret.back() *= 2; + } + return ret; +} + +bwc_precision GetBWCType(DataType type) +{ + bwc_precision bwcType; //= bwc_precision_none; + + if (type == helper::GetDataType()) + { + bwcType = bwc_precision_double; + } + else if (type == helper::GetDataType()) + { + bwcType = bwc_precision_single; + } + else if (type == helper::GetDataType>()) + { + bwcType = bwc_precision_single; + } + else if (type == helper::GetDataType>()) + { + bwcType = bwc_precision_double; + } + else + { + helper::Throw("Operator", "CompressBWC", "GetBWCType", + "invalid data type " + ToString(type)); + } + + return bwcType; +} + +} // end namespace compress +} // end namespace core +} // end namespace adios2 diff --git a/source/adios2/operator/compress/CompressBigWhoop.h b/source/adios2/operator/compress/CompressBigWhoop.h new file mode 100644 index 0000000000..b3ce1edd62 --- /dev/null +++ b/source/adios2/operator/compress/CompressBigWhoop.h @@ -0,0 +1,74 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * CompressBigWhoop.h : wrapper to BigWhoop compression library + * + * Created on: Tue Jan 26, 2024 + * Author: Gregor Weiss gregor.weiss@hlrs.de + */ + +#ifndef ADIOS2_OPERATOR_COMPRESS_COMPRESS_BIGWHOOP_H_ +#define ADIOS2_OPERATOR_COMPRESS_COMPRESS_BIGWHOOP_H_ + +#include "adios2/core/Operator.h" + +namespace adios2 +{ +namespace core +{ +namespace compress +{ + +class CompressBigWhoop : public Operator +{ + +public: + /** + * Unique constructor + */ + CompressBigWhoop(const Params ¶meters); + + ~CompressBigWhoop() = default; + + /** + * @param dataIn + * @param blockStart + * @param blockCount + * @param type + * @param bufferOut + * @return size of compressed buffer + */ + size_t Operate(const char *dataIn, const Dims &blockStart, const Dims &blockCount, + const DataType type, char *bufferOut) final; + + /** + * @param bufferIn + * @param sizeIn + * @param dataOut + * @return size of decompressed buffer + */ + size_t InverseOperate(const char *bufferIn, const size_t sizeIn, char *dataOut) final; + + bool IsDataTypeValid(const DataType type) const final; + +private: + /** + * Decompress function for V1 buffer. Do NOT remove even if the buffer + * version is updated. Data might be still in legacy formats. This function + * must be kept for backward compatibility. + * @param bufferIn : compressed data buffer (V1 only) + * @param sizeIn : number of bytes in bufferIn + * @param dataOut : decompressed data buffer + * @return : number of bytes in dataOut + */ + size_t DecompressV1(const char *bufferIn, const size_t sizeIn, char *dataOut); + + std::string m_VersionInfo; +}; + +} // end namespace compress +} // end namespace core +} // end namespace adios2 + +#endif /* ADIOS2_TRANSFORM_COMPRESSION_COMPRESS_BIGWHOOP_H_ */