From 629953f6e9a0f8ab44e017d8112279edb6795882 Mon Sep 17 00:00:00 2001 From: Thomas Padioleau Date: Fri, 20 Sep 2024 22:20:10 +0200 Subject: [PATCH] Rebase on KokkosFFT --- .github/workflows/tests.yml | 12 + .gitmodules | 3 + CMakeLists.txt | 55 ++-- LICENSES/BSD-3-Clause.txt | 11 - cmake/DDCConfig.cmake.in | 5 + cmake/FindFFTW.cmake | 419 ------------------------- include/ddc/kernels/fft.hpp | 604 ++++++++---------------------------- tests/fft/fft.cpp | 16 +- vendor/kokkos-fft | 1 + 9 files changed, 177 insertions(+), 949 deletions(-) delete mode 100644 LICENSES/BSD-3-Clause.txt delete mode 100644 cmake/FindFFTW.cmake create mode 160000 vendor/kokkos-fft diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7535ea934..4dcd4c11e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -159,6 +159,7 @@ jobs: export benchmark_ROOT=$PWD/opt/benchmark export GTest_ROOT=$PWD/opt/gtest export Kokkos_ROOT=$PWD/opt/kokkos + export KokkosFFT_ROOT=$PWD/opt/kokkos-fft export KokkosKernels_ROOT=$PWD/opt/kokkos-kernels export mdspan_ROOT=$PWD/opt/mdspan @@ -197,6 +198,16 @@ jobs: cmake --install build --prefix $Kokkos_ROOT rm -rf build + cmake \ + -DCMAKE_BUILD_TYPE=${{matrix.cmake_build_type}} \ + -DCMAKE_CXX_STANDARD=${{matrix.cxx_version}} \ + -DKokkosFFT_ENABLE_HOST_AND_DEVICE=ON \ + -B build \ + -S /src/vendor/kokkos-fft + cmake --build build --parallel 2 + cmake --install build --prefix $KokkosFFT_ROOT + rm -rf build + cmake \ -DCMAKE_BUILD_TYPE=${{matrix.cmake_build_type}} \ -DCMAKE_CXX_STANDARD=${{matrix.cxx_version}} \ @@ -234,6 +245,7 @@ jobs: -DDDC_benchmark_DEPENDENCY_POLICY=INSTALLED \ -DDDC_GTest_DEPENDENCY_POLICY=INSTALLED \ -DDDC_Kokkos_DEPENDENCY_POLICY=INSTALLED \ + -DDDC_KokkosFFT_DEPENDENCY_POLICY=INSTALLED \ -DDDC_mdspan_DEPENDENCY_POLICY=INSTALLED \ -B build \ -S /src diff --git a/.gitmodules b/.gitmodules index ffc884cc4..a808a56ac 100644 --- a/.gitmodules +++ b/.gitmodules @@ -21,3 +21,6 @@ [submodule "vendor/doxygen-awesome-css"] path = vendor/doxygen-awesome-css url = https://github.com/jothepro/doxygen-awesome-css.git +[submodule "vendor/kokkos-fft"] + path = vendor/kokkos-fft + url = https://github.com/kokkos/kokkos-fft.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 7041e6e93..daea5bc99 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -78,19 +78,6 @@ endif() # Custom cmake modules list( APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake" ) -# FFTW -if("${DDC_BUILD_KERNELS_FFT}" AND NOT FFTW_FOUND) - find_package( FFTW MODULE REQUIRED ) -endif() - -## CUDA + CUDAToolkit -if("${DDC_BUILD_KERNELS_FFT}" AND "${Kokkos_ENABLE_CUDA}") - find_package( CUDAToolkit MODULE REQUIRED ) - if( NOT(CUDAToolkit_FOUND) ) - message(FATAL_ERROR "CUDAToolkit not found." ) - endif() -endif() - ## PDI @@ -196,30 +183,26 @@ install( # Link library to DDC -if( FFTW_FOUND ) - target_link_libraries( DDC INTERFACE FFTW::Float ) - target_link_libraries( DDC INTERFACE FFTW::Double ) -endif() - -if( FFTW_FOUND AND "${Kokkos_ENABLE_SERIAL}") - target_compile_definitions(DDC INTERFACE fftw_serial_AVAIL) -endif() - -if( FFTW_FOUND AND "${Kokkos_ENABLE_OPENMP}") - target_link_libraries( DDC INTERFACE FFTW::FloatOpenMP ) - target_link_libraries( DDC INTERFACE FFTW::DoubleOpenMP ) - target_compile_definitions(DDC INTERFACE fftw_omp_AVAIL) -endif() - -if( CUDAToolkit_FOUND AND "${Kokkos_ENABLE_CUDA}") - target_link_libraries( DDC INTERFACE CUDA::cufft ) - target_compile_definitions(DDC INTERFACE cufft_AVAIL) -endif() +if("${DDC_BUILD_KERNELS_FFT}") + # Kokkos-fft + set(DDC_KokkosFFT_DEPENDENCY_POLICY "AUTO" CACHE STRING "Policy to find the `Kokkos-fft` package. Options: ${DDC_DEPENDENCY_POLICIES}") + set_property(CACHE DDC_KokkosFFT_DEPENDENCY_POLICY PROPERTY STRINGS "${DDC_DEPENDENCY_POLICIES}") + if("${DDC_KokkosFFT_DEPENDENCY_POLICY}" STREQUAL "AUTO") + if(NOT TARGET KokkosFFT::fft) + find_package(KokkosFFT CONFIG QUIET) + if(NOT KokkosFFT_FOUND) + set(KokkosFFT_ENABLE_HOST_AND_DEVICE ON) + add_subdirectory(vendor/kokkos-fft) + endif() + endif() + elseif("${DDC_KokkosFFT_DEPENDENCY_POLICY}" STREQUAL "EMBEDDED") + set(KokkosFFT_ENABLE_HOST_AND_DEVICE ON) + add_subdirectory(vendor/kokkos-fft) + elseif("${DDC_KokkosFFT_DEPENDENCY_POLICY}" STREQUAL "INSTALLED") + find_package(KokkosFFT CONFIG REQUIRED) + endif() -if("${Kokkos_ENABLE_HIP}") - find_package( hipfft REQUIRED ) # Not compatible with nvidia - target_link_libraries( DDC INTERFACE hip::hipfft ) - target_compile_definitions(DDC INTERFACE hipfft_AVAIL) + target_link_libraries(DDC INTERFACE KokkosFFT::fft) endif() if("${DDC_BUILD_KERNELS_SPLINES}") diff --git a/LICENSES/BSD-3-Clause.txt b/LICENSES/BSD-3-Clause.txt deleted file mode 100644 index ea890afbc..000000000 --- a/LICENSES/BSD-3-Clause.txt +++ /dev/null @@ -1,11 +0,0 @@ -Copyright (c) . - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/cmake/DDCConfig.cmake.in b/cmake/DDCConfig.cmake.in index c346562eb..36ff0b17c 100644 --- a/cmake/DDCConfig.cmake.in +++ b/cmake/DDCConfig.cmake.in @@ -7,6 +7,11 @@ include(CMakeFindDependencyMacro) find_dependency(mdspan) + +if(@DDC_BUILD_KERNELS_FFT@) + find_dependency(KokkosFFT) +endif() + if(@DDC_BUILD_PDI_WRAPPER@) find_dependency(PDI COMPONENTS C) endif() diff --git a/cmake/FindFFTW.cmake b/cmake/FindFFTW.cmake deleted file mode 100644 index 8a43d3270..000000000 --- a/cmake/FindFFTW.cmake +++ /dev/null @@ -1,419 +0,0 @@ -# Copyright (C) 2015, Wenzel Jakob -# Copyright (C) 2017, Patrick Bos -# Copyright (C) The DDC development team, see COPYRIGHT.md file -# -# SPDX-License-Identifier: BSD-3-Clause - -# - Find the FFTW library -# -# Usage: -# find_package(FFTW [REQUIRED] [QUIET] [COMPONENTS component1 ... componentX] ) -# -# It sets the following variables: -# FFTW_FOUND ... true if fftw is found on the system -# FFTW_[component]_LIB_FOUND ... true if the component is found on the system (see components below) -# FFTW_LIBRARIES ... full paths to all found fftw libraries -# FFTW_[component]_LIB ... full path to one of the components (see below) -# FFTW_INCLUDE_DIRS ... fftw include directory paths -# -# The following variables will be checked by the function -# FFTW_USE_STATIC_LIBS ... if true, only static libraries are found, otherwise both static and shared. -# FFTW_ROOT ... if set, the libraries are exclusively searched -# under this path -# -# This package supports the following components: -# FLOAT_LIB -# DOUBLE_LIB -# LONGDOUBLE_LIB -# FLOAT_THREADS_LIB -# DOUBLE_THREADS_LIB -# LONGDOUBLE_THREADS_LIB -# FLOAT_OPENMP_LIB -# DOUBLE_OPENMP_LIB -# LONGDOUBLE_OPENMP_LIB -# - -# TODO (maybe): extend with ExternalProject download + build option -# TODO: put on conda-forge - - -if( NOT FFTW_ROOT AND DEFINED ENV{FFTWDIR} ) - set( FFTW_ROOT $ENV{FFTWDIR} ) -endif() - -# Check if we can use PkgConfig -find_package(PkgConfig) - -#Determine from PKG -if( PKG_CONFIG_FOUND AND NOT FFTW_ROOT ) - pkg_check_modules( PKG_FFTW QUIET "fftw3" ) -endif() - -#Check whether to search static or dynamic libs -set( CMAKE_FIND_LIBRARY_SUFFIXES_SAV ${CMAKE_FIND_LIBRARY_SUFFIXES} ) - -if( ${FFTW_USE_STATIC_LIBS} ) - set( CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_STATIC_LIBRARY_SUFFIX} ) -else() - set( CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES_SAV} ) -endif() - -if( FFTW_ROOT ) - # find libs - - find_library( - FFTW_DOUBLE_LIB - NAMES "fftw3" libfftw3-3 - PATHS ${FFTW_ROOT} - PATH_SUFFIXES "lib" "lib64" - NO_DEFAULT_PATH - ) - - find_library( - FFTW_DOUBLE_THREADS_LIB - NAMES "fftw3_threads" - PATHS ${FFTW_ROOT} - PATH_SUFFIXES "lib" "lib64" - NO_DEFAULT_PATH - ) - - find_library( - FFTW_DOUBLE_OPENMP_LIB - NAMES "fftw3_omp" - PATHS ${FFTW_ROOT} - PATH_SUFFIXES "lib" "lib64" - NO_DEFAULT_PATH - ) - - find_library( - FFTW_DOUBLE_MPI_LIB - NAMES "fftw3_mpi" - PATHS ${FFTW_ROOT} - PATH_SUFFIXES "lib" "lib64" - NO_DEFAULT_PATH - ) - - find_library( - FFTW_FLOAT_LIB - NAMES "fftw3f" libfftw3f-3 - PATHS ${FFTW_ROOT} - PATH_SUFFIXES "lib" "lib64" - NO_DEFAULT_PATH - ) - - find_library( - FFTW_FLOAT_THREADS_LIB - NAMES "fftw3f_threads" - PATHS ${FFTW_ROOT} - PATH_SUFFIXES "lib" "lib64" - NO_DEFAULT_PATH - ) - - find_library( - FFTW_FLOAT_OPENMP_LIB - NAMES "fftw3f_omp" - PATHS ${FFTW_ROOT} - PATH_SUFFIXES "lib" "lib64" - NO_DEFAULT_PATH - ) - - find_library( - FFTW_FLOAT_MPI_LIB - NAMES "fftw3f_mpi" - PATHS ${FFTW_ROOT} - PATH_SUFFIXES "lib" "lib64" - NO_DEFAULT_PATH - ) - - find_library( - FFTW_LONGDOUBLE_LIB - NAMES "fftw3l" libfftw3l-3 - PATHS ${FFTW_ROOT} - PATH_SUFFIXES "lib" "lib64" - NO_DEFAULT_PATH - ) - - find_library( - FFTW_LONGDOUBLE_THREADS_LIB - NAMES "fftw3l_threads" - PATHS ${FFTW_ROOT} - PATH_SUFFIXES "lib" "lib64" - NO_DEFAULT_PATH - ) - - find_library( - FFTW_LONGDOUBLE_OPENMP_LIB - NAMES "fftw3l_omp" - PATHS ${FFTW_ROOT} - PATH_SUFFIXES "lib" "lib64" - NO_DEFAULT_PATH - ) - - find_library( - FFTW_LONGDOUBLE_MPI_LIB - NAMES "fftw3l_mpi" - PATHS ${FFTW_ROOT} - PATH_SUFFIXES "lib" "lib64" - NO_DEFAULT_PATH - ) - - #find includes - find_path(FFTW_INCLUDE_DIRS - NAMES "fftw3.h" - PATHS ${FFTW_ROOT} - PATH_SUFFIXES "include" - NO_DEFAULT_PATH - ) - -else() - - find_library( - FFTW_DOUBLE_LIB - NAMES "fftw3" - PATHS ${PKG_FFTW_LIBRARY_DIRS} ${LIB_INSTALL_DIR} - ) - - find_library( - FFTW_DOUBLE_THREADS_LIB - NAMES "fftw3_threads" - PATHS ${PKG_FFTW_LIBRARY_DIRS} ${LIB_INSTALL_DIR} - ) - - find_library( - FFTW_DOUBLE_OPENMP_LIB - NAMES "fftw3_omp" - PATHS ${PKG_FFTW_LIBRARY_DIRS} ${LIB_INSTALL_DIR} - ) - - find_library( - FFTW_DOUBLE_MPI_LIB - NAMES "fftw3_mpi" - PATHS ${PKG_FFTW_LIBRARY_DIRS} ${LIB_INSTALL_DIR} - ) - - find_library( - FFTW_FLOAT_LIB - NAMES "fftw3f" - PATHS ${PKG_FFTW_LIBRARY_DIRS} ${LIB_INSTALL_DIR} - ) - - find_library( - FFTW_FLOAT_THREADS_LIB - NAMES "fftw3f_threads" - PATHS ${PKG_FFTW_LIBRARY_DIRS} ${LIB_INSTALL_DIR} - ) - - find_library( - FFTW_FLOAT_OPENMP_LIB - NAMES "fftw3f_omp" - PATHS ${PKG_FFTW_LIBRARY_DIRS} ${LIB_INSTALL_DIR} - ) - - find_library( - FFTW_FLOAT_MPI_LIB - NAMES "fftw3f_mpi" - PATHS ${PKG_FFTW_LIBRARY_DIRS} ${LIB_INSTALL_DIR} - ) - - find_library( - FFTW_LONGDOUBLE_LIB - NAMES "fftw3l" - PATHS ${PKG_FFTW_LIBRARY_DIRS} ${LIB_INSTALL_DIR} - ) - - find_library( - FFTW_LONGDOUBLE_THREADS_LIB - NAMES "fftw3l_threads" - PATHS ${PKG_FFTW_LIBRARY_DIRS} ${LIB_INSTALL_DIR} - ) - - find_library(FFTW_LONGDOUBLE_OPENMP_LIB - NAMES "fftw3l_omp" - PATHS ${PKG_FFTW_LIBRARY_DIRS} ${LIB_INSTALL_DIR} - ) - - find_library(FFTW_LONGDOUBLE_MPI_LIB - NAMES "fftw3l_mpi" - PATHS ${PKG_FFTW_LIBRARY_DIRS} ${LIB_INSTALL_DIR} - ) - - find_path(FFTW_INCLUDE_DIRS - NAMES "fftw3.h" - PATHS ${PKG_FFTW_INCLUDE_DIRS} ${INCLUDE_INSTALL_DIR} - ) - -endif( FFTW_ROOT ) - -#--------------------------------------- components - -if (FFTW_DOUBLE_LIB) - set(FFTW_DOUBLE_LIB_FOUND TRUE) - set(FFTW_LIBRARIES ${FFTW_LIBRARIES} ${FFTW_DOUBLE_LIB}) - add_library(FFTW::Double INTERFACE IMPORTED) - set_target_properties(FFTW::Double - PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${FFTW_INCLUDE_DIRS}" - INTERFACE_LINK_LIBRARIES "${FFTW_DOUBLE_LIB}" - ) -else() - set(FFTW_DOUBLE_LIB_FOUND FALSE) -endif() - -if (FFTW_FLOAT_LIB) - set(FFTW_FLOAT_LIB_FOUND TRUE) - set(FFTW_LIBRARIES ${FFTW_LIBRARIES} ${FFTW_FLOAT_LIB}) - add_library(FFTW::Float INTERFACE IMPORTED) - set_target_properties(FFTW::Float - PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${FFTW_INCLUDE_DIRS}" - INTERFACE_LINK_LIBRARIES "${FFTW_FLOAT_LIB}" - ) -else() - set(FFTW_FLOAT_LIB_FOUND FALSE) -endif() - -if (FFTW_LONGDOUBLE_LIB) - set(FFTW_LONGDOUBLE_LIB_FOUND TRUE) - set(FFTW_LIBRARIES ${FFTW_LIBRARIES} ${FFTW_LONGDOUBLE_LIB}) - add_library(FFTW::LongDouble INTERFACE IMPORTED) - set_target_properties(FFTW::LongDouble - PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${FFTW_INCLUDE_DIRS}" - INTERFACE_LINK_LIBRARIES "${FFTW_LONGDOUBLE_LIB}" - ) -else() - set(FFTW_LONGDOUBLE_LIB_FOUND FALSE) -endif() - -if (FFTW_DOUBLE_THREADS_LIB) - set(FFTW_DOUBLE_THREADS_LIB_FOUND TRUE) - set(FFTW_LIBRARIES ${FFTW_LIBRARIES} ${FFTW_DOUBLE_THREADS_LIB}) - add_library(FFTW::DoubleThreads INTERFACE IMPORTED) - set_target_properties(FFTW::DoubleThreads - PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${FFTW_INCLUDE_DIRS}" - INTERFACE_LINK_LIBRARIES "${FFTW_DOUBLE_THREADS_LIB}" - ) -else() - set(FFTW_DOUBLE_THREADS_LIB_FOUND FALSE) -endif() - -if (FFTW_FLOAT_THREADS_LIB) - set(FFTW_FLOAT_THREADS_LIB_FOUND TRUE) - set(FFTW_LIBRARIES ${FFTW_LIBRARIES} ${FFTW_FLOAT_THREADS_LIB}) - add_library(FFTW::FloatThreads INTERFACE IMPORTED) - set_target_properties(FFTW::FloatThreads - PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${FFTW_INCLUDE_DIRS}" - INTERFACE_LINK_LIBRARIES "${FFTW_FLOAT_THREADS_LIB}" - ) -else() - set(FFTW_FLOAT_THREADS_LIB_FOUND FALSE) -endif() - -if (FFTW_LONGDOUBLE_THREADS_LIB) - set(FFTW_LONGDOUBLE_THREADS_LIB_FOUND TRUE) - set(FFTW_LIBRARIES ${FFTW_LIBRARIES} ${FFTW_LONGDOUBLE_THREADS_LIB}) - add_library(FFTW::LongDoubleThreads INTERFACE IMPORTED) - set_target_properties(FFTW::LongDoubleThreads - PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${FFTW_INCLUDE_DIRS}" - INTERFACE_LINK_LIBRARIES "${FFTW_LONGDOUBLE_THREADS_LIB}" - ) -else() - set(FFTW_LONGDOUBLE_THREADS_LIB_FOUND FALSE) -endif() - -if (FFTW_DOUBLE_OPENMP_LIB) - set(FFTW_DOUBLE_OPENMP_LIB_FOUND TRUE) - set(FFTW_LIBRARIES ${FFTW_LIBRARIES} ${FFTW_DOUBLE_OPENMP_LIB}) - add_library(FFTW::DoubleOpenMP INTERFACE IMPORTED) - set_target_properties(FFTW::DoubleOpenMP - PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${FFTW_INCLUDE_DIRS}" - INTERFACE_LINK_LIBRARIES "${FFTW_DOUBLE_OPENMP_LIB}" - ) -else() - set(FFTW_DOUBLE_OPENMP_LIB_FOUND FALSE) -endif() - -if (FFTW_FLOAT_OPENMP_LIB) - set(FFTW_FLOAT_OPENMP_LIB_FOUND TRUE) - set(FFTW_LIBRARIES ${FFTW_LIBRARIES} ${FFTW_FLOAT_OPENMP_LIB}) - add_library(FFTW::FloatOpenMP INTERFACE IMPORTED) - set_target_properties(FFTW::FloatOpenMP - PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${FFTW_INCLUDE_DIRS}" - INTERFACE_LINK_LIBRARIES "${FFTW_FLOAT_OPENMP_LIB}" - ) -else() - set(FFTW_FLOAT_OPENMP_LIB_FOUND FALSE) -endif() - -if (FFTW_LONGDOUBLE_OPENMP_LIB) - set(FFTW_LONGDOUBLE_OPENMP_LIB_FOUND TRUE) - set(FFTW_LIBRARIES ${FFTW_LIBRARIES} ${FFTW_LONGDOUBLE_OPENMP_LIB}) - add_library(FFTW::LongDoubleOpenMP INTERFACE IMPORTED) - set_target_properties(FFTW::LongDoubleOpenMP - PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${FFTW_INCLUDE_DIRS}" - INTERFACE_LINK_LIBRARIES "${FFTW_LONGDOUBLE_OPENMP_LIB}" - ) -else() - set(FFTW_LONGDOUBLE_OPENMP_LIB_FOUND FALSE) -endif() - -if (FFTW_DOUBLE_MPI_LIB) - set(FFTW_DOUBLE_MPI_LIB_FOUND TRUE) - set(FFTW_LIBRARIES ${FFTW_LIBRARIES} ${FFTW_DOUBLE_MPI_LIB}) - add_library(FFTW::DoubleMPI INTERFACE IMPORTED) - set_target_properties(FFTW::DoubleMPI - PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${FFTW_INCLUDE_DIRS}" - INTERFACE_LINK_LIBRARIES "${FFTW_DOUBLE_MPI_LIB}" - ) -else() - set(FFTW_DOUBLE_MPI_LIB_FOUND FALSE) -endif() - -if (FFTW_FLOAT_MPI_LIB) - set(FFTW_FLOAT_MPI_LIB_FOUND TRUE) - set(FFTW_LIBRARIES ${FFTW_LIBRARIES} ${FFTW_FLOAT_MPI_LIB}) - add_library(FFTW::FloatMPI INTERFACE IMPORTED) - set_target_properties(FFTW::FloatMPI - PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${FFTW_INCLUDE_DIRS}" - INTERFACE_LINK_LIBRARIES "${FFTW_FLOAT_MPI_LIB}" - ) -else() - set(FFTW_FLOAT_MPI_LIB_FOUND FALSE) -endif() - -if (FFTW_LONGDOUBLE_MPI_LIB) - set(FFTW_LONGDOUBLE_MPI_LIB_FOUND TRUE) - set(FFTW_LIBRARIES ${FFTW_LIBRARIES} ${FFTW_LONGDOUBLE_MPI_LIB}) - add_library(FFTW::LongDoubleMPI INTERFACE IMPORTED) - set_target_properties(FFTW::LongDoubleMPI - PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${FFTW_INCLUDE_DIRS}" - INTERFACE_LINK_LIBRARIES "${FFTW_LONGDOUBLE_MPI_LIB}" - ) -else() - set(FFTW_LONGDOUBLE_MPI_LIB_FOUND FALSE) -endif() - -#--------------------------------------- end components - -set( CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES_SAV} ) - -include(FindPackageHandleStandardArgs) - -find_package_handle_standard_args(FFTW - REQUIRED_VARS FFTW_INCLUDE_DIRS - HANDLE_COMPONENTS - ) - -mark_as_advanced( - FFTW_INCLUDE_DIRS - FFTW_LIBRARIES - FFTW_FLOAT_LIB - FFTW_DOUBLE_LIB - FFTW_LONGDOUBLE_LIB - FFTW_FLOAT_THREADS_LIB - FFTW_DOUBLE_THREADS_LIB - FFTW_LONGDOUBLE_THREADS_LIB - FFTW_FLOAT_OPENMP_LIB - FFTW_DOUBLE_OPENMP_LIB - FFTW_LONGDOUBLE_OPENMP_LIB - FFTW_FLOAT_MPI_LIB - FFTW_DOUBLE_MPI_LIB - FFTW_LONGDOUBLE_MPI_LIB - ) diff --git a/include/ddc/kernels/fft.hpp b/include/ddc/kernels/fft.hpp index eaa5047df..aab1ea117 100644 --- a/include/ddc/kernels/fft.hpp +++ b/include/ddc/kernels/fft.hpp @@ -4,8 +4,6 @@ #pragma once -#include -#include #include #include @@ -13,57 +11,9 @@ #include +#include #include -#if fftw_serial_AVAIL || fftw_omp_AVAIL -#include -#endif - -#if cufft_AVAIL -#include -#include -#include - -#include -#include -#endif - -#if hipfft_AVAIL -#include -#include -#include - -#include -#include -#endif - -#if fftw_serial_AVAIL || fftw_omp_AVAIL -static_assert(sizeof(fftwf_complex) == sizeof(Kokkos::complex)); -static_assert(alignof(fftwf_complex) <= alignof(Kokkos::complex)); - -static_assert(sizeof(fftw_complex) == sizeof(Kokkos::complex)); -static_assert(alignof(fftw_complex) <= alignof(Kokkos::complex)); - -static_assert(sizeof(fftwl_complex) == sizeof(Kokkos::complex)); -static_assert(alignof(fftwl_complex) <= alignof(Kokkos::complex)); -#endif - -#if cufft_AVAIL -static_assert(sizeof(cufftComplex) == sizeof(Kokkos::complex)); -static_assert(alignof(cufftComplex) <= alignof(Kokkos::complex)); - -static_assert(sizeof(cufftDoubleComplex) == sizeof(Kokkos::complex)); -static_assert(alignof(cufftDoubleComplex) <= alignof(Kokkos::complex)); -#endif - -#if hipfft_AVAIL -static_assert(sizeof(hipfftComplex) == sizeof(Kokkos::complex)); -static_assert(alignof(hipfftComplex) <= alignof(Kokkos::complex)); - -static_assert(sizeof(hipfftDoubleComplex) == sizeof(Kokkos::complex)); -static_assert(alignof(hipfftDoubleComplex) <= alignof(Kokkos::complex)); -#endif - namespace ddc { /** @@ -147,222 +97,6 @@ KOKKOS_FUNCTION constexpr T LastSelector(const T a, const T b) return LastSelector(a, b); } -/** - * @brief A trait to identify the type of transformation (R2C, C2R, C2C...). - * - * It does not contain the information about the base type (float or double). - */ -enum class TransformType { R2R, R2C, C2R, C2C }; - -template -struct transform_type -{ - static constexpr TransformType value = TransformType::R2R; -}; - -template -struct transform_type> -{ - static constexpr TransformType value = TransformType::R2C; -}; - -template -struct transform_type, T2> -{ - static constexpr TransformType value = TransformType::C2R; -}; - -template -struct transform_type, Kokkos::complex> -{ - static constexpr TransformType value = TransformType::C2C; -}; - -/** - * @brief A trait to get the TransformType for the input and output types. - * - * Internally check if T1 and T2 are Kokkos::complex or not. - * - * @tparam T1 The input type. - * @tparam T2 The output type. - */ -template -constexpr TransformType transform_type_v = transform_type::value; - -#if fftw_serial_AVAIL || fftw_omp_AVAIL -// _fftw_type : compatible with both single and double precision -template -struct _fftw_type -{ - using type = T; -}; - -template -struct _fftw_type> -{ - using type = std:: - conditional_t, float>, fftwf_complex, fftw_complex>; -}; - -// _fftw_plan : compatible with both single and double precision -template -using _fftw_plan = std::conditional_t, float>, fftwf_plan, fftw_plan>; - -// _fftw_plan_many_dft : templated function working for all types of transformation -template -_fftw_plan _fftw_plan_many_dft( - [[maybe_unused]] PenultArg penultArg, - LastArg lastArg, - Args... args) -{ // Ugly, penultArg and lastArg are passed before the rest because of a limitation of C++ (parameter packs must be last arguments) - const TransformType transformType = transform_type_v; - if constexpr (transformType == TransformType::R2C && std::is_same_v) { - return fftwf_plan_many_dft_r2c(args..., lastArg); - } else if constexpr (transformType == TransformType::R2C && std::is_same_v) { - return fftw_plan_many_dft_r2c(args..., lastArg); - } else if constexpr (transformType == TransformType::C2R && std::is_same_v) { - return fftwf_plan_many_dft_c2r(args..., lastArg); - } else if constexpr (transformType == TransformType::C2R && std::is_same_v) { - return fftw_plan_many_dft_c2r(args..., lastArg); - } else if constexpr ( - transformType == TransformType::C2C && std::is_same_v>) { - return fftwf_plan_many_dft(args..., penultArg, lastArg); - } else if constexpr ( - transformType == TransformType::C2C && std::is_same_v>) { - return fftw_plan_many_dft(args..., penultArg, lastArg); - } - // else constexpr - // static_assert(false, "Transform type not supported"); -} - -#endif -#if cufft_AVAIL -// _cufft_type : compatible with both single and double precision -template -struct _cufft_type -{ - using type = std::conditional_t, cufftReal, cufftDoubleReal>; -}; - -template -struct _cufft_type> -{ - using type = std:: - conditional_t, float>, cufftComplex, cufftDoubleComplex>; -}; - -// cufft_transform_type : argument passed in the cufftMakePlan function -template -constexpr auto cufft_transform_type() -{ - const TransformType transformType = transform_type_v; - if constexpr (transformType == TransformType::R2C && std::is_same_v) - return CUFFT_R2C; - else if constexpr (transformType == TransformType::R2C && std::is_same_v) - return CUFFT_D2Z; - else if constexpr (transformType == TransformType::C2R && std::is_same_v) - return CUFFT_C2R; - else if constexpr (transformType == TransformType::C2R && std::is_same_v) - return CUFFT_Z2D; - else if constexpr ( - transformType == TransformType::C2C && std::is_same_v>) - return CUFFT_C2C; - else if constexpr ( - transformType == TransformType::C2C && std::is_same_v>) - return CUFFT_Z2Z; - // else constexpr - // static_assert(false, "Transform type not supported"); -} - -// cufftExec : argument passed in the cufftMakePlan function -// _fftw_plan_many_dft : templated function working for all types of transformation -template -cufftResult _cufftExec([[maybe_unused]] LastArg lastArg, Args... args) -{ // Ugly for same reason as fftw - const TransformType transformType = transform_type_v; - if constexpr (transformType == TransformType::R2C && std::is_same_v) - return cufftExecR2C(args...); - else if constexpr (transformType == TransformType::R2C && std::is_same_v) - return cufftExecD2Z(args...); - else if constexpr (transformType == TransformType::C2R && std::is_same_v) - return cufftExecC2R(args...); - else if constexpr (transformType == TransformType::C2R && std::is_same_v) - return cufftExecZ2D(args...); - else if constexpr ( - transformType == TransformType::C2C && std::is_same_v>) - return cufftExecC2C(args..., lastArg); - else if constexpr ( - transformType == TransformType::C2C && std::is_same_v>) - return cufftExecZ2Z(args..., lastArg); - // else constexpr - // static_assert(false, "Transform type not supported"); -} -#endif -#if hipfft_AVAIL -// _hipfft_type : compatible with both single and double precision -template -struct _hipfft_type -{ - using type = std::conditional_t, hipfftReal, hipfftDoubleReal>; -}; - -template -struct _hipfft_type> -{ - using type = std::conditional_t< - std::is_same_v, float>, - hipfftComplex, - hipfftDoubleComplex>; -}; - -// hipfft_transform_type : argument passed in the hipfftMakePlan function -template -constexpr auto hipfft_transform_type() -{ - const TransformType transformType = transform_type_v; - if constexpr (transformType == TransformType::R2C && std::is_same_v) - return HIPFFT_R2C; - else if constexpr (transformType == TransformType::R2C && std::is_same_v) - return HIPFFT_D2Z; - else if constexpr (transformType == TransformType::C2R && std::is_same_v) - return HIPFFT_C2R; - else if constexpr (transformType == TransformType::C2R && std::is_same_v) - return HIPFFT_Z2D; - else if constexpr ( - transformType == TransformType::C2C && std::is_same_v>) - return HIPFFT_C2C; - else if constexpr ( - transformType == TransformType::C2C && std::is_same_v>) - return HIPFFT_Z2Z; - // else constexpr - // static_assert(false, "Transform type not supported"); -} - -// hipfftExec : argument passed in the hipfftMakePlan function -// _fftw_plan_many_dft : templated function working for all types of transformation -template -hipfftResult _hipfftExec([[maybe_unused]] LastArg lastArg, Args... args) -{ - const TransformType transformType = transform_type_v; - if constexpr (transformType == TransformType::R2C && std::is_same_v) - return hipfftExecR2C(args...); - else if constexpr (transformType == TransformType::R2C && std::is_same_v) - return hipfftExecD2Z(args...); - else if constexpr (transformType == TransformType::C2R && std::is_same_v) - return hipfftExecC2R(args...); - else if constexpr (transformType == TransformType::C2R && std::is_same_v) - return hipfftExecZ2D(args...); - else if constexpr ( - transformType == TransformType::C2C && std::is_same_v>) - return hipfftExecC2C(args..., lastArg); - else if constexpr ( - transformType == TransformType::C2C && std::is_same_v>) - return hipfftExecZ2Z(args..., lastArg); - // else constexpr - // static_assert(false, "Transform type not supported"); -} -#endif - /* * @brief A structure embedding the configuration of the impl FFT function: direction and type of normalization. * @@ -392,14 +126,72 @@ int N(ddc::DiscreteDomain x_mesh) return static_cast(x_mesh.template extent()); } +template +KokkosFFT::axis_type axes() +{ + return KokkosFFT::axis_type { + static_cast(ddc::type_seq_rank_v>)...}; +} + +inline KokkosFFT::Normalization ddc_fft_normalization_to_kokkos_fft( + FFT_Normalization const ddc_fft_normalization) +{ + if (ddc_fft_normalization == ddc::FFT_Normalization::OFF) { + return KokkosFFT::Normalization::none; + } + + if (ddc_fft_normalization == ddc::FFT_Normalization::FORWARD) { + return KokkosFFT::Normalization::forward; + } + + if (ddc_fft_normalization == ddc::FFT_Normalization::BACKWARD) { + return KokkosFFT::Normalization::backward; + } + + if (ddc_fft_normalization == ddc::FFT_Normalization::ORTHO) { + return KokkosFFT::Normalization::ortho; + } + + // Last case is FULL which is mesh-dependant and thus handled by DDC. + return KokkosFFT::Normalization::none; +} + +template < + typename ExecSpace, + typename ElementType, + typename DDom, + typename Layout, + typename MemorySpace, + typename T> +void rescale( + ExecSpace const& exec_space, + ddc::ChunkSpan const& chunk_span, + T const& value) +{ + ddc::parallel_for_each( + "ddc_fft_normalization", + exec_space, + chunk_span.domain(), + KOKKOS_LAMBDA(typename DDom::discrete_element_type const i) { + chunk_span(i) *= value; + }); +} + /// @brief Core internal function to perform the FFT. -template +template < + typename Tin, + typename Tout, + typename ExecSpace, + typename MemorySpace, + typename LayoutIn, + typename LayoutOut, + typename... DDimIn, + typename... DDimOut> void impl( ExecSpace const& exec_space, - Tout* out_data, - Tin* in_data, - ddc::DiscreteDomain mesh, - const kwArgs_impl& kwargs) + ddc::ChunkSpan, LayoutIn, MemorySpace> const& in, + ddc::ChunkSpan, LayoutOut, MemorySpace> const& out, + kwArgs_impl const& kwargs) { static_assert( std::is_same_v, float> || std::is_same_v, double>, @@ -410,204 +202,74 @@ void impl( static_assert( Kokkos::SpaceAccessibility::accessible, "MemorySpace has to be accessible for ExecutionSpace."); - static_assert( - (is_uniform_point_sampling_v && ...), - "DDimX dimensions should derive from UniformPointSampling"); - std::array n = {static_cast(ddc::get(mesh.extents()))...}; - int idist = 1; - int odist = 1; - for (std::size_t i = 0; i < sizeof...(DDimX); ++i) { - idist = transform_type_v == TransformType::C2R && i == sizeof...(DDimX) - 1 - ? idist * (n[i] / 2 + 1) - : idist * n[i]; - odist = transform_type_v == TransformType::R2C && i == sizeof...(DDimX) - 1 - ? odist * (n[i] / 2 + 1) - : odist * n[i]; - } - -#if fftw_serial_AVAIL - if constexpr (std::is_same_v) { - _fftw_plan plan = _fftw_plan_many_dft( - kwargs.direction == ddc::FFT_Direction::FORWARD ? FFTW_FORWARD : FFTW_BACKWARD, - FFTW_ESTIMATE, - static_cast(sizeof...(DDimX)), - n.data(), - 1, - reinterpret_cast::type*>(in_data), - static_cast(nullptr), - 1, - idist, - reinterpret_cast::type*>(out_data), - static_cast(nullptr), - 1, - odist); - if constexpr (std::is_same_v, float>) { - fftwf_execute(plan); - fftwf_destroy_plan(plan); - } else { - fftw_execute(plan); - fftw_destroy_plan(plan); - } - } -#endif -#if fftw_omp_AVAIL - if constexpr (std::is_same_v) { - if constexpr (std::is_same_v, float>) { - fftwf_init_threads(); - fftwf_plan_with_nthreads(exec_space.concurrency()); + Kokkos::View< + ddc::detail::mdspan_to_kokkos_element_t, + ddc::detail::mdspan_to_kokkos_layout_t, + MemorySpace> const in_view + = in.allocation_kokkos_view(); + Kokkos::View< + ddc::detail::mdspan_to_kokkos_element_t, + ddc::detail::mdspan_to_kokkos_layout_t, + MemorySpace> const out_view + = out.allocation_kokkos_view(); + KokkosFFT::Normalization const kokkos_fft_normalization + = ddc_fft_normalization_to_kokkos_fft(kwargs.normalization); + + // C2C + if constexpr (std::is_same_v) { + if (kwargs.direction == ddc::FFT_Direction::FORWARD) { + KokkosFFT:: + fftn(exec_space, + in_view, + out_view, + axes(), + kokkos_fft_normalization); } else { - fftw_init_threads(); - fftw_plan_with_nthreads(exec_space.concurrency()); + KokkosFFT:: + ifftn(exec_space, + in_view, + out_view, + axes(), + kokkos_fft_normalization); } - _fftw_plan plan = _fftw_plan_many_dft( - kwargs.direction == ddc::FFT_Direction::FORWARD ? FFTW_FORWARD : FFTW_BACKWARD, - FFTW_ESTIMATE, - static_cast(sizeof...(DDimX)), - n.data(), - 1, - reinterpret_cast::type*>(in_data), - static_cast(nullptr), - 1, - idist, - reinterpret_cast::type*>(out_data), - static_cast(nullptr), - 1, - odist); - if constexpr (std::is_same_v, float>) { - fftwf_execute(plan); - fftwf_destroy_plan(plan); + // R2C & C2R + } else { + if constexpr (is_complex_v) { + assert(kwargs.direction == ddc::FFT_Direction::FORWARD); + KokkosFFT:: + rfftn(exec_space, + in_view, + out_view, + axes(), + kokkos_fft_normalization); } else { - fftw_execute(plan); - fftw_destroy_plan(plan); + assert(kwargs.direction == ddc::FFT_Direction::BACKWARD); + KokkosFFT:: + irfftn(exec_space, + in_view, + out_view, + axes(), + kokkos_fft_normalization); } } -#endif -#if cufft_AVAIL - if constexpr (std::is_same_v) { - cudaStream_t stream = exec_space.cuda_stream(); - - cufftHandle unmanaged_plan = -1; - cufftResult cufft_rt = cufftCreate(&unmanaged_plan); - - if (cufft_rt != CUFFT_SUCCESS) - throw std::runtime_error("cufftCreate failed"); - - std::unique_ptr> const - managed_plan(&unmanaged_plan, [](cufftHandle* handle) { cufftDestroy(*handle); }); - - cufftSetStream(unmanaged_plan, stream); - cufft_rt = cufftPlanMany( - &unmanaged_plan, // plan handle - sizeof...(DDimX), - n.data(), // Nx, Ny... - nullptr, - 1, - idist, - nullptr, - 1, - odist, - cufft_transform_type(), - 1); - - if (cufft_rt != CUFFT_SUCCESS) - throw std::runtime_error("cufftPlan failed"); - - cufft_rt = _cufftExec( - kwargs.direction == ddc::FFT_Direction::FORWARD ? CUFFT_FORWARD : CUFFT_INVERSE, - unmanaged_plan, - reinterpret_cast::type*>(in_data), - reinterpret_cast::type*>(out_data)); - if (cufft_rt != CUFFT_SUCCESS) - throw std::runtime_error("cufftExec failed"); - } -#endif -#if hipfft_AVAIL - if constexpr (std::is_same_v) { - hipStream_t stream = exec_space.hip_stream(); - - hipfftHandle unmanaged_plan; - hipfftResult hipfft_rt = hipfftCreate(&unmanaged_plan); - - if (hipfft_rt != HIPFFT_SUCCESS) - throw std::runtime_error("hipfftCreate failed"); - - std::unique_ptr> const - managed_plan(&unmanaged_plan, [](hipfftHandle* handle) { hipfftDestroy(*handle); }); - - hipfftSetStream(unmanaged_plan, stream); - hipfft_rt = hipfftPlanMany( - &unmanaged_plan, // plan handle - sizeof...(DDimX), - n.data(), // Nx, Ny... - nullptr, - 1, - idist, - nullptr, - 1, - odist, - hipfft_transform_type(), - 1); - - if (hipfft_rt != HIPFFT_SUCCESS) - throw std::runtime_error("hipfftPlan failed"); - - hipfft_rt = _hipfftExec( - kwargs.direction == ddc::FFT_Direction::FORWARD ? HIPFFT_FORWARD : HIPFFT_BACKWARD, - unmanaged_plan, - reinterpret_cast::type*>(in_data), - reinterpret_cast::type*>(out_data)); - if (hipfft_rt != HIPFFT_SUCCESS) - throw std::runtime_error("hipfftExec failed"); - } -#endif - - if (kwargs.normalization != ddc::FFT_Normalization::OFF) { - real_type_t norm_coef = 1; - switch (kwargs.normalization) { - case ddc::FFT_Normalization::OFF: - break; - case ddc::FFT_Normalization::FORWARD: - norm_coef = kwargs.direction == ddc::FFT_Direction::FORWARD - ? 1. / (ddc::get(mesh.extents()) * ...) - : 1.; - break; - case ddc::FFT_Normalization::BACKWARD: - norm_coef = kwargs.direction == ddc::FFT_Direction::BACKWARD - ? 1. / (ddc::get(mesh.extents()) * ...) - : 1.; - break; - case ddc::FFT_Normalization::ORTHO: - norm_coef = 1. / Kokkos::sqrt((ddc::get(mesh.extents()) * ...)); - break; - case ddc::FFT_Normalization::FULL: - norm_coef = kwargs.direction == ddc::FFT_Direction::FORWARD - ? (((coordinate(ddc::select(mesh).back()) - - coordinate(ddc::select(mesh).front())) - / (ddc::get(mesh.extents()) - 1) - / Kokkos::sqrt(2 * Kokkos::numbers::pi)) - * ...) - : ((Kokkos::sqrt(2 * Kokkos::numbers::pi) - / (coordinate(ddc::select(mesh).back()) - - coordinate(ddc::select(mesh).front())) - * (ddc::get(mesh.extents()) - 1) - / ddc::get(mesh.extents())) - * ...); - break; - } - Kokkos::parallel_for( - "ddc_fft_normalization", - Kokkos::RangePolicy( - exec_space, - 0, - is_complex_v && transform_type_v != TransformType::C2C - ? (LastSelector( - ddc::get(mesh.extents()) / 2 + 1, - ddc::get(mesh.extents())) - * ...) - : (ddc::get(mesh.extents()) * ...)), - KOKKOS_LAMBDA(const int& i) { out_data[i] = out_data[i] * norm_coef; }); + // The FULL normalization is mesh-dependant and thus handled by DDC + if (kwargs.normalization == ddc::FFT_Normalization::FULL) { + real_type_t const norm_coef + = kwargs.direction == ddc::FFT_Direction::FORWARD + ? (((coordinate(ddc::select(in.domain()).back()) + - coordinate(ddc::select(in.domain()).front())) + / (ddc::get(in.domain().extents()) - 1) + / Kokkos::sqrt(2 * Kokkos::numbers::pi)) + * ...) + : ((Kokkos::sqrt(2 * Kokkos::numbers::pi) + / (coordinate(ddc::select(out.domain()).back()) + - coordinate(ddc::select(out.domain()).front())) + * (ddc::get(out.domain().extents()) - 1) + / ddc::get(out.domain().extents())) + * ...); + rescale(exec_space, out, norm_coef); } } @@ -749,12 +411,8 @@ void fft( (is_periodic_sampling_v && ...), "DDimFx dimensions should derive from PeriodicPointSampling"); - ddc::detail::fft::impl( - exec_space, - out.data_handle(), - in.data_handle(), - in.domain(), - {ddc::FFT_Direction::FORWARD, kwargs.normalization}); + ddc::detail::fft:: + impl(exec_space, in, out, {ddc::FFT_Direction::FORWARD, kwargs.normalization}); } /** @@ -806,12 +464,8 @@ void ifft( (is_periodic_sampling_v && ...), "DDimFx dimensions should derive from PeriodicPointSampling"); - ddc::detail::fft::impl( - exec_space, - out.data_handle(), - in.data_handle(), - out.domain(), - {ddc::FFT_Direction::BACKWARD, kwargs.normalization}); + ddc::detail::fft:: + impl(exec_space, in, out, {ddc::FFT_Direction::BACKWARD, kwargs.normalization}); } } // namespace ddc diff --git a/tests/fft/fft.cpp b/tests/fft/fft.cpp index f174df09a..8425cdf22 100644 --- a/tests/fft/fft.cpp +++ b/tests/fft/fft.cpp @@ -240,7 +240,7 @@ TEST(FourierMesh, Odd) test_fourier_mesh(17); } -#if fftw_serial_AVAIL +#if defined(KOKKOS_ENABLE_SERIAL) TEST(FftNorm, Off) { test_fft_norm< @@ -320,7 +320,7 @@ TEST(FftSerialHost, R2cIn3d) } #endif -#if fftw_omp_AVAIL +#if defined(KOKKOS_ENABLE_OPENMP) TEST(FftParallelHost, R2cIn1d) { test_fft, RDimX>(); @@ -383,7 +383,7 @@ TEST(FftParallelDevice, R2cIn3d) RDimZ>(); } -#if fftw_serial_AVAIL +#if defined(KOKKOS_ENABLE_SERIAL) TEST(FftSerialHost, C2cIn1d) { test_fft< @@ -418,7 +418,7 @@ TEST(FftSerialHost, C2cIn3d) } #endif -#if fftw_omp_AVAIL +#if defined(KOKKOS_ENABLE_OPENMP) TEST(FftParallelHost, C2cIn1d) { test_fft< @@ -486,7 +486,7 @@ TEST(FftParallelDevice, C2cIn3d) RDimZ>(); } -#if fftw_serial_AVAIL +#if defined(KOKKOS_ENABLE_SERIAL) TEST(FftSerialHost, D2zIn1d) { test_fft< @@ -521,7 +521,7 @@ TEST(FftSerialHost, D2zIn3d) } #endif -#if fftw_omp_AVAIL +#if defined(KOKKOS_ENABLE_OPENMP) TEST(FftParallelHost, D2zIn1d) { test_fft< @@ -589,7 +589,7 @@ TEST(FftParallelDevice, D2zIn3d) RDimZ>(); } -#if fftw_serial_AVAIL +#if defined(KOKKOS_ENABLE_SERIAL) TEST(FftSerialHost, Z2zIn1d) { test_fft< @@ -624,7 +624,7 @@ TEST(FftSerialHost, Z2zIn3d) } #endif -#if fftw_omp_AVAIL +#if defined(KOKKOS_ENABLE_OPENMP) TEST(FftParallelHost, Z2zIn1d) { test_fft< diff --git a/vendor/kokkos-fft b/vendor/kokkos-fft new file mode 160000 index 000000000..2fc134da4 --- /dev/null +++ b/vendor/kokkos-fft @@ -0,0 +1 @@ +Subproject commit 2fc134da494d81894a84db41d818e15e989f976f