diff --git a/benchmarks/deepcopy.cpp b/benchmarks/deepcopy.cpp index 2edafe00b..cb6c67668 100644 --- a/benchmarks/deepcopy.cpp +++ b/benchmarks/deepcopy.cpp @@ -133,4 +133,16 @@ BENCHMARK(memcpy_2d)->Args({large_dim1_2D, large_dim2_2D}); BENCHMARK(deepcopy_2d)->Args({large_dim1_2D, large_dim2_2D}); BENCHMARK(deepcopy_subchunk_2d)->Args({large_dim1_2D, large_dim2_2D}); -BENCHMARK_MAIN(); +int main(int argc, char** argv) +{ + ::benchmark::Initialize(&argc, argv); + if (::benchmark::ReportUnrecognizedArguments(argc, argv)) { + return 1; + } + { + ddc::ScopeGuard const guard; + ::benchmark::RunSpecifiedBenchmarks(); + } + ::benchmark::Shutdown(); + return 0; +} diff --git a/include/ddc/chunk_common.hpp b/include/ddc/chunk_common.hpp index afedaf428..6b91ddf22 100644 --- a/include/ddc/chunk_common.hpp +++ b/include/ddc/chunk_common.hpp @@ -132,43 +132,43 @@ class ChunkCommon, LayoutStridedPolicy> mdomain_type m_domain; public: - static constexpr int rank() noexcept + static KOKKOS_FUNCTION constexpr int rank() noexcept { return extents_type::rank(); } - static constexpr int rank_dynamic() noexcept + static KOKKOS_FUNCTION constexpr int rank_dynamic() noexcept { return extents_type::rank_dynamic(); } - static constexpr size_type static_extent(std::size_t r) noexcept + static KOKKOS_FUNCTION constexpr size_type static_extent(std::size_t r) noexcept { return extents_type::static_extent(r); } - static constexpr bool is_always_unique() noexcept + static KOKKOS_FUNCTION constexpr bool is_always_unique() noexcept { return mapping_type::is_always_unique(); } - static constexpr bool is_always_exhaustive() noexcept + static KOKKOS_FUNCTION constexpr bool is_always_exhaustive() noexcept { return mapping_type::is_always_exhaustive(); } - static constexpr bool is_always_strided() noexcept + static KOKKOS_FUNCTION constexpr bool is_always_strided() noexcept { return mapping_type::is_always_strided(); } public: - constexpr accessor_type accessor() const + KOKKOS_FUNCTION constexpr accessor_type accessor() const { return m_internal_mdspan.accessor(); } - constexpr DiscreteVector extents() const noexcept + KOKKOS_FUNCTION constexpr DiscreteVector extents() const noexcept { return DiscreteVector( (m_internal_mdspan.extent(type_seq_rank_v>) @@ -176,39 +176,39 @@ class ChunkCommon, LayoutStridedPolicy> } template - constexpr size_type extent() const noexcept + KOKKOS_FUNCTION constexpr size_type extent() const noexcept { return m_internal_mdspan.extent(type_seq_rank_v>) - front(m_domain).uid(); } - constexpr size_type size() const noexcept + KOKKOS_FUNCTION constexpr size_type size() const noexcept { return allocation_mdspan().size(); } - constexpr mapping_type mapping() const noexcept + KOKKOS_FUNCTION constexpr mapping_type mapping() const noexcept { return allocation_mdspan().mapping(); } - constexpr bool is_unique() const noexcept + KOKKOS_FUNCTION constexpr bool is_unique() const noexcept { return allocation_mdspan().is_unique(); } - constexpr bool is_exhaustive() const noexcept + KOKKOS_FUNCTION constexpr bool is_exhaustive() const noexcept { return allocation_mdspan().is_exhaustive(); } - constexpr bool is_strided() const noexcept + KOKKOS_FUNCTION constexpr bool is_strided() const noexcept { return allocation_mdspan().is_strided(); } template - constexpr size_type stride() const + KOKKOS_FUNCTION constexpr size_type stride() const { return m_internal_mdspan.stride(type_seq_rank_v>); } @@ -216,7 +216,7 @@ class ChunkCommon, LayoutStridedPolicy> /** Provide access to the domain on which this chunk is defined * @return the domain on which this chunk is defined */ - constexpr mdomain_type domain() const noexcept + KOKKOS_FUNCTION constexpr mdomain_type domain() const noexcept { return m_domain; } @@ -225,20 +225,22 @@ class ChunkCommon, LayoutStridedPolicy> * @return the domain on which this chunk is defined */ template - constexpr DiscreteDomain domain() const noexcept + KOKKOS_FUNCTION constexpr DiscreteDomain domain() const noexcept { return select(domain()); } protected: /// Empty ChunkCommon - constexpr ChunkCommon() = default; + KOKKOS_DEFAULTED_FUNCTION constexpr ChunkCommon() = default; /** Constructs a new ChunkCommon from scratch * @param internal_mdspan * @param domain */ - constexpr ChunkCommon(internal_mdspan_type internal_mdspan, mdomain_type const& domain) noexcept + KOKKOS_FUNCTION constexpr ChunkCommon( + internal_mdspan_type internal_mdspan, + mdomain_type const& domain) noexcept : m_internal_mdspan(std::move(internal_mdspan)) , m_domain(domain) { @@ -251,7 +253,7 @@ class ChunkCommon, LayoutStridedPolicy> template < class Mapping = mapping_type, std::enable_if_t, int> = 0> - constexpr ChunkCommon(ElementType* ptr, mdomain_type const& domain) + KOKKOS_FUNCTION constexpr ChunkCommon(ElementType* ptr, mdomain_type const& domain) : ChunkCommon {ptr, domain, make_mapping_for(domain)} { } @@ -259,29 +261,31 @@ class ChunkCommon, LayoutStridedPolicy> /** Constructs a new ChunkCommon by copy, yields a new view to the same data * @param other the ChunkCommon to copy */ - constexpr ChunkCommon(ChunkCommon const& other) = default; + KOKKOS_DEFAULTED_FUNCTION constexpr ChunkCommon(ChunkCommon const& other) = default; /** Constructs a new ChunkCommon by move * @param other the ChunkCommon to move */ - constexpr ChunkCommon(ChunkCommon&& other) = default; + KOKKOS_DEFAULTED_FUNCTION constexpr ChunkCommon(ChunkCommon&& other) = default; + + KOKKOS_DEFAULTED_FUNCTION ~ChunkCommon() = default; /** Copy-assigns a new value to this ChunkCommon, yields a new view to the same data * @param other the ChunkCommon to copy * @return *this */ - constexpr ChunkCommon& operator=(ChunkCommon const& other) = default; + KOKKOS_DEFAULTED_FUNCTION constexpr ChunkCommon& operator=(ChunkCommon const& other) = default; /** Move-assigns a new value to this ChunkCommon * @param other the ChunkCommon to move * @return *this */ - constexpr ChunkCommon& operator=(ChunkCommon&& other) = default; + KOKKOS_DEFAULTED_FUNCTION constexpr ChunkCommon& operator=(ChunkCommon&& other) = default; /** Access to the underlying allocation pointer * @return allocation pointer */ - constexpr ElementType* data_handle() const + KOKKOS_FUNCTION constexpr ElementType* data_handle() const { return &m_internal_mdspan(front(m_domain).uid()...); } @@ -289,7 +293,7 @@ class ChunkCommon, LayoutStridedPolicy> /** Provide a modifiable view of the data * @return a modifiable view of the data */ - constexpr internal_mdspan_type internal_mdspan() const + KOKKOS_FUNCTION constexpr internal_mdspan_type internal_mdspan() const { return m_internal_mdspan; } @@ -297,7 +301,7 @@ class ChunkCommon, LayoutStridedPolicy> /** Provide a modifiable view of the data * @return a modifiable view of the data */ - constexpr allocation_mdspan_type allocation_mdspan() const + KOKKOS_FUNCTION constexpr allocation_mdspan_type allocation_mdspan() const { DDC_IF_NVCC_THEN_PUSH_AND_SUPPRESS(implicit_return_from_non_void_function) extents_type extents_s(::ddc::extents(m_domain).value()...); @@ -316,7 +320,7 @@ class ChunkCommon, LayoutStridedPolicy> * @param domain the domain that sustains the view */ template - constexpr std::enable_if_t< + KOKKOS_FUNCTION constexpr std::enable_if_t< std::is_constructible_v, std::experimental::layout_stride::mapping> make_mapping_for(mdomain_type const& domain) @@ -334,7 +338,7 @@ class ChunkCommon, LayoutStridedPolicy> * @param ptr the allocation pointer to the data_handle() * @param domain the domain that sustains the view */ - constexpr ChunkCommon( + KOKKOS_FUNCTION constexpr ChunkCommon( ElementType* ptr, mdomain_type const& domain, std::experimental::layout_stride::mapping&& s_domain) diff --git a/include/ddc/chunk_span.hpp b/include/ddc/chunk_span.hpp index 6c4188798..6f10c1794 100644 --- a/include/ddc/chunk_span.hpp +++ b/include/ddc/chunk_span.hpp @@ -89,7 +89,7 @@ class ChunkSpan, LayoutStridedPolicy, Memo protected: template - constexpr auto get_slicer_for(DiscreteElement const& c) const + KOKKOS_FUNCTION constexpr auto get_slicer_for(DiscreteElement const& c) const { DDC_IF_NVCC_THEN_PUSH_AND_SUPPRESS(implicit_return_from_non_void_function) if constexpr (in_tags_v>) { @@ -101,7 +101,7 @@ class ChunkSpan, LayoutStridedPolicy, Memo } template - constexpr auto get_slicer_for(DiscreteDomain const& c) const + KOKKOS_FUNCTION constexpr auto get_slicer_for(DiscreteDomain const& c) const { DDC_IF_NVCC_THEN_PUSH_AND_SUPPRESS(implicit_return_from_non_void_function) if constexpr (in_tags_v>) { @@ -116,23 +116,24 @@ class ChunkSpan, LayoutStridedPolicy, Memo public: /// Empty ChunkSpan - constexpr ChunkSpan() = default; + KOKKOS_DEFAULTED_FUNCTION constexpr ChunkSpan() = default; /** Constructs a new ChunkSpan by copy, yields a new view to the same data * @param other the ChunkSpan to copy */ - constexpr ChunkSpan(ChunkSpan const& other) = default; + KOKKOS_DEFAULTED_FUNCTION constexpr ChunkSpan(ChunkSpan const& other) = default; /** Constructs a new ChunkSpan by move * @param other the ChunkSpan to move */ - constexpr ChunkSpan(ChunkSpan&& other) = default; + KOKKOS_DEFAULTED_FUNCTION constexpr ChunkSpan(ChunkSpan&& other) = default; /** Constructs a new ChunkSpan from a Chunk, yields a new view to the same data * @param other the Chunk to view */ template - constexpr ChunkSpan(Chunk& other) noexcept + KOKKOS_FUNCTION constexpr ChunkSpan( + Chunk& other) noexcept : base_type(other.m_internal_mdspan, other.m_domain) { } @@ -146,7 +147,8 @@ class ChunkSpan, LayoutStridedPolicy, Memo class SFINAEElementType = ElementType, class = std::enable_if_t>, class Allocator> - constexpr ChunkSpan(Chunk const& other) noexcept + KOKKOS_FUNCTION constexpr ChunkSpan( + Chunk const& other) noexcept : base_type(other.m_internal_mdspan, other.m_domain) { } @@ -155,7 +157,7 @@ class ChunkSpan, LayoutStridedPolicy, Memo * @param other the ChunkSpan to move */ template - constexpr ChunkSpan( + KOKKOS_FUNCTION constexpr ChunkSpan( ChunkSpan const& other) noexcept : base_type(other.m_internal_mdspan, other.m_domain) { @@ -168,7 +170,8 @@ class ChunkSpan, LayoutStridedPolicy, Memo template < class Mapping = mapping_type, std::enable_if_t, int> = 0> - constexpr ChunkSpan(ElementType* const ptr, mdomain_type const& domain) : base_type(ptr, domain) + KOKKOS_FUNCTION constexpr ChunkSpan(ElementType* const ptr, mdomain_type const& domain) + : base_type(ptr, domain) { } @@ -196,29 +199,32 @@ class ChunkSpan, LayoutStridedPolicy, Memo * @param domain the domain that sustains the view */ template ::value>> - constexpr ChunkSpan(KokkosView const& view, mdomain_type const& domain) noexcept + KOKKOS_FUNCTION constexpr ChunkSpan(KokkosView const& view, mdomain_type const& domain) noexcept : ChunkSpan( detail::build_mdspan(view, std::make_index_sequence {}), domain) { } + KOKKOS_DEFAULTED_FUNCTION ~ChunkSpan() = default; + /** Copy-assigns a new value to this ChunkSpan, yields a new view to the same data * @param other the ChunkSpan to copy * @return *this */ - constexpr ChunkSpan& operator=(ChunkSpan const& other) = default; + KOKKOS_DEFAULTED_FUNCTION constexpr ChunkSpan& operator=(ChunkSpan const& other) = default; /** Move-assigns a new value to this ChunkSpan * @param other the ChunkSpan to move * @return *this */ - constexpr ChunkSpan& operator=(ChunkSpan&& other) = default; + KOKKOS_DEFAULTED_FUNCTION constexpr ChunkSpan& operator=(ChunkSpan&& other) = default; /** Slice out some dimensions */ template - constexpr auto operator[](DiscreteElement const& slice_spec) const + KOKKOS_FUNCTION constexpr auto operator[]( + DiscreteElement const& slice_spec) const { auto subview = std::experimental:: submdspan(allocation_mdspan(), get_slicer_for(slice_spec)...); @@ -234,7 +240,7 @@ class ChunkSpan, LayoutStridedPolicy, Memo /** Restrict to a subdomain */ template - constexpr auto operator[](DiscreteDomain const& odomain) const + KOKKOS_FUNCTION constexpr auto operator[](DiscreteDomain const& odomain) const { auto subview = std::experimental:: submdspan(allocation_mdspan(), get_slicer_for(odomain)...); @@ -259,7 +265,8 @@ class ChunkSpan, LayoutStridedPolicy, Memo * @return const-reference to this element */ template - constexpr reference operator()(DiscreteElement const& delems) const noexcept + KOKKOS_FUNCTION constexpr reference operator()( + DiscreteElement const& delems) const noexcept { static_assert(sizeof...(ODDims) == sizeof...(DDims), "Invalid number of dimensions"); assert(((select(delems) >= front(this->m_domain)) && ...)); @@ -283,7 +290,7 @@ class ChunkSpan, LayoutStridedPolicy, Memo /** Access to the underlying allocation pointer * @return allocation pointer */ - constexpr ElementType* data_handle() const + KOKKOS_FUNCTION constexpr ElementType* data_handle() const { return base_type::data_handle(); } @@ -291,7 +298,7 @@ class ChunkSpan, LayoutStridedPolicy, Memo /** Provide a mdspan on the memory allocation * @return allocation mdspan */ - constexpr allocation_mdspan_type allocation_mdspan() const + KOKKOS_FUNCTION constexpr allocation_mdspan_type allocation_mdspan() const { return base_type::allocation_mdspan(); } @@ -299,7 +306,7 @@ class ChunkSpan, LayoutStridedPolicy, Memo /** Provide a mdspan on the memory allocation * @return allocation mdspan */ - constexpr auto allocation_kokkos_view() const + KOKKOS_FUNCTION constexpr auto allocation_kokkos_view() const { auto s = this->allocation_mdspan(); auto kokkos_layout = detail::build_kokkos_layout( @@ -312,12 +319,12 @@ class ChunkSpan, LayoutStridedPolicy, Memo MemorySpace>(s.data_handle(), kokkos_layout); } - constexpr view_type span_cview() const + KOKKOS_FUNCTION constexpr view_type span_cview() const { return view_type(*this); } - constexpr span_type span_view() const + KOKKOS_FUNCTION constexpr span_type span_view() const { return *this; } diff --git a/include/ddc/coordinate_md.hpp b/include/ddc/coordinate_md.hpp index 05bbdf672..658910965 100644 --- a/include/ddc/coordinate_md.hpp +++ b/include/ddc/coordinate_md.hpp @@ -9,7 +9,7 @@ namespace ddc { template 1), int> = 0> -DDC_INLINE_FUNCTION Coordinate coordinate( +KOKKOS_FUNCTION Coordinate coordinate( DiscreteElement const& c) { return Coordinate(coordinate(select(c))...); diff --git a/include/ddc/detail/kokkos.hpp b/include/ddc/detail/kokkos.hpp index 57fdbbdb2..6de7d2771 100644 --- a/include/ddc/detail/kokkos.hpp +++ b/include/ddc/detail/kokkos.hpp @@ -105,7 +105,7 @@ using kokkos_to_mdspan_element_t = typename kokkos_to_mdspan_element::type; template -Kokkos::LayoutStride make_layout_stride( +KOKKOS_FUNCTION Kokkos::LayoutStride make_layout_stride( std::array const& interleaved_extents_strides, std::index_sequence) { @@ -113,7 +113,7 @@ Kokkos::LayoutStride make_layout_stride( } template -mdspan_to_kokkos_layout_t build_kokkos_layout( +KOKKOS_FUNCTION mdspan_to_kokkos_layout_t build_kokkos_layout( EP const& ep, MP const& mapping, std::index_sequence) @@ -138,7 +138,9 @@ mdspan_to_kokkos_layout_t build_kokkos_layout( } template -auto build_mdspan(Kokkos::View const view, std::index_sequence) +KOKKOS_FUNCTION auto build_mdspan( + Kokkos::View const view, + std::index_sequence) { DDC_IF_NVCC_THEN_PUSH_AND_SUPPRESS(implicit_return_from_non_void_function) using element_type = kokkos_to_mdspan_element_t; diff --git a/include/ddc/detail/tagged_vector.hpp b/include/ddc/detail/tagged_vector.hpp index dace14d5b..705309015 100644 --- a/include/ddc/detail/tagged_vector.hpp +++ b/include/ddc/detail/tagged_vector.hpp @@ -35,20 +35,21 @@ inline constexpr bool is_tagged_vector_v = IsTaggedVector::value; template -inline constexpr ElementType const& get( +KOKKOS_FUNCTION constexpr ElementType const& get( detail::TaggedVector const& tuple) noexcept { return tuple.template get(); } template -inline constexpr ElementType& get(detail::TaggedVector& tuple) noexcept +KOKKOS_FUNCTION constexpr ElementType& get( + detail::TaggedVector& tuple) noexcept { return tuple.template get(); } template -inline constexpr ElementType const& get_or( +KOKKOS_FUNCTION constexpr ElementType const& get_or( detail::TaggedVector const& tuple, ElementType const& default_value) noexcept { @@ -60,14 +61,14 @@ namespace detail { /// Unary operators: +, - template -constexpr inline detail::TaggedVector operator+( +KOKKOS_FUNCTION constexpr detail::TaggedVector operator+( detail::TaggedVector const& x) { return x; } template -constexpr inline detail::TaggedVector operator-( +KOKKOS_FUNCTION constexpr detail::TaggedVector operator-( detail::TaggedVector const& x) { return detail::TaggedVector((-get(x))...); @@ -76,7 +77,7 @@ constexpr inline detail::TaggedVector operator-( /// Internal binary operators: +, - template -constexpr inline auto operator+( +KOKKOS_FUNCTION constexpr auto operator+( detail::TaggedVector const& lhs, detail::TaggedVector const& rhs) { @@ -91,7 +92,7 @@ template < class OElementType, class = std::enable_if_t>, class = std::enable_if_t>> -constexpr inline auto operator+( +KOKKOS_FUNCTION constexpr auto operator+( detail::TaggedVector const& lhs, OElementType const& rhs) { @@ -105,7 +106,7 @@ template < class OElementType, class = std::enable_if_t>, class = std::enable_if_t>> -constexpr inline auto operator+( +KOKKOS_FUNCTION constexpr auto operator+( OElementType const& lhs, detail::TaggedVector const& rhs) { @@ -114,7 +115,7 @@ constexpr inline auto operator+( } template -constexpr inline auto operator-( +KOKKOS_FUNCTION constexpr auto operator-( detail::TaggedVector const& lhs, detail::TaggedVector const& rhs) { @@ -129,7 +130,7 @@ template < class OElementType, class = std::enable_if_t>, class = std::enable_if_t>> -constexpr inline auto operator-( +KOKKOS_FUNCTION constexpr auto operator-( detail::TaggedVector const& lhs, OElementType const& rhs) { @@ -143,7 +144,7 @@ template < class OElementType, class = std::enable_if_t>, class = std::enable_if_t>> -constexpr inline auto operator-( +KOKKOS_FUNCTION constexpr auto operator-( OElementType const& lhs, detail::TaggedVector const& rhs) { @@ -159,7 +160,7 @@ template < class... Tags, class = std::enable_if_t>, class = std::enable_if_t>> -constexpr inline auto operator*( +KOKKOS_FUNCTION constexpr auto operator*( ElementType const& lhs, detail::TaggedVector const& rhs) { @@ -170,14 +171,14 @@ constexpr inline auto operator*( } // namespace detail template -inline constexpr detail::TaggedVector select( +KOKKOS_FUNCTION constexpr detail::TaggedVector select( detail::TaggedVector const& arr) noexcept { return detail::TaggedVector(arr); } template -inline constexpr detail::TaggedVector select( +KOKKOS_FUNCTION constexpr detail::TaggedVector select( detail::TaggedVector&& arr) noexcept { return detail::TaggedVector(std::move(arr)); @@ -186,7 +187,7 @@ inline constexpr detail::TaggedVector select( namespace detail { template -constexpr detail::TaggedVector const& take_impl( +KOKKOS_FUNCTION constexpr detail::TaggedVector const& take_impl( detail::TaggedVector const& head, detail::TaggedVector const&... tags) { @@ -204,7 +205,7 @@ constexpr detail::TaggedVector const& take_impl( } template -constexpr detail::TaggedVector const& take( +KOKKOS_FUNCTION constexpr detail::TaggedVector const& take( detail::TaggedVector const&... tags) { return take_impl(tags...); @@ -220,12 +221,12 @@ template class ConversionOperators> { public: - constexpr inline operator ElementType const &() const noexcept + KOKKOS_FUNCTION constexpr operator ElementType const &() const noexcept { return static_cast const*>(this)->m_values[0]; } - constexpr inline operator ElementType&() noexcept + KOKKOS_FUNCTION constexpr operator ElementType&() noexcept { return static_cast*>(this)->m_values[0]; } @@ -242,26 +243,29 @@ class TaggedVector : public ConversionOperators m_values; public: - static constexpr std::size_t size() noexcept + using value_type = ElementType; + + static KOKKOS_FUNCTION constexpr std::size_t size() noexcept { return sizeof...(Tags); } public: - inline constexpr TaggedVector() = default; + KOKKOS_DEFAULTED_FUNCTION constexpr TaggedVector() = default; - inline constexpr TaggedVector(TaggedVector const&) = default; + KOKKOS_DEFAULTED_FUNCTION constexpr TaggedVector(TaggedVector const&) = default; - inline constexpr TaggedVector(TaggedVector&&) = default; + KOKKOS_DEFAULTED_FUNCTION constexpr TaggedVector(TaggedVector&&) = default; template - inline constexpr TaggedVector(TaggedVector const&... other) noexcept + KOKKOS_FUNCTION constexpr TaggedVector( + TaggedVector const&... other) noexcept : m_values {take(other...).value()...} { } template - explicit inline constexpr TaggedVector( + explicit KOKKOS_FUNCTION constexpr TaggedVector( TaggedVector const& other) noexcept : m_values {(static_cast(other.template get()))...} { @@ -272,17 +276,19 @@ class TaggedVector : public ConversionOperators && ...)>, class = std::enable_if_t<(!is_tagged_vector_v && ...)>, class = std::enable_if_t> - explicit inline constexpr TaggedVector(Params const&... params) noexcept + explicit KOKKOS_FUNCTION constexpr TaggedVector(Params const&... params) noexcept : m_values {static_cast(params)...} { } - constexpr inline TaggedVector& operator=(TaggedVector const& other) = default; + KOKKOS_DEFAULTED_FUNCTION ~TaggedVector() = default; + + KOKKOS_DEFAULTED_FUNCTION TaggedVector& operator=(TaggedVector const& other) = default; - constexpr inline TaggedVector& operator=(TaggedVector&& other) = default; + KOKKOS_DEFAULTED_FUNCTION TaggedVector& operator=(TaggedVector&& other) = default; template - constexpr inline TaggedVector& operator=( + KOKKOS_FUNCTION constexpr TaggedVector& operator=( TaggedVector const& other) noexcept { m_values = other.m_values; @@ -290,48 +296,51 @@ class TaggedVector : public ConversionOperators - constexpr inline TaggedVector& operator=(TaggedVector&& other) noexcept + KOKKOS_FUNCTION constexpr TaggedVector& operator=( + TaggedVector&& other) noexcept { m_values = std::move(other.m_values); return *this; } /// Returns a reference to the underlying `std::array` - constexpr inline std::array& array() noexcept + KOKKOS_FUNCTION constexpr std::array& array() noexcept { return m_values; } /// Returns a const reference to the underlying `std::array` - constexpr inline std::array const& array() const noexcept + KOKKOS_FUNCTION constexpr std::array const& array() const noexcept { return m_values; } - constexpr inline ElementType& operator[](size_t pos) + KOKKOS_FUNCTION constexpr ElementType& operator[](size_t pos) { return m_values[pos]; } - constexpr inline ElementType const& operator[](size_t pos) const + KOKKOS_FUNCTION constexpr ElementType const& operator[](size_t pos) const { return m_values[pos]; } template - constexpr inline bool operator==(TaggedVector const& rhs) const noexcept + KOKKOS_FUNCTION constexpr bool operator==( + TaggedVector const& rhs) const noexcept { return ((m_values[type_seq_rank_v] == rhs.template get()) && ...); } template - constexpr inline bool operator!=(TaggedVector const& rhs) const noexcept + KOKKOS_FUNCTION constexpr bool operator!=( + TaggedVector const& rhs) const noexcept { return !(*this == rhs); } template - inline constexpr ElementType& get() noexcept + KOKKOS_FUNCTION constexpr ElementType& get() noexcept { using namespace detail; static_assert(in_tags_v, "requested Tag absent from TaggedVector"); @@ -339,7 +348,7 @@ class TaggedVector : public ConversionOperators - inline constexpr ElementType const& get() const noexcept + KOKKOS_FUNCTION constexpr ElementType const& get() const noexcept { using namespace detail; static_assert(in_tags_v, "requested Tag absent from TaggedVector"); @@ -347,7 +356,7 @@ class TaggedVector : public ConversionOperators - ElementType const& get_or(ElementType const& default_value) const& + KOKKOS_FUNCTION constexpr ElementType const& get_or(ElementType const& default_value) const& { DDC_IF_NVCC_THEN_PUSH_AND_SUPPRESS(implicit_return_from_non_void_function) if constexpr (in_tags_v) { @@ -359,13 +368,14 @@ class TaggedVector : public ConversionOperators - constexpr inline std::enable_if_t value() const noexcept + KOKKOS_FUNCTION constexpr std::enable_if_t value() const noexcept { return m_values[0]; } template - constexpr inline TaggedVector& operator+=(TaggedVector const& rhs) + KOKKOS_FUNCTION constexpr TaggedVector& operator+=( + TaggedVector const& rhs) { static_assert(type_seq_same_v>); ((m_values[type_seq_rank_v] += rhs.template get()), ...); @@ -373,14 +383,15 @@ class TaggedVector : public ConversionOperators - constexpr inline TaggedVector& operator+=(OElementType const& rhs) + KOKKOS_FUNCTION constexpr TaggedVector& operator+=(OElementType const& rhs) { ((m_values[type_seq_rank_v] += rhs), ...); return *this; } template - constexpr inline TaggedVector& operator-=(TaggedVector const& rhs) + KOKKOS_FUNCTION constexpr TaggedVector& operator-=( + TaggedVector const& rhs) { static_assert(type_seq_same_v>); ((m_values[type_seq_rank_v] -= rhs.template get()), ...); @@ -388,14 +399,15 @@ class TaggedVector : public ConversionOperators - constexpr inline TaggedVector& operator-=(OElementType const& rhs) + KOKKOS_FUNCTION constexpr TaggedVector& operator-=(OElementType const& rhs) { ((m_values[type_seq_rank_v] -= rhs), ...); return *this; } template - constexpr inline TaggedVector& operator*=(TaggedVector const& rhs) + KOKKOS_FUNCTION constexpr TaggedVector& operator*=( + TaggedVector const& rhs) { static_assert(type_seq_same_v>); ((m_values[type_seq_rank_v] *= rhs.template get()), ...); diff --git a/include/ddc/discrete_domain.hpp b/include/ddc/discrete_domain.hpp index 7d2f2e2b9..af96066b6 100644 --- a/include/ddc/discrete_domain.hpp +++ b/include/ddc/discrete_domain.hpp @@ -36,16 +36,16 @@ class DiscreteDomain DiscreteElement m_element_end; public: - static constexpr std::size_t rank() + static KOKKOS_FUNCTION constexpr std::size_t rank() { return sizeof...(DDims); } - DiscreteDomain() = default; + KOKKOS_DEFAULTED_FUNCTION DiscreteDomain() = default; /// Construct a DiscreteDomain by copies and merge of domains template - explicit constexpr DiscreteDomain(DD const&... domains) + explicit KOKKOS_FUNCTION constexpr DiscreteDomain(DD const&... domains) : m_element_begin(domains.front()...) , m_element_end((domains.front() + domains.extents())...) { @@ -55,24 +55,26 @@ class DiscreteDomain * @param element_begin the lower bound in each direction * @param size the number of points in each direction */ - constexpr DiscreteDomain(discrete_element_type const& element_begin, mlength_type const& size) + KOKKOS_FUNCTION constexpr DiscreteDomain( + discrete_element_type const& element_begin, + mlength_type const& size) : m_element_begin(element_begin) , m_element_end(element_begin + size) { } - DiscreteDomain(DiscreteDomain const& x) = default; + KOKKOS_DEFAULTED_FUNCTION DiscreteDomain(DiscreteDomain const& x) = default; - DiscreteDomain(DiscreteDomain&& x) = default; + KOKKOS_DEFAULTED_FUNCTION DiscreteDomain(DiscreteDomain&& x) = default; - ~DiscreteDomain() = default; + KOKKOS_DEFAULTED_FUNCTION ~DiscreteDomain() = default; - DiscreteDomain& operator=(DiscreteDomain const& x) = default; + KOKKOS_DEFAULTED_FUNCTION DiscreteDomain& operator=(DiscreteDomain const& x) = default; - DiscreteDomain& operator=(DiscreteDomain&& x) = default; + KOKKOS_DEFAULTED_FUNCTION DiscreteDomain& operator=(DiscreteDomain&& x) = default; template - constexpr bool operator==(DiscreteDomain const& other) const + KOKKOS_FUNCTION constexpr bool operator==(DiscreteDomain const& other) const { return m_element_begin == other.m_element_begin && m_element_end == other.m_element_end; } @@ -80,66 +82,66 @@ class DiscreteDomain #if __cplusplus <= 201703L // Shall not be necessary anymore in C++20 // `a!=b` shall be translated by the compiler to `!(a==b)` - constexpr bool operator!=(DiscreteDomain const& other) const + KOKKOS_FUNCTION constexpr bool operator!=(DiscreteDomain const& other) const { return !(*this == other); } #endif - constexpr std::size_t size() const + KOKKOS_FUNCTION constexpr std::size_t size() const { return (1ul * ... * (uid(m_element_end) - uid(m_element_begin))); } - constexpr mlength_type extents() const noexcept + KOKKOS_FUNCTION constexpr mlength_type extents() const noexcept { return mlength_type((uid(m_element_end) - uid(m_element_begin))...); } template - inline constexpr DiscreteVector extent() const noexcept + KOKKOS_FUNCTION constexpr DiscreteVector extent() const noexcept { return DiscreteVector( uid(m_element_end) - uid(m_element_begin)); } - constexpr discrete_element_type front() const noexcept + KOKKOS_FUNCTION constexpr discrete_element_type front() const noexcept { return m_element_begin; } - constexpr discrete_element_type back() const noexcept + KOKKOS_FUNCTION constexpr discrete_element_type back() const noexcept { return discrete_element_type((uid(m_element_end) - 1)...); } - constexpr DiscreteDomain take_first(mlength_type n) const + KOKKOS_FUNCTION constexpr DiscreteDomain take_first(mlength_type n) const { return DiscreteDomain(front(), n); } - constexpr DiscreteDomain take_last(mlength_type n) const + KOKKOS_FUNCTION constexpr DiscreteDomain take_last(mlength_type n) const { return DiscreteDomain(front() + (extents() - n), n); } - constexpr DiscreteDomain remove_first(mlength_type n) const + KOKKOS_FUNCTION constexpr DiscreteDomain remove_first(mlength_type n) const { return DiscreteDomain(front() + n, extents() - n); } - constexpr DiscreteDomain remove_last(mlength_type n) const + KOKKOS_FUNCTION constexpr DiscreteDomain remove_last(mlength_type n) const { return DiscreteDomain(front(), extents() - n); } - constexpr DiscreteDomain remove(mlength_type n1, mlength_type n2) const + KOKKOS_FUNCTION constexpr DiscreteDomain remove(mlength_type n1, mlength_type n2) const { return DiscreteDomain(front() + n1, extents() - n1 - n2); } template - constexpr auto restrict(DiscreteDomain const& odomain) const + KOKKOS_FUNCTION constexpr auto restrict(DiscreteDomain const& odomain) const { assert(((uid(m_element_begin) <= uid(odomain.m_element_begin)) && ...)); assert(((uid(m_element_end) >= uid(odomain.m_element_end)) && ...)); @@ -151,12 +153,12 @@ class DiscreteDomain DiscreteVector((get_or(oextents, get(myextents)))...)); } - constexpr bool empty() const noexcept + KOKKOS_FUNCTION constexpr bool empty() const noexcept { return size() == 0; } - constexpr explicit operator bool() + KOKKOS_FUNCTION constexpr explicit operator bool() { return !empty(); } @@ -164,7 +166,7 @@ class DiscreteDomain template < std::size_t N = sizeof...(DDims), class DDim0 = std::enable_if_t>>> - auto begin() const + KOKKOS_FUNCTION auto begin() const { return DiscreteDomainIterator(front()); } @@ -172,7 +174,7 @@ class DiscreteDomain template < std::size_t N = sizeof...(DDims), class DDim0 = std::enable_if_t>>> - auto end() const + KOKKOS_FUNCTION auto end() const { return DiscreteDomainIterator(m_element_end); } @@ -180,7 +182,7 @@ class DiscreteDomain template < std::size_t N = sizeof...(DDims), class DDim0 = std::enable_if_t>>> - auto cbegin() const + KOKKOS_FUNCTION auto cbegin() const { return DiscreteDomainIterator(front()); } @@ -188,7 +190,7 @@ class DiscreteDomain template < std::size_t N = sizeof...(DDims), class DDim0 = std::enable_if_t>>> - auto cend() const + KOKKOS_FUNCTION auto cend() const { return DiscreteDomainIterator(m_element_end); } @@ -196,7 +198,7 @@ class DiscreteDomain template < std::size_t N = sizeof...(DDims), class = std::enable_if_t>>> - constexpr decltype(auto) operator[](std::size_t n) + KOKKOS_FUNCTION constexpr decltype(auto) operator[](std::size_t n) { return begin()[n]; } @@ -204,7 +206,7 @@ class DiscreteDomain template < std::size_t N = sizeof...(DDims), class = std::enable_if_t>>> - constexpr decltype(auto) operator[](std::size_t n) const + KOKKOS_FUNCTION constexpr decltype(auto) operator[](std::size_t n) const { return begin()[n]; } @@ -221,7 +223,7 @@ class DiscreteDomain<> using mlength_type = DiscreteVector<>; - static constexpr std::size_t rank() + static KOKKOS_FUNCTION constexpr std::size_t rank() { return 0; } @@ -230,7 +232,8 @@ class DiscreteDomain<> // Construct a DiscreteDomain from a reordered copy of `domain` template - explicit constexpr DiscreteDomain([[maybe_unused]] DiscreteDomain const& domain) + explicit KOKKOS_FUNCTION constexpr DiscreteDomain( + [[maybe_unused]] DiscreteDomain const& domain) { } @@ -238,23 +241,23 @@ class DiscreteDomain<> * @param element_begin the lower bound in each direction * @param size the number of points in each direction */ - constexpr DiscreteDomain( + KOKKOS_FUNCTION constexpr DiscreteDomain( [[maybe_unused]] discrete_element_type const& element_begin, [[maybe_unused]] mlength_type const& size) { } - constexpr DiscreteDomain(DiscreteDomain const& x) = default; + KOKKOS_DEFAULTED_FUNCTION DiscreteDomain(DiscreteDomain const& x) = default; - constexpr DiscreteDomain(DiscreteDomain&& x) = default; + KOKKOS_DEFAULTED_FUNCTION DiscreteDomain(DiscreteDomain&& x) = default; - ~DiscreteDomain() = default; + KOKKOS_DEFAULTED_FUNCTION ~DiscreteDomain() = default; - DiscreteDomain& operator=(DiscreteDomain const& x) = default; + KOKKOS_DEFAULTED_FUNCTION DiscreteDomain& operator=(DiscreteDomain const& x) = default; - DiscreteDomain& operator=(DiscreteDomain&& x) = default; + KOKKOS_DEFAULTED_FUNCTION DiscreteDomain& operator=(DiscreteDomain&& x) = default; - constexpr bool operator==([[maybe_unused]] DiscreteDomain const& other) const + KOKKOS_FUNCTION constexpr bool operator==([[maybe_unused]] DiscreteDomain const& other) const { return true; } @@ -262,53 +265,53 @@ class DiscreteDomain<> #if __cplusplus <= 201703L // Shall not be necessary anymore in C++20 // `a!=b` shall be translated by the compiler to `!(a==b)` - constexpr bool operator!=(DiscreteDomain const& other) const + KOKKOS_FUNCTION constexpr bool operator!=(DiscreteDomain const& other) const { return !(*this == other); } #endif - constexpr std::size_t size() const + KOKKOS_FUNCTION constexpr std::size_t size() const { return 1; } - constexpr mlength_type extents() const noexcept + KOKKOS_FUNCTION constexpr mlength_type extents() const noexcept { return {}; } - constexpr discrete_element_type front() const noexcept + KOKKOS_FUNCTION constexpr discrete_element_type front() const noexcept { return {}; } - constexpr discrete_element_type back() const noexcept + KOKKOS_FUNCTION constexpr discrete_element_type back() const noexcept { return {}; } - constexpr DiscreteDomain take_first([[maybe_unused]] mlength_type n) const + KOKKOS_FUNCTION constexpr DiscreteDomain take_first([[maybe_unused]] mlength_type n) const { return *this; } - constexpr DiscreteDomain take_last([[maybe_unused]] mlength_type n) const + KOKKOS_FUNCTION constexpr DiscreteDomain take_last([[maybe_unused]] mlength_type n) const { return *this; } - constexpr DiscreteDomain remove_first([[maybe_unused]] mlength_type n) const + KOKKOS_FUNCTION constexpr DiscreteDomain remove_first([[maybe_unused]] mlength_type n) const { return *this; } - constexpr DiscreteDomain remove_last([[maybe_unused]] mlength_type n) const + KOKKOS_FUNCTION constexpr DiscreteDomain remove_last([[maybe_unused]] mlength_type n) const { return *this; } - constexpr DiscreteDomain remove( + KOKKOS_FUNCTION constexpr DiscreteDomain remove( [[maybe_unused]] mlength_type n1, [[maybe_unused]] mlength_type n2) const { @@ -316,24 +319,25 @@ class DiscreteDomain<> } template - constexpr DiscreteDomain restrict(DiscreteDomain const&) const + KOKKOS_FUNCTION constexpr DiscreteDomain restrict(DiscreteDomain const&) const { return *this; } - constexpr bool empty() const noexcept + KOKKOS_FUNCTION constexpr bool empty() const noexcept { return false; } - constexpr explicit operator bool() + KOKKOS_FUNCTION constexpr explicit operator bool() { return true; } }; template -constexpr DiscreteDomain select(DiscreteDomain const& domain) +KOKKOS_FUNCTION constexpr DiscreteDomain select( + DiscreteDomain const& domain) { return DiscreteDomain( select(domain.front()), @@ -385,7 +389,7 @@ using cartesian_prod_t = typename cartesian_prod::type; // Computes the substraction DDom_a - DDom_b in the sense of linear spaces(retained dimensions are those in DDom_a which are not in DDom_b) template -constexpr auto remove_dims_of( +KOKKOS_FUNCTION constexpr auto remove_dims_of( DiscreteDomain const& DDom_a, [[maybe_unused]] DiscreteDomain const& DDom_b) noexcept { @@ -399,7 +403,7 @@ constexpr auto remove_dims_of( // Checks if dimension of DDom_a is DDim1. If not, returns restriction to DDim2 of DDom_b. May not be usefull in its own, it helps for replace_dim_of template -constexpr std::conditional_t< +KOKKOS_FUNCTION constexpr std::conditional_t< std::is_same_v, ddc::DiscreteDomain, ddc::DiscreteDomain> @@ -416,7 +420,7 @@ replace_dim_of_1d( // Replace in DDom_a the dimension Dim1 by the dimension Dim2 of DDom_b template -constexpr auto replace_dim_of( +KOKKOS_FUNCTION constexpr auto replace_dim_of( DiscreteDomain const& DDom_a, [[maybe_unused]] DiscreteDomain const& DDom_b) noexcept { @@ -435,19 +439,22 @@ constexpr auto replace_dim_of( } template -constexpr DiscreteVector extents(DiscreteDomain const& domain) noexcept +KOKKOS_FUNCTION constexpr DiscreteVector extents( + DiscreteDomain const& domain) noexcept { return DiscreteVector(select(domain).size()...); } template -constexpr DiscreteElement front(DiscreteDomain const& domain) noexcept +KOKKOS_FUNCTION constexpr DiscreteElement front( + DiscreteDomain const& domain) noexcept { return DiscreteElement(select(domain).front()...); } template -constexpr DiscreteElement back(DiscreteDomain const& domain) noexcept +KOKKOS_FUNCTION constexpr DiscreteElement back( + DiscreteDomain const& domain) noexcept { return DiscreteElement(select(domain).back()...); } @@ -461,7 +468,7 @@ template struct Selection> { template - static constexpr auto select(Domain const& domain) + static KOKKOS_FUNCTION constexpr auto select(Domain const& domain) { return ddc::select(domain); } @@ -470,7 +477,7 @@ struct Selection> } // namespace detail template -constexpr auto select_by_type_seq(DiscreteDomain const& domain) +KOKKOS_FUNCTION constexpr auto select_by_type_seq(DiscreteDomain const& domain) { return detail::Selection::select(domain); } @@ -488,42 +495,45 @@ struct DiscreteDomainIterator using difference_type = std::ptrdiff_t; - DiscreteDomainIterator() = default; + KOKKOS_DEFAULTED_FUNCTION DiscreteDomainIterator() = default; - constexpr explicit DiscreteDomainIterator(DiscreteElement value) : m_value(value) {} + KOKKOS_FUNCTION constexpr explicit DiscreteDomainIterator(DiscreteElement value) + : m_value(value) + { + } - constexpr DiscreteElement operator*() const noexcept + KOKKOS_FUNCTION constexpr DiscreteElement operator*() const noexcept { return m_value; } - constexpr DiscreteDomainIterator& operator++() + KOKKOS_FUNCTION constexpr DiscreteDomainIterator& operator++() { ++m_value.uid(); return *this; } - constexpr DiscreteDomainIterator operator++(int) + KOKKOS_FUNCTION constexpr DiscreteDomainIterator operator++(int) { auto tmp = *this; ++*this; return tmp; } - constexpr DiscreteDomainIterator& operator--() + KOKKOS_FUNCTION constexpr DiscreteDomainIterator& operator--() { --m_value.uid(); return *this; } - constexpr DiscreteDomainIterator operator--(int) + KOKKOS_FUNCTION constexpr DiscreteDomainIterator operator--(int) { auto tmp = *this; --*this; return tmp; } - constexpr DiscreteDomainIterator& operator+=(difference_type n) + KOKKOS_FUNCTION constexpr DiscreteDomainIterator& operator+=(difference_type n) { if (n >= difference_type(0)) m_value.uid() += static_cast(n); @@ -532,7 +542,7 @@ struct DiscreteDomainIterator return *this; } - constexpr DiscreteDomainIterator& operator-=(difference_type n) + KOKKOS_FUNCTION constexpr DiscreteDomainIterator& operator-=(difference_type n) { if (n >= difference_type(0)) m_value.uid() -= static_cast(n); @@ -541,69 +551,75 @@ struct DiscreteDomainIterator return *this; } - constexpr DiscreteElement operator[](difference_type n) const + KOKKOS_FUNCTION constexpr DiscreteElement operator[](difference_type n) const { return m_value + n; } - friend constexpr bool operator==( + friend KOKKOS_FUNCTION constexpr bool operator==( DiscreteDomainIterator const& xx, DiscreteDomainIterator const& yy) { return xx.m_value == yy.m_value; } - friend constexpr bool operator!=( + friend KOKKOS_FUNCTION constexpr bool operator!=( DiscreteDomainIterator const& xx, DiscreteDomainIterator const& yy) { return xx.m_value != yy.m_value; } - friend constexpr bool operator<( + friend KOKKOS_FUNCTION constexpr bool operator<( DiscreteDomainIterator const& xx, DiscreteDomainIterator const& yy) { return xx.m_value < yy.m_value; } - friend constexpr bool operator>( + friend KOKKOS_FUNCTION constexpr bool operator>( DiscreteDomainIterator const& xx, DiscreteDomainIterator const& yy) { return yy < xx; } - friend constexpr bool operator<=( + friend KOKKOS_FUNCTION constexpr bool operator<=( DiscreteDomainIterator const& xx, DiscreteDomainIterator const& yy) { return !(yy < xx); } - friend constexpr bool operator>=( + friend KOKKOS_FUNCTION constexpr bool operator>=( DiscreteDomainIterator const& xx, DiscreteDomainIterator const& yy) { return !(xx < yy); } - friend constexpr DiscreteDomainIterator operator+(DiscreteDomainIterator i, difference_type n) + friend KOKKOS_FUNCTION constexpr DiscreteDomainIterator operator+( + DiscreteDomainIterator i, + difference_type n) { return i += n; } - friend constexpr DiscreteDomainIterator operator+(difference_type n, DiscreteDomainIterator i) + friend KOKKOS_FUNCTION constexpr DiscreteDomainIterator operator+( + difference_type n, + DiscreteDomainIterator i) { return i += n; } - friend constexpr DiscreteDomainIterator operator-(DiscreteDomainIterator i, difference_type n) + friend KOKKOS_FUNCTION constexpr DiscreteDomainIterator operator-( + DiscreteDomainIterator i, + difference_type n) { return i -= n; } - friend constexpr difference_type operator-( + friend KOKKOS_FUNCTION constexpr difference_type operator-( DiscreteDomainIterator const& xx, DiscreteDomainIterator const& yy) { diff --git a/include/ddc/discrete_element.hpp b/include/ddc/discrete_element.hpp index 0d0e438e8..3907f1b40 100644 --- a/include/ddc/discrete_element.hpp +++ b/include/ddc/discrete_element.hpp @@ -35,31 +35,32 @@ inline constexpr bool is_discrete_element_v = IsDiscreteElement::value; using DiscreteElementType = std::size_t; template -inline constexpr DiscreteElementType const& uid(DiscreteElement const& tuple) noexcept +KOKKOS_FUNCTION constexpr DiscreteElementType const& uid(DiscreteElement const& tuple) noexcept { return tuple.uid(); } template -inline constexpr DiscreteElementType& uid(DiscreteElement& tuple) noexcept +KOKKOS_FUNCTION constexpr DiscreteElementType& uid(DiscreteElement& tuple) noexcept { return tuple.uid(); } template -inline constexpr DiscreteElementType const& uid(DiscreteElement const& tuple) noexcept +KOKKOS_FUNCTION constexpr DiscreteElementType const& uid( + DiscreteElement const& tuple) noexcept { return tuple.template uid(); } template -inline constexpr DiscreteElementType& uid(DiscreteElement& tuple) noexcept +KOKKOS_FUNCTION constexpr DiscreteElementType& uid(DiscreteElement& tuple) noexcept { return tuple.template uid(); } template -inline constexpr DiscreteElementType const& uid_or( +KOKKOS_FUNCTION constexpr DiscreteElementType const& uid_or( DiscreteElement const& tuple, DiscreteElementType const& default_value) noexcept { @@ -67,19 +68,21 @@ inline constexpr DiscreteElementType const& uid_or( } template -inline constexpr DiscreteElement select(DiscreteElement const& arr) noexcept +KOKKOS_FUNCTION constexpr DiscreteElement select( + DiscreteElement const& arr) noexcept { return DiscreteElement(arr); } template -inline constexpr DiscreteElement select(DiscreteElement&& arr) noexcept +KOKKOS_FUNCTION constexpr DiscreteElement select( + DiscreteElement&& arr) noexcept { return DiscreteElement(std::move(arr)); } template -constexpr DiscreteElement take( +KOKKOS_FUNCTION constexpr DiscreteElement take( DiscreteElement const& head, DETail const&... tail) { @@ -101,7 +104,7 @@ namespace detail { /// Returns a reference to the underlying `std::array` template -constexpr inline std::array& array( +KOKKOS_FUNCTION constexpr std::array& array( DiscreteElement& v) noexcept { return v.m_values; @@ -109,7 +112,7 @@ constexpr inline std::array& array( /// Returns a reference to the underlying `std::array` template -constexpr inline std::array const& array( +KOKKOS_FUNCTION constexpr std::array const& array( DiscreteElement const& v) noexcept { return v.m_values; @@ -126,11 +129,11 @@ class DiscreteElement { using tags_seq = detail::TypeSeq; - friend constexpr std::array& detail::array( - DiscreteElement& v) noexcept; + friend KOKKOS_FUNCTION constexpr std::array& detail:: + array(DiscreteElement& v) noexcept; - friend constexpr std::array const& detail::array( - DiscreteElement const& v) noexcept; + friend KOKKOS_FUNCTION constexpr std::array const& + detail::array(DiscreteElement const& v) noexcept; private: std::array m_values; @@ -138,26 +141,27 @@ class DiscreteElement public: using value_type = DiscreteElementType; - static constexpr std::size_t size() noexcept + static KOKKOS_FUNCTION constexpr std::size_t size() noexcept { return sizeof...(Tags); } public: - inline constexpr DiscreteElement() = default; + KOKKOS_DEFAULTED_FUNCTION constexpr DiscreteElement() = default; - inline constexpr DiscreteElement(DiscreteElement const&) = default; + KOKKOS_DEFAULTED_FUNCTION constexpr DiscreteElement(DiscreteElement const&) = default; - inline constexpr DiscreteElement(DiscreteElement&&) = default; + KOKKOS_DEFAULTED_FUNCTION constexpr DiscreteElement(DiscreteElement&&) = default; template - explicit inline constexpr DiscreteElement(DiscreteElement const& other) noexcept + explicit KOKKOS_FUNCTION constexpr DiscreteElement( + DiscreteElement const& other) noexcept : m_values {other.template uid()...} { } template && ...)>> - explicit inline constexpr DiscreteElement(DE const&... other) noexcept + explicit KOKKOS_FUNCTION constexpr DiscreteElement(DE const&... other) noexcept : m_values {take(other...).uid()...} { } @@ -167,31 +171,34 @@ class DiscreteElement class = std::enable_if_t<(std::is_integral_v && ...)>, class = std::enable_if_t<(!is_discrete_element_v && ...)>, class = std::enable_if_t> - explicit inline constexpr DiscreteElement(Params const&... params) noexcept + explicit KOKKOS_FUNCTION constexpr DiscreteElement(Params const&... params) noexcept : m_values {static_cast(params)...} { } - constexpr inline DiscreteElement& operator=(DiscreteElement const& other) = default; + KOKKOS_DEFAULTED_FUNCTION ~DiscreteElement() = default; - constexpr inline DiscreteElement& operator=(DiscreteElement&& other) = default; + KOKKOS_DEFAULTED_FUNCTION DiscreteElement& operator=(DiscreteElement const& other) = default; + + KOKKOS_DEFAULTED_FUNCTION DiscreteElement& operator=(DiscreteElement&& other) = default; template - constexpr inline DiscreteElement& operator=(DiscreteElement const& other) noexcept + KOKKOS_FUNCTION constexpr DiscreteElement& operator=( + DiscreteElement const& other) noexcept { m_values = other.m_values; return *this; } template - constexpr inline DiscreteElement& operator=(DiscreteElement&& other) noexcept + KOKKOS_FUNCTION constexpr DiscreteElement& operator=(DiscreteElement&& other) noexcept { m_values = std::move(other.m_values); return *this; } template - value_type const& uid_or(value_type const& default_value) const& + KOKKOS_FUNCTION constexpr value_type const& uid_or(value_type const& default_value) const& { DDC_IF_NVCC_THEN_PUSH_AND_SUPPRESS(implicit_return_from_non_void_function) if constexpr (in_tags_v) { @@ -203,7 +210,7 @@ class DiscreteElement } template - inline constexpr value_type& uid() noexcept + KOKKOS_FUNCTION constexpr value_type& uid() noexcept { using namespace detail; static_assert(in_tags_v, "requested Tag absent from DiscreteElement"); @@ -211,7 +218,7 @@ class DiscreteElement } template - inline constexpr value_type const& uid() const noexcept + KOKKOS_FUNCTION constexpr value_type const& uid() const noexcept { using namespace detail; static_assert(in_tags_v, "requested Tag absent from DiscreteElement"); @@ -219,26 +226,26 @@ class DiscreteElement } template - constexpr inline std::enable_if_t uid() noexcept + KOKKOS_FUNCTION constexpr std::enable_if_t uid() noexcept { return m_values[0]; } template - constexpr inline std::enable_if_t uid() const noexcept + KOKKOS_FUNCTION constexpr std::enable_if_t uid() const noexcept { return m_values[0]; } template > - constexpr inline DiscreteElement& operator++() + KOKKOS_FUNCTION constexpr DiscreteElement& operator++() { ++m_values[0]; return *this; } template > - constexpr inline DiscreteElement operator++(int) + KOKKOS_FUNCTION constexpr DiscreteElement operator++(int) { DiscreteElement const tmp = *this; ++m_values[0]; @@ -246,14 +253,14 @@ class DiscreteElement } template > - constexpr inline DiscreteElement& operator--() + KOKKOS_FUNCTION constexpr DiscreteElement& operator--() { ++m_values[0]; return *this; } template > - constexpr inline DiscreteElement operator--(int) + KOKKOS_FUNCTION constexpr DiscreteElement operator--(int) { DiscreteElement const tmp = *this; ++m_values[0]; @@ -261,7 +268,7 @@ class DiscreteElement } template - constexpr inline DiscreteElement& operator+=(DiscreteVector const& rhs) + KOKKOS_FUNCTION constexpr DiscreteElement& operator+=(DiscreteVector const& rhs) { static_assert(((type_seq_contains_v, tags_seq>)&&...)); ((m_values[type_seq_rank_v] += rhs.template get()), ...); @@ -273,14 +280,14 @@ class DiscreteElement std::size_t N = sizeof...(Tags), class = std::enable_if_t, class = std::enable_if_t>> - constexpr inline DiscreteElement& operator+=(IntegralType const& rhs) + KOKKOS_FUNCTION constexpr DiscreteElement& operator+=(IntegralType const& rhs) { m_values[0] += rhs; return *this; } template - constexpr inline DiscreteElement& operator-=(DiscreteVector const& rhs) + KOKKOS_FUNCTION constexpr DiscreteElement& operator-=(DiscreteVector const& rhs) { static_assert(((type_seq_contains_v, tags_seq>)&&...)); ((m_values[type_seq_rank_v] -= rhs.template get()), ...); @@ -292,7 +299,7 @@ class DiscreteElement std::size_t N = sizeof...(Tags), class = std::enable_if_t, class = std::enable_if_t>> - constexpr inline DiscreteElement& operator-=(IntegralType const& rhs) + KOKKOS_FUNCTION constexpr DiscreteElement& operator-=(IntegralType const& rhs) { m_values[0] -= rhs; return *this; @@ -317,7 +324,7 @@ std::ostream& operator<<(std::ostream& out, DiscreteElement const template -constexpr inline bool operator==( +KOKKOS_FUNCTION constexpr bool operator==( DiscreteElement const& lhs, DiscreteElement const& rhs) noexcept { @@ -325,7 +332,7 @@ constexpr inline bool operator==( } template -constexpr inline bool operator!=( +KOKKOS_FUNCTION constexpr bool operator!=( DiscreteElement const& lhs, DiscreteElement const& rhs) noexcept { @@ -333,25 +340,33 @@ constexpr inline bool operator!=( } template -constexpr inline bool operator<(DiscreteElement const& lhs, DiscreteElement const& rhs) +KOKKOS_FUNCTION constexpr bool operator<( + DiscreteElement const& lhs, + DiscreteElement const& rhs) { return lhs.uid() < rhs.uid(); } template -constexpr inline bool operator<=(DiscreteElement const& lhs, DiscreteElement const& rhs) +KOKKOS_FUNCTION constexpr bool operator<=( + DiscreteElement const& lhs, + DiscreteElement const& rhs) { return lhs.uid() <= rhs.uid(); } template -constexpr inline bool operator>(DiscreteElement const& lhs, DiscreteElement const& rhs) +KOKKOS_FUNCTION constexpr bool operator>( + DiscreteElement const& lhs, + DiscreteElement const& rhs) { return lhs.uid() > rhs.uid(); } template -constexpr inline bool operator>=(DiscreteElement const& lhs, DiscreteElement const& rhs) +KOKKOS_FUNCTION constexpr bool operator>=( + DiscreteElement const& lhs, + DiscreteElement const& rhs) { return lhs.uid() >= rhs.uid(); } @@ -359,7 +374,7 @@ constexpr inline bool operator>=(DiscreteElement const& lhs, DiscreteElemen /// right external binary operators: +, - template -constexpr inline DiscreteElement operator+( +KOKKOS_FUNCTION constexpr DiscreteElement operator+( DiscreteElement const& lhs, DiscreteVector const& rhs) { @@ -375,7 +390,7 @@ template < class IntegralType, class = std::enable_if_t>, class = std::enable_if_t>> -constexpr inline DiscreteElement operator+( +KOKKOS_FUNCTION constexpr DiscreteElement operator+( DiscreteElement const& lhs, IntegralType const& rhs) { @@ -383,7 +398,7 @@ constexpr inline DiscreteElement operator+( } template -constexpr inline DiscreteElement operator-( +KOKKOS_FUNCTION constexpr DiscreteElement operator-( DiscreteElement const& lhs, DiscreteVector const& rhs) { @@ -399,7 +414,7 @@ template < class IntegralType, class = std::enable_if_t>, class = std::enable_if_t>> -constexpr inline DiscreteElement operator-( +KOKKOS_FUNCTION constexpr DiscreteElement operator-( DiscreteElement const& lhs, IntegralType const& rhs) { @@ -409,7 +424,7 @@ constexpr inline DiscreteElement operator-( /// binary operator: - template -constexpr inline DiscreteVector operator-( +KOKKOS_FUNCTION constexpr DiscreteVector operator-( DiscreteElement const& lhs, DiscreteElement const& rhs) { diff --git a/include/ddc/discrete_space.hpp b/include/ddc/discrete_space.hpp index d3276bbdf..54e0df3b4 100644 --- a/include/ddc/discrete_space.hpp +++ b/include/ddc/discrete_space.hpp @@ -75,19 +75,19 @@ class gpu_proxy alignas(T) Kokkos::Array m_data; public: - DDC_INLINE_FUNCTION + KOKKOS_FORCEINLINE_FUNCTION T* operator->() { return reinterpret_cast(m_data.data()); } - DDC_INLINE_FUNCTION + KOKKOS_FORCEINLINE_FUNCTION T& operator*() { return *reinterpret_cast(m_data.data()); } - DDC_INLINE_FUNCTION + KOKKOS_FORCEINLINE_FUNCTION T* data() { return reinterpret_cast(m_data.data()); @@ -187,7 +187,7 @@ std::enable_if_t<2 <= sizeof...(Args), std::tuple> init_discrete_space( } template -DDC_INLINE_FUNCTION detail::ddim_impl_t const& discrete_space() +KOKKOS_FORCEINLINE_FUNCTION detail::ddim_impl_t const& discrete_space() { if constexpr (std::is_same_v) { return detail::g_discrete_space_dual->get_host(); diff --git a/include/ddc/discrete_vector.hpp b/include/ddc/discrete_vector.hpp index acea572ec..09ee7d898 100644 --- a/include/ddc/discrete_vector.hpp +++ b/include/ddc/discrete_vector.hpp @@ -7,6 +7,8 @@ #include #include +#include + #include "ddc/detail/type_seq.hpp" namespace ddc { @@ -33,19 +35,20 @@ inline constexpr bool is_discrete_vector_v = IsDiscreteVector::value; using DiscreteVectorElement = std::ptrdiff_t; template -inline constexpr DiscreteVectorElement const& get(DiscreteVector const& tuple) noexcept +KOKKOS_FUNCTION constexpr DiscreteVectorElement const& get( + DiscreteVector const& tuple) noexcept { return tuple.template get(); } template -inline constexpr DiscreteVectorElement& get(DiscreteVector& tuple) noexcept +KOKKOS_FUNCTION constexpr DiscreteVectorElement& get(DiscreteVector& tuple) noexcept { return tuple.template get(); } template -inline constexpr DiscreteVectorElement const& get_or( +KOKKOS_FUNCTION constexpr DiscreteVectorElement const& get_or( DiscreteVector const& tuple, DiscreteVectorElement const& default_value) noexcept { @@ -55,13 +58,13 @@ inline constexpr DiscreteVectorElement const& get_or( /// Unary operators: +, - template -constexpr inline DiscreteVector operator+(DiscreteVector const& x) +KOKKOS_FUNCTION constexpr DiscreteVector operator+(DiscreteVector const& x) { return x; } template -constexpr inline DiscreteVector operator-(DiscreteVector const& x) +KOKKOS_FUNCTION constexpr DiscreteVector operator-(DiscreteVector const& x) { return DiscreteVector((-get(x))...); } @@ -69,7 +72,7 @@ constexpr inline DiscreteVector operator-(DiscreteVector const /// Internal binary operators: +, - template -constexpr inline auto operator+( +KOKKOS_FUNCTION constexpr auto operator+( DiscreteVector const& lhs, DiscreteVector const& rhs) { @@ -88,7 +91,7 @@ constexpr inline auto operator+( } template -constexpr inline auto operator-( +KOKKOS_FUNCTION constexpr auto operator-( DiscreteVector const& lhs, DiscreteVector const& rhs) { @@ -107,7 +110,7 @@ constexpr inline auto operator-( } template >> -constexpr inline DiscreteVector operator+( +KOKKOS_FUNCTION constexpr DiscreteVector operator+( DiscreteVector const& lhs, IntegralType const& rhs) { @@ -115,7 +118,7 @@ constexpr inline DiscreteVector operator+( } template >> -constexpr inline DiscreteVector operator+( +KOKKOS_FUNCTION constexpr DiscreteVector operator+( IntegralType const& lhs, DiscreteVector const& rhs) { @@ -123,7 +126,7 @@ constexpr inline DiscreteVector operator+( } template >> -constexpr inline DiscreteVector operator-( +KOKKOS_FUNCTION constexpr DiscreteVector operator-( DiscreteVector const& lhs, IntegralType const& rhs) { @@ -131,7 +134,7 @@ constexpr inline DiscreteVector operator-( } template >> -constexpr inline DiscreteVector operator-( +KOKKOS_FUNCTION constexpr DiscreteVector operator-( IntegralType const& lhs, DiscreteVector const& rhs) { @@ -144,25 +147,29 @@ template < class IntegralType, class... Tags, class = std::enable_if_t>> -constexpr inline auto operator*(IntegralType const& lhs, DiscreteVector const& rhs) +KOKKOS_FUNCTION constexpr auto operator*( + IntegralType const& lhs, + DiscreteVector const& rhs) { return DiscreteVector((lhs * get(rhs))...); } template -inline constexpr DiscreteVector select(DiscreteVector const& arr) noexcept +KOKKOS_FUNCTION constexpr DiscreteVector select( + DiscreteVector const& arr) noexcept { return DiscreteVector(arr); } template -inline constexpr DiscreteVector select(DiscreteVector&& arr) noexcept +KOKKOS_FUNCTION constexpr DiscreteVector select( + DiscreteVector&& arr) noexcept { return DiscreteVector(std::move(arr)); } template -constexpr DiscreteVector const& take( +KOKKOS_FUNCTION constexpr DiscreteVector const& take( DiscreteVector const& head, DiscreteVector const&... tags) { @@ -188,12 +195,12 @@ template class ConversionOperators> { public: - constexpr inline operator DiscreteVectorElement const &() const noexcept + KOKKOS_FUNCTION constexpr operator DiscreteVectorElement const &() const noexcept { return static_cast const*>(this)->m_values[0]; } - constexpr inline operator DiscreteVectorElement&() noexcept + KOKKOS_FUNCTION constexpr operator DiscreteVectorElement&() noexcept { return static_cast*>(this)->m_values[0]; } @@ -203,7 +210,7 @@ namespace detail { /// Returns a reference to the underlying `std::array` template -constexpr inline std::array& array( +KOKKOS_FUNCTION constexpr std::array& array( DiscreteVector& v) noexcept { return v.m_values; @@ -211,7 +218,7 @@ constexpr inline std::array& array( /// Returns a reference to the underlying `std::array` template -constexpr inline std::array const& array( +KOKKOS_FUNCTION constexpr std::array const& array( DiscreteVector const& v) noexcept { return v.m_values; @@ -228,10 +235,10 @@ class DiscreteVector : public ConversionOperators> { friend class ConversionOperators>; - friend constexpr std::array& detail::array( - DiscreteVector& v) noexcept; - friend constexpr std::array const& detail::array< - Tags...>(DiscreteVector const& v) noexcept; + friend KOKKOS_FUNCTION constexpr std::array& detail:: + array(DiscreteVector& v) noexcept; + friend KOKKOS_FUNCTION constexpr std::array const& + detail::array(DiscreteVector const& v) noexcept; using tags_seq = detail::TypeSeq; @@ -239,26 +246,28 @@ class DiscreteVector : public ConversionOperators> std::array m_values; public: - static constexpr std::size_t size() noexcept + static KOKKOS_FUNCTION constexpr std::size_t size() noexcept { return sizeof...(Tags); } public: - inline constexpr DiscreteVector() = default; + KOKKOS_DEFAULTED_FUNCTION constexpr DiscreteVector() = default; - inline constexpr DiscreteVector(DiscreteVector const&) = default; + KOKKOS_DEFAULTED_FUNCTION constexpr DiscreteVector(DiscreteVector const&) = default; - inline constexpr DiscreteVector(DiscreteVector&&) = default; + KOKKOS_DEFAULTED_FUNCTION constexpr DiscreteVector(DiscreteVector&&) = default; template - explicit inline constexpr DiscreteVector(DiscreteVector const&... other) noexcept + explicit KOKKOS_FUNCTION constexpr DiscreteVector( + DiscreteVector const&... other) noexcept : m_values {take(other...).value()...} { } template - explicit inline constexpr DiscreteVector(DiscreteVector const& other) noexcept + explicit KOKKOS_FUNCTION constexpr DiscreteVector( + DiscreteVector const& other) noexcept : m_values {other.template get()...} { } @@ -268,43 +277,47 @@ class DiscreteVector : public ConversionOperators> class = std::enable_if_t<(std::is_convertible_v && ...)>, class = std::enable_if_t<(!is_discrete_vector_v && ...)>, class = std::enable_if_t> - explicit inline constexpr DiscreteVector(Params const&... params) noexcept + explicit KOKKOS_FUNCTION constexpr DiscreteVector(Params const&... params) noexcept : m_values {static_cast(params)...} { } - constexpr inline DiscreteVector& operator=(DiscreteVector const& other) = default; + KOKKOS_DEFAULTED_FUNCTION ~DiscreteVector() = default; + + KOKKOS_DEFAULTED_FUNCTION constexpr DiscreteVector& operator=(DiscreteVector const& other) + = default; - constexpr inline DiscreteVector& operator=(DiscreteVector&& other) = default; + KOKKOS_DEFAULTED_FUNCTION constexpr DiscreteVector& operator=(DiscreteVector&& other) = default; template - constexpr inline DiscreteVector& operator=(DiscreteVector const& other) noexcept + KOKKOS_FUNCTION constexpr DiscreteVector& operator=( + DiscreteVector const& other) noexcept { m_values = other.m_values; return *this; } template - constexpr inline DiscreteVector& operator=(DiscreteVector&& other) noexcept + KOKKOS_FUNCTION constexpr DiscreteVector& operator=(DiscreteVector&& other) noexcept { m_values = std::move(other.m_values); return *this; } template - constexpr inline bool operator==(DiscreteVector const& rhs) const noexcept + KOKKOS_FUNCTION constexpr bool operator==(DiscreteVector const& rhs) const noexcept { return ((m_values[type_seq_rank_v] == rhs.template get()) && ...); } template - constexpr inline bool operator!=(DiscreteVector const& rhs) const noexcept + KOKKOS_FUNCTION constexpr bool operator!=(DiscreteVector const& rhs) const noexcept { return !(*this == rhs); } template - inline constexpr DiscreteVectorElement& get() noexcept + KOKKOS_FUNCTION constexpr DiscreteVectorElement& get() noexcept { using namespace detail; static_assert(in_tags_v, "requested Tag absent from DiscreteVector"); @@ -312,7 +325,7 @@ class DiscreteVector : public ConversionOperators> } template - inline constexpr DiscreteVectorElement const& get() const noexcept + KOKKOS_FUNCTION constexpr DiscreteVectorElement const& get() const noexcept { using namespace detail; static_assert(in_tags_v, "requested Tag absent from DiscreteVector"); @@ -320,7 +333,8 @@ class DiscreteVector : public ConversionOperators> } template - DiscreteVectorElement const& get_or(DiscreteVectorElement const& default_value) const& + KOKKOS_FUNCTION constexpr DiscreteVectorElement const& get_or( + DiscreteVectorElement const& default_value) const& { DDC_IF_NVCC_THEN_PUSH_AND_SUPPRESS(implicit_return_from_non_void_function) if constexpr (in_tags_v) { @@ -332,20 +346,21 @@ class DiscreteVector : public ConversionOperators> } template - constexpr inline std::enable_if_t value() const noexcept + KOKKOS_FUNCTION constexpr std::enable_if_t value() + const noexcept { return m_values[0]; } template > - constexpr inline DiscreteVector& operator++() + KOKKOS_FUNCTION constexpr DiscreteVector& operator++() { ++m_values[0]; return *this; } template > - constexpr inline DiscreteVector operator++(int) + KOKKOS_FUNCTION constexpr DiscreteVector operator++(int) { DiscreteVector const tmp = *this; ++m_values[0]; @@ -353,14 +368,14 @@ class DiscreteVector : public ConversionOperators> } template > - constexpr inline DiscreteVector& operator--() + KOKKOS_FUNCTION constexpr DiscreteVector& operator--() { ++m_values[0]; return *this; } template > - constexpr inline DiscreteVector operator--(int) + KOKKOS_FUNCTION constexpr DiscreteVector operator--(int) { DiscreteVector const tmp = *this; ++m_values[0]; @@ -368,7 +383,7 @@ class DiscreteVector : public ConversionOperators> } template - constexpr inline DiscreteVector& operator+=(DiscreteVector const& rhs) + KOKKOS_FUNCTION constexpr DiscreteVector& operator+=(DiscreteVector const& rhs) { static_assert(((type_seq_contains_v, tags_seq>)&&...)); ((m_values[type_seq_rank_v] += rhs.template get()), ...); @@ -380,14 +395,14 @@ class DiscreteVector : public ConversionOperators> std::size_t N = sizeof...(Tags), class = std::enable_if_t, class = std::enable_if_t>> - constexpr inline DiscreteVector& operator+=(IntegralType const& rhs) + KOKKOS_FUNCTION constexpr DiscreteVector& operator+=(IntegralType const& rhs) { m_values[0] += rhs; return *this; } template - constexpr inline DiscreteVector& operator-=(DiscreteVector const& rhs) + KOKKOS_FUNCTION constexpr DiscreteVector& operator-=(DiscreteVector const& rhs) { static_assert(((type_seq_contains_v, tags_seq>)&&...)); ((m_values[type_seq_rank_v] -= rhs.template get()), ...); @@ -399,14 +414,14 @@ class DiscreteVector : public ConversionOperators> std::size_t N = sizeof...(Tags), class = std::enable_if_t, class = std::enable_if_t>> - constexpr inline DiscreteVector& operator-=(IntegralType const& rhs) + KOKKOS_FUNCTION constexpr DiscreteVector& operator-=(IntegralType const& rhs) { m_values[0] -= rhs; return *this; } template - constexpr inline DiscreteVector& operator*=(DiscreteVector const& rhs) + KOKKOS_FUNCTION constexpr DiscreteVector& operator*=(DiscreteVector const& rhs) { static_assert(((type_seq_contains_v, tags_seq>)&&...)); ((m_values[type_seq_rank_v] *= rhs.template get()), ...); @@ -415,13 +430,15 @@ class DiscreteVector : public ConversionOperators> }; template -constexpr inline bool operator<(DiscreteVector const& lhs, DiscreteVector const& rhs) +KOKKOS_FUNCTION constexpr bool operator<( + DiscreteVector const& lhs, + DiscreteVector const& rhs) { return lhs.value() < rhs.value(); } template -constexpr inline bool operator<(DiscreteVector const& lhs, IntegralType const& rhs) +KOKKOS_FUNCTION constexpr bool operator<(DiscreteVector const& lhs, IntegralType const& rhs) { return lhs.value() < rhs; } @@ -441,4 +458,5 @@ std::ostream& operator<<(std::ostream& out, DiscreteVector const& out << ")"; return out; } + } // namespace ddc diff --git a/include/ddc/dual_discretization.hpp b/include/ddc/dual_discretization.hpp index 0ca16a1be..265d9afeb 100644 --- a/include/ddc/dual_discretization.hpp +++ b/include/ddc/dual_discretization.hpp @@ -48,7 +48,7 @@ class DualDiscretization } template - DDC_INLINE_FUNCTION typename DDim::template Impl const& get() + KOKKOS_FORCEINLINE_FUNCTION typename DDim::template Impl const& get() { if constexpr (std::is_same_v) { return m_host; @@ -67,12 +67,12 @@ class DualDiscretization } } - DDC_INLINE_FUNCTION DDimImplHost const& get_host() + KOKKOS_FORCEINLINE_FUNCTION DDimImplHost const& get_host() { return m_host; } - DDC_INLINE_FUNCTION DDimImplDevice const& get_device() + KOKKOS_FORCEINLINE_FUNCTION DDimImplDevice const& get_device() { #if defined(__CUDACC__) || defined(__HIPCC__) return m_device_on_host; diff --git a/include/ddc/kernels/fft.hpp b/include/ddc/kernels/fft.hpp index 9807bd550..5db1e2664 100644 --- a/include/ddc/kernels/fft.hpp +++ b/include/ddc/kernels/fft.hpp @@ -91,13 +91,13 @@ constexpr bool is_complex_v = is_complex::value; // LastSelector: returns a if Dim==Last, else b template -constexpr T LastSelector(const T a, const T b) +KOKKOS_FUNCTION constexpr T LastSelector(const T a, const T b) { return std::is_same_v ? a : b; } template -constexpr T LastSelector(const T a, const T b) +KOKKOS_FUNCTION constexpr T LastSelector(const T a, const T b) { return LastSelector(a, b); } @@ -401,10 +401,10 @@ void core( else if constexpr (std::is_same_v) { if constexpr (std::is_same_v, float>) { fftwf_init_threads(); - fftwf_plan_with_nthreads(ExecSpace::concurrency()); + fftwf_plan_with_nthreads(execSpace.concurrency()); } else { fftw_init_threads(); - fftw_plan_with_nthreads(ExecSpace::concurrency()); + fftw_plan_with_nthreads(execSpace.concurrency()); } _fftw_plan plan = _fftw_plan_many_dft( kwargs.direction == ddc::FFT_Direction::FORWARD ? FFTW_FORWARD : FFTW_BACKWARD, diff --git a/include/ddc/non_uniform_point_sampling.hpp b/include/ddc/non_uniform_point_sampling.hpp index 2f1a849ca..648bd2102 100644 --- a/include/ddc/non_uniform_point_sampling.hpp +++ b/include/ddc/non_uniform_point_sampling.hpp @@ -60,7 +60,7 @@ class NonUniformPointSampling /// @brief Construct a `NonUniformPointSampling` using a C++20 "common range". template - explicit inline constexpr Impl(InputRange const& points) + explicit Impl(InputRange const& points) { if constexpr (Kokkos::is_view::value) { Kokkos::deep_copy(m_points, points); @@ -75,7 +75,7 @@ class NonUniformPointSampling /// @brief Construct a `NonUniformPointSampling` using a pair of iterators. template - inline constexpr Impl(InputIt points_begin, InputIt points_end) + Impl(InputIt points_begin, InputIt points_end) { std::vector host_points(points_begin, points_end); Kokkos::View @@ -96,14 +96,14 @@ class NonUniformPointSampling ~Impl() = default; - constexpr std::size_t size() const + KOKKOS_FUNCTION std::size_t size() const { return m_points.size(); } /// @brief Convert a mesh index into a position in `CDim` - constexpr continuous_element_type coordinate( - discrete_element_type const& icoord) const noexcept + KOKKOS_FUNCTION continuous_element_type + coordinate(discrete_element_type const& icoord) const noexcept { return m_points(icoord.uid()); } @@ -134,40 +134,37 @@ std::ostream& operator<<(std::ostream& out, DDimImpl const& mesh) } template -DDC_INLINE_FUNCTION Coordinate coordinate( - DiscreteElement> const& c) +KOKKOS_FUNCTION Coordinate coordinate(DiscreteElement> const& c) { return discrete_space>().coordinate(c); } template -DDC_INLINE_FUNCTION Coordinate distance_at_left( - DiscreteElement> i) +KOKKOS_FUNCTION Coordinate distance_at_left(DiscreteElement> i) { return coordinate(i) - coordinate(i - 1); } template -DDC_INLINE_FUNCTION Coordinate distance_at_right( - DiscreteElement> i) +KOKKOS_FUNCTION Coordinate distance_at_right(DiscreteElement> i) { return coordinate(i + 1) - coordinate(i); } template -DDC_INLINE_FUNCTION Coordinate rmin(DiscreteDomain> const& d) +KOKKOS_FUNCTION Coordinate rmin(DiscreteDomain> const& d) { return coordinate(d.front()); } template -DDC_INLINE_FUNCTION Coordinate rmax(DiscreteDomain> const& d) +KOKKOS_FUNCTION Coordinate rmax(DiscreteDomain> const& d) { return coordinate(d.back()); } template -DDC_INLINE_FUNCTION Coordinate rlength(DiscreteDomain> const& d) +KOKKOS_FUNCTION Coordinate rlength(DiscreteDomain> const& d) { return rmax(d) - rmin(d); } diff --git a/include/ddc/periodic_sampling.hpp b/include/ddc/periodic_sampling.hpp index 0335159e7..42dc02ad7 100644 --- a/include/ddc/periodic_sampling.hpp +++ b/include/ddc/periodic_sampling.hpp @@ -73,7 +73,7 @@ class PeriodicSampling * @param step the real distance between two points of mesh distance 1 * @param n_period the number of steps in a period */ - constexpr Impl(continuous_element_type origin, Real step, std::size_t n_period) + Impl(continuous_element_type origin, Real step, std::size_t n_period) : m_origin(origin) , m_step(step) , m_n_period(n_period) @@ -85,32 +85,32 @@ class PeriodicSampling ~Impl() = default; /// @brief Lower bound index of the mesh - constexpr continuous_element_type origin() const noexcept + KOKKOS_FUNCTION continuous_element_type origin() const noexcept { return m_origin; } /// @brief Lower bound index of the mesh - constexpr discrete_element_type front() const noexcept + KOKKOS_FUNCTION discrete_element_type front() const noexcept { return discrete_element_type {0}; } /// @brief Spacing step of the mesh - constexpr Real step() const + KOKKOS_FUNCTION Real step() const { return m_step; } /// @brief Number of steps in a period - constexpr std::size_t n_period() const + KOKKOS_FUNCTION std::size_t n_period() const { return m_n_period; } /// @brief Convert a mesh index into a position in `CDim` - constexpr continuous_element_type coordinate( - discrete_element_type const& icoord) const noexcept + KOKKOS_FUNCTION continuous_element_type + coordinate(discrete_element_type const& icoord) const noexcept { return m_origin + continuous_element_type( @@ -243,7 +243,7 @@ std::ostream& operator<<(std::ostream& out, DDimImpl const& mesh) /// @brief Lower bound index of the mesh template -DDC_INLINE_FUNCTION std:: +KOKKOS_FUNCTION std:: enable_if_t, typename DDim::continuous_element_type> origin() noexcept { @@ -252,53 +252,51 @@ DDC_INLINE_FUNCTION std:: /// @brief Lower bound index of the mesh template -DDC_INLINE_FUNCTION std:: - enable_if_t, typename DDim::discrete_element_type> - front() noexcept +KOKKOS_FUNCTION std::enable_if_t, typename DDim::discrete_element_type> +front() noexcept { return discrete_space().front(); } /// @brief Spacing step of the mesh template -DDC_INLINE_FUNCTION std::enable_if_t, Real> step() noexcept +KOKKOS_FUNCTION std::enable_if_t, Real> step() noexcept { return discrete_space().step(); } template -DDC_INLINE_FUNCTION constexpr Coordinate coordinate( - DiscreteElement> const& c) +KOKKOS_FUNCTION Coordinate coordinate(DiscreteElement> const& c) { return discrete_space>().coordinate(c); } template -DDC_INLINE_FUNCTION Coordinate distance_at_left(DiscreteElement>) +KOKKOS_FUNCTION Coordinate distance_at_left(DiscreteElement>) { return Coordinate(step>()); } template -DDC_INLINE_FUNCTION Coordinate distance_at_right(DiscreteElement>) +KOKKOS_FUNCTION Coordinate distance_at_right(DiscreteElement>) { return Coordinate(step>()); } template -DDC_INLINE_FUNCTION Coordinate rmin(DiscreteDomain> const& d) +KOKKOS_FUNCTION Coordinate rmin(DiscreteDomain> const& d) { return coordinate(d.front()); } template -DDC_INLINE_FUNCTION Coordinate rmax(DiscreteDomain> const& d) +KOKKOS_FUNCTION Coordinate rmax(DiscreteDomain> const& d) { return coordinate(d.back()); } template -DDC_INLINE_FUNCTION Coordinate rlength(DiscreteDomain> const& d) +KOKKOS_FUNCTION Coordinate rlength(DiscreteDomain> const& d) { return rmax(d) - rmin(d); } diff --git a/include/ddc/reducer.hpp b/include/ddc/reducer.hpp index b70e9d998..820459e5f 100644 --- a/include/ddc/reducer.hpp +++ b/include/ddc/reducer.hpp @@ -3,6 +3,8 @@ #include #include +#include + namespace ddc::reducer { template @@ -10,7 +12,8 @@ struct sum { using value_type = T; - constexpr value_type operator()(value_type const& lhs, value_type const& rhs) const noexcept + KOKKOS_FUNCTION constexpr value_type operator()(value_type const& lhs, value_type const& rhs) + const noexcept { return lhs + rhs; } @@ -21,7 +24,8 @@ struct prod { using value_type = T; - constexpr value_type operator()(value_type const& lhs, value_type const& rhs) const noexcept + KOKKOS_FUNCTION constexpr value_type operator()(value_type const& lhs, value_type const& rhs) + const noexcept { return lhs * rhs; } @@ -31,7 +35,8 @@ struct land { using value_type = bool; - constexpr value_type operator()(value_type const& lhs, value_type const& rhs) const noexcept + KOKKOS_FUNCTION constexpr value_type operator()(value_type const& lhs, value_type const& rhs) + const noexcept { return lhs && rhs; } @@ -41,7 +46,8 @@ struct lor { using value_type = bool; - constexpr value_type operator()(value_type const& lhs, value_type const& rhs) const noexcept + KOKKOS_FUNCTION constexpr value_type operator()(value_type const& lhs, value_type const& rhs) + const noexcept { return lhs || rhs; } @@ -52,7 +58,8 @@ struct band { using value_type = T; - constexpr value_type operator()(value_type const& lhs, value_type const& rhs) const noexcept + KOKKOS_FUNCTION constexpr value_type operator()(value_type const& lhs, value_type const& rhs) + const noexcept { return lhs & rhs; } @@ -63,7 +70,8 @@ struct bor { using value_type = T; - constexpr value_type operator()(value_type const& lhs, value_type const& rhs) const noexcept + KOKKOS_FUNCTION constexpr value_type operator()(value_type const& lhs, value_type const& rhs) + const noexcept { return lhs | rhs; } @@ -74,7 +82,8 @@ struct bxor { using value_type = T; - constexpr value_type operator()(value_type const& lhs, value_type const& rhs) const noexcept + KOKKOS_FUNCTION constexpr value_type operator()(value_type const& lhs, value_type const& rhs) + const noexcept { return lhs ^ rhs; } @@ -85,7 +94,8 @@ struct min { using value_type = T; - constexpr value_type operator()(value_type const& lhs, value_type const& rhs) const noexcept + KOKKOS_FUNCTION constexpr value_type operator()(value_type const& lhs, value_type const& rhs) + const noexcept { return lhs < rhs ? lhs : rhs; } @@ -96,7 +106,8 @@ struct max { using value_type = T; - constexpr value_type operator()(value_type const& lhs, value_type const& rhs) const noexcept + KOKKOS_FUNCTION constexpr value_type operator()(value_type const& lhs, value_type const& rhs) + const noexcept { return lhs > rhs ? lhs : rhs; } @@ -107,7 +118,8 @@ struct minmax { using value_type = std::pair; - constexpr value_type operator()(value_type const& lhs, value_type const& rhs) const noexcept + KOKKOS_FUNCTION constexpr value_type operator()(value_type const& lhs, value_type const& rhs) + const noexcept { return value_type( lhs.first < rhs.first ? lhs.first : rhs.first, diff --git a/include/ddc/transform_reduce.hpp b/include/ddc/transform_reduce.hpp index aa9708126..3b0d358cd 100644 --- a/include/ddc/transform_reduce.hpp +++ b/include/ddc/transform_reduce.hpp @@ -133,7 +133,6 @@ class TransformReducerKokkosLambdaAdapter Functor functor; public: - DDC_FORCEINLINE_FUNCTION TransformReducerKokkosLambdaAdapter(Reducer const& r, Functor const& f) : reducer(r), functor(f) { } diff --git a/include/ddc/uniform_point_sampling.hpp b/include/ddc/uniform_point_sampling.hpp index 514ae1691..b5f21012f 100644 --- a/include/ddc/uniform_point_sampling.hpp +++ b/include/ddc/uniform_point_sampling.hpp @@ -69,7 +69,7 @@ class UniformPointSampling * @param origin the real coordinate of mesh coordinate 0 * @param step the real distance between two points of mesh distance 1 */ - constexpr Impl(continuous_element_type origin, Real step) : m_origin(origin), m_step(step) + Impl(continuous_element_type origin, Real step) : m_origin(origin), m_step(step) { assert(step > 0); } @@ -77,26 +77,26 @@ class UniformPointSampling ~Impl() = default; /// @brief Lower bound index of the mesh - constexpr continuous_element_type origin() const noexcept + KOKKOS_FUNCTION continuous_element_type origin() const noexcept { return m_origin; } /// @brief Lower bound index of the mesh - constexpr discrete_element_type front() const noexcept + KOKKOS_FUNCTION discrete_element_type front() const noexcept { return discrete_element_type {0}; } /// @brief Spacing step of the mesh - constexpr Real step() const + KOKKOS_FUNCTION Real step() const { return m_step; } /// @brief Convert a mesh index into a position in `CDim` - constexpr continuous_element_type coordinate( - discrete_element_type const& icoord) const noexcept + KOKKOS_FUNCTION continuous_element_type + coordinate(discrete_element_type const& icoord) const noexcept { return m_origin + continuous_element_type(icoord.uid()) * m_step; } @@ -215,15 +215,14 @@ std::ostream& operator<<(std::ostream& out, DDimImpl const& mesh) } template -DDC_INLINE_FUNCTION Coordinate coordinate( - DiscreteElement> const& c) +KOKKOS_FUNCTION Coordinate coordinate(DiscreteElement> const& c) { return discrete_space>().coordinate(c); } /// @brief Lower bound index of the mesh template -DDC_INLINE_FUNCTION std:: +KOKKOS_FUNCTION std:: enable_if_t, typename DDim::continuous_element_type> origin() noexcept { @@ -232,46 +231,45 @@ DDC_INLINE_FUNCTION std:: /// @brief Lower bound index of the mesh template -DDC_INLINE_FUNCTION std:: - enable_if_t, typename DDim::discrete_element_type> - front() noexcept +KOKKOS_FUNCTION std::enable_if_t, typename DDim::discrete_element_type> +front() noexcept { return discrete_space().front(); } /// @brief Spacing step of the mesh template -DDC_INLINE_FUNCTION std::enable_if_t, Real> step() noexcept +KOKKOS_FUNCTION std::enable_if_t, Real> step() noexcept { return discrete_space().step(); } template -DDC_INLINE_FUNCTION Coordinate distance_at_left(DiscreteElement>) +KOKKOS_FUNCTION Coordinate distance_at_left(DiscreteElement>) { return Coordinate(step>()); } template -DDC_INLINE_FUNCTION Coordinate distance_at_right(DiscreteElement>) +KOKKOS_FUNCTION Coordinate distance_at_right(DiscreteElement>) { return Coordinate(step>()); } template -DDC_INLINE_FUNCTION Coordinate rmin(DiscreteDomain> const& d) +KOKKOS_FUNCTION Coordinate rmin(DiscreteDomain> const& d) { return coordinate(d.front()); } template -DDC_INLINE_FUNCTION Coordinate rmax(DiscreteDomain> const& d) +KOKKOS_FUNCTION Coordinate rmax(DiscreteDomain> const& d) { return coordinate(d.back()); } template -DDC_INLINE_FUNCTION Coordinate rlength(DiscreteDomain> const& d) +KOKKOS_FUNCTION Coordinate rlength(DiscreteDomain> const& d) { return rmax(d) - rmin(d); } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 004468464..fb09849f9 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,5 +1,8 @@ # SPDX-License-Identifier: MIT cmake_minimum_required(VERSION 3.15) + +add_subdirectory(discrete_space) + add_executable(ddc_tests main.cpp aligned_allocator.cpp diff --git a/tests/discrete_space/CMakeLists.txt b/tests/discrete_space/CMakeLists.txt new file mode 100644 index 000000000..c9b0a6049 --- /dev/null +++ b/tests/discrete_space/CMakeLists.txt @@ -0,0 +1,16 @@ +add_library(discrete_space_tests_lib STATIC discrete_space.cpp) +target_link_libraries(discrete_space_tests_lib + PUBLIC + GTest::gtest + DDC::DDC +) + +add_executable(discrete_space_tests main.cpp) +target_link_libraries(discrete_space_tests + PUBLIC + discrete_space_tests_lib + GTest::gtest + DDC::DDC +) + +gtest_discover_tests(discrete_space_tests) diff --git a/tests/discrete_space/discrete_space.cpp b/tests/discrete_space/discrete_space.cpp new file mode 100644 index 000000000..a30e4c5b9 --- /dev/null +++ b/tests/discrete_space/discrete_space.cpp @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: MIT + +#include + +#include + +#include "discrete_space.hpp" + +struct DimX; +using DDimX = ddc::UniformPointSampling; + +void do_not_optimize_away_discrete_space_tests() {} + +TEST(DiscreteSpace, Initialization) +{ + // ddc::init_discrete_space(std::vector>()); + ddc::init_discrete_space(DDimX:: + init(ddc::Coordinate(0), + ddc::Coordinate(1), + ddc::DiscreteVector(2))); +} diff --git a/tests/discrete_space/discrete_space.hpp b/tests/discrete_space/discrete_space.hpp new file mode 100644 index 000000000..26f61aece --- /dev/null +++ b/tests/discrete_space/discrete_space.hpp @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: MIT + +#pragma once + +void do_not_optimize_away_discrete_space_tests(); diff --git a/tests/discrete_space/main.cpp b/tests/discrete_space/main.cpp new file mode 100644 index 000000000..379b55161 --- /dev/null +++ b/tests/discrete_space/main.cpp @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: MIT + +#include + +#include + +#include "discrete_space.hpp" + +namespace testing::internal { +// accessing gtest internals is not very clean, but gtest provides no public access... +extern bool g_help_flag; +} // namespace testing::internal + +int main(int argc, char** argv) +{ + ::testing::InitGoogleTest(&argc, argv); + do_not_optimize_away_discrete_space_tests(); + if (::testing::GTEST_FLAG(list_tests) || ::testing::internal::g_help_flag) { + // do not initialize DDC just to list tests or show help so as to be able to run + // a Cuda/Hip/etc. enabled DDC with no device available + return RUN_ALL_TESTS(); + } else { + ddc::ScopeGuard scope(argc, argv); + return RUN_ALL_TESTS(); + } +}