#ifndef PYTHONIC_UTILS_BROADCAST_COPY_HPP #define PYTHONIC_UTILS_BROADCAST_COPY_HPP #include "pythonic/include/utils/broadcast_copy.hpp" #include "pythonic/types/tuple.hpp" #ifdef _OPENMP #include // as a macro so that an enlightened user can modify this variable :-) #ifndef PYTHRAN_OPENMP_MIN_ITERATION_COUNT #define PYTHRAN_OPENMP_MIN_ITERATION_COUNT 1000 #endif #endif PYTHONIC_NS_BEGIN namespace utils { /* helper for specialization of the broadcasting, vectorizing copy operator * due to expression templates, this may also triggers a lot of *computations! * * ``vector_form'' is set to true if the operation can be done using *Boost.SIMD * * the call operator has four template parameters: * * template * void operator()(E &&self, F const &other, utils::int_, utils::int_) * * ``E'' is the type of the object to which the data are copied * * ``F'' is the type of the object from which the data are copied * * ``N'' is the depth of the loop nest. When it reaches ``1'', we have a raw *loop * that may be vectorizable * * ``D'' is the delta between the number of dimensions of E && F. When set *to a * value greater than ``0'', some broadcasting is needed */ template struct _broadcast_copy; struct fast_novectorize { }; template <> struct _broadcast_copy { template void helper(E &&self, F const &other, SelfIndices &&self_indices, OtherIndices &&other_indices, utils::index_sequence) { std::forward(self) .store((typename std::decay::type::dtype)other.load( (long)std::get(other_indices)...), (long)std::get(self_indices)...); } template void operator()(E &&self, F const &other, SelfIndices &&self_indices, OtherIndices &&other_indices) { helper(std::forward(self), other, self_indices, other_indices, utils::make_index_sequence::type>::value>()); } }; template struct _broadcast_copy { template void operator()(E &&self, F const &other, SelfIndices &&self_indices, OtherIndices &&other_indices) { long const other_size = other.template shape::type::value - N>(); long const self_size = self.template shape::type::value - N>(); if (self_size == other_size) for (long i = 0; i < self_size; ++i) _broadcast_copy{}( std::forward(self), other, std::tuple_cat(self_indices, std::make_tuple(i)), std::tuple_cat(other_indices, std::make_tuple(i))); else for (long i = 0; i < self_size; ++i) _broadcast_copy{}( std::forward(self), other, std::tuple_cat(self_indices, std::make_tuple(i)), std::tuple_cat(other_indices, std::make_tuple(0))); } }; template struct _broadcast_copy { template void operator()(E &&self, F const &other, SelfIndices &&self_indices, OtherIndices &&other_indices) { using broadcaster = typename std::conditional< types::is_dtype::value, types::broadcast::type::dtype>, types::broadcasted>::type; _broadcast_copy{}( std::forward(self), broadcaster(other), std::forward(self_indices), std::forward(other_indices)); } }; template struct _broadcast_copy { template void operator()(E &&self, F const &other, Indices... indices) { long self_size = std::distance(self.begin(), self.end()), other_size = std::distance(other.begin(), other.end()); #ifdef _OPENMP if (other_size >= PYTHRAN_OPENMP_MIN_ITERATION_COUNT) { auto siter = self.begin(); auto oiter = other.begin(); #pragma omp parallel for for (long i = 0; i < other_size; ++i) *(siter + i) = *(oiter + i); } else #endif std::copy(other.begin(), other.end(), self.begin()); // eventually repeat the pattern #ifdef _OPENMP if (self_size >= PYTHRAN_OPENMP_MIN_ITERATION_COUNT * other_size) #pragma omp parallel for for (long i = other_size; i < self_size; i += other_size) std::copy_n(self.begin(), other_size, self.begin() + i); else #endif for (long i = other_size; i < self_size; i += other_size) std::copy_n(self.begin(), other_size, self.begin() + i); } }; // ``D'' is not ``0'' so we should broadcast template struct _broadcast_copy { template void operator()(E &&self, F const &other) { if (types::is_dtype::value) { std::fill(self.begin(), self.end(), other); } else { auto sfirst = self.begin(); *sfirst = other; #ifdef _OPENMP auto siter = sfirst; long n = self.template shape<0>(); if (n >= PYTHRAN_OPENMP_MIN_ITERATION_COUNT) #pragma omp parallel for for (long i = 1; i < n; ++i) *(siter + i) = *sfirst; else #endif std::fill(self.begin() + 1, self.end(), *sfirst); } } template void operator()(E &&self, F const &other, ES, FS) { if (types::is_dtype::value) { std::fill(self.begin(), self.end(), other); } else { auto sfirst = self.begin(); *sfirst = other; #ifdef _OPENMP auto siter = sfirst; long n = self.template shape<0>(); if (n >= PYTHRAN_OPENMP_MIN_ITERATION_COUNT) #pragma omp parallel for for (long i = 1; i < n; ++i) *(siter + i) = *sfirst; else #endif std::fill(self.begin() + 1, self.end(), *sfirst); } } }; #ifdef USE_XSIMD // specialize for SIMD only if available // otherwise use the std::copy fallback template void vbroadcast_copy(E &&self, F const &other) { using T = typename F::dtype; using vT = xsimd::simd_type; static const std::size_t vN = vT::size; long self_size = std::distance(self.begin(), self.end()), other_size = std::distance(other.begin(), other.end()); auto oiter = vectorizer::vbegin(other); const long bound = std::distance(vectorizer::vbegin(other), vectorizer::vend(other)); #ifdef _OPENMP if (bound >= PYTHRAN_OPENMP_MIN_ITERATION_COUNT) { auto iter = vectorizer::vbegin(self); #pragma omp parallel for for (long i = 0; i < bound; ++i) { (iter + i).store(*(oiter + i)); } } else #endif for (auto iter = vectorizer::vbegin(self), end = vectorizer::vend(self); iter != end; ++iter, ++oiter) { iter.store(*oiter); } // tail { auto siter = self.begin(); auto oiter = other.begin(); for (long i = bound * vN; i < other_size; ++i) *(siter + i) = *(oiter + i); } #ifdef _OPENMP if (self_size >= PYTHRAN_OPENMP_MIN_ITERATION_COUNT * other_size) #pragma omp parallel for for (long i = other_size; i < self_size; i += other_size) std::copy_n(self.begin(), other_size, self.begin() + i); else #endif for (long i = other_size; i < self_size; i += other_size) std::copy_n(self.begin(), other_size, self.begin() + i); } template <> struct _broadcast_copy { template void operator()(E &&self, F const &other) { return vbroadcast_copy(std::forward(self), other); } }; template <> struct _broadcast_copy { template void operator()(E &&self, F const &other) { return vbroadcast_copy( std::forward(self), other); } }; #endif template struct broadcast_copy_dispatcher; template struct broadcast_copy_dispatcher { void operator()(E &self, F const &other) { if (utils::no_broadcast_ex(other)) _broadcast_copy{}( self, other, std::make_tuple(), std::make_tuple()); else _broadcast_copy{}(self, other); } }; template struct broadcast_copy_dispatcher { void operator()(E &self, F const &other) { if (utils::no_broadcast_ex(other)) _broadcast_copy{}( self, other, std::make_tuple(), std::make_tuple()); else _broadcast_copy{}(self, other); } }; template E &broadcast_copy(E &self, F const &other) { if (self.size()) broadcast_copy_dispatcher{}(self, other); return self; } /* update */ // ``D'' is not ``0'' so we should broadcast template struct _broadcast_update { template void operator()(E &&self, F const &other) { long n = self.template shape<0>(); auto siter = self.begin(); #ifdef _OPENMP if (n >= PYTHRAN_OPENMP_MIN_ITERATION_COUNT) #pragma omp parallel for for (long i = 0; i < n; ++i) Op{}(*(siter + i), other); else #endif for (long i = 0; i < n; ++i) Op{}(*(siter + i), other); } }; template struct _broadcast_update { template void operator()(E &&self, F const &other) { long other_size = std::distance(other.begin(), other.end()); auto siter = self.begin(); auto oiter = other.begin(); #ifdef _OPENMP if (other_size >= PYTHRAN_OPENMP_MIN_ITERATION_COUNT) #pragma omp parallel for for (long i = 0; i < other_size; ++i) Op{}(*(siter + i), *(oiter + i)); else #endif if (other_size == 1) { auto value = *oiter; for (auto send = self.end(); siter != send; ++siter) Op{}(*siter, value); } else for (auto send = self.end(); siter != send;) { auto ooiter = oiter; for (long i = 0; i < other_size; ++i, ++siter, ++ooiter) Op{}(*siter, *ooiter); } } template void operator()(E &&self, types::broadcast const &other) { auto value = *other.begin(); for (auto siter = self.begin(), send = self.end(); siter != send; ++siter) Op{}(*siter, value); } template void operator()(E &&self, types::broadcasted const &other) { auto value = *other.end(); for (auto siter = self.begin(), send = self.end(); siter != send; ++siter) Op{}(*siter, value); } }; template struct _broadcast_update { template void helper(E &&self, F const &other, SelfIndices &&self_indices, OtherIndices &&other_indices, utils::index_sequence) { self.template update(other.load((long)std::get(other_indices)...), (long)std::get(self_indices)...); } template void operator()(E &&self, F const &other, SelfIndices &&self_indices, OtherIndices &&other_indices) { helper(std::forward(self), other, self_indices, other_indices, utils::make_index_sequence::type>::value>()); } }; template struct _broadcast_update { template void operator()(E &&self, F const &other, SelfIndices &&self_indices, OtherIndices &&other_indices) { auto const other_size = other.template shape::type::value - N>(); auto const self_size = self.template shape::type::value - N>(); if (self_size == other_size) for (long i = 0; i < self_size; ++i) _broadcast_update{}( std::forward(self), other, std::tuple_cat(self_indices, std::make_tuple(i)), std::tuple_cat(other_indices, std::make_tuple(i))); else for (long i = 0; i < self_size; ++i) _broadcast_update{}( std::forward(self), other, std::tuple_cat(self_indices, std::make_tuple(i)), std::tuple_cat(other_indices, std::make_tuple(0))); } }; template struct _broadcast_update { template void operator()(E &&self, F const &other, SelfIndices &&self_indices, OtherIndices &&other_indices) { using broadcaster = typename std::conditional< types::is_dtype::value, types::broadcast::type::dtype>, types::broadcasted>::type; _broadcast_update{}( std::forward(self), broadcaster(other), std::forward(self_indices), std::forward(other_indices)); } }; #ifdef USE_XSIMD // specialize for SIMD only if available // otherwise use the std::copy fallback template void vbroadcast_update(E &&self, F const &other) { using T = typename F::dtype; using vT = typename xsimd::simd_type; long other_size = std::distance(other.begin(), other.end()); static const std::size_t vN = vT::size; auto oiter = vectorizer::vbegin(other); auto iter = vectorizer::vbegin(self); const long bound = std::distance(vectorizer::vbegin(other), vectorizer::vend(other)); #ifdef _OPENMP if (bound >= PYTHRAN_OPENMP_MIN_ITERATION_COUNT) #pragma omp parallel for for (long i = 0; i < bound; i++) { (iter + i).store(Op{}(*(iter + i), *(oiter + i))); } else #endif for (auto end = vectorizer::vend(self); iter != end; ++iter, ++oiter) { iter.store(Op{}(*iter, *oiter)); } // tail { auto siter = self.begin(); auto oiter = other.begin(); for (long i = bound * vN; i < other_size; ++i) Op{}(*(siter + i), *(oiter + i)); } } template void vbroadcast_update(E &&self, types::broadcast const &other) { auto value = *other.begin(); for (auto siter = self.begin(), send = self.end(); siter != send; ++siter) Op{}(*siter, value); } template void vbroadcast_update(E &&self, types::broadcasted const &other) { auto value = *other.end(); for (auto siter = self.begin(), send = self.end(); siter != send; ++siter) Op{}(*siter, value); } template struct _broadcast_update { template void operator()(Args &&... args) { vbroadcast_update(std::forward(args)...); } }; template struct _broadcast_update { template void operator()(Args &&... args) { vbroadcast_update( std::forward(args)...); } }; #endif template struct broadcast_update_dispatcher; template struct broadcast_update_dispatcher { void operator()(E &self, F const &other) { if (utils::no_broadcast_ex(other)) _broadcast_update{}( self, other, std::make_tuple(), std::make_tuple()); else _broadcast_update{}(self, other); } }; template struct broadcast_update_dispatcher { void operator()(E &self, F const &other) { if (utils::no_broadcast_ex(other)) _broadcast_update{}( self, other, std::make_tuple(), std::make_tuple()); else _broadcast_update{}(self, other); } }; template E &broadcast_update(E &self, F const &other) { if (self.size()) broadcast_update_dispatcher{}(self, other); return self; } } PYTHONIC_NS_END #endif