algebramix: include/algebramix/vector

00001 
00002 /******************************************************************************
00003 * MODULE     : vector_sse.hpp
00004 * DESCRIPTION: SSE instructions support for aligned vectors
00005 * COPYRIGHT  : (C) 2008  Joris van der Hoeven and Gregoire Lecerf
00006 *******************************************************************************
00007 * This software falls under the GNU general public license and comes WITHOUT
00008 * ANY WARRANTY WHATSOEVER. See the file $TEXMACS_PATH/LICENSE for more details.
00009 * If you don't have this file, write to the Free Software Foundation, Inc.,
00010 * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
00011 ******************************************************************************/
00012 
00013 #ifndef __MMX__VECTOR_SSE__HPP
00014 #define __MMX__VECTOR_SSE__HPP
00015 
00016 #include <basix/operators.hpp>
00017 #include <basix/double.hpp>
00018 #include <basix/int.hpp>
00019 #include <numerix/sse.hpp>
00020 #include <algebramix/algebramix-config.hpp>
00021 #include <algebramix/vector_aligned.hpp>
00022 
00023 #ifndef NUMERIX_ENABLE_SIMD
00024 #undef ALGEBRAMIX_ENABLE_SIMD
00025 #endif
00026 
00027 namespace mmx {
00028 #ifdef ALGEBRAMIX_ENABLE_SIMD
00029 
00030 /******************************************************************************
00031 * SSE features helpers
00032 ******************************************************************************/
00033 
00034 // According to basix/fast_new.hpp, any allocation of size
00035 // being a multiple of 16 bytes is aligned.
00036 
00037 #ifdef __SSE2__
00038 
00039 template<typename Op, typename C>
00040 struct sse_has_helper { static const bool value= false;};
00041 
00042 STMPL
00043 struct sse_has_helper<id_op, double> {static const bool value= true;};
00044 STMPL
00045 struct sse_has_helper<id_op, int8_t> {static const bool value= true;};
00046 STMPL
00047 struct sse_has_helper<id_op, uint8_t> {static const bool value= true;};
00048 STMPL
00049 struct sse_has_helper<id_op, int16_t> {static const bool value= true;};
00050 STMPL
00051 struct sse_has_helper<id_op, uint16_t> {static const bool value= true;};
00052 STMPL
00053 struct sse_has_helper<id_op, int32_t> {static const bool value= true;};
00054 STMPL
00055 struct sse_has_helper<id_op, uint32_t> {static const bool value= true;};
00056 STMPL
00057 struct sse_has_helper<id_op, int64_t> {static const bool value= true;};
00058 STMPL
00059 struct sse_has_helper<id_op, uint64_t> {static const bool value= true;};
00060 
00061 STMPL
00062 struct sse_has_helper<neg_op, double> {static const bool value= true;};
00063 STMPL
00064 struct sse_has_helper<neg_op, int8_t> {static const bool value= true;};
00065 STMPL
00066 struct sse_has_helper<neg_op, uint8_t> {static const bool value= true;};
00067 STMPL
00068 struct sse_has_helper<neg_op, int16_t> {static const bool value= true;};
00069 STMPL
00070 struct sse_has_helper<neg_op, uint16_t> {static const bool value= true;};
00071 STMPL
00072 struct sse_has_helper<neg_op, int32_t> {static const bool value= true;};
00073 STMPL
00074 struct sse_has_helper<neg_op, uint32_t> {static const bool value= true;};
00075 STMPL
00076 struct sse_has_helper<neg_op, int64_t> {static const bool value= true;};
00077 STMPL
00078 struct sse_has_helper<neg_op, uint64_t> {static const bool value= true;};
00079 
00080 STMPL
00081 struct sse_has_helper<add_op, double> {static const bool value= true;};
00082 STMPL
00083 struct sse_has_helper<add_op, int8_t> {static const bool value= true;};
00084 STMPL
00085 struct sse_has_helper<add_op, uint8_t> {static const bool value= true;};
00086 STMPL
00087 struct sse_has_helper<add_op, int16_t> {static const bool value= true;};
00088 STMPL
00089 struct sse_has_helper<add_op, uint16_t> {static const bool value= true;};
00090 STMPL
00091 struct sse_has_helper<add_op, int32_t> {static const bool value= true;};
00092 STMPL
00093 struct sse_has_helper<add_op, uint32_t> {static const bool value= true;};
00094 STMPL
00095 struct sse_has_helper<add_op, int64_t> {static const bool value= true;};
00096 STMPL
00097 struct sse_has_helper<add_op, uint64_t> {static const bool value= true;};
00098 
00099 STMPL
00100 struct sse_has_helper<sub_op, double> {static const bool value= true;};
00101 STMPL
00102 struct sse_has_helper<sub_op, int8_t> {static const bool value= true;};
00103 STMPL
00104 struct sse_has_helper<sub_op, uint8_t> {static const bool value= true;};
00105 STMPL
00106 struct sse_has_helper<sub_op, int16_t> {static const bool value= true;};
00107 STMPL
00108 struct sse_has_helper<sub_op, uint16_t> {static const bool value= true;};
00109 STMPL
00110 struct sse_has_helper<sub_op, int32_t> {static const bool value= true;};
00111 STMPL
00112 struct sse_has_helper<sub_op, uint32_t> {static const bool value= true;};
00113 STMPL
00114 struct sse_has_helper<sub_op, int64_t> {static const bool value= true;};
00115 STMPL
00116 struct sse_has_helper<sub_op, uint64_t> {static const bool value= true;};
00117 
00118 STMPL
00119 struct sse_has_helper<mul_op, double> {static const bool value= true;};
00120 STMPL
00121 struct sse_has_helper<mul_op, int8_t> {static const bool value= true;};
00122 STMPL
00123 struct sse_has_helper<mul_op, uint8_t> {static const bool value= true;};
00124 STMPL
00125 struct sse_has_helper<mul_op, int16_t> {static const bool value= true;};
00126 STMPL
00127 struct sse_has_helper<mul_op, uint16_t> {static const bool value= true;};
00128 #ifdef __SSE2__
00129 STMPL
00130 struct sse_has_helper<mul_op, uint32_t> {static const bool value= true;};
00131 #endif
00132 #ifdef __SSE4_1__
00133 STMPL
00134 struct sse_has_helper<mul_op, int32_t> {static const bool value= true;};
00135 #endif
00136 
00137 template<typename C>
00138 struct sse_has_helper<lmul_op, C> {
00139   static const bool value= sse_has_helper<mul_op,C>::value; };
00140 
00141 template<typename C>
00142 struct sse_has_helper<rmul_op, C> {
00143   static const bool value= sse_has_helper<mul_op,C>::value; };
00144 
00145 STMPL
00146 struct sse_has_helper<div_op, double> {static const bool value= true;};
00147 
00148 template<typename C>
00149 struct sse_has_helper<ldiv_op, C> {
00150   static const bool value= sse_has_helper<div_op,C>::value; };
00151 
00152 template<typename C>
00153 struct sse_has_helper<rdiv_op, C> {
00154   static const bool value= sse_has_helper<div_op,C>::value; };
00155 
00156 template<typename C>
00157 struct sse_has_helper<mul_add_op, C> {
00158   static const bool value= sse_has_helper<add_op,C>::value &&
00159                            sse_has_helper<mul_op,C>::value; };
00160 
00161 STMPL
00162 struct sse_has_helper<min_op, double> {static const bool value= true;};
00163 STMPL
00164 struct sse_has_helper<min_op, uint8_t> {static const bool value= true;};
00165 #ifdef __SSE4_1__
00166 STMPL
00167 struct sse_has_helper<min_op, int8_t> {static const bool value= true;};
00168 STMPL
00169 struct sse_has_helper<min_op, uint16_t> {static const bool value= true;};
00170 #endif
00171 STMPL
00172 struct sse_has_helper<min_op, int16_t> {static const bool value= true;};
00173 #ifdef __SSE4_1__
00174 STMPL
00175 struct sse_has_helper<min_op, uint32_t> {static const bool value= true;};
00176 STMPL
00177 struct sse_has_helper<min_op, int32_t> {static const bool value= true;};
00178 #endif
00179 
00180 STMPL
00181 struct sse_has_helper<max_op, double> {static const bool value= true;};
00182 STMPL
00183 struct sse_has_helper<max_op, uint8_t> {static const bool value= true;};
00184 #ifdef __SSE4_1__
00185 STMPL
00186 struct sse_has_helper<max_op, int8_t> {static const bool value= true;};
00187 STMPL
00188 struct sse_has_helper<max_op, uint16_t> {static const bool value= true;};
00189 #endif
00190 STMPL
00191 struct sse_has_helper<max_op, int16_t> {static const bool value= true;};
00192 #ifdef __SSE4_1__
00193 STMPL
00194 struct sse_has_helper<max_op, uint32_t> {static const bool value= true;};
00195 STMPL
00196 struct sse_has_helper<max_op, int32_t> {static const bool value= true;};
00197 #endif
00198 
00199 /******************************************************************************
00200 * SSE support for aligned vectors
00201 ******************************************************************************/
00202 
00203 // Unary
00204 template<bool b, typename V, typename W, typename Op, typename C>
00205 struct vec_unary_simd_helper {
00206   typedef implementation<vector_abstractions,V> Vec;
00207   static inline void op (C* dest, const C* s, nat n) {
00208     Vec::template vec_unary<Op, C, C> (dest, s, n); }
00209 };
00210 
00211 template<typename V, typename W, typename Op, typename C>
00212 struct vec_unary_simd_helper <true,V,W,Op,C> {
00213   static inline void op (C* dest, const C* s, nat n) {
00214     typedef typename Simd_type (C) simd_C;
00215     typedef implementation<vector_abstractions,W> SVec;
00216     static const nat m= Simd_size (C);
00217     SVec::template vec_unary<Op, simd_C, simd_C>
00218       ((simd_C*) dest, (const simd_C*) s, n / m); }
00219 };
00220 
00221 template<typename V, typename W, typename Op, typename C>
00222 struct vec_unary_aligned_helper <V,W,Op,C,C> :
00223     vec_unary_simd_helper <sse_has_helper<Op, C>::value,V,W,Op,C> {};
00224 
00225 // Binary
00226 template<bool b, typename V, typename W, typename Op, typename C>
00227 struct vec_binary_simd_helper {
00228   static inline void op (C* dest, const C* s1, const C* s2, nat n) {
00229     typedef implementation<vector_abstractions,V> Vec;
00230       Vec::template vec_binary<Op, C, C, C> (dest, s1, s2, n); }
00231 };
00232 
00233 template<typename V, typename W, typename Op, typename C>
00234 struct vec_binary_simd_helper <true,V,W,Op,C> {
00235   static inline void op (C* dest, const C* s1, const C* s2, nat n) {
00236     typedef typename Simd_type (C) simd_C;
00237     typedef implementation<vector_abstractions,W> SVec;
00238     static const nat m= Simd_size (C);
00239     SVec::template vec_binary<Op, simd_C, simd_C, simd_C>
00240       ((simd_C*) dest, (const simd_C*) s1, (const simd_C*) s2, n / m); }
00241 };
00242 
00243 template<typename V, typename W,typename Op, typename C>
00244 struct vec_binary_aligned_helper <V,W,Op,C,C,C> :
00245     vec_binary_simd_helper <sse_has_helper<Op, C>::value,V,W,Op,C> {};
00246 
00247 // Binary scalar
00248 template<bool b, typename V, typename W, typename Op, typename C>
00249 struct vec_binary_scalar_simd_helper {
00250   static inline void op (C* dest, const C* s, const C& x, nat n) {
00251     typedef implementation<vector_abstractions,V> Vec;
00252     Vec::template vec_binary_scalar<Op, C, C, C > (dest, s, x, n); }
00253 };
00254 
00255 template<typename V, typename W, typename Op, typename C>
00256 struct vec_binary_scalar_simd_helper <true,V,W,Op,C> {
00257   static inline void op (C* dest, const C* s, const C& x, nat n) {
00258     typedef typename Simd_type (C) simd_C;
00259     typedef implementation<vector_abstractions,W> SVec;
00260     static const nat m= Simd_size (C);
00261     simd_C vx= simd_set_duplicate (x);
00262     SVec::template vec_binary_scalar<Op, simd_C, simd_C, simd_C >
00263       ((simd_C*) dest, (const simd_C*) s, vx, n / m); }
00264 };
00265 
00266 template<typename V, typename W, typename Op, typename C>
00267 struct vec_binary_scalar_aligned_helper <V,W,Op,C,C,C> :
00268     vec_binary_scalar_simd_helper <sse_has_helper<Op, C>::value,V,W,Op,C> {};
00269 
00270 // Ternary scalar
00271 template<bool b, typename V, typename W, typename Op, typename C>
00272 struct vec_ternary_scalar_simd_helper {
00273   static inline void op (C* dest, const C* s1, const C* s2, const C& x, nat n) {
00274     typedef implementation<vector_abstractions,V> Vec;
00275     Vec::template vec_ternary_scalar<Op, C, C, C, C> (dest, s1, s2, x, n); }
00276 };
00277 
00278 template<typename V, typename W, typename Op, typename C>
00279 struct vec_ternary_scalar_simd_helper <true,V,W,Op,C> {
00280   static inline void op (C* dest, const C* s1, const C* s2, const C& x, nat n) {
00281     typedef typename Simd_type (C) simd_C;
00282     typedef implementation<vector_abstractions,W> SVec;
00283     static const nat m= Simd_size (C);
00284     simd_C vx= simd_set_duplicate (x);
00285     SVec::template vec_ternary_scalar<Op, simd_C, simd_C, simd_C, simd_C >
00286       ((simd_C*) dest, (const simd_C*) s1, (const simd_C*) s2, vx, n / m); }
00287 };
00288 
00289 template<typename V, typename W, typename Op, typename C>
00290 struct vec_ternary_scalar_aligned_helper <V,W,Op,C,C,C,C> :
00291     vec_ternary_scalar_simd_helper <sse_has_helper<Op, C>::value,V,W,Op,C> {};
00292 
00293 // Ternary scalar scalar
00294 template<bool b, typename V, typename W, typename Op, typename C>
00295 struct vec_ternary_scalar_scalar_simd_helper {
00296   static inline void op (C* dest, const C* s, const C& x1, const C& x2, nat n) {
00297     typedef implementation<vector_abstractions,V> Vec;
00298     Vec::template vec_ternary_scalar_scalar<Op, C, C, C, C>
00299       (dest, s, x1, x2, n); }
00300 };
00301 
00302 template<typename V, typename W, typename Op, typename C>
00303 struct vec_ternary_scalar_scalar_simd_helper <true,V,W,Op,C> {
00304   static inline void op (C* dest, const C* s, const C& x1, const C& x2, nat n) {
00305     typedef typename Simd_type (C) simd_C;
00306     typedef implementation<vector_abstractions,W> SVec;
00307     static const nat m= Simd_size (C);
00308     simd_C vx1= simd_set_duplicate (x1);
00309     simd_C vx2= simd_set_duplicate (x2);
00310     SVec::template vec_ternary_scalar_scalar
00311       <Op, simd_C, simd_C, simd_C, simd_C >
00312       ((simd_C*) dest, (const simd_C*) s, vx1, vx2, n / m); }
00313 };
00314 
00315 template<typename V, typename W, typename Op, typename C>
00316 struct vec_ternary_scalar_scalar_aligned_helper <V,W,Op,C,C,C,C> :
00317     vec_ternary_scalar_scalar_simd_helper
00318       <sse_has_helper<Op, C>::value,V,W,Op,C> {};
00319 
00320 // Binary test
00321 template<bool b, typename V, typename W, typename Op, typename C>
00322 struct vec_binary_test_simd_helper {
00323   static inline bool op (const C* s1, const C* s2, nat n) {
00324     typedef implementation<vector_abstractions,V> Vec;
00325     return Vec::template vec_binary_test<Op, C, C> (s1, s2, n); }
00326 };
00327 
00328 template<typename V, typename W, typename Op, typename C>
00329 struct vec_binary_test_simd_helper <true,V,W,Op,C> {
00330   static inline bool op (const C* s1, const C* s2, nat n) {
00331     typedef typename Simd_type (C) simd_C;
00332     typedef implementation<vector_abstractions,W> SVec;
00333     static const nat m= Simd_size (C);
00334     return SVec::template vec_binary_test<Op, simd_C, simd_C>
00335       ((const simd_C*) s1, (const simd_C*) s2, n / m); }
00336 };
00337 
00338 template<typename V, typename W, typename Op, typename C>
00339 struct vec_binary_test_aligned_helper <V,W,Op,C,C> :
00340   vec_binary_test_simd_helper <sse_has_helper<Op, C>::value,V,W,Op,C> {};
00341 
00342 #if 0
00343   // problem with neutral on simd types. FIXME
00344 template<typename V, typename W, typename C>
00345 struct vec_binary_big_aligned_helper<V,W,mul_add_op,C,C> {
00346   static inline C op (const C* s1, const C* s2, nat n) {
00347     typedef typename Simd_type (C) simd_C;
00348     typedef implementation<vector_abstractions,V> Vec;
00349     typedef implementation<vector_abstractions,W> SVec;
00350     static const nat m= Simd_size (C);
00351     if (sse_has_helper<mul_add_op, C>::value && m > 1) {
00352       simd_C vr= SVec::template vec_binary_big<mul_add_op, simd_C, simd_C>
00353         ((simd_C*) s1, (const simd_C*) s2, n / m);
00354       return simd_big_add<C> (vr);
00355     }
00356     else
00357       return Vec::template vec_binary_big<mul_add_op, C, C> (s1, s2, n); }
00358 };
00359 #endif
00360 #endif // __SSE2__
00361 #endif // ALGEBRAMIX_ENABLE_SIMD
00362 } // namespace mmx
00363 
00364 #endif // __MMX__VECTOR_SSE__HPP
include/algebramix/vector_sse.hpp
