30 #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("sse4.1")))
33 #define _MM_FROUND_TO_NEAREST_INT 0x00
34 #define _MM_FROUND_TO_NEG_INF 0x01
35 #define _MM_FROUND_TO_POS_INF 0x02
36 #define _MM_FROUND_TO_ZERO 0x03
37 #define _MM_FROUND_CUR_DIRECTION 0x04
39 #define _MM_FROUND_RAISE_EXC 0x00
40 #define _MM_FROUND_NO_EXC 0x08
42 #define _MM_FROUND_NINT (_MM_FROUND_RAISE_EXC | _MM_FROUND_TO_NEAREST_INT)
43 #define _MM_FROUND_FLOOR (_MM_FROUND_RAISE_EXC | _MM_FROUND_TO_NEG_INF)
44 #define _MM_FROUND_CEIL (_MM_FROUND_RAISE_EXC | _MM_FROUND_TO_POS_INF)
45 #define _MM_FROUND_TRUNC (_MM_FROUND_RAISE_EXC | _MM_FROUND_TO_ZERO)
46 #define _MM_FROUND_RINT (_MM_FROUND_RAISE_EXC | _MM_FROUND_CUR_DIRECTION)
47 #define _MM_FROUND_NEARBYINT (_MM_FROUND_NO_EXC | _MM_FROUND_CUR_DIRECTION)
49 #define _mm_ceil_ps(X) _mm_round_ps((X), _MM_FROUND_CEIL)
50 #define _mm_ceil_pd(X) _mm_round_pd((X), _MM_FROUND_CEIL)
51 #define _mm_ceil_ss(X, Y) _mm_round_ss((X), (Y), _MM_FROUND_CEIL)
52 #define _mm_ceil_sd(X, Y) _mm_round_sd((X), (Y), _MM_FROUND_CEIL)
54 #define _mm_floor_ps(X) _mm_round_ps((X), _MM_FROUND_FLOOR)
55 #define _mm_floor_pd(X) _mm_round_pd((X), _MM_FROUND_FLOOR)
56 #define _mm_floor_ss(X, Y) _mm_round_ss((X), (Y), _MM_FROUND_FLOOR)
57 #define _mm_floor_sd(X, Y) _mm_round_sd((X), (Y), _MM_FROUND_FLOOR)
59 #define _mm_round_ps(X, M) __extension__ ({ \
60 (__m128)__builtin_ia32_roundps((__v4sf)(__m128)(X), (M)); })
62 #define _mm_round_ss(X, Y, M) __extension__ ({ \
63 (__m128)__builtin_ia32_roundss((__v4sf)(__m128)(X), \
64 (__v4sf)(__m128)(Y), (M)); })
66 #define _mm_round_pd(X, M) __extension__ ({ \
67 (__m128d)__builtin_ia32_roundpd((__v2df)(__m128d)(X), (M)); })
69 #define _mm_round_sd(X, Y, M) __extension__ ({ \
70 (__m128d)__builtin_ia32_roundsd((__v2df)(__m128d)(X), \
71 (__v2df)(__m128d)(Y), (M)); })
74 #define _mm_blend_pd(V1, V2, M) __extension__ ({ \
75 (__m128d)__builtin_shufflevector((__v2df)(__m128d)(V1), \
76 (__v2df)(__m128d)(V2), \
77 (((M) & 0x01) ? 2 : 0), \
78 (((M) & 0x02) ? 3 : 1)); })
80 #define _mm_blend_ps(V1, V2, M) __extension__ ({ \
81 (__m128)__builtin_shufflevector((__v4sf)(__m128)(V1), (__v4sf)(__m128)(V2), \
82 (((M) & 0x01) ? 4 : 0), \
83 (((M) & 0x02) ? 5 : 1), \
84 (((M) & 0x04) ? 6 : 2), \
85 (((M) & 0x08) ? 7 : 3)); })
90 return (__m128d) __builtin_ia32_blendvpd ((__v2df)__V1, (__v2df)__V2,
97 return (__m128) __builtin_ia32_blendvps ((__v4sf)__V1, (__v4sf)__V2,
104 return (__m128i) __builtin_ia32_pblendvb128 ((__v16qi)__V1, (__v16qi)__V2,
108 #define _mm_blend_epi16(V1, V2, M) __extension__ ({ \
109 (__m128i)__builtin_shufflevector((__v8hi)(__m128i)(V1), \
110 (__v8hi)(__m128i)(V2), \
111 (((M) & 0x01) ? 8 : 0), \
112 (((M) & 0x02) ? 9 : 1), \
113 (((M) & 0x04) ? 10 : 2), \
114 (((M) & 0x08) ? 11 : 3), \
115 (((M) & 0x10) ? 12 : 4), \
116 (((M) & 0x20) ? 13 : 5), \
117 (((M) & 0x40) ? 14 : 6), \
118 (((M) & 0x80) ? 15 : 7)); })
124 return (__m128i) ((__v4si)__V1 * (__v4si)__V2);
130 return (__m128i) __builtin_ia32_pmuldq128 ((__v4si)__V1, (__v4si)__V2);
134 #define _mm_dp_ps(X, Y, M) __extension__ ({ \
135 (__m128) __builtin_ia32_dpps((__v4sf)(__m128)(X), \
136 (__v4sf)(__m128)(Y), (M)); })
138 #define _mm_dp_pd(X, Y, M) __extension__ ({\
139 (__m128d) __builtin_ia32_dppd((__v2df)(__m128d)(X), \
140 (__v2df)(__m128d)(Y), (M)); })
146 return (__m128i) __builtin_ia32_movntdqa ((
const __v2di *) __V);
153 return (__m128i) __builtin_ia32_pminsb128 ((__v16qi) __V1, (__v16qi) __V2);
159 return (__m128i) __builtin_ia32_pmaxsb128 ((__v16qi) __V1, (__v16qi) __V2);
165 return (__m128i) __builtin_ia32_pminuw128 ((__v8hi) __V1, (__v8hi) __V2);
171 return (__m128i) __builtin_ia32_pmaxuw128 ((__v8hi) __V1, (__v8hi) __V2);
177 return (__m128i) __builtin_ia32_pminsd128 ((__v4si) __V1, (__v4si) __V2);
183 return (__m128i) __builtin_ia32_pmaxsd128 ((__v4si) __V1, (__v4si) __V2);
189 return (__m128i) __builtin_ia32_pminud128((__v4si) __V1, (__v4si) __V2);
195 return (__m128i) __builtin_ia32_pmaxud128((__v4si) __V1, (__v4si) __V2);
199 #define _mm_insert_ps(X, Y, N) __builtin_ia32_insertps128((X), (Y), (N))
200 #define _mm_extract_ps(X, N) (__extension__ \
201 ({ union { int __i; float __f; } __t; \
202 __v4sf __a = (__v4sf)(__m128)(X); \
203 __t.__f = __a[(N) & 3]; \
208 #define _MM_EXTRACT_FLOAT(D, X, N) (__extension__ ({ __v4sf __a = (__v4sf)(X); \
213 #define _MM_MK_INSERTPS_NDX(X, Y, Z) (((X) << 6) | ((Y) << 4) | (Z))
216 #define _MM_PICK_OUT_PS(X, N) _mm_insert_ps (_mm_setzero_ps(), (X), \
217 _MM_MK_INSERTPS_NDX((N), 0, 0x0e))
220 #define _mm_insert_epi8(X, I, N) (__extension__ \
221 ({ __v16qi __a = (__v16qi)(__m128i)(X); \
222 __a[(N) & 15] = (I); \
224 #define _mm_insert_epi32(X, I, N) (__extension__ \
225 ({ __v4si __a = (__v4si)(__m128i)(X); \
226 __a[(N) & 3] = (I); \
229 #define _mm_insert_epi64(X, I, N) (__extension__ \
230 ({ __v2di __a = (__v2di)(__m128i)(X); \
231 __a[(N) & 1] = (I); \
238 #define _mm_extract_epi8(X, N) (__extension__ \
239 ({ __v16qi __a = (__v16qi)(__m128i)(X); \
240 (int)(unsigned char) __a[(N) & 15];}))
241 #define _mm_extract_epi32(X, N) (__extension__ \
242 ({ __v4si __a = (__v4si)(__m128i)(X); \
243 (int)__a[(N) & 3];}))
245 #define _mm_extract_epi64(X, N) (__extension__ \
246 ({ __v2di __a = (__v2di)(__m128i)(X); \
247 (long long)__a[(N) & 1];}))
254 return __builtin_ia32_ptestz128((__v2di)__M, (__v2di)__V);
260 return __builtin_ia32_ptestc128((__v2di)__M, (__v2di)__V);
266 return __builtin_ia32_ptestnzc128((__v2di)__M, (__v2di)__V);
269 #define _mm_test_all_ones(V) _mm_testc_si128((V), _mm_cmpeq_epi32((V), (V)))
270 #define _mm_test_mix_ones_zeros(M, V) _mm_testnzc_si128((M), (V))
271 #define _mm_test_all_zeros(M, V) _mm_testz_si128 ((M), (V))
277 return (__m128i)((__v2di)__V1 == (__v2di)__V2);
286 return (__m128i)__builtin_convertvector(__builtin_shufflevector((__v16qs)__V, (__v16qs)__V, 0, 1, 2, 3, 4, 5, 6, 7), __v8hi);
294 return (__m128i)__builtin_convertvector(__builtin_shufflevector((__v16qs)__V, (__v16qs)__V, 0, 1, 2, 3), __v4si);
302 typedef signed char __v16qs
__attribute__((__vector_size__(16)));
303 return (__m128i)__builtin_convertvector(__builtin_shufflevector((__v16qs)__V, (__v16qs)__V, 0, 1), __v2di);
309 return (__m128i)__builtin_convertvector(__builtin_shufflevector((__v8hi)__V, (__v8hi)__V, 0, 1, 2, 3), __v4si);
315 return (__m128i)__builtin_convertvector(__builtin_shufflevector((__v8hi)__V, (__v8hi)__V, 0, 1), __v2di);
321 return (__m128i)__builtin_convertvector(__builtin_shufflevector((__v4si)__V, (__v4si)__V, 0, 1), __v2di);
328 return (__m128i) __builtin_ia32_pmovzxbw128((__v16qi) __V);
334 return (__m128i) __builtin_ia32_pmovzxbd128((__v16qi)__V);
340 return (__m128i) __builtin_ia32_pmovzxbq128((__v16qi)__V);
346 return (__m128i) __builtin_ia32_pmovzxwd128((__v8hi)__V);
352 return (__m128i) __builtin_ia32_pmovzxwq128((__v8hi)__V);
358 return (__m128i) __builtin_ia32_pmovzxdq128((__v4si)__V);
365 return (__m128i) __builtin_ia32_packusdw128((__v4si)__V1, (__v4si)__V2);
369 #define _mm_mpsadbw_epu8(X, Y, M) __extension__ ({ \
370 (__m128i) __builtin_ia32_mpsadbw128((__v16qi)(__m128i)(X), \
371 (__v16qi)(__m128i)(Y), (M)); })
376 return (__m128i) __builtin_ia32_phminposuw128((__v8hi)__V);
384 #undef __DEFAULT_FN_ATTRS
385 #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__, __target__("sse4.2")))
388 #define _SIDD_UBYTE_OPS 0x00
389 #define _SIDD_UWORD_OPS 0x01
390 #define _SIDD_SBYTE_OPS 0x02
391 #define _SIDD_SWORD_OPS 0x03
394 #define _SIDD_CMP_EQUAL_ANY 0x00
395 #define _SIDD_CMP_RANGES 0x04
396 #define _SIDD_CMP_EQUAL_EACH 0x08
397 #define _SIDD_CMP_EQUAL_ORDERED 0x0c
400 #define _SIDD_POSITIVE_POLARITY 0x00
401 #define _SIDD_NEGATIVE_POLARITY 0x10
402 #define _SIDD_MASKED_POSITIVE_POLARITY 0x20
403 #define _SIDD_MASKED_NEGATIVE_POLARITY 0x30
406 #define _SIDD_LEAST_SIGNIFICANT 0x00
407 #define _SIDD_MOST_SIGNIFICANT 0x40
410 #define _SIDD_BIT_MASK 0x00
411 #define _SIDD_UNIT_MASK 0x40
414 #define _mm_cmpistrm(A, B, M) \
415 (__m128i)__builtin_ia32_pcmpistrm128((__v16qi)(__m128i)(A), \
416 (__v16qi)(__m128i)(B), (int)(M))
417 #define _mm_cmpistri(A, B, M) \
418 (int)__builtin_ia32_pcmpistri128((__v16qi)(__m128i)(A), \
419 (__v16qi)(__m128i)(B), (int)(M))
421 #define _mm_cmpestrm(A, LA, B, LB, M) \
422 (__m128i)__builtin_ia32_pcmpestrm128((__v16qi)(__m128i)(A), (int)(LA), \
423 (__v16qi)(__m128i)(B), (int)(LB), \
425 #define _mm_cmpestri(A, LA, B, LB, M) \
426 (int)__builtin_ia32_pcmpestri128((__v16qi)(__m128i)(A), (int)(LA), \
427 (__v16qi)(__m128i)(B), (int)(LB), \
431 #define _mm_cmpistra(A, B, M) \
432 (int)__builtin_ia32_pcmpistria128((__v16qi)(__m128i)(A), \
433 (__v16qi)(__m128i)(B), (int)(M))
434 #define _mm_cmpistrc(A, B, M) \
435 (int)__builtin_ia32_pcmpistric128((__v16qi)(__m128i)(A), \
436 (__v16qi)(__m128i)(B), (int)(M))
437 #define _mm_cmpistro(A, B, M) \
438 (int)__builtin_ia32_pcmpistrio128((__v16qi)(__m128i)(A), \
439 (__v16qi)(__m128i)(B), (int)(M))
440 #define _mm_cmpistrs(A, B, M) \
441 (int)__builtin_ia32_pcmpistris128((__v16qi)(__m128i)(A), \
442 (__v16qi)(__m128i)(B), (int)(M))
443 #define _mm_cmpistrz(A, B, M) \
444 (int)__builtin_ia32_pcmpistriz128((__v16qi)(__m128i)(A), \
445 (__v16qi)(__m128i)(B), (int)(M))
447 #define _mm_cmpestra(A, LA, B, LB, M) \
448 (int)__builtin_ia32_pcmpestria128((__v16qi)(__m128i)(A), (int)(LA), \
449 (__v16qi)(__m128i)(B), (int)(LB), \
451 #define _mm_cmpestrc(A, LA, B, LB, M) \
452 (int)__builtin_ia32_pcmpestric128((__v16qi)(__m128i)(A), (int)(LA), \
453 (__v16qi)(__m128i)(B), (int)(LB), \
455 #define _mm_cmpestro(A, LA, B, LB, M) \
456 (int)__builtin_ia32_pcmpestrio128((__v16qi)(__m128i)(A), (int)(LA), \
457 (__v16qi)(__m128i)(B), (int)(LB), \
459 #define _mm_cmpestrs(A, LA, B, LB, M) \
460 (int)__builtin_ia32_pcmpestris128((__v16qi)(__m128i)(A), (int)(LA), \
461 (__v16qi)(__m128i)(B), (int)(LB), \
463 #define _mm_cmpestrz(A, LA, B, LB, M) \
464 (int)__builtin_ia32_pcmpestriz128((__v16qi)(__m128i)(A), (int)(LA), \
465 (__v16qi)(__m128i)(B), (int)(LB), \
472 return (__m128i)((__v2di)__V1 > (__v2di)__V2);
479 return __builtin_ia32_crc32qi(__C, __D);
485 return __builtin_ia32_crc32hi(__C, __D);
491 return __builtin_ia32_crc32si(__C, __D);
496 _mm_crc32_u64(
unsigned long long __C,
unsigned long long __D)
498 return __builtin_ia32_crc32di(__C, __D);
502 #undef __DEFAULT_FN_ATTRS
static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_min_epi32(__m128i __V1, __m128i __V2)
static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepi16_epi64(__m128i __V)
static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_max_epu32(__m128i __V1, __m128i __V2)
static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmpeq_epi64(__m128i __V1, __m128i __V2)
static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_packus_epi32(__m128i __V1, __m128i __V2)
static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_max_epi32(__m128i __V1, __m128i __V2)
static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_stream_load_si128(__m128i const *__V)
static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepu16_epi64(__m128i __V)
static __inline__ unsigned int __DEFAULT_FN_ATTRS _mm_crc32_u16(unsigned int __C, unsigned short __D)
static __inline__ __m128d __DEFAULT_FN_ATTRS _mm_blendv_pd(__m128d __V1, __m128d __V2, __m128d __M)
static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_min_epi8(__m128i __V1, __m128i __V2)
static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepi32_epi64(__m128i __V)
static __inline__ unsigned int __DEFAULT_FN_ATTRS _mm_crc32_u32(unsigned int __C, unsigned int __D)
static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_mullo_epi32(__m128i __V1, __m128i __V2)
static __inline__ int __DEFAULT_FN_ATTRS _mm_testz_si128(__m128i __M, __m128i __V)
static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_blendv_epi8(__m128i __V1, __m128i __V2, __m128i __M)
static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cmpgt_epi64(__m128i __V1, __m128i __V2)
static __inline__ int __DEFAULT_FN_ATTRS _mm_testnzc_si128(__m128i __M, __m128i __V)
static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepu8_epi16(__m128i __V)
static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_min_epu16(__m128i __V1, __m128i __V2)
static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_max_epu16(__m128i __V1, __m128i __V2)
static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_mul_epi32(__m128i __V1, __m128i __V2)
static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepu8_epi64(__m128i __V)
static __inline__ __m128 __DEFAULT_FN_ATTRS _mm_blendv_ps(__m128 __V1, __m128 __V2, __m128 __M)
static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_max_epi8(__m128i __V1, __m128i __V2)
static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepu16_epi32(__m128i __V)
static __inline__ unsigned int __DEFAULT_FN_ATTRS _mm_crc32_u8(unsigned int __C, unsigned char __D)
static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepi8_epi64(__m128i __V)
static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_minpos_epu16(__m128i __V)
static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_min_epu32(__m128i __V1, __m128i __V2)
#define __DEFAULT_FN_ATTRS
char __v64qi __attribute__((__vector_size__(64)))
static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepi16_epi32(__m128i __V)
static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepi8_epi32(__m128i __V)
static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepu32_epi64(__m128i __V)
static __inline__ int __DEFAULT_FN_ATTRS _mm_testc_si128(__m128i __M, __m128i __V)
static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepu8_epi32(__m128i __V)
static __inline__ __m128i __DEFAULT_FN_ATTRS _mm_cvtepi8_epi16(__m128i __V)