master
1/*===------------- avx512vlvbmi2intrin.h - VBMI2 intrinsics -----------------===
2 *
3 *
4 * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5 * See https://llvm.org/LICENSE.txt for license information.
6 * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 *
8 *===-----------------------------------------------------------------------===
9 */
10#ifndef __IMMINTRIN_H
11#error "Never use <avx512vlvbmi2intrin.h> directly; include <immintrin.h> instead."
12#endif
13
14#ifndef __AVX512VLVBMI2INTRIN_H
15#define __AVX512VLVBMI2INTRIN_H
16
17/* Define the default attributes for the functions in this file. */
18#define __DEFAULT_FN_ATTRS128 \
19 __attribute__((__always_inline__, __nodebug__, \
20 __target__("avx512vl,avx512vbmi2,no-evex512"), \
21 __min_vector_width__(128)))
22#define __DEFAULT_FN_ATTRS256 \
23 __attribute__((__always_inline__, __nodebug__, \
24 __target__("avx512vl,avx512vbmi2,no-evex512"), \
25 __min_vector_width__(256)))
26
27static __inline__ __m128i __DEFAULT_FN_ATTRS128
28_mm_mask_compress_epi16(__m128i __S, __mmask8 __U, __m128i __D)
29{
30 return (__m128i) __builtin_ia32_compresshi128_mask ((__v8hi) __D,
31 (__v8hi) __S,
32 __U);
33}
34
35static __inline__ __m128i __DEFAULT_FN_ATTRS128
36_mm_maskz_compress_epi16(__mmask8 __U, __m128i __D)
37{
38 return (__m128i) __builtin_ia32_compresshi128_mask ((__v8hi) __D,
39 (__v8hi) _mm_setzero_si128(),
40 __U);
41}
42
43static __inline__ __m128i __DEFAULT_FN_ATTRS128
44_mm_mask_compress_epi8(__m128i __S, __mmask16 __U, __m128i __D)
45{
46 return (__m128i) __builtin_ia32_compressqi128_mask ((__v16qi) __D,
47 (__v16qi) __S,
48 __U);
49}
50
51static __inline__ __m128i __DEFAULT_FN_ATTRS128
52_mm_maskz_compress_epi8(__mmask16 __U, __m128i __D)
53{
54 return (__m128i) __builtin_ia32_compressqi128_mask ((__v16qi) __D,
55 (__v16qi) _mm_setzero_si128(),
56 __U);
57}
58
59static __inline__ void __DEFAULT_FN_ATTRS128
60_mm_mask_compressstoreu_epi16(void *__P, __mmask8 __U, __m128i __D)
61{
62 __builtin_ia32_compressstorehi128_mask ((__v8hi *) __P, (__v8hi) __D,
63 __U);
64}
65
66static __inline__ void __DEFAULT_FN_ATTRS128
67_mm_mask_compressstoreu_epi8(void *__P, __mmask16 __U, __m128i __D)
68{
69 __builtin_ia32_compressstoreqi128_mask ((__v16qi *) __P, (__v16qi) __D,
70 __U);
71}
72
73static __inline__ __m128i __DEFAULT_FN_ATTRS128
74_mm_mask_expand_epi16(__m128i __S, __mmask8 __U, __m128i __D)
75{
76 return (__m128i) __builtin_ia32_expandhi128_mask ((__v8hi) __D,
77 (__v8hi) __S,
78 __U);
79}
80
81static __inline__ __m128i __DEFAULT_FN_ATTRS128
82_mm_maskz_expand_epi16(__mmask8 __U, __m128i __D)
83{
84 return (__m128i) __builtin_ia32_expandhi128_mask ((__v8hi) __D,
85 (__v8hi) _mm_setzero_si128(),
86 __U);
87}
88
89static __inline__ __m128i __DEFAULT_FN_ATTRS128
90_mm_mask_expand_epi8(__m128i __S, __mmask16 __U, __m128i __D)
91{
92 return (__m128i) __builtin_ia32_expandqi128_mask ((__v16qi) __D,
93 (__v16qi) __S,
94 __U);
95}
96
97static __inline__ __m128i __DEFAULT_FN_ATTRS128
98_mm_maskz_expand_epi8(__mmask16 __U, __m128i __D)
99{
100 return (__m128i) __builtin_ia32_expandqi128_mask ((__v16qi) __D,
101 (__v16qi) _mm_setzero_si128(),
102 __U);
103}
104
105static __inline__ __m128i __DEFAULT_FN_ATTRS128
106_mm_mask_expandloadu_epi16(__m128i __S, __mmask8 __U, void const *__P)
107{
108 return (__m128i) __builtin_ia32_expandloadhi128_mask ((const __v8hi *)__P,
109 (__v8hi) __S,
110 __U);
111}
112
113static __inline__ __m128i __DEFAULT_FN_ATTRS128
114_mm_maskz_expandloadu_epi16(__mmask8 __U, void const *__P)
115{
116 return (__m128i) __builtin_ia32_expandloadhi128_mask ((const __v8hi *)__P,
117 (__v8hi) _mm_setzero_si128(),
118 __U);
119}
120
121static __inline__ __m128i __DEFAULT_FN_ATTRS128
122_mm_mask_expandloadu_epi8(__m128i __S, __mmask16 __U, void const *__P)
123{
124 return (__m128i) __builtin_ia32_expandloadqi128_mask ((const __v16qi *)__P,
125 (__v16qi) __S,
126 __U);
127}
128
129static __inline__ __m128i __DEFAULT_FN_ATTRS128
130_mm_maskz_expandloadu_epi8(__mmask16 __U, void const *__P)
131{
132 return (__m128i) __builtin_ia32_expandloadqi128_mask ((const __v16qi *)__P,
133 (__v16qi) _mm_setzero_si128(),
134 __U);
135}
136
137static __inline__ __m256i __DEFAULT_FN_ATTRS256
138_mm256_mask_compress_epi16(__m256i __S, __mmask16 __U, __m256i __D)
139{
140 return (__m256i) __builtin_ia32_compresshi256_mask ((__v16hi) __D,
141 (__v16hi) __S,
142 __U);
143}
144
145static __inline__ __m256i __DEFAULT_FN_ATTRS256
146_mm256_maskz_compress_epi16(__mmask16 __U, __m256i __D)
147{
148 return (__m256i) __builtin_ia32_compresshi256_mask ((__v16hi) __D,
149 (__v16hi) _mm256_setzero_si256(),
150 __U);
151}
152
153static __inline__ __m256i __DEFAULT_FN_ATTRS256
154_mm256_mask_compress_epi8(__m256i __S, __mmask32 __U, __m256i __D)
155{
156 return (__m256i) __builtin_ia32_compressqi256_mask ((__v32qi) __D,
157 (__v32qi) __S,
158 __U);
159}
160
161static __inline__ __m256i __DEFAULT_FN_ATTRS256
162_mm256_maskz_compress_epi8(__mmask32 __U, __m256i __D)
163{
164 return (__m256i) __builtin_ia32_compressqi256_mask ((__v32qi) __D,
165 (__v32qi) _mm256_setzero_si256(),
166 __U);
167}
168
169static __inline__ void __DEFAULT_FN_ATTRS256
170_mm256_mask_compressstoreu_epi16(void *__P, __mmask16 __U, __m256i __D)
171{
172 __builtin_ia32_compressstorehi256_mask ((__v16hi *) __P, (__v16hi) __D,
173 __U);
174}
175
176static __inline__ void __DEFAULT_FN_ATTRS256
177_mm256_mask_compressstoreu_epi8(void *__P, __mmask32 __U, __m256i __D)
178{
179 __builtin_ia32_compressstoreqi256_mask ((__v32qi *) __P, (__v32qi) __D,
180 __U);
181}
182
183static __inline__ __m256i __DEFAULT_FN_ATTRS256
184_mm256_mask_expand_epi16(__m256i __S, __mmask16 __U, __m256i __D)
185{
186 return (__m256i) __builtin_ia32_expandhi256_mask ((__v16hi) __D,
187 (__v16hi) __S,
188 __U);
189}
190
191static __inline__ __m256i __DEFAULT_FN_ATTRS256
192_mm256_maskz_expand_epi16(__mmask16 __U, __m256i __D)
193{
194 return (__m256i) __builtin_ia32_expandhi256_mask ((__v16hi) __D,
195 (__v16hi) _mm256_setzero_si256(),
196 __U);
197}
198
199static __inline__ __m256i __DEFAULT_FN_ATTRS256
200_mm256_mask_expand_epi8(__m256i __S, __mmask32 __U, __m256i __D)
201{
202 return (__m256i) __builtin_ia32_expandqi256_mask ((__v32qi) __D,
203 (__v32qi) __S,
204 __U);
205}
206
207static __inline__ __m256i __DEFAULT_FN_ATTRS256
208_mm256_maskz_expand_epi8(__mmask32 __U, __m256i __D)
209{
210 return (__m256i) __builtin_ia32_expandqi256_mask ((__v32qi) __D,
211 (__v32qi) _mm256_setzero_si256(),
212 __U);
213}
214
215static __inline__ __m256i __DEFAULT_FN_ATTRS256
216_mm256_mask_expandloadu_epi16(__m256i __S, __mmask16 __U, void const *__P)
217{
218 return (__m256i) __builtin_ia32_expandloadhi256_mask ((const __v16hi *)__P,
219 (__v16hi) __S,
220 __U);
221}
222
223static __inline__ __m256i __DEFAULT_FN_ATTRS256
224_mm256_maskz_expandloadu_epi16(__mmask16 __U, void const *__P)
225{
226 return (__m256i) __builtin_ia32_expandloadhi256_mask ((const __v16hi *)__P,
227 (__v16hi) _mm256_setzero_si256(),
228 __U);
229}
230
231static __inline__ __m256i __DEFAULT_FN_ATTRS256
232_mm256_mask_expandloadu_epi8(__m256i __S, __mmask32 __U, void const *__P)
233{
234 return (__m256i) __builtin_ia32_expandloadqi256_mask ((const __v32qi *)__P,
235 (__v32qi) __S,
236 __U);
237}
238
239static __inline__ __m256i __DEFAULT_FN_ATTRS256
240_mm256_maskz_expandloadu_epi8(__mmask32 __U, void const *__P)
241{
242 return (__m256i) __builtin_ia32_expandloadqi256_mask ((const __v32qi *)__P,
243 (__v32qi) _mm256_setzero_si256(),
244 __U);
245}
246
247#define _mm256_shldi_epi64(A, B, I) \
248 ((__m256i)__builtin_ia32_vpshldq256((__v4di)(__m256i)(A), \
249 (__v4di)(__m256i)(B), (int)(I)))
250
251#define _mm256_mask_shldi_epi64(S, U, A, B, I) \
252 ((__m256i)__builtin_ia32_selectq_256((__mmask8)(U), \
253 (__v4di)_mm256_shldi_epi64((A), (B), (I)), \
254 (__v4di)(__m256i)(S)))
255
256#define _mm256_maskz_shldi_epi64(U, A, B, I) \
257 ((__m256i)__builtin_ia32_selectq_256((__mmask8)(U), \
258 (__v4di)_mm256_shldi_epi64((A), (B), (I)), \
259 (__v4di)_mm256_setzero_si256()))
260
261#define _mm_shldi_epi64(A, B, I) \
262 ((__m128i)__builtin_ia32_vpshldq128((__v2di)(__m128i)(A), \
263 (__v2di)(__m128i)(B), (int)(I)))
264
265#define _mm_mask_shldi_epi64(S, U, A, B, I) \
266 ((__m128i)__builtin_ia32_selectq_128((__mmask8)(U), \
267 (__v2di)_mm_shldi_epi64((A), (B), (I)), \
268 (__v2di)(__m128i)(S)))
269
270#define _mm_maskz_shldi_epi64(U, A, B, I) \
271 ((__m128i)__builtin_ia32_selectq_128((__mmask8)(U), \
272 (__v2di)_mm_shldi_epi64((A), (B), (I)), \
273 (__v2di)_mm_setzero_si128()))
274
275#define _mm256_shldi_epi32(A, B, I) \
276 ((__m256i)__builtin_ia32_vpshldd256((__v8si)(__m256i)(A), \
277 (__v8si)(__m256i)(B), (int)(I)))
278
279#define _mm256_mask_shldi_epi32(S, U, A, B, I) \
280 ((__m256i)__builtin_ia32_selectd_256((__mmask8)(U), \
281 (__v8si)_mm256_shldi_epi32((A), (B), (I)), \
282 (__v8si)(__m256i)(S)))
283
284#define _mm256_maskz_shldi_epi32(U, A, B, I) \
285 ((__m256i)__builtin_ia32_selectd_256((__mmask8)(U), \
286 (__v8si)_mm256_shldi_epi32((A), (B), (I)), \
287 (__v8si)_mm256_setzero_si256()))
288
289#define _mm_shldi_epi32(A, B, I) \
290 ((__m128i)__builtin_ia32_vpshldd128((__v4si)(__m128i)(A), \
291 (__v4si)(__m128i)(B), (int)(I)))
292
293#define _mm_mask_shldi_epi32(S, U, A, B, I) \
294 ((__m128i)__builtin_ia32_selectd_128((__mmask8)(U), \
295 (__v4si)_mm_shldi_epi32((A), (B), (I)), \
296 (__v4si)(__m128i)(S)))
297
298#define _mm_maskz_shldi_epi32(U, A, B, I) \
299 ((__m128i)__builtin_ia32_selectd_128((__mmask8)(U), \
300 (__v4si)_mm_shldi_epi32((A), (B), (I)), \
301 (__v4si)_mm_setzero_si128()))
302
303#define _mm256_shldi_epi16(A, B, I) \
304 ((__m256i)__builtin_ia32_vpshldw256((__v16hi)(__m256i)(A), \
305 (__v16hi)(__m256i)(B), (int)(I)))
306
307#define _mm256_mask_shldi_epi16(S, U, A, B, I) \
308 ((__m256i)__builtin_ia32_selectw_256((__mmask16)(U), \
309 (__v16hi)_mm256_shldi_epi16((A), (B), (I)), \
310 (__v16hi)(__m256i)(S)))
311
312#define _mm256_maskz_shldi_epi16(U, A, B, I) \
313 ((__m256i)__builtin_ia32_selectw_256((__mmask16)(U), \
314 (__v16hi)_mm256_shldi_epi16((A), (B), (I)), \
315 (__v16hi)_mm256_setzero_si256()))
316
317#define _mm_shldi_epi16(A, B, I) \
318 ((__m128i)__builtin_ia32_vpshldw128((__v8hi)(__m128i)(A), \
319 (__v8hi)(__m128i)(B), (int)(I)))
320
321#define _mm_mask_shldi_epi16(S, U, A, B, I) \
322 ((__m128i)__builtin_ia32_selectw_128((__mmask8)(U), \
323 (__v8hi)_mm_shldi_epi16((A), (B), (I)), \
324 (__v8hi)(__m128i)(S)))
325
326#define _mm_maskz_shldi_epi16(U, A, B, I) \
327 ((__m128i)__builtin_ia32_selectw_128((__mmask8)(U), \
328 (__v8hi)_mm_shldi_epi16((A), (B), (I)), \
329 (__v8hi)_mm_setzero_si128()))
330
331#define _mm256_shrdi_epi64(A, B, I) \
332 ((__m256i)__builtin_ia32_vpshrdq256((__v4di)(__m256i)(A), \
333 (__v4di)(__m256i)(B), (int)(I)))
334
335#define _mm256_mask_shrdi_epi64(S, U, A, B, I) \
336 ((__m256i)__builtin_ia32_selectq_256((__mmask8)(U), \
337 (__v4di)_mm256_shrdi_epi64((A), (B), (I)), \
338 (__v4di)(__m256i)(S)))
339
340#define _mm256_maskz_shrdi_epi64(U, A, B, I) \
341 ((__m256i)__builtin_ia32_selectq_256((__mmask8)(U), \
342 (__v4di)_mm256_shrdi_epi64((A), (B), (I)), \
343 (__v4di)_mm256_setzero_si256()))
344
345#define _mm_shrdi_epi64(A, B, I) \
346 ((__m128i)__builtin_ia32_vpshrdq128((__v2di)(__m128i)(A), \
347 (__v2di)(__m128i)(B), (int)(I)))
348
349#define _mm_mask_shrdi_epi64(S, U, A, B, I) \
350 ((__m128i)__builtin_ia32_selectq_128((__mmask8)(U), \
351 (__v2di)_mm_shrdi_epi64((A), (B), (I)), \
352 (__v2di)(__m128i)(S)))
353
354#define _mm_maskz_shrdi_epi64(U, A, B, I) \
355 ((__m128i)__builtin_ia32_selectq_128((__mmask8)(U), \
356 (__v2di)_mm_shrdi_epi64((A), (B), (I)), \
357 (__v2di)_mm_setzero_si128()))
358
359#define _mm256_shrdi_epi32(A, B, I) \
360 ((__m256i)__builtin_ia32_vpshrdd256((__v8si)(__m256i)(A), \
361 (__v8si)(__m256i)(B), (int)(I)))
362
363#define _mm256_mask_shrdi_epi32(S, U, A, B, I) \
364 ((__m256i)__builtin_ia32_selectd_256((__mmask8)(U), \
365 (__v8si)_mm256_shrdi_epi32((A), (B), (I)), \
366 (__v8si)(__m256i)(S)))
367
368#define _mm256_maskz_shrdi_epi32(U, A, B, I) \
369 ((__m256i)__builtin_ia32_selectd_256((__mmask8)(U), \
370 (__v8si)_mm256_shrdi_epi32((A), (B), (I)), \
371 (__v8si)_mm256_setzero_si256()))
372
373#define _mm_shrdi_epi32(A, B, I) \
374 ((__m128i)__builtin_ia32_vpshrdd128((__v4si)(__m128i)(A), \
375 (__v4si)(__m128i)(B), (int)(I)))
376
377#define _mm_mask_shrdi_epi32(S, U, A, B, I) \
378 ((__m128i)__builtin_ia32_selectd_128((__mmask8)(U), \
379 (__v4si)_mm_shrdi_epi32((A), (B), (I)), \
380 (__v4si)(__m128i)(S)))
381
382#define _mm_maskz_shrdi_epi32(U, A, B, I) \
383 ((__m128i)__builtin_ia32_selectd_128((__mmask8)(U), \
384 (__v4si)_mm_shrdi_epi32((A), (B), (I)), \
385 (__v4si)_mm_setzero_si128()))
386
387#define _mm256_shrdi_epi16(A, B, I) \
388 ((__m256i)__builtin_ia32_vpshrdw256((__v16hi)(__m256i)(A), \
389 (__v16hi)(__m256i)(B), (int)(I)))
390
391#define _mm256_mask_shrdi_epi16(S, U, A, B, I) \
392 ((__m256i)__builtin_ia32_selectw_256((__mmask16)(U), \
393 (__v16hi)_mm256_shrdi_epi16((A), (B), (I)), \
394 (__v16hi)(__m256i)(S)))
395
396#define _mm256_maskz_shrdi_epi16(U, A, B, I) \
397 ((__m256i)__builtin_ia32_selectw_256((__mmask16)(U), \
398 (__v16hi)_mm256_shrdi_epi16((A), (B), (I)), \
399 (__v16hi)_mm256_setzero_si256()))
400
401#define _mm_shrdi_epi16(A, B, I) \
402 ((__m128i)__builtin_ia32_vpshrdw128((__v8hi)(__m128i)(A), \
403 (__v8hi)(__m128i)(B), (int)(I)))
404
405#define _mm_mask_shrdi_epi16(S, U, A, B, I) \
406 ((__m128i)__builtin_ia32_selectw_128((__mmask8)(U), \
407 (__v8hi)_mm_shrdi_epi16((A), (B), (I)), \
408 (__v8hi)(__m128i)(S)))
409
410#define _mm_maskz_shrdi_epi16(U, A, B, I) \
411 ((__m128i)__builtin_ia32_selectw_128((__mmask8)(U), \
412 (__v8hi)_mm_shrdi_epi16((A), (B), (I)), \
413 (__v8hi)_mm_setzero_si128()))
414
415static __inline__ __m256i __DEFAULT_FN_ATTRS256
416_mm256_shldv_epi64(__m256i __A, __m256i __B, __m256i __C)
417{
418 return (__m256i)__builtin_ia32_vpshldvq256((__v4di)__A, (__v4di)__B,
419 (__v4di)__C);
420}
421
422static __inline__ __m256i __DEFAULT_FN_ATTRS256
423_mm256_mask_shldv_epi64(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C)
424{
425 return (__m256i)__builtin_ia32_selectq_256(__U,
426 (__v4di)_mm256_shldv_epi64(__A, __B, __C),
427 (__v4di)__A);
428}
429
430static __inline__ __m256i __DEFAULT_FN_ATTRS256
431_mm256_maskz_shldv_epi64(__mmask8 __U, __m256i __A, __m256i __B, __m256i __C)
432{
433 return (__m256i)__builtin_ia32_selectq_256(__U,
434 (__v4di)_mm256_shldv_epi64(__A, __B, __C),
435 (__v4di)_mm256_setzero_si256());
436}
437
438static __inline__ __m128i __DEFAULT_FN_ATTRS128
439_mm_shldv_epi64(__m128i __A, __m128i __B, __m128i __C)
440{
441 return (__m128i)__builtin_ia32_vpshldvq128((__v2di)__A, (__v2di)__B,
442 (__v2di)__C);
443}
444
445static __inline__ __m128i __DEFAULT_FN_ATTRS128
446_mm_mask_shldv_epi64(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C)
447{
448 return (__m128i)__builtin_ia32_selectq_128(__U,
449 (__v2di)_mm_shldv_epi64(__A, __B, __C),
450 (__v2di)__A);
451}
452
453static __inline__ __m128i __DEFAULT_FN_ATTRS128
454_mm_maskz_shldv_epi64(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C)
455{
456 return (__m128i)__builtin_ia32_selectq_128(__U,
457 (__v2di)_mm_shldv_epi64(__A, __B, __C),
458 (__v2di)_mm_setzero_si128());
459}
460
461static __inline__ __m256i __DEFAULT_FN_ATTRS256
462_mm256_shldv_epi32(__m256i __A, __m256i __B, __m256i __C)
463{
464 return (__m256i)__builtin_ia32_vpshldvd256((__v8si)__A, (__v8si)__B,
465 (__v8si)__C);
466}
467
468static __inline__ __m256i __DEFAULT_FN_ATTRS256
469_mm256_mask_shldv_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C)
470{
471 return (__m256i)__builtin_ia32_selectd_256(__U,
472 (__v8si)_mm256_shldv_epi32(__A, __B, __C),
473 (__v8si)__A);
474}
475
476static __inline__ __m256i __DEFAULT_FN_ATTRS256
477_mm256_maskz_shldv_epi32(__mmask8 __U, __m256i __A, __m256i __B, __m256i __C)
478{
479 return (__m256i)__builtin_ia32_selectd_256(__U,
480 (__v8si)_mm256_shldv_epi32(__A, __B, __C),
481 (__v8si)_mm256_setzero_si256());
482}
483
484static __inline__ __m128i __DEFAULT_FN_ATTRS128
485_mm_shldv_epi32(__m128i __A, __m128i __B, __m128i __C)
486{
487 return (__m128i)__builtin_ia32_vpshldvd128((__v4si)__A, (__v4si)__B,
488 (__v4si)__C);
489}
490
491static __inline__ __m128i __DEFAULT_FN_ATTRS128
492_mm_mask_shldv_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C)
493{
494 return (__m128i)__builtin_ia32_selectd_128(__U,
495 (__v4si)_mm_shldv_epi32(__A, __B, __C),
496 (__v4si)__A);
497}
498
499static __inline__ __m128i __DEFAULT_FN_ATTRS128
500_mm_maskz_shldv_epi32(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C)
501{
502 return (__m128i)__builtin_ia32_selectd_128(__U,
503 (__v4si)_mm_shldv_epi32(__A, __B, __C),
504 (__v4si)_mm_setzero_si128());
505}
506
507static __inline__ __m256i __DEFAULT_FN_ATTRS256
508_mm256_shldv_epi16(__m256i __A, __m256i __B, __m256i __C)
509{
510 return (__m256i)__builtin_ia32_vpshldvw256((__v16hi)__A, (__v16hi)__B,
511 (__v16hi)__C);
512}
513
514static __inline__ __m256i __DEFAULT_FN_ATTRS256
515_mm256_mask_shldv_epi16(__m256i __A, __mmask16 __U, __m256i __B, __m256i __C)
516{
517 return (__m256i)__builtin_ia32_selectw_256(__U,
518 (__v16hi)_mm256_shldv_epi16(__A, __B, __C),
519 (__v16hi)__A);
520}
521
522static __inline__ __m256i __DEFAULT_FN_ATTRS256
523_mm256_maskz_shldv_epi16(__mmask16 __U, __m256i __A, __m256i __B, __m256i __C)
524{
525 return (__m256i)__builtin_ia32_selectw_256(__U,
526 (__v16hi)_mm256_shldv_epi16(__A, __B, __C),
527 (__v16hi)_mm256_setzero_si256());
528}
529
530static __inline__ __m128i __DEFAULT_FN_ATTRS128
531_mm_shldv_epi16(__m128i __A, __m128i __B, __m128i __C)
532{
533 return (__m128i)__builtin_ia32_vpshldvw128((__v8hi)__A, (__v8hi)__B,
534 (__v8hi)__C);
535}
536
537static __inline__ __m128i __DEFAULT_FN_ATTRS128
538_mm_mask_shldv_epi16(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C)
539{
540 return (__m128i)__builtin_ia32_selectw_128(__U,
541 (__v8hi)_mm_shldv_epi16(__A, __B, __C),
542 (__v8hi)__A);
543}
544
545static __inline__ __m128i __DEFAULT_FN_ATTRS128
546_mm_maskz_shldv_epi16(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C)
547{
548 return (__m128i)__builtin_ia32_selectw_128(__U,
549 (__v8hi)_mm_shldv_epi16(__A, __B, __C),
550 (__v8hi)_mm_setzero_si128());
551}
552
553static __inline__ __m256i __DEFAULT_FN_ATTRS256
554_mm256_shrdv_epi64(__m256i __A, __m256i __B, __m256i __C)
555{
556 return (__m256i)__builtin_ia32_vpshrdvq256((__v4di)__A, (__v4di)__B,
557 (__v4di)__C);
558}
559
560static __inline__ __m256i __DEFAULT_FN_ATTRS256
561_mm256_mask_shrdv_epi64(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C)
562{
563 return (__m256i)__builtin_ia32_selectq_256(__U,
564 (__v4di)_mm256_shrdv_epi64(__A, __B, __C),
565 (__v4di)__A);
566}
567
568static __inline__ __m256i __DEFAULT_FN_ATTRS256
569_mm256_maskz_shrdv_epi64(__mmask8 __U, __m256i __A, __m256i __B, __m256i __C)
570{
571 return (__m256i)__builtin_ia32_selectq_256(__U,
572 (__v4di)_mm256_shrdv_epi64(__A, __B, __C),
573 (__v4di)_mm256_setzero_si256());
574}
575
576static __inline__ __m128i __DEFAULT_FN_ATTRS128
577_mm_shrdv_epi64(__m128i __A, __m128i __B, __m128i __C)
578{
579 return (__m128i)__builtin_ia32_vpshrdvq128((__v2di)__A, (__v2di)__B,
580 (__v2di)__C);
581}
582
583static __inline__ __m128i __DEFAULT_FN_ATTRS128
584_mm_mask_shrdv_epi64(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C)
585{
586 return (__m128i)__builtin_ia32_selectq_128(__U,
587 (__v2di)_mm_shrdv_epi64(__A, __B, __C),
588 (__v2di)__A);
589}
590
591static __inline__ __m128i __DEFAULT_FN_ATTRS128
592_mm_maskz_shrdv_epi64(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C)
593{
594 return (__m128i)__builtin_ia32_selectq_128(__U,
595 (__v2di)_mm_shrdv_epi64(__A, __B, __C),
596 (__v2di)_mm_setzero_si128());
597}
598
599static __inline__ __m256i __DEFAULT_FN_ATTRS256
600_mm256_shrdv_epi32(__m256i __A, __m256i __B, __m256i __C)
601{
602 return (__m256i)__builtin_ia32_vpshrdvd256((__v8si)__A, (__v8si)__B,
603 (__v8si)__C);
604}
605
606static __inline__ __m256i __DEFAULT_FN_ATTRS256
607_mm256_mask_shrdv_epi32(__m256i __A, __mmask8 __U, __m256i __B, __m256i __C)
608{
609 return (__m256i)__builtin_ia32_selectd_256(__U,
610 (__v8si)_mm256_shrdv_epi32(__A, __B, __C),
611 (__v8si)__A);
612}
613
614static __inline__ __m256i __DEFAULT_FN_ATTRS256
615_mm256_maskz_shrdv_epi32(__mmask8 __U, __m256i __A, __m256i __B, __m256i __C)
616{
617 return (__m256i)__builtin_ia32_selectd_256(__U,
618 (__v8si)_mm256_shrdv_epi32(__A, __B, __C),
619 (__v8si)_mm256_setzero_si256());
620}
621
622static __inline__ __m128i __DEFAULT_FN_ATTRS128
623_mm_shrdv_epi32(__m128i __A, __m128i __B, __m128i __C)
624{
625 return (__m128i)__builtin_ia32_vpshrdvd128((__v4si)__A, (__v4si)__B,
626 (__v4si)__C);
627}
628
629static __inline__ __m128i __DEFAULT_FN_ATTRS128
630_mm_mask_shrdv_epi32(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C)
631{
632 return (__m128i)__builtin_ia32_selectd_128(__U,
633 (__v4si)_mm_shrdv_epi32(__A, __B, __C),
634 (__v4si)__A);
635}
636
637static __inline__ __m128i __DEFAULT_FN_ATTRS128
638_mm_maskz_shrdv_epi32(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C)
639{
640 return (__m128i)__builtin_ia32_selectd_128(__U,
641 (__v4si)_mm_shrdv_epi32(__A, __B, __C),
642 (__v4si)_mm_setzero_si128());
643}
644
645static __inline__ __m256i __DEFAULT_FN_ATTRS256
646_mm256_shrdv_epi16(__m256i __A, __m256i __B, __m256i __C)
647{
648 return (__m256i)__builtin_ia32_vpshrdvw256((__v16hi)__A, (__v16hi)__B,
649 (__v16hi)__C);
650}
651
652static __inline__ __m256i __DEFAULT_FN_ATTRS256
653_mm256_mask_shrdv_epi16(__m256i __A, __mmask16 __U, __m256i __B, __m256i __C)
654{
655 return (__m256i)__builtin_ia32_selectw_256(__U,
656 (__v16hi)_mm256_shrdv_epi16(__A, __B, __C),
657 (__v16hi)__A);
658}
659
660static __inline__ __m256i __DEFAULT_FN_ATTRS256
661_mm256_maskz_shrdv_epi16(__mmask16 __U, __m256i __A, __m256i __B, __m256i __C)
662{
663 return (__m256i)__builtin_ia32_selectw_256(__U,
664 (__v16hi)_mm256_shrdv_epi16(__A, __B, __C),
665 (__v16hi)_mm256_setzero_si256());
666}
667
668static __inline__ __m128i __DEFAULT_FN_ATTRS128
669_mm_shrdv_epi16(__m128i __A, __m128i __B, __m128i __C)
670{
671 return (__m128i)__builtin_ia32_vpshrdvw128((__v8hi)__A, (__v8hi)__B,
672 (__v8hi)__C);
673}
674
675static __inline__ __m128i __DEFAULT_FN_ATTRS128
676_mm_mask_shrdv_epi16(__m128i __A, __mmask8 __U, __m128i __B, __m128i __C)
677{
678 return (__m128i)__builtin_ia32_selectw_128(__U,
679 (__v8hi)_mm_shrdv_epi16(__A, __B, __C),
680 (__v8hi)__A);
681}
682
683static __inline__ __m128i __DEFAULT_FN_ATTRS128
684_mm_maskz_shrdv_epi16(__mmask8 __U, __m128i __A, __m128i __B, __m128i __C)
685{
686 return (__m128i)__builtin_ia32_selectw_128(__U,
687 (__v8hi)_mm_shrdv_epi16(__A, __B, __C),
688 (__v8hi)_mm_setzero_si128());
689}
690
691
692#undef __DEFAULT_FN_ATTRS128
693#undef __DEFAULT_FN_ATTRS256
694
695#endif