1 // x86 specific conversion optimizations -*- C++ -*- 2 3 // Copyright (C) 2020-2021 Free Software Foundation, Inc. 4 // 5 // This file is part of the GNU ISO C++ Library. This library is free 6 // software; you can redistribute it and/or modify it under the 7 // terms of the GNU General Public License as published by the 8 // Free Software Foundation; either version 3, or (at your option) 9 // any later version. 10 11 // This library is distributed in the hope that it will be useful, 12 // but WITHOUT ANY WARRANTY; without even the implied warranty of 13 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 // GNU General Public License for more details. 15 16 // Under Section 7 of GPL version 3, you are granted additional 17 // permissions described in the GCC Runtime Library Exception, version 18 // 3.1, as published by the Free Software Foundation. 19 20 // You should have received a copy of the GNU General Public License and 21 // a copy of the GCC Runtime Library Exception along with this program; 22 // see the files COPYING3 and COPYING.RUNTIME respectively. If not, see 23 // <http://www.gnu.org/licenses/>. 24 25 #ifndef _GLIBCXX_EXPERIMENTAL_SIMD_X86_CONVERSIONS_H 26 #define _GLIBCXX_EXPERIMENTAL_SIMD_X86_CONVERSIONS_H 27 28 #if __cplusplus >= 201703L 29 30 // work around PR85827 31 // 1-arg __convert_x86 {{{1 32 template <typename _To, typename _V, typename _Traits> 33 _GLIBCXX_SIMD_INTRINSIC _To __convert_x86(_V __v)34 __convert_x86(_V __v) 35 { 36 static_assert(__is_vector_type_v<_V>); 37 using _Tp = typename _Traits::value_type; 38 constexpr size_t _Np = _Traits::_S_full_size; 39 [[maybe_unused]] const auto __intrin = __to_intrin(__v); 40 using _Up = typename _VectorTraits<_To>::value_type; 41 constexpr size_t _M = _VectorTraits<_To>::_S_full_size; 42 43 // [xyz]_to_[xyz] {{{2 44 [[maybe_unused]] constexpr bool __x_to_x 45 = sizeof(__v) <= 16 && sizeof(_To) <= 16; 46 [[maybe_unused]] constexpr bool __x_to_y 47 = sizeof(__v) <= 16 && sizeof(_To) == 32; 48 [[maybe_unused]] constexpr bool __x_to_z 49 = sizeof(__v) <= 16 && sizeof(_To) == 64; 50 [[maybe_unused]] constexpr bool __y_to_x 51 = sizeof(__v) == 32 && sizeof(_To) <= 16; 52 [[maybe_unused]] constexpr bool __y_to_y 53 = sizeof(__v) == 32 && sizeof(_To) == 32; 54 [[maybe_unused]] constexpr bool __y_to_z 55 = sizeof(__v) == 32 && sizeof(_To) == 64; 56 [[maybe_unused]] constexpr bool __z_to_x 57 = sizeof(__v) == 64 && sizeof(_To) <= 16; 58 [[maybe_unused]] constexpr bool __z_to_y 59 = sizeof(__v) == 64 && sizeof(_To) == 32; 60 [[maybe_unused]] constexpr bool __z_to_z 61 = sizeof(__v) == 64 && sizeof(_To) == 64; 62 63 // iX_to_iX {{{2 64 [[maybe_unused]] constexpr bool __i_to_i 65 = is_integral_v<_Up> && is_integral_v<_Tp>; 66 [[maybe_unused]] constexpr bool __i8_to_i16 67 = __i_to_i && sizeof(_Tp) == 1 && sizeof(_Up) == 2; 68 [[maybe_unused]] constexpr bool __i8_to_i32 69 = __i_to_i && sizeof(_Tp) == 1 && sizeof(_Up) == 4; 70 [[maybe_unused]] constexpr bool __i8_to_i64 71 = __i_to_i && sizeof(_Tp) == 1 && sizeof(_Up) == 8; 72 [[maybe_unused]] constexpr bool __i16_to_i8 73 = __i_to_i && sizeof(_Tp) == 2 && sizeof(_Up) == 1; 74 [[maybe_unused]] constexpr bool __i16_to_i32 75 = __i_to_i && sizeof(_Tp) == 2 && sizeof(_Up) == 4; 76 [[maybe_unused]] constexpr bool __i16_to_i64 77 = __i_to_i && sizeof(_Tp) == 2 && sizeof(_Up) == 8; 78 [[maybe_unused]] constexpr bool __i32_to_i8 79 = __i_to_i && sizeof(_Tp) == 4 && sizeof(_Up) == 1; 80 [[maybe_unused]] constexpr bool __i32_to_i16 81 = __i_to_i && sizeof(_Tp) == 4 && sizeof(_Up) == 2; 82 [[maybe_unused]] constexpr bool __i32_to_i64 83 = __i_to_i && sizeof(_Tp) == 4 && sizeof(_Up) == 8; 84 [[maybe_unused]] constexpr bool __i64_to_i8 85 = __i_to_i && sizeof(_Tp) == 8 && sizeof(_Up) == 1; 86 [[maybe_unused]] constexpr bool __i64_to_i16 87 = __i_to_i && sizeof(_Tp) == 8 && sizeof(_Up) == 2; 88 [[maybe_unused]] constexpr bool __i64_to_i32 89 = __i_to_i && sizeof(_Tp) == 8 && sizeof(_Up) == 4; 90 91 // [fsu]X_to_[fsu]X {{{2 92 // ibw = integral && byte or word, i.e. char and short with any signedness 93 [[maybe_unused]] constexpr bool __s64_to_f32 94 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 8 95 && is_floating_point_v<_Up> && sizeof(_Up) == 4; 96 [[maybe_unused]] constexpr bool __s32_to_f32 97 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 4 98 && is_floating_point_v<_Up> && sizeof(_Up) == 4; 99 [[maybe_unused]] constexpr bool __s16_to_f32 100 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 2 101 && is_floating_point_v<_Up> && sizeof(_Up) == 4; 102 [[maybe_unused]] constexpr bool __s8_to_f32 103 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 1 104 && is_floating_point_v<_Up> && sizeof(_Up) == 4; 105 [[maybe_unused]] constexpr bool __u64_to_f32 106 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 8 107 && is_floating_point_v<_Up> && sizeof(_Up) == 4; 108 [[maybe_unused]] constexpr bool __u32_to_f32 109 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 4 110 && is_floating_point_v<_Up> && sizeof(_Up) == 4; 111 [[maybe_unused]] constexpr bool __u16_to_f32 112 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 2 113 && is_floating_point_v<_Up> && sizeof(_Up) == 4; 114 [[maybe_unused]] constexpr bool __u8_to_f32 115 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 1 116 && is_floating_point_v<_Up> && sizeof(_Up) == 4; 117 [[maybe_unused]] constexpr bool __s64_to_f64 118 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 8 119 && is_floating_point_v<_Up> && sizeof(_Up) == 8; 120 [[maybe_unused]] constexpr bool __s32_to_f64 121 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 4 122 && is_floating_point_v<_Up> && sizeof(_Up) == 8; 123 [[maybe_unused]] constexpr bool __u64_to_f64 124 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 8 125 && is_floating_point_v<_Up> && sizeof(_Up) == 8; 126 [[maybe_unused]] constexpr bool __u32_to_f64 127 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 4 128 && is_floating_point_v<_Up> && sizeof(_Up) == 8; 129 [[maybe_unused]] constexpr bool __f32_to_s64 130 = is_integral_v<_Up> && is_signed_v<_Up> && sizeof(_Up) == 8 131 && is_floating_point_v<_Tp> && sizeof(_Tp) == 4; 132 [[maybe_unused]] constexpr bool __f32_to_s32 133 = is_integral_v<_Up> && is_signed_v<_Up> && sizeof(_Up) == 4 134 && is_floating_point_v<_Tp> && sizeof(_Tp) == 4; 135 [[maybe_unused]] constexpr bool __f32_to_u64 136 = is_integral_v<_Up> && is_unsigned_v<_Up> && sizeof(_Up) == 8 137 && is_floating_point_v<_Tp> && sizeof(_Tp) == 4; 138 [[maybe_unused]] constexpr bool __f32_to_u32 139 = is_integral_v<_Up> && is_unsigned_v<_Up> && sizeof(_Up) == 4 140 && is_floating_point_v<_Tp> && sizeof(_Tp) == 4; 141 [[maybe_unused]] constexpr bool __f64_to_s64 142 = is_integral_v<_Up> && is_signed_v<_Up> && sizeof(_Up) == 8 143 && is_floating_point_v<_Tp> && sizeof(_Tp) == 8; 144 [[maybe_unused]] constexpr bool __f64_to_s32 145 = is_integral_v<_Up> && is_signed_v<_Up> && sizeof(_Up) == 4 146 && is_floating_point_v<_Tp> && sizeof(_Tp) == 8; 147 [[maybe_unused]] constexpr bool __f64_to_u64 148 = is_integral_v<_Up> && is_unsigned_v<_Up> && sizeof(_Up) == 8 149 && is_floating_point_v<_Tp> && sizeof(_Tp) == 8; 150 [[maybe_unused]] constexpr bool __f64_to_u32 151 = is_integral_v<_Up> && is_unsigned_v<_Up> && sizeof(_Up) == 4 152 && is_floating_point_v<_Tp> && sizeof(_Tp) == 8; 153 [[maybe_unused]] constexpr bool __ibw_to_f32 154 = is_integral_v<_Tp> && sizeof(_Tp) <= 2 155 && is_floating_point_v<_Up> && sizeof(_Up) == 4; 156 [[maybe_unused]] constexpr bool __ibw_to_f64 157 = is_integral_v<_Tp> && sizeof(_Tp) <= 2 158 && is_floating_point_v<_Up> && sizeof(_Up) == 8; 159 [[maybe_unused]] constexpr bool __f32_to_ibw 160 = is_integral_v<_Up> && sizeof(_Up) <= 2 161 && is_floating_point_v<_Tp> && sizeof(_Tp) == 4; 162 [[maybe_unused]] constexpr bool __f64_to_ibw 163 = is_integral_v<_Up> && sizeof(_Up) <= 2 164 && is_floating_point_v<_Tp> && sizeof(_Tp) == 8; 165 [[maybe_unused]] constexpr bool __f32_to_f64 166 = is_floating_point_v<_Tp> && sizeof(_Tp) == 4 167 && is_floating_point_v<_Up> && sizeof(_Up) == 8; 168 [[maybe_unused]] constexpr bool __f64_to_f32 169 = is_floating_point_v<_Tp> && sizeof(_Tp) == 8 170 && is_floating_point_v<_Up> && sizeof(_Up) == 4; 171 172 if constexpr (__i_to_i && __y_to_x && !__have_avx2) //{{{2 173 return __convert_x86<_To>(__lo128(__v), __hi128(__v)); 174 else if constexpr (__i_to_i && __x_to_y && !__have_avx2) //{{{2 175 return __concat(__convert_x86<__vector_type_t<_Up, _M / 2>>(__v), 176 __convert_x86<__vector_type_t<_Up, _M / 2>>( 177 __extract_part<1, _Np / _M * 2>(__v))); 178 else if constexpr (__i_to_i) //{{{2 179 { 180 static_assert(__x_to_x || __have_avx2, 181 "integral conversions with ymm registers require AVX2"); 182 static_assert(__have_avx512bw 183 || ((sizeof(_Tp) >= 4 || sizeof(__v) < 64) 184 && (sizeof(_Up) >= 4 || sizeof(_To) < 64)), 185 "8/16-bit integers in zmm registers require AVX512BW"); 186 static_assert((sizeof(__v) < 64 && sizeof(_To) < 64) || __have_avx512f, 187 "integral conversions with ymm registers require AVX2"); 188 } 189 if constexpr (is_floating_point_v<_Tp> == is_floating_point_v<_Up> && //{{{2 190 sizeof(_Tp) == sizeof(_Up)) 191 { 192 // conversion uses simple bit reinterpretation (or no conversion at all) 193 if constexpr (_Np >= _M) 194 return __intrin_bitcast<_To>(__v); 195 else 196 return __zero_extend(__vector_bitcast<_Up>(__v)); 197 } 198 else if constexpr (_Np < _M && sizeof(_To) > 16) //{{{2 199 // zero extend (eg. xmm -> ymm) 200 return __zero_extend( 201 __convert_x86<__vector_type_t< 202 _Up, (16 / sizeof(_Up) > _Np) ? 16 / sizeof(_Up) : _Np>>(__v)); 203 else if constexpr (_Np > _M && sizeof(__v) > 16) //{{{2 204 // partial input (eg. ymm -> xmm) 205 return __convert_x86<_To>(__extract_part<0, _Np / _M>(__v)); 206 else if constexpr (__i64_to_i32) //{{{2 207 { 208 if constexpr (__x_to_x && __have_avx512vl) 209 return __intrin_bitcast<_To>(_mm_cvtepi64_epi32(__intrin)); 210 else if constexpr (__x_to_x) 211 return __auto_bitcast( 212 _mm_shuffle_ps(__vector_bitcast<float>(__v), __m128(), 8)); 213 else if constexpr (__y_to_x && __have_avx512vl) 214 return __intrin_bitcast<_To>(_mm256_cvtepi64_epi32(__intrin)); 215 else if constexpr (__y_to_x && __have_avx512f) 216 return __intrin_bitcast<_To>( 217 __lo128(_mm512_cvtepi64_epi32(__auto_bitcast(__v)))); 218 else if constexpr (__y_to_x) 219 return __intrin_bitcast<_To>( 220 __lo128(_mm256_permute4x64_epi64(_mm256_shuffle_epi32(__intrin, 8), 221 0 + 4 * 2))); 222 else if constexpr (__z_to_y) 223 return __intrin_bitcast<_To>(_mm512_cvtepi64_epi32(__intrin)); 224 } 225 else if constexpr (__i64_to_i16) //{{{2 226 { 227 if constexpr (__x_to_x && __have_avx512vl) 228 return __intrin_bitcast<_To>(_mm_cvtepi64_epi16(__intrin)); 229 else if constexpr (__x_to_x && __have_avx512f) 230 return __intrin_bitcast<_To>( 231 __lo128(_mm512_cvtepi64_epi16(__auto_bitcast(__v)))); 232 else if constexpr (__x_to_x && __have_ssse3) 233 { 234 return __intrin_bitcast<_To>( 235 _mm_shuffle_epi8(__intrin, 236 _mm_setr_epi8(0, 1, 8, 9, -0x80, -0x80, -0x80, 237 -0x80, -0x80, -0x80, -0x80, -0x80, 238 -0x80, -0x80, -0x80, -0x80))); 239 // fallback without SSSE3 240 } 241 else if constexpr (__y_to_x && __have_avx512vl) 242 return __intrin_bitcast<_To>(_mm256_cvtepi64_epi16(__intrin)); 243 else if constexpr (__y_to_x && __have_avx512f) 244 return __intrin_bitcast<_To>( 245 __lo128(_mm512_cvtepi64_epi16(__auto_bitcast(__v)))); 246 else if constexpr (__y_to_x) 247 { 248 const auto __a = _mm256_shuffle_epi8( 249 __intrin, 250 _mm256_setr_epi8(0, 1, 8, 9, -0x80, -0x80, -0x80, -0x80, -0x80, 251 -0x80, -0x80, -0x80, -0x80, -0x80, -0x80, -0x80, 252 -0x80, -0x80, -0x80, -0x80, 0, 1, 8, 9, -0x80, 253 -0x80, -0x80, -0x80, -0x80, -0x80, -0x80, 254 -0x80)); 255 return __intrin_bitcast<_To>(__lo128(__a) | __hi128(__a)); 256 } 257 else if constexpr (__z_to_x) 258 return __intrin_bitcast<_To>(_mm512_cvtepi64_epi16(__intrin)); 259 } 260 else if constexpr (__i64_to_i8) //{{{2 261 { 262 if constexpr (__x_to_x && __have_avx512vl) 263 return __intrin_bitcast<_To>(_mm_cvtepi64_epi8(__intrin)); 264 else if constexpr (__x_to_x && __have_avx512f) 265 return __intrin_bitcast<_To>( 266 __lo128(_mm512_cvtepi64_epi8(__zero_extend(__intrin)))); 267 else if constexpr (__y_to_x && __have_avx512vl) 268 return __intrin_bitcast<_To>(_mm256_cvtepi64_epi8(__intrin)); 269 else if constexpr (__y_to_x && __have_avx512f) 270 return __intrin_bitcast<_To>( 271 _mm512_cvtepi64_epi8(__zero_extend(__intrin))); 272 else if constexpr (__z_to_x) 273 return __intrin_bitcast<_To>(_mm512_cvtepi64_epi8(__intrin)); 274 } 275 else if constexpr (__i32_to_i64) //{{{2 276 { 277 if constexpr (__have_sse4_1 && __x_to_x) 278 return __intrin_bitcast<_To>(is_signed_v<_Tp> 279 ? _mm_cvtepi32_epi64(__intrin) 280 : _mm_cvtepu32_epi64(__intrin)); 281 else if constexpr (__x_to_x) 282 { 283 return __intrin_bitcast<_To>( 284 _mm_unpacklo_epi32(__intrin, is_signed_v<_Tp> 285 ? _mm_srai_epi32(__intrin, 31) 286 : __m128i())); 287 } 288 else if constexpr (__x_to_y) 289 return __intrin_bitcast<_To>(is_signed_v<_Tp> 290 ? _mm256_cvtepi32_epi64(__intrin) 291 : _mm256_cvtepu32_epi64(__intrin)); 292 else if constexpr (__y_to_z) 293 return __intrin_bitcast<_To>(is_signed_v<_Tp> 294 ? _mm512_cvtepi32_epi64(__intrin) 295 : _mm512_cvtepu32_epi64(__intrin)); 296 } 297 else if constexpr (__i32_to_i16) //{{{2 298 { 299 if constexpr (__x_to_x && __have_avx512vl) 300 return __intrin_bitcast<_To>(_mm_cvtepi32_epi16(__intrin)); 301 else if constexpr (__x_to_x && __have_avx512f) 302 return __intrin_bitcast<_To>( 303 __lo128(_mm512_cvtepi32_epi16(__auto_bitcast(__v)))); 304 else if constexpr (__x_to_x && __have_ssse3) 305 return __intrin_bitcast<_To>(_mm_shuffle_epi8( 306 __intrin, _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, -0x80, -0x80, 307 -0x80, -0x80, -0x80, -0x80, -0x80, -0x80))); 308 else if constexpr (__x_to_x) 309 { 310 auto __a = _mm_unpacklo_epi16(__intrin, __m128i()); // 0o.o 1o.o 311 auto __b = _mm_unpackhi_epi16(__intrin, __m128i()); // 2o.o 3o.o 312 auto __c = _mm_unpacklo_epi16(__a, __b); // 02oo ..oo 313 auto __d = _mm_unpackhi_epi16(__a, __b); // 13oo ..oo 314 return __intrin_bitcast<_To>( 315 _mm_unpacklo_epi16(__c, __d)); // 0123 oooo 316 } 317 else if constexpr (__y_to_x && __have_avx512vl) 318 return __intrin_bitcast<_To>(_mm256_cvtepi32_epi16(__intrin)); 319 else if constexpr (__y_to_x && __have_avx512f) 320 return __intrin_bitcast<_To>( 321 __lo128(_mm512_cvtepi32_epi16(__auto_bitcast(__v)))); 322 else if constexpr (__y_to_x) 323 { 324 auto __a = _mm256_shuffle_epi8( 325 __intrin, 326 _mm256_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, -0x80, -0x80, -0x80, 327 -0x80, -0x80, -0x80, -0x80, -0x80, 0, 1, 4, 5, 8, 328 9, 12, 13, -0x80, -0x80, -0x80, -0x80, -0x80, 329 -0x80, -0x80, -0x80)); 330 return __intrin_bitcast<_To>(__lo128( 331 _mm256_permute4x64_epi64(__a, 332 0xf8))); // __a[0] __a[2] | __a[3] __a[3] 333 } 334 else if constexpr (__z_to_y) 335 return __intrin_bitcast<_To>(_mm512_cvtepi32_epi16(__intrin)); 336 } 337 else if constexpr (__i32_to_i8) //{{{2 338 { 339 if constexpr (__x_to_x && __have_avx512vl) 340 return __intrin_bitcast<_To>(_mm_cvtepi32_epi8(__intrin)); 341 else if constexpr (__x_to_x && __have_avx512f) 342 return __intrin_bitcast<_To>( 343 __lo128(_mm512_cvtepi32_epi8(__zero_extend(__intrin)))); 344 else if constexpr (__x_to_x && __have_ssse3) 345 { 346 return __intrin_bitcast<_To>( 347 _mm_shuffle_epi8(__intrin, 348 _mm_setr_epi8(0, 4, 8, 12, -0x80, -0x80, -0x80, 349 -0x80, -0x80, -0x80, -0x80, -0x80, 350 -0x80, -0x80, -0x80, -0x80))); 351 } 352 else if constexpr (__x_to_x) 353 { 354 const auto __a 355 = _mm_unpacklo_epi8(__intrin, __intrin); // 0... .... 1... .... 356 const auto __b 357 = _mm_unpackhi_epi8(__intrin, __intrin); // 2... .... 3... .... 358 const auto __c = _mm_unpacklo_epi8(__a, __b); // 02.. .... .... .... 359 const auto __d = _mm_unpackhi_epi8(__a, __b); // 13.. .... .... .... 360 const auto __e = _mm_unpacklo_epi8(__c, __d); // 0123 .... .... .... 361 return __intrin_bitcast<_To>(__e & _mm_cvtsi32_si128(-1)); 362 } 363 else if constexpr (__y_to_x && __have_avx512vl) 364 return __intrin_bitcast<_To>(_mm256_cvtepi32_epi8(__intrin)); 365 else if constexpr (__y_to_x && __have_avx512f) 366 return __intrin_bitcast<_To>( 367 _mm512_cvtepi32_epi8(__zero_extend(__intrin))); 368 else if constexpr (__z_to_x) 369 return __intrin_bitcast<_To>(_mm512_cvtepi32_epi8(__intrin)); 370 } 371 else if constexpr (__i16_to_i64) //{{{2 372 { 373 if constexpr (__x_to_x && __have_sse4_1) 374 return __intrin_bitcast<_To>(is_signed_v<_Tp> 375 ? _mm_cvtepi16_epi64(__intrin) 376 : _mm_cvtepu16_epi64(__intrin)); 377 else if constexpr (__x_to_x && is_signed_v<_Tp>) 378 { 379 auto __x = _mm_srai_epi16(__intrin, 15); 380 auto __y = _mm_unpacklo_epi16(__intrin, __x); 381 __x = _mm_unpacklo_epi16(__x, __x); 382 return __intrin_bitcast<_To>(_mm_unpacklo_epi32(__y, __x)); 383 } 384 else if constexpr (__x_to_x) 385 return __intrin_bitcast<_To>( 386 _mm_unpacklo_epi32(_mm_unpacklo_epi16(__intrin, __m128i()), 387 __m128i())); 388 else if constexpr (__x_to_y) 389 return __intrin_bitcast<_To>(is_signed_v<_Tp> 390 ? _mm256_cvtepi16_epi64(__intrin) 391 : _mm256_cvtepu16_epi64(__intrin)); 392 else if constexpr (__x_to_z) 393 return __intrin_bitcast<_To>(is_signed_v<_Tp> 394 ? _mm512_cvtepi16_epi64(__intrin) 395 : _mm512_cvtepu16_epi64(__intrin)); 396 } 397 else if constexpr (__i16_to_i32) //{{{2 398 { 399 if constexpr (__x_to_x && __have_sse4_1) 400 return __intrin_bitcast<_To>(is_signed_v<_Tp> 401 ? _mm_cvtepi16_epi32(__intrin) 402 : _mm_cvtepu16_epi32(__intrin)); 403 else if constexpr (__x_to_x && is_signed_v<_Tp>) 404 return __intrin_bitcast<_To>( 405 _mm_srai_epi32(_mm_unpacklo_epi16(__intrin, __intrin), 16)); 406 else if constexpr (__x_to_x && is_unsigned_v<_Tp>) 407 return __intrin_bitcast<_To>(_mm_unpacklo_epi16(__intrin, __m128i())); 408 else if constexpr (__x_to_y) 409 return __intrin_bitcast<_To>(is_signed_v<_Tp> 410 ? _mm256_cvtepi16_epi32(__intrin) 411 : _mm256_cvtepu16_epi32(__intrin)); 412 else if constexpr (__y_to_z) 413 return __intrin_bitcast<_To>(is_signed_v<_Tp> 414 ? _mm512_cvtepi16_epi32(__intrin) 415 : _mm512_cvtepu16_epi32(__intrin)); 416 } 417 else if constexpr (__i16_to_i8) //{{{2 418 { 419 if constexpr (__x_to_x && __have_avx512bw_vl) 420 return __intrin_bitcast<_To>(_mm_cvtepi16_epi8(__intrin)); 421 else if constexpr (__x_to_x && __have_avx512bw) 422 return __intrin_bitcast<_To>( 423 __lo128(_mm512_cvtepi16_epi8(__zero_extend(__intrin)))); 424 else if constexpr (__x_to_x && __have_ssse3) 425 return __intrin_bitcast<_To>(_mm_shuffle_epi8( 426 __intrin, _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, -0x80, -0x80, 427 -0x80, -0x80, -0x80, -0x80, -0x80, -0x80))); 428 else if constexpr (__x_to_x) 429 { 430 auto __a 431 = _mm_unpacklo_epi8(__intrin, __intrin); // 00.. 11.. 22.. 33.. 432 auto __b 433 = _mm_unpackhi_epi8(__intrin, __intrin); // 44.. 55.. 66.. 77.. 434 auto __c = _mm_unpacklo_epi8(__a, __b); // 0404 .... 1515 .... 435 auto __d = _mm_unpackhi_epi8(__a, __b); // 2626 .... 3737 .... 436 auto __e = _mm_unpacklo_epi8(__c, __d); // 0246 0246 .... .... 437 auto __f = _mm_unpackhi_epi8(__c, __d); // 1357 1357 .... .... 438 return __intrin_bitcast<_To>(_mm_unpacklo_epi8(__e, __f)); 439 } 440 else if constexpr (__y_to_x && __have_avx512bw_vl) 441 return __intrin_bitcast<_To>(_mm256_cvtepi16_epi8(__intrin)); 442 else if constexpr (__y_to_x && __have_avx512bw) 443 return __intrin_bitcast<_To>( 444 __lo256(_mm512_cvtepi16_epi8(__zero_extend(__intrin)))); 445 else if constexpr (__y_to_x) 446 { 447 auto __a = _mm256_shuffle_epi8( 448 __intrin, 449 _mm256_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, -0x80, -0x80, -0x80, 450 -0x80, -0x80, -0x80, -0x80, -0x80, -0x80, -0x80, 451 -0x80, -0x80, -0x80, -0x80, -0x80, -0x80, 0, 2, 452 4, 6, 8, 10, 12, 14)); 453 return __intrin_bitcast<_To>(__lo128(__a) | __hi128(__a)); 454 } 455 else if constexpr (__z_to_y && __have_avx512bw) 456 return __intrin_bitcast<_To>(_mm512_cvtepi16_epi8(__intrin)); 457 else if constexpr (__z_to_y) 458 __assert_unreachable<_Tp>(); 459 } 460 else if constexpr (__i8_to_i64) //{{{2 461 { 462 if constexpr (__x_to_x && __have_sse4_1) 463 return __intrin_bitcast<_To>(is_signed_v<_Tp> 464 ? _mm_cvtepi8_epi64(__intrin) 465 : _mm_cvtepu8_epi64(__intrin)); 466 else if constexpr (__x_to_x && is_signed_v<_Tp>) 467 { 468 if constexpr (__have_ssse3) 469 { 470 auto __dup = _mm_unpacklo_epi8(__intrin, __intrin); 471 auto __epi16 = _mm_srai_epi16(__dup, 8); 472 _mm_shuffle_epi8(__epi16, 473 _mm_setr_epi8(0, 1, 1, 1, 1, 1, 1, 1, 2, 3, 3, 474 3, 3, 3, 3, 3)); 475 } 476 else 477 { 478 auto __x = _mm_unpacklo_epi8(__intrin, __intrin); 479 __x = _mm_unpacklo_epi16(__x, __x); 480 return __intrin_bitcast<_To>( 481 _mm_unpacklo_epi32(_mm_srai_epi32(__x, 24), 482 _mm_srai_epi32(__x, 31))); 483 } 484 } 485 else if constexpr (__x_to_x) 486 { 487 return __intrin_bitcast<_To>(_mm_unpacklo_epi32( 488 _mm_unpacklo_epi16(_mm_unpacklo_epi8(__intrin, __m128i()), 489 __m128i()), 490 __m128i())); 491 } 492 else if constexpr (__x_to_y) 493 return __intrin_bitcast<_To>(is_signed_v<_Tp> 494 ? _mm256_cvtepi8_epi64(__intrin) 495 : _mm256_cvtepu8_epi64(__intrin)); 496 else if constexpr (__x_to_z) 497 return __intrin_bitcast<_To>(is_signed_v<_Tp> 498 ? _mm512_cvtepi8_epi64(__intrin) 499 : _mm512_cvtepu8_epi64(__intrin)); 500 } 501 else if constexpr (__i8_to_i32) //{{{2 502 { 503 if constexpr (__x_to_x && __have_sse4_1) 504 return __intrin_bitcast<_To>(is_signed_v<_Tp> 505 ? _mm_cvtepi8_epi32(__intrin) 506 : _mm_cvtepu8_epi32(__intrin)); 507 else if constexpr (__x_to_x && is_signed_v<_Tp>) 508 { 509 const auto __x = _mm_unpacklo_epi8(__intrin, __intrin); 510 return __intrin_bitcast<_To>( 511 _mm_srai_epi32(_mm_unpacklo_epi16(__x, __x), 24)); 512 } 513 else if constexpr (__x_to_x && is_unsigned_v<_Tp>) 514 return __intrin_bitcast<_To>( 515 _mm_unpacklo_epi16(_mm_unpacklo_epi8(__intrin, __m128i()), 516 __m128i())); 517 else if constexpr (__x_to_y) 518 return __intrin_bitcast<_To>(is_signed_v<_Tp> 519 ? _mm256_cvtepi8_epi32(__intrin) 520 : _mm256_cvtepu8_epi32(__intrin)); 521 else if constexpr (__x_to_z) 522 return __intrin_bitcast<_To>(is_signed_v<_Tp> 523 ? _mm512_cvtepi8_epi32(__intrin) 524 : _mm512_cvtepu8_epi32(__intrin)); 525 } 526 else if constexpr (__i8_to_i16) //{{{2 527 { 528 if constexpr (__x_to_x && __have_sse4_1) 529 return __intrin_bitcast<_To>(is_signed_v<_Tp> 530 ? _mm_cvtepi8_epi16(__intrin) 531 : _mm_cvtepu8_epi16(__intrin)); 532 else if constexpr (__x_to_x && is_signed_v<_Tp>) 533 return __intrin_bitcast<_To>( 534 _mm_srai_epi16(_mm_unpacklo_epi8(__intrin, __intrin), 8)); 535 else if constexpr (__x_to_x && is_unsigned_v<_Tp>) 536 return __intrin_bitcast<_To>(_mm_unpacklo_epi8(__intrin, __m128i())); 537 else if constexpr (__x_to_y) 538 return __intrin_bitcast<_To>(is_signed_v<_Tp> 539 ? _mm256_cvtepi8_epi16(__intrin) 540 : _mm256_cvtepu8_epi16(__intrin)); 541 else if constexpr (__y_to_z && __have_avx512bw) 542 return __intrin_bitcast<_To>(is_signed_v<_Tp> 543 ? _mm512_cvtepi8_epi16(__intrin) 544 : _mm512_cvtepu8_epi16(__intrin)); 545 else if constexpr (__y_to_z) 546 __assert_unreachable<_Tp>(); 547 } 548 else if constexpr (__f32_to_s64) //{{{2 549 { 550 if constexpr (__have_avx512dq_vl && __x_to_x) 551 return __intrin_bitcast<_To>(_mm_cvttps_epi64(__intrin)); 552 else if constexpr (__have_avx512dq_vl && __x_to_y) 553 return __intrin_bitcast<_To>(_mm256_cvttps_epi64(__intrin)); 554 else if constexpr (__have_avx512dq && __y_to_z) 555 return __intrin_bitcast<_To>(_mm512_cvttps_epi64(__intrin)); 556 // else use scalar fallback 557 } 558 else if constexpr (__f32_to_u64) //{{{2 559 { 560 if constexpr (__have_avx512dq_vl && __x_to_x) 561 return __intrin_bitcast<_To>(_mm_cvttps_epu64(__intrin)); 562 else if constexpr (__have_avx512dq_vl && __x_to_y) 563 return __intrin_bitcast<_To>(_mm256_cvttps_epu64(__intrin)); 564 else if constexpr (__have_avx512dq && __y_to_z) 565 return __intrin_bitcast<_To>(_mm512_cvttps_epu64(__intrin)); 566 // else use scalar fallback 567 } 568 else if constexpr (__f32_to_s32) //{{{2 569 { 570 if constexpr (__x_to_x || __y_to_y || __z_to_z) 571 { 572 // go to fallback, it does the right thing 573 } 574 else 575 __assert_unreachable<_Tp>(); 576 } 577 else if constexpr (__f32_to_u32) //{{{2 578 { 579 if constexpr (__have_avx512vl && __x_to_x) 580 return __auto_bitcast(_mm_cvttps_epu32(__intrin)); 581 else if constexpr (__have_avx512f && __x_to_x) 582 return __auto_bitcast( 583 __lo128(_mm512_cvttps_epu32(__auto_bitcast(__v)))); 584 else if constexpr (__have_avx512vl && __y_to_y) 585 return __vector_bitcast<_Up>(_mm256_cvttps_epu32(__intrin)); 586 else if constexpr (__have_avx512f && __y_to_y) 587 return __vector_bitcast<_Up>( 588 __lo256(_mm512_cvttps_epu32(__auto_bitcast(__v)))); 589 else if constexpr (__x_to_x || __y_to_y || __z_to_z) 590 { 591 // go to fallback, it does the right thing. We can't use the 592 // _mm_floor_ps - 0x8000'0000 trick for f32->u32 because it would 593 // discard small input values (only 24 mantissa bits) 594 } 595 else 596 __assert_unreachable<_Tp>(); 597 } 598 else if constexpr (__f32_to_ibw) //{{{2 599 return __convert_x86<_To>(__convert_x86<__vector_type_t<int, _Np>>(__v)); 600 else if constexpr (__f64_to_s64) //{{{2 601 { 602 if constexpr (__have_avx512dq_vl && __x_to_x) 603 return __intrin_bitcast<_To>(_mm_cvttpd_epi64(__intrin)); 604 else if constexpr (__have_avx512dq_vl && __y_to_y) 605 return __intrin_bitcast<_To>(_mm256_cvttpd_epi64(__intrin)); 606 else if constexpr (__have_avx512dq && __z_to_z) 607 return __intrin_bitcast<_To>(_mm512_cvttpd_epi64(__intrin)); 608 // else use scalar fallback 609 } 610 else if constexpr (__f64_to_u64) //{{{2 611 { 612 if constexpr (__have_avx512dq_vl && __x_to_x) 613 return __intrin_bitcast<_To>(_mm_cvttpd_epu64(__intrin)); 614 else if constexpr (__have_avx512dq_vl && __y_to_y) 615 return __intrin_bitcast<_To>(_mm256_cvttpd_epu64(__intrin)); 616 else if constexpr (__have_avx512dq && __z_to_z) 617 return __intrin_bitcast<_To>(_mm512_cvttpd_epu64(__intrin)); 618 // else use scalar fallback 619 } 620 else if constexpr (__f64_to_s32) //{{{2 621 { 622 if constexpr (__x_to_x) 623 return __intrin_bitcast<_To>(_mm_cvttpd_epi32(__intrin)); 624 else if constexpr (__y_to_x) 625 return __intrin_bitcast<_To>(_mm256_cvttpd_epi32(__intrin)); 626 else if constexpr (__z_to_y) 627 return __intrin_bitcast<_To>(_mm512_cvttpd_epi32(__intrin)); 628 } 629 else if constexpr (__f64_to_u32) //{{{2 630 { 631 if constexpr (__have_avx512vl && __x_to_x) 632 return __intrin_bitcast<_To>(_mm_cvttpd_epu32(__intrin)); 633 else if constexpr (__have_sse4_1 && __x_to_x) 634 return __vector_bitcast<_Up, _M>( 635 _mm_cvttpd_epi32(_mm_floor_pd(__intrin) - 0x8000'0000u)) 636 ^ 0x8000'0000u; 637 else if constexpr (__x_to_x) 638 { 639 // use scalar fallback: it's only 2 values to convert, can't get 640 // much better than scalar decomposition 641 } 642 else if constexpr (__have_avx512vl && __y_to_x) 643 return __intrin_bitcast<_To>(_mm256_cvttpd_epu32(__intrin)); 644 else if constexpr (__y_to_x) 645 { 646 return __intrin_bitcast<_To>( 647 __vector_bitcast<_Up>( 648 _mm256_cvttpd_epi32(_mm256_floor_pd(__intrin) - 0x8000'0000u)) 649 ^ 0x8000'0000u); 650 } 651 else if constexpr (__z_to_y) 652 return __intrin_bitcast<_To>(_mm512_cvttpd_epu32(__intrin)); 653 } 654 else if constexpr (__f64_to_ibw) //{{{2 655 { 656 return __convert_x86<_To>( 657 __convert_x86<__vector_type_t<int, (_Np < 4 ? 4 : _Np)>>(__v)); 658 } 659 else if constexpr (__s64_to_f32) //{{{2 660 { 661 if constexpr (__x_to_x && __have_avx512dq_vl) 662 return __intrin_bitcast<_To>(_mm_cvtepi64_ps(__intrin)); 663 else if constexpr (__y_to_x && __have_avx512dq_vl) 664 return __intrin_bitcast<_To>(_mm256_cvtepi64_ps(__intrin)); 665 else if constexpr (__z_to_y && __have_avx512dq) 666 return __intrin_bitcast<_To>(_mm512_cvtepi64_ps(__intrin)); 667 else if constexpr (__z_to_y) 668 return __intrin_bitcast<_To>( 669 _mm512_cvtpd_ps(__convert_x86<__vector_type_t<double, 8>>(__v))); 670 } 671 else if constexpr (__u64_to_f32) //{{{2 672 { 673 if constexpr (__x_to_x && __have_avx512dq_vl) 674 return __intrin_bitcast<_To>(_mm_cvtepu64_ps(__intrin)); 675 else if constexpr (__y_to_x && __have_avx512dq_vl) 676 return __intrin_bitcast<_To>(_mm256_cvtepu64_ps(__intrin)); 677 else if constexpr (__z_to_y && __have_avx512dq) 678 return __intrin_bitcast<_To>(_mm512_cvtepu64_ps(__intrin)); 679 else if constexpr (__z_to_y) 680 { 681 return __intrin_bitcast<_To>( 682 __lo256(_mm512_cvtepu32_ps(__auto_bitcast( 683 _mm512_cvtepi64_epi32(_mm512_srai_epi64(__intrin, 32))))) 684 * 0x100000000LL 685 + __lo256(_mm512_cvtepu32_ps( 686 __auto_bitcast(_mm512_cvtepi64_epi32(__intrin))))); 687 } 688 } 689 else if constexpr (__s32_to_f32) //{{{2 690 { 691 // use fallback (builtin conversion) 692 } 693 else if constexpr (__u32_to_f32) //{{{2 694 { 695 if constexpr (__x_to_x && __have_avx512vl) 696 { 697 // use fallback 698 } 699 else if constexpr (__x_to_x && __have_avx512f) 700 return __intrin_bitcast<_To>( 701 __lo128(_mm512_cvtepu32_ps(__auto_bitcast(__v)))); 702 else if constexpr (__x_to_x && (__have_fma || __have_fma4)) 703 // work around PR85819 704 return __auto_bitcast(0x10000 705 * _mm_cvtepi32_ps(__to_intrin(__v >> 16)) 706 + _mm_cvtepi32_ps(__to_intrin(__v & 0xffff))); 707 else if constexpr (__y_to_y && __have_avx512vl) 708 { 709 // use fallback 710 } 711 else if constexpr (__y_to_y && __have_avx512f) 712 return __intrin_bitcast<_To>( 713 __lo256(_mm512_cvtepu32_ps(__auto_bitcast(__v)))); 714 else if constexpr (__y_to_y) 715 // work around PR85819 716 return 0x10000 * _mm256_cvtepi32_ps(__to_intrin(__v >> 16)) 717 + _mm256_cvtepi32_ps(__to_intrin(__v & 0xffff)); 718 // else use fallback (builtin conversion) 719 } 720 else if constexpr (__ibw_to_f32) //{{{2 721 { 722 if constexpr (_M <= 4 || __have_avx2) 723 return __convert_x86<_To>( 724 __convert_x86<__vector_type_t<int, _M>>(__v)); 725 else 726 { 727 static_assert(__x_to_y); 728 __m128i __a, __b; 729 if constexpr (__have_sse4_1) 730 { 731 __a = sizeof(_Tp) == 2 732 ? (is_signed_v<_Tp> ? _mm_cvtepi16_epi32(__intrin) 733 : _mm_cvtepu16_epi32(__intrin)) 734 : (is_signed_v<_Tp> ? _mm_cvtepi8_epi32(__intrin) 735 : _mm_cvtepu8_epi32(__intrin)); 736 const auto __w 737 = _mm_shuffle_epi32(__intrin, sizeof(_Tp) == 2 ? 0xee : 0xe9); 738 __b = sizeof(_Tp) == 2 739 ? (is_signed_v<_Tp> ? _mm_cvtepi16_epi32(__w) 740 : _mm_cvtepu16_epi32(__w)) 741 : (is_signed_v<_Tp> ? _mm_cvtepi8_epi32(__w) 742 : _mm_cvtepu8_epi32(__w)); 743 } 744 else 745 { 746 __m128i __tmp; 747 if constexpr (sizeof(_Tp) == 1) 748 { 749 __tmp = is_signed_v<_Tp> 750 ? _mm_srai_epi16(_mm_unpacklo_epi8(__intrin, 751 __intrin), 752 8) 753 : _mm_unpacklo_epi8(__intrin, __m128i()); 754 } 755 else 756 { 757 static_assert(sizeof(_Tp) == 2); 758 __tmp = __intrin; 759 } 760 __a = is_signed_v<_Tp> 761 ? _mm_srai_epi32(_mm_unpacklo_epi16(__tmp, __tmp), 16) 762 : _mm_unpacklo_epi16(__tmp, __m128i()); 763 __b = is_signed_v<_Tp> 764 ? _mm_srai_epi32(_mm_unpackhi_epi16(__tmp, __tmp), 16) 765 : _mm_unpackhi_epi16(__tmp, __m128i()); 766 } 767 return __convert_x86<_To>(__vector_bitcast<int>(__a), 768 __vector_bitcast<int>(__b)); 769 } 770 } 771 else if constexpr (__s64_to_f64) //{{{2 772 { 773 if constexpr (__x_to_x && __have_avx512dq_vl) 774 return __intrin_bitcast<_To>(_mm_cvtepi64_pd(__intrin)); 775 else if constexpr (__y_to_y && __have_avx512dq_vl) 776 return __intrin_bitcast<_To>(_mm256_cvtepi64_pd(__intrin)); 777 else if constexpr (__z_to_z && __have_avx512dq) 778 return __intrin_bitcast<_To>(_mm512_cvtepi64_pd(__intrin)); 779 else if constexpr (__z_to_z) 780 { 781 return __intrin_bitcast<_To>( 782 _mm512_cvtepi32_pd(_mm512_cvtepi64_epi32(__to_intrin(__v >> 32))) 783 * 0x100000000LL 784 + _mm512_cvtepu32_pd(_mm512_cvtepi64_epi32(__intrin))); 785 } 786 } 787 else if constexpr (__u64_to_f64) //{{{2 788 { 789 if constexpr (__x_to_x && __have_avx512dq_vl) 790 return __intrin_bitcast<_To>(_mm_cvtepu64_pd(__intrin)); 791 else if constexpr (__y_to_y && __have_avx512dq_vl) 792 return __intrin_bitcast<_To>(_mm256_cvtepu64_pd(__intrin)); 793 else if constexpr (__z_to_z && __have_avx512dq) 794 return __intrin_bitcast<_To>(_mm512_cvtepu64_pd(__intrin)); 795 else if constexpr (__z_to_z) 796 { 797 return __intrin_bitcast<_To>( 798 _mm512_cvtepu32_pd(_mm512_cvtepi64_epi32(__to_intrin(__v >> 32))) 799 * 0x100000000LL 800 + _mm512_cvtepu32_pd(_mm512_cvtepi64_epi32(__intrin))); 801 } 802 } 803 else if constexpr (__s32_to_f64) //{{{2 804 { 805 if constexpr (__x_to_x) 806 return __intrin_bitcast<_To>(_mm_cvtepi32_pd(__intrin)); 807 else if constexpr (__x_to_y) 808 return __intrin_bitcast<_To>(_mm256_cvtepi32_pd(__intrin)); 809 else if constexpr (__y_to_z) 810 return __intrin_bitcast<_To>(_mm512_cvtepi32_pd(__intrin)); 811 } 812 else if constexpr (__u32_to_f64) //{{{2 813 { 814 if constexpr (__x_to_x && __have_avx512vl) 815 return __intrin_bitcast<_To>(_mm_cvtepu32_pd(__intrin)); 816 else if constexpr (__x_to_x && __have_avx512f) 817 return __intrin_bitcast<_To>( 818 __lo128(_mm512_cvtepu32_pd(__auto_bitcast(__v)))); 819 else if constexpr (__x_to_x) 820 return __intrin_bitcast<_To>( 821 _mm_cvtepi32_pd(__to_intrin(__v ^ 0x8000'0000u)) + 0x8000'0000u); 822 else if constexpr (__x_to_y && __have_avx512vl) 823 return __intrin_bitcast<_To>(_mm256_cvtepu32_pd(__intrin)); 824 else if constexpr (__x_to_y && __have_avx512f) 825 return __intrin_bitcast<_To>( 826 __lo256(_mm512_cvtepu32_pd(__auto_bitcast(__v)))); 827 else if constexpr (__x_to_y) 828 return __intrin_bitcast<_To>( 829 _mm256_cvtepi32_pd(__to_intrin(__v ^ 0x8000'0000u)) + 0x8000'0000u); 830 else if constexpr (__y_to_z) 831 return __intrin_bitcast<_To>(_mm512_cvtepu32_pd(__intrin)); 832 } 833 else if constexpr (__ibw_to_f64) //{{{2 834 { 835 return __convert_x86<_To>( 836 __convert_x86<__vector_type_t<int, std::max(size_t(4), _M)>>(__v)); 837 } 838 else if constexpr (__f32_to_f64) //{{{2 839 { 840 if constexpr (__x_to_x) 841 return __intrin_bitcast<_To>(_mm_cvtps_pd(__intrin)); 842 else if constexpr (__x_to_y) 843 return __intrin_bitcast<_To>(_mm256_cvtps_pd(__intrin)); 844 else if constexpr (__y_to_z) 845 return __intrin_bitcast<_To>(_mm512_cvtps_pd(__intrin)); 846 } 847 else if constexpr (__f64_to_f32) //{{{2 848 { 849 if constexpr (__x_to_x) 850 return __intrin_bitcast<_To>(_mm_cvtpd_ps(__intrin)); 851 else if constexpr (__y_to_x) 852 return __intrin_bitcast<_To>(_mm256_cvtpd_ps(__intrin)); 853 else if constexpr (__z_to_y) 854 return __intrin_bitcast<_To>(_mm512_cvtpd_ps(__intrin)); 855 } 856 else //{{{2 857 __assert_unreachable<_Tp>(); 858 859 // fallback:{{{2 860 return __vector_convert<_To>(__v, make_index_sequence<std::min(_M, _Np)>()); 861 //}}} 862 } 863 864 // }}} 865 // 2-arg __convert_x86 {{{1 866 template <typename _To, typename _V, typename _Traits> 867 _GLIBCXX_SIMD_INTRINSIC _To __convert_x86(_V __v0,_V __v1)868 __convert_x86(_V __v0, _V __v1) 869 { 870 static_assert(__is_vector_type_v<_V>); 871 using _Tp = typename _Traits::value_type; 872 constexpr size_t _Np = _Traits::_S_full_size; 873 [[maybe_unused]] const auto __i0 = __to_intrin(__v0); 874 [[maybe_unused]] const auto __i1 = __to_intrin(__v1); 875 using _Up = typename _VectorTraits<_To>::value_type; 876 constexpr size_t _M = _VectorTraits<_To>::_S_full_size; 877 878 static_assert(2 * _Np <= _M, 879 "__v1 would be discarded; use the one-argument " 880 "__convert_x86 overload instead"); 881 882 // [xyz]_to_[xyz] {{{2 883 [[maybe_unused]] constexpr bool __x_to_x 884 = sizeof(__v0) <= 16 && sizeof(_To) <= 16; 885 [[maybe_unused]] constexpr bool __x_to_y 886 = sizeof(__v0) <= 16 && sizeof(_To) == 32; 887 [[maybe_unused]] constexpr bool __x_to_z 888 = sizeof(__v0) <= 16 && sizeof(_To) == 64; 889 [[maybe_unused]] constexpr bool __y_to_x 890 = sizeof(__v0) == 32 && sizeof(_To) <= 16; 891 [[maybe_unused]] constexpr bool __y_to_y 892 = sizeof(__v0) == 32 && sizeof(_To) == 32; 893 [[maybe_unused]] constexpr bool __y_to_z 894 = sizeof(__v0) == 32 && sizeof(_To) == 64; 895 [[maybe_unused]] constexpr bool __z_to_x 896 = sizeof(__v0) == 64 && sizeof(_To) <= 16; 897 [[maybe_unused]] constexpr bool __z_to_y 898 = sizeof(__v0) == 64 && sizeof(_To) == 32; 899 [[maybe_unused]] constexpr bool __z_to_z 900 = sizeof(__v0) == 64 && sizeof(_To) == 64; 901 902 // iX_to_iX {{{2 903 [[maybe_unused]] constexpr bool __i_to_i 904 = is_integral_v<_Up> && is_integral_v<_Tp>; 905 [[maybe_unused]] constexpr bool __i8_to_i16 906 = __i_to_i && sizeof(_Tp) == 1 && sizeof(_Up) == 2; 907 [[maybe_unused]] constexpr bool __i8_to_i32 908 = __i_to_i && sizeof(_Tp) == 1 && sizeof(_Up) == 4; 909 [[maybe_unused]] constexpr bool __i8_to_i64 910 = __i_to_i && sizeof(_Tp) == 1 && sizeof(_Up) == 8; 911 [[maybe_unused]] constexpr bool __i16_to_i8 912 = __i_to_i && sizeof(_Tp) == 2 && sizeof(_Up) == 1; 913 [[maybe_unused]] constexpr bool __i16_to_i32 914 = __i_to_i && sizeof(_Tp) == 2 && sizeof(_Up) == 4; 915 [[maybe_unused]] constexpr bool __i16_to_i64 916 = __i_to_i && sizeof(_Tp) == 2 && sizeof(_Up) == 8; 917 [[maybe_unused]] constexpr bool __i32_to_i8 918 = __i_to_i && sizeof(_Tp) == 4 && sizeof(_Up) == 1; 919 [[maybe_unused]] constexpr bool __i32_to_i16 920 = __i_to_i && sizeof(_Tp) == 4 && sizeof(_Up) == 2; 921 [[maybe_unused]] constexpr bool __i32_to_i64 922 = __i_to_i && sizeof(_Tp) == 4 && sizeof(_Up) == 8; 923 [[maybe_unused]] constexpr bool __i64_to_i8 924 = __i_to_i && sizeof(_Tp) == 8 && sizeof(_Up) == 1; 925 [[maybe_unused]] constexpr bool __i64_to_i16 926 = __i_to_i && sizeof(_Tp) == 8 && sizeof(_Up) == 2; 927 [[maybe_unused]] constexpr bool __i64_to_i32 928 = __i_to_i && sizeof(_Tp) == 8 && sizeof(_Up) == 4; 929 930 // [fsu]X_to_[fsu]X {{{2 931 // ibw = integral && byte or word, i.e. char and short with any signedness 932 [[maybe_unused]] constexpr bool __i64_to_f32 933 = is_integral_v<_Tp> && sizeof(_Tp) == 8 934 && is_floating_point_v<_Up> && sizeof(_Up) == 4; 935 [[maybe_unused]] constexpr bool __s32_to_f32 936 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 4 937 && is_floating_point_v<_Up> && sizeof(_Up) == 4; 938 [[maybe_unused]] constexpr bool __s16_to_f32 939 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 2 940 && is_floating_point_v<_Up> && sizeof(_Up) == 4; 941 [[maybe_unused]] constexpr bool __s8_to_f32 942 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 1 943 && is_floating_point_v<_Up> && sizeof(_Up) == 4; 944 [[maybe_unused]] constexpr bool __u32_to_f32 945 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 4 946 && is_floating_point_v<_Up> && sizeof(_Up) == 4; 947 [[maybe_unused]] constexpr bool __u16_to_f32 948 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 2 949 && is_floating_point_v<_Up> && sizeof(_Up) == 4; 950 [[maybe_unused]] constexpr bool __u8_to_f32 951 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 1 952 && is_floating_point_v<_Up> && sizeof(_Up) == 4; 953 [[maybe_unused]] constexpr bool __s64_to_f64 954 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 8 955 && is_floating_point_v<_Up> && sizeof(_Up) == 8; 956 [[maybe_unused]] constexpr bool __s32_to_f64 957 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 4 958 && is_floating_point_v<_Up> && sizeof(_Up) == 8; 959 [[maybe_unused]] constexpr bool __s16_to_f64 960 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 2 961 && is_floating_point_v<_Up> && sizeof(_Up) == 8; 962 [[maybe_unused]] constexpr bool __s8_to_f64 963 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 1 964 && is_floating_point_v<_Up> && sizeof(_Up) == 8; 965 [[maybe_unused]] constexpr bool __u64_to_f64 966 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 8 967 && is_floating_point_v<_Up> && sizeof(_Up) == 8; 968 [[maybe_unused]] constexpr bool __u32_to_f64 969 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 4 970 && is_floating_point_v<_Up> && sizeof(_Up) == 8; 971 [[maybe_unused]] constexpr bool __u16_to_f64 972 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 2 973 && is_floating_point_v<_Up> && sizeof(_Up) == 8; 974 [[maybe_unused]] constexpr bool __u8_to_f64 975 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 1 976 && is_floating_point_v<_Up> && sizeof(_Up) == 8; 977 [[maybe_unused]] constexpr bool __f32_to_s64 978 = is_integral_v<_Up> && is_signed_v<_Up> && sizeof(_Up) == 8 979 && is_floating_point_v<_Tp> && sizeof(_Tp) == 4; 980 [[maybe_unused]] constexpr bool __f32_to_s32 981 = is_integral_v<_Up> && is_signed_v<_Up> && sizeof(_Up) == 4 982 && is_floating_point_v<_Tp> && sizeof(_Tp) == 4; 983 [[maybe_unused]] constexpr bool __f32_to_u64 984 = is_integral_v<_Up> && is_unsigned_v<_Up> && sizeof(_Up) == 8 985 && is_floating_point_v<_Tp> && sizeof(_Tp) == 4; 986 [[maybe_unused]] constexpr bool __f32_to_u32 987 = is_integral_v<_Up> && is_unsigned_v<_Up> && sizeof(_Up) == 4 988 && is_floating_point_v<_Tp> && sizeof(_Tp) == 4; 989 [[maybe_unused]] constexpr bool __f64_to_s64 990 = is_integral_v<_Up> && is_signed_v<_Up> && sizeof(_Up) == 8 991 && is_floating_point_v<_Tp> && sizeof(_Tp) == 8; 992 [[maybe_unused]] constexpr bool __f64_to_s32 993 = is_integral_v<_Up> && is_signed_v<_Up> && sizeof(_Up) == 4 994 && is_floating_point_v<_Tp> && sizeof(_Tp) == 8; 995 [[maybe_unused]] constexpr bool __f64_to_u64 996 = is_integral_v<_Up> && is_unsigned_v<_Up> && sizeof(_Up) == 8 997 && is_floating_point_v<_Tp> && sizeof(_Tp) == 8; 998 [[maybe_unused]] constexpr bool __f64_to_u32 999 = is_integral_v<_Up> && is_unsigned_v<_Up> && sizeof(_Up) == 4 1000 && is_floating_point_v<_Tp> && sizeof(_Tp) == 8; 1001 [[maybe_unused]] constexpr bool __f32_to_ibw 1002 = is_integral_v<_Up> && sizeof(_Up) <= 2 1003 && is_floating_point_v<_Tp> && sizeof(_Tp) == 4; 1004 [[maybe_unused]] constexpr bool __f64_to_ibw 1005 = is_integral_v<_Up> && sizeof(_Up) <= 2 1006 && is_floating_point_v<_Tp> && sizeof(_Tp) == 8; 1007 [[maybe_unused]] constexpr bool __f32_to_f64 1008 = is_floating_point_v<_Tp> && sizeof(_Tp) == 4 1009 && is_floating_point_v<_Up> && sizeof(_Up) == 8; 1010 [[maybe_unused]] constexpr bool __f64_to_f32 1011 = is_floating_point_v<_Tp> && sizeof(_Tp) == 8 1012 && is_floating_point_v<_Up> && sizeof(_Up) == 4; 1013 1014 if constexpr (__i_to_i && __y_to_x && !__have_avx2) //{{{2 1015 // <double, 4>, <double, 4> => <short, 8> 1016 return __convert_x86<_To>(__lo128(__v0), __hi128(__v0), __lo128(__v1), 1017 __hi128(__v1)); 1018 else if constexpr (__i_to_i) // assert ISA {{{2 1019 { 1020 static_assert(__x_to_x || __have_avx2, 1021 "integral conversions with ymm registers require AVX2"); 1022 static_assert(__have_avx512bw 1023 || ((sizeof(_Tp) >= 4 || sizeof(__v0) < 64) 1024 && (sizeof(_Up) >= 4 || sizeof(_To) < 64)), 1025 "8/16-bit integers in zmm registers require AVX512BW"); 1026 static_assert((sizeof(__v0) < 64 && sizeof(_To) < 64) || __have_avx512f, 1027 "integral conversions with ymm registers require AVX2"); 1028 } 1029 // concat => use 1-arg __convert_x86 {{{2 1030 if constexpr (sizeof(__v0) < 16 || (sizeof(__v0) == 16 && __have_avx2) 1031 || (sizeof(__v0) == 16 && __have_avx 1032 && is_floating_point_v<_Tp>) 1033 || (sizeof(__v0) == 32 && __have_avx512f 1034 && (sizeof(_Tp) >= 4 || __have_avx512bw))) 1035 { 1036 // The ISA can handle wider input registers, so concat and use one-arg 1037 // implementation. This reduces code duplication considerably. 1038 return __convert_x86<_To>(__concat(__v0, __v1)); 1039 } 1040 else //{{{2 1041 { 1042 // conversion using bit reinterpretation (or no conversion at all) 1043 // should all go through the concat branch above: 1044 static_assert( 1045 !(is_floating_point_v< 1046 _Tp> == is_floating_point_v<_Up> && sizeof(_Tp) == sizeof(_Up))); 1047 // handle all zero extension{{{2 1048 if constexpr (2 * _Np < _M && sizeof(_To) > 16) 1049 { 1050 constexpr size_t Min = 16 / sizeof(_Up); 1051 return __zero_extend( 1052 __convert_x86< 1053 __vector_type_t<_Up, (Min > 2 * _Np) ? Min : 2 * _Np>>(__v0, 1054 __v1)); 1055 } 1056 else if constexpr (__i64_to_i32) //{{{2 1057 { 1058 if constexpr (__x_to_x) 1059 return __auto_bitcast(_mm_shuffle_ps(__auto_bitcast(__v0), 1060 __auto_bitcast(__v1), 0x88)); 1061 else if constexpr (__y_to_y) 1062 { 1063 // AVX512F is not available (would concat otherwise) 1064 return __auto_bitcast( 1065 __xzyw(_mm256_shuffle_ps(__auto_bitcast(__v0), 1066 __auto_bitcast(__v1), 0x88))); 1067 // alternative: 1068 // const auto v0_abxxcdxx = _mm256_shuffle_epi32(__v0, 8); 1069 // const auto v1_efxxghxx = _mm256_shuffle_epi32(__v1, 8); 1070 // const auto v_abefcdgh = _mm256_unpacklo_epi64(v0_abxxcdxx, 1071 // v1_efxxghxx); return _mm256_permute4x64_epi64(v_abefcdgh, 1072 // 0x01 * 0 + 0x04 * 2 + 0x10 * 1 + 0x40 * 3); // abcdefgh 1073 } 1074 else if constexpr (__z_to_z) 1075 return __intrin_bitcast<_To>( 1076 __concat(_mm512_cvtepi64_epi32(__i0), 1077 _mm512_cvtepi64_epi32(__i1))); 1078 } 1079 else if constexpr (__i64_to_i16) //{{{2 1080 { 1081 if constexpr (__x_to_x) 1082 { 1083 // AVX2 is not available (would concat otherwise) 1084 if constexpr (__have_sse4_1) 1085 { 1086 return __intrin_bitcast<_To>(_mm_shuffle_epi8( 1087 _mm_blend_epi16(__i0, _mm_slli_si128(__i1, 4), 0x44), 1088 _mm_setr_epi8(0, 1, 8, 9, 4, 5, 12, 13, -0x80, -0x80, 1089 -0x80, -0x80, -0x80, -0x80, -0x80, -0x80))); 1090 } 1091 else 1092 { 1093 return __vector_type_t<_Up, _M>{_Up(__v0[0]), _Up(__v0[1]), 1094 _Up(__v1[0]), _Up(__v1[1])}; 1095 } 1096 } 1097 else if constexpr (__y_to_x) 1098 { 1099 auto __a 1100 = _mm256_unpacklo_epi16(__i0, __i1); // 04.. .... 26.. .... 1101 auto __b 1102 = _mm256_unpackhi_epi16(__i0, __i1); // 15.. .... 37.. .... 1103 auto __c 1104 = _mm256_unpacklo_epi16(__a, __b); // 0145 .... 2367 .... 1105 return __intrin_bitcast<_To>( 1106 _mm_unpacklo_epi32(__lo128(__c), __hi128(__c))); // 0123 4567 1107 } 1108 else if constexpr (__z_to_y) 1109 return __intrin_bitcast<_To>( 1110 __concat(_mm512_cvtepi64_epi16(__i0), 1111 _mm512_cvtepi64_epi16(__i1))); 1112 } 1113 else if constexpr (__i64_to_i8) //{{{2 1114 { 1115 if constexpr (__x_to_x && __have_sse4_1) 1116 { 1117 return __intrin_bitcast<_To>(_mm_shuffle_epi8( 1118 _mm_blend_epi16(__i0, _mm_slli_si128(__i1, 4), 0x44), 1119 _mm_setr_epi8(0, 8, 4, 12, -0x80, -0x80, -0x80, -0x80, -0x80, 1120 -0x80, -0x80, -0x80, -0x80, -0x80, -0x80, 1121 -0x80))); 1122 } 1123 else if constexpr (__x_to_x && __have_ssse3) 1124 { 1125 return __intrin_bitcast<_To>(_mm_unpacklo_epi16( 1126 _mm_shuffle_epi8( 1127 __i0, _mm_setr_epi8(0, 8, -0x80, -0x80, -0x80, -0x80, -0x80, 1128 -0x80, -0x80, -0x80, -0x80, -0x80, 1129 -0x80, -0x80, -0x80, -0x80)), 1130 _mm_shuffle_epi8( 1131 __i1, _mm_setr_epi8(0, 8, -0x80, -0x80, -0x80, -0x80, -0x80, 1132 -0x80, -0x80, -0x80, -0x80, -0x80, 1133 -0x80, -0x80, -0x80, -0x80)))); 1134 } 1135 else if constexpr (__x_to_x) 1136 { 1137 return __vector_type_t<_Up, _M>{_Up(__v0[0]), _Up(__v0[1]), 1138 _Up(__v1[0]), _Up(__v1[1])}; 1139 } 1140 else if constexpr (__y_to_x) 1141 { 1142 const auto __a = _mm256_shuffle_epi8( 1143 _mm256_blend_epi32(__i0, _mm256_slli_epi64(__i1, 32), 0xAA), 1144 _mm256_setr_epi8(0, 8, -0x80, -0x80, 4, 12, -0x80, -0x80, 1145 -0x80, -0x80, -0x80, -0x80, -0x80, -0x80, 1146 -0x80, -0x80, -0x80, -0x80, 0, 8, -0x80, 1147 -0x80, 4, 12, -0x80, -0x80, -0x80, -0x80, 1148 -0x80, -0x80, -0x80, -0x80)); 1149 return __intrin_bitcast<_To>(__lo128(__a) | __hi128(__a)); 1150 } // __z_to_x uses concat fallback 1151 } 1152 else if constexpr (__i32_to_i16) //{{{2 1153 { 1154 if constexpr (__x_to_x) 1155 { 1156 // AVX2 is not available (would concat otherwise) 1157 if constexpr (__have_sse4_1) 1158 { 1159 return __intrin_bitcast<_To>(_mm_shuffle_epi8( 1160 _mm_blend_epi16(__i0, _mm_slli_si128(__i1, 2), 0xaa), 1161 _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 1162 11, 14, 15))); 1163 } 1164 else if constexpr (__have_ssse3) 1165 { 1166 return __intrin_bitcast<_To>( 1167 _mm_hadd_epi16(__to_intrin(__v0 << 16), 1168 __to_intrin(__v1 << 16))); 1169 /* 1170 return _mm_unpacklo_epi64( 1171 _mm_shuffle_epi8(__i0, _mm_setr_epi8(0, 1, 4, 5, 8, 9, 1172 12, 13, 8, 9, 12, 13, 12, 13, 14, 15)), 1173 _mm_shuffle_epi8(__i1, _mm_setr_epi8(0, 1, 4, 5, 8, 9, 12, 1174 13, 8, 9, 12, 13, 12, 13, 14, 15))); 1175 */ 1176 } 1177 else 1178 { 1179 auto __a = _mm_unpacklo_epi16(__i0, __i1); // 04.. 15.. 1180 auto __b = _mm_unpackhi_epi16(__i0, __i1); // 26.. 37.. 1181 auto __c = _mm_unpacklo_epi16(__a, __b); // 0246 .... 1182 auto __d = _mm_unpackhi_epi16(__a, __b); // 1357 .... 1183 return __intrin_bitcast<_To>( 1184 _mm_unpacklo_epi16(__c, __d)); // 0123 4567 1185 } 1186 } 1187 else if constexpr (__y_to_y) 1188 { 1189 const auto __shuf 1190 = _mm256_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, -0x80, -0x80, 1191 -0x80, -0x80, -0x80, -0x80, -0x80, -0x80, 1192 0, 1, 4, 5, 8, 9, 12, 13, -0x80, -0x80, 1193 -0x80, -0x80, -0x80, -0x80, -0x80, -0x80); 1194 auto __a = _mm256_shuffle_epi8(__i0, __shuf); 1195 auto __b = _mm256_shuffle_epi8(__i1, __shuf); 1196 return __intrin_bitcast<_To>( 1197 __xzyw(_mm256_unpacklo_epi64(__a, __b))); 1198 } // __z_to_z uses concat fallback 1199 } 1200 else if constexpr (__i32_to_i8) //{{{2 1201 { 1202 if constexpr (__x_to_x && __have_ssse3) 1203 { 1204 const auto shufmask 1205 = _mm_setr_epi8(0, 4, 8, 12, -0x80, -0x80, -0x80, -0x80, 1206 -0x80, -0x80, -0x80, -0x80, -0x80, -0x80, 1207 -0x80, -0x80); 1208 return __intrin_bitcast<_To>( 1209 _mm_unpacklo_epi32(_mm_shuffle_epi8(__i0, shufmask), 1210 _mm_shuffle_epi8(__i1, shufmask))); 1211 } 1212 else if constexpr (__x_to_x) 1213 { 1214 auto __a = _mm_unpacklo_epi8(__i0, __i1); // 04.. .... 15.. .... 1215 auto __b = _mm_unpackhi_epi8(__i0, __i1); // 26.. .... 37.. .... 1216 auto __c = _mm_unpacklo_epi8(__a, __b); // 0246 .... .... .... 1217 auto __d = _mm_unpackhi_epi8(__a, __b); // 1357 .... .... .... 1218 auto __e = _mm_unpacklo_epi8(__c, __d); // 0123 4567 .... .... 1219 return __intrin_bitcast<_To>(__e & __m128i{-1, 0}); 1220 } 1221 else if constexpr (__y_to_x) 1222 { 1223 const auto __a = _mm256_shuffle_epi8( 1224 _mm256_blend_epi16(__i0, _mm256_slli_epi32(__i1, 16), 0xAA), 1225 _mm256_setr_epi8(0, 4, 8, 12, -0x80, -0x80, -0x80, -0x80, 2, 1226 6, 10, 14, -0x80, -0x80, -0x80, -0x80, -0x80, 1227 -0x80, -0x80, -0x80, 0, 4, 8, 12, -0x80, 1228 -0x80, -0x80, -0x80, 2, 6, 10, 14)); 1229 return __intrin_bitcast<_To>(__lo128(__a) | __hi128(__a)); 1230 } // __z_to_y uses concat fallback 1231 } 1232 else if constexpr (__i16_to_i8) //{{{2 1233 { 1234 if constexpr (__x_to_x && __have_ssse3) 1235 { 1236 const auto __shuf = reinterpret_cast<__m128i>( 1237 __vector_type_t<_UChar, 16>{0, 2, 4, 6, 8, 10, 12, 14, 0x80, 1238 0x80, 0x80, 0x80, 0x80, 0x80, 1239 0x80, 0x80}); 1240 return __intrin_bitcast<_To>( 1241 _mm_unpacklo_epi64(_mm_shuffle_epi8(__i0, __shuf), 1242 _mm_shuffle_epi8(__i1, __shuf))); 1243 } 1244 else if constexpr (__x_to_x) 1245 { 1246 auto __a = _mm_unpacklo_epi8(__i0, __i1); // 08.. 19.. 2A.. 3B.. 1247 auto __b = _mm_unpackhi_epi8(__i0, __i1); // 4C.. 5D.. 6E.. 7F.. 1248 auto __c = _mm_unpacklo_epi8(__a, __b); // 048C .... 159D .... 1249 auto __d = _mm_unpackhi_epi8(__a, __b); // 26AE .... 37BF .... 1250 auto __e = _mm_unpacklo_epi8(__c, __d); // 0246 8ACE .... .... 1251 auto __f = _mm_unpackhi_epi8(__c, __d); // 1357 9BDF .... .... 1252 return __intrin_bitcast<_To>(_mm_unpacklo_epi8(__e, __f)); 1253 } 1254 else if constexpr (__y_to_y) 1255 { 1256 return __intrin_bitcast<_To>(__xzyw(_mm256_shuffle_epi8( 1257 (__to_intrin(__v0) & _mm256_set1_epi32(0x00ff00ff)) 1258 | _mm256_slli_epi16(__i1, 8), 1259 _mm256_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 1260 13, 15, 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 1261 7, 9, 11, 13, 15)))); 1262 } // __z_to_z uses concat fallback 1263 } 1264 else if constexpr (__i64_to_f32) //{{{2 1265 { 1266 if constexpr (__x_to_x) 1267 return __make_wrapper<float>(__v0[0], __v0[1], __v1[0], __v1[1]); 1268 else if constexpr (__y_to_y) 1269 { 1270 static_assert(__y_to_y && __have_avx2); 1271 const auto __a = _mm256_unpacklo_epi32(__i0, __i1); // aeAE cgCG 1272 const auto __b = _mm256_unpackhi_epi32(__i0, __i1); // bfBF dhDH 1273 const auto __lo32 1274 = _mm256_unpacklo_epi32(__a, __b); // abef cdgh 1275 const auto __hi32 = __vector_bitcast< 1276 conditional_t<is_signed_v<_Tp>, int, _UInt>>( 1277 _mm256_unpackhi_epi32(__a, __b)); // ABEF CDGH 1278 const auto __hi 1279 = 0x100000000LL 1280 * __convert_x86<__vector_type_t<float, 8>>(__hi32); 1281 const auto __mid 1282 = 0x10000 * _mm256_cvtepi32_ps(_mm256_srli_epi32(__lo32, 16)); 1283 const auto __lo 1284 = _mm256_cvtepi32_ps(_mm256_set1_epi32(0x0000ffffu) & __lo32); 1285 return __xzyw((__hi + __mid) + __lo); 1286 } 1287 else if constexpr (__z_to_z && __have_avx512dq) 1288 { 1289 return is_signed_v<_Tp> ? __concat(_mm512_cvtepi64_ps(__i0), 1290 _mm512_cvtepi64_ps(__i1)) 1291 : __concat(_mm512_cvtepu64_ps(__i0), 1292 _mm512_cvtepu64_ps(__i1)); 1293 } 1294 else if constexpr (__z_to_z && is_signed_v<_Tp>) 1295 { 1296 const __m512 __hi32 = _mm512_cvtepi32_ps( 1297 __concat(_mm512_cvtepi64_epi32(__to_intrin(__v0 >> 32)), 1298 _mm512_cvtepi64_epi32(__to_intrin(__v1 >> 32)))); 1299 const __m512i __lo32 = __concat(_mm512_cvtepi64_epi32(__i0), 1300 _mm512_cvtepi64_epi32(__i1)); 1301 // split low 32-bits, because if __hi32 is a small negative 1302 // number, the 24-bit mantissa may lose important information if 1303 // any of the high 8 bits of __lo32 is set, leading to 1304 // catastrophic cancelation in the FMA 1305 const __m512 __hi16 1306 = _mm512_cvtepu32_ps(_mm512_set1_epi32(0xffff0000u) & __lo32); 1307 const __m512 __lo16 1308 = _mm512_cvtepi32_ps(_mm512_set1_epi32(0x0000ffffu) & __lo32); 1309 return (__hi32 * 0x100000000LL + __hi16) + __lo16; 1310 } 1311 else if constexpr (__z_to_z && is_unsigned_v<_Tp>) 1312 { 1313 return __intrin_bitcast<_To>( 1314 _mm512_cvtepu32_ps(__concat( 1315 _mm512_cvtepi64_epi32(_mm512_srai_epi64(__i0, 32)), 1316 _mm512_cvtepi64_epi32(_mm512_srai_epi64(__i1, 32)))) 1317 * 0x100000000LL 1318 + _mm512_cvtepu32_ps(__concat(_mm512_cvtepi64_epi32(__i0), 1319 _mm512_cvtepi64_epi32(__i1)))); 1320 } 1321 } 1322 else if constexpr (__f64_to_s32) //{{{2 1323 { 1324 // use concat fallback 1325 } 1326 else if constexpr (__f64_to_u32) //{{{2 1327 { 1328 if constexpr (__x_to_x && __have_sse4_1) 1329 { 1330 return __vector_bitcast<_Up, _M>(_mm_unpacklo_epi64( 1331 _mm_cvttpd_epi32(_mm_floor_pd(__i0) - 0x8000'0000u), 1332 _mm_cvttpd_epi32(_mm_floor_pd(__i1) - 0x8000'0000u))) 1333 ^ 0x8000'0000u; 1334 // without SSE4.1 just use the scalar fallback, it's only four 1335 // values 1336 } 1337 else if constexpr (__y_to_y) 1338 { 1339 return __vector_bitcast<_Up>( 1340 __concat(_mm256_cvttpd_epi32(_mm256_floor_pd(__i0) 1341 - 0x8000'0000u), 1342 _mm256_cvttpd_epi32(_mm256_floor_pd(__i1) 1343 - 0x8000'0000u))) 1344 ^ 0x8000'0000u; 1345 } // __z_to_z uses fallback 1346 } 1347 else if constexpr (__f64_to_ibw) //{{{2 1348 { 1349 // one-arg __f64_to_ibw goes via _SimdWrapper<int, ?>. The fallback 1350 // would go via two independet conversions to _SimdWrapper<_To> and 1351 // subsequent interleaving. This is better, because f64->__i32 1352 // allows to combine __v0 and __v1 into one register: if constexpr 1353 // (__z_to_x || __y_to_x) { 1354 return __convert_x86<_To>( 1355 __convert_x86<__vector_type_t<int, _Np * 2>>(__v0, __v1)); 1356 //} 1357 } 1358 else if constexpr (__f32_to_ibw) //{{{2 1359 { 1360 return __convert_x86<_To>( 1361 __convert_x86<__vector_type_t<int, _Np>>(__v0), 1362 __convert_x86<__vector_type_t<int, _Np>>(__v1)); 1363 } //}}} 1364 1365 // fallback: {{{2 1366 if constexpr (sizeof(_To) >= 32) 1367 // if _To is ymm or zmm, then _SimdWrapper<_Up, _M / 2> is xmm or ymm 1368 return __concat(__convert_x86<__vector_type_t<_Up, _M / 2>>(__v0), 1369 __convert_x86<__vector_type_t<_Up, _M / 2>>(__v1)); 1370 else if constexpr (sizeof(_To) == 16) 1371 { 1372 const auto __lo = __to_intrin(__convert_x86<_To>(__v0)); 1373 const auto __hi = __to_intrin(__convert_x86<_To>(__v1)); 1374 if constexpr (sizeof(_Up) * _Np == 8) 1375 { 1376 if constexpr (is_floating_point_v<_Up>) 1377 return __auto_bitcast( 1378 _mm_unpacklo_pd(__vector_bitcast<double>(__lo), 1379 __vector_bitcast<double>(__hi))); 1380 else 1381 return __intrin_bitcast<_To>(_mm_unpacklo_epi64(__lo, __hi)); 1382 } 1383 else if constexpr (sizeof(_Up) * _Np == 4) 1384 { 1385 if constexpr (is_floating_point_v<_Up>) 1386 return __auto_bitcast( 1387 _mm_unpacklo_ps(__vector_bitcast<float>(__lo), 1388 __vector_bitcast<float>(__hi))); 1389 else 1390 return __intrin_bitcast<_To>(_mm_unpacklo_epi32(__lo, __hi)); 1391 } 1392 else if constexpr (sizeof(_Up) * _Np == 2) 1393 return __intrin_bitcast<_To>(_mm_unpacklo_epi16(__lo, __hi)); 1394 else 1395 __assert_unreachable<_Tp>(); 1396 } 1397 else 1398 return __vector_convert<_To>(__v0, __v1, make_index_sequence<_Np>()); 1399 //}}} 1400 } 1401 } 1402 1403 //}}}1 1404 // 4-arg __convert_x86 {{{1 1405 template <typename _To, typename _V, typename _Traits> 1406 _GLIBCXX_SIMD_INTRINSIC _To 1407 __convert_x86(_V __v0, _V __v1, _V __v2, _V __v3) 1408 { 1409 static_assert(__is_vector_type_v<_V>); 1410 using _Tp = typename _Traits::value_type; 1411 constexpr size_t _Np = _Traits::_S_full_size; 1412 [[maybe_unused]] const auto __i0 = __to_intrin(__v0); 1413 [[maybe_unused]] const auto __i1 = __to_intrin(__v1); 1414 [[maybe_unused]] const auto __i2 = __to_intrin(__v2); 1415 [[maybe_unused]] const auto __i3 = __to_intrin(__v3); 1416 using _Up = typename _VectorTraits<_To>::value_type; 1417 constexpr size_t _M = _VectorTraits<_To>::_S_full_size; 1418 1419 static_assert(4 * _Np <= _M, 1420 "__v2/__v3 would be discarded; use the two/one-argument " 1421 "__convert_x86 overload instead"); 1422 1423 // [xyz]_to_[xyz] {{{2 1424 [[maybe_unused]] constexpr bool __x_to_x 1425 = sizeof(__v0) <= 16 && sizeof(_To) <= 16; 1426 [[maybe_unused]] constexpr bool __x_to_y 1427 = sizeof(__v0) <= 16 && sizeof(_To) == 32; 1428 [[maybe_unused]] constexpr bool __x_to_z 1429 = sizeof(__v0) <= 16 && sizeof(_To) == 64; 1430 [[maybe_unused]] constexpr bool __y_to_x 1431 = sizeof(__v0) == 32 && sizeof(_To) <= 16; 1432 [[maybe_unused]] constexpr bool __y_to_y 1433 = sizeof(__v0) == 32 && sizeof(_To) == 32; 1434 [[maybe_unused]] constexpr bool __y_to_z 1435 = sizeof(__v0) == 32 && sizeof(_To) == 64; 1436 [[maybe_unused]] constexpr bool __z_to_x 1437 = sizeof(__v0) == 64 && sizeof(_To) <= 16; 1438 [[maybe_unused]] constexpr bool __z_to_y 1439 = sizeof(__v0) == 64 && sizeof(_To) == 32; 1440 [[maybe_unused]] constexpr bool __z_to_z 1441 = sizeof(__v0) == 64 && sizeof(_To) == 64; 1442 1443 // iX_to_iX {{{2 1444 [[maybe_unused]] constexpr bool __i_to_i 1445 = is_integral_v<_Up> && is_integral_v<_Tp>; 1446 [[maybe_unused]] constexpr bool __i8_to_i16 1447 = __i_to_i && sizeof(_Tp) == 1 && sizeof(_Up) == 2; 1448 [[maybe_unused]] constexpr bool __i8_to_i32 1449 = __i_to_i && sizeof(_Tp) == 1 && sizeof(_Up) == 4; 1450 [[maybe_unused]] constexpr bool __i8_to_i64 1451 = __i_to_i && sizeof(_Tp) == 1 && sizeof(_Up) == 8; 1452 [[maybe_unused]] constexpr bool __i16_to_i8 1453 = __i_to_i && sizeof(_Tp) == 2 && sizeof(_Up) == 1; 1454 [[maybe_unused]] constexpr bool __i16_to_i32 1455 = __i_to_i && sizeof(_Tp) == 2 && sizeof(_Up) == 4; 1456 [[maybe_unused]] constexpr bool __i16_to_i64 1457 = __i_to_i && sizeof(_Tp) == 2 && sizeof(_Up) == 8; 1458 [[maybe_unused]] constexpr bool __i32_to_i8 1459 = __i_to_i && sizeof(_Tp) == 4 && sizeof(_Up) == 1; 1460 [[maybe_unused]] constexpr bool __i32_to_i16 1461 = __i_to_i && sizeof(_Tp) == 4 && sizeof(_Up) == 2; 1462 [[maybe_unused]] constexpr bool __i32_to_i64 1463 = __i_to_i && sizeof(_Tp) == 4 && sizeof(_Up) == 8; 1464 [[maybe_unused]] constexpr bool __i64_to_i8 1465 = __i_to_i && sizeof(_Tp) == 8 && sizeof(_Up) == 1; 1466 [[maybe_unused]] constexpr bool __i64_to_i16 1467 = __i_to_i && sizeof(_Tp) == 8 && sizeof(_Up) == 2; 1468 [[maybe_unused]] constexpr bool __i64_to_i32 1469 = __i_to_i && sizeof(_Tp) == 8 && sizeof(_Up) == 4; 1470 1471 // [fsu]X_to_[fsu]X {{{2 1472 // ibw = integral && byte or word, i.e. char and short with any signedness 1473 [[maybe_unused]] constexpr bool __i64_to_f32 1474 = is_integral_v<_Tp> && sizeof(_Tp) == 8 1475 && is_floating_point_v<_Up> && sizeof(_Up) == 4; 1476 [[maybe_unused]] constexpr bool __s32_to_f32 1477 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 4 1478 && is_floating_point_v<_Up> && sizeof(_Up) == 4; 1479 [[maybe_unused]] constexpr bool __s16_to_f32 1480 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 2 1481 && is_floating_point_v<_Up> && sizeof(_Up) == 4; 1482 [[maybe_unused]] constexpr bool __s8_to_f32 1483 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 1 1484 && is_floating_point_v<_Up> && sizeof(_Up) == 4; 1485 [[maybe_unused]] constexpr bool __u32_to_f32 1486 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 4 1487 && is_floating_point_v<_Up> && sizeof(_Up) == 4; 1488 [[maybe_unused]] constexpr bool __u16_to_f32 1489 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 2 1490 && is_floating_point_v<_Up> && sizeof(_Up) == 4; 1491 [[maybe_unused]] constexpr bool __u8_to_f32 1492 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 1 1493 && is_floating_point_v<_Up> && sizeof(_Up) == 4; 1494 [[maybe_unused]] constexpr bool __s64_to_f64 1495 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 8 1496 && is_floating_point_v<_Up> && sizeof(_Up) == 8; 1497 [[maybe_unused]] constexpr bool __s32_to_f64 1498 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 4 1499 && is_floating_point_v<_Up> && sizeof(_Up) == 8; 1500 [[maybe_unused]] constexpr bool __s16_to_f64 1501 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 2 1502 && is_floating_point_v<_Up> && sizeof(_Up) == 8; 1503 [[maybe_unused]] constexpr bool __s8_to_f64 1504 = is_integral_v<_Tp> && is_signed_v<_Tp> && sizeof(_Tp) == 1 1505 && is_floating_point_v<_Up> && sizeof(_Up) == 8; 1506 [[maybe_unused]] constexpr bool __u64_to_f64 1507 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 8 1508 && is_floating_point_v<_Up> && sizeof(_Up) == 8; 1509 [[maybe_unused]] constexpr bool __u32_to_f64 1510 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 4 1511 && is_floating_point_v<_Up> && sizeof(_Up) == 8; 1512 [[maybe_unused]] constexpr bool __u16_to_f64 1513 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 2 1514 && is_floating_point_v<_Up> && sizeof(_Up) == 8; 1515 [[maybe_unused]] constexpr bool __u8_to_f64 1516 = is_integral_v<_Tp> && is_unsigned_v<_Tp> && sizeof(_Tp) == 1 1517 && is_floating_point_v<_Up> && sizeof(_Up) == 8; 1518 [[maybe_unused]] constexpr bool __f32_to_s64 1519 = is_integral_v<_Up> && is_signed_v<_Up> && sizeof(_Up) == 8 1520 && is_floating_point_v<_Tp> && sizeof(_Tp) == 4; 1521 [[maybe_unused]] constexpr bool __f32_to_s32 1522 = is_integral_v<_Up> && is_signed_v<_Up> && sizeof(_Up) == 4 1523 && is_floating_point_v<_Tp> && sizeof(_Tp) == 4; 1524 [[maybe_unused]] constexpr bool __f32_to_u64 1525 = is_integral_v<_Up> && is_unsigned_v<_Up> && sizeof(_Up) == 8 1526 && is_floating_point_v<_Tp> && sizeof(_Tp) == 4; 1527 [[maybe_unused]] constexpr bool __f32_to_u32 1528 = is_integral_v<_Up> && is_unsigned_v<_Up> && sizeof(_Up) == 4 1529 && is_floating_point_v<_Tp> && sizeof(_Tp) == 4; 1530 [[maybe_unused]] constexpr bool __f64_to_s64 1531 = is_integral_v<_Up> && is_signed_v<_Up> && sizeof(_Up) == 8 1532 && is_floating_point_v<_Tp> && sizeof(_Tp) == 8; 1533 [[maybe_unused]] constexpr bool __f64_to_s32 1534 = is_integral_v<_Up> && is_signed_v<_Up> && sizeof(_Up) == 4 1535 && is_floating_point_v<_Tp> && sizeof(_Tp) == 8; 1536 [[maybe_unused]] constexpr bool __f64_to_u64 1537 = is_integral_v<_Up> && is_unsigned_v<_Up> && sizeof(_Up) == 8 1538 && is_floating_point_v<_Tp> && sizeof(_Tp) == 8; 1539 [[maybe_unused]] constexpr bool __f64_to_u32 1540 = is_integral_v<_Up> && is_unsigned_v<_Up> && sizeof(_Up) == 4 1541 && is_floating_point_v<_Tp> && sizeof(_Tp) == 8; 1542 [[maybe_unused]] constexpr bool __f32_to_ibw 1543 = is_integral_v<_Up> && sizeof(_Up) <= 2 1544 && is_floating_point_v<_Tp> && sizeof(_Tp) == 4; 1545 [[maybe_unused]] constexpr bool __f64_to_ibw 1546 = is_integral_v<_Up> && sizeof(_Up) <= 2 1547 && is_floating_point_v<_Tp> && sizeof(_Tp) == 8; 1548 [[maybe_unused]] constexpr bool __f32_to_f64 1549 = is_floating_point_v<_Tp> && sizeof(_Tp) == 4 1550 && is_floating_point_v<_Up> && sizeof(_Up) == 8; 1551 [[maybe_unused]] constexpr bool __f64_to_f32 1552 = is_floating_point_v<_Tp> && sizeof(_Tp) == 8 1553 && is_floating_point_v<_Up> && sizeof(_Up) == 4; 1554 1555 if constexpr (__i_to_i && __y_to_x && !__have_avx2) //{{{2 1556 { 1557 // <double, 4>, <double, 4>, <double, 4>, <double, 4> => <char, 16> 1558 return __convert_x86<_To>(__lo128(__v0), __hi128(__v0), __lo128(__v1), 1559 __hi128(__v1), __lo128(__v2), __hi128(__v2), 1560 __lo128(__v3), __hi128(__v3)); 1561 } 1562 else if constexpr (__i_to_i) // assert ISA {{{2 1563 { 1564 static_assert(__x_to_x || __have_avx2, 1565 "integral conversions with ymm registers require AVX2"); 1566 static_assert(__have_avx512bw 1567 || ((sizeof(_Tp) >= 4 || sizeof(__v0) < 64) 1568 && (sizeof(_Up) >= 4 || sizeof(_To) < 64)), 1569 "8/16-bit integers in zmm registers require AVX512BW"); 1570 static_assert((sizeof(__v0) < 64 && sizeof(_To) < 64) || __have_avx512f, 1571 "integral conversions with ymm registers require AVX2"); 1572 } 1573 // concat => use 2-arg __convert_x86 {{{2 1574 if constexpr (sizeof(__v0) < 16 || (sizeof(__v0) == 16 && __have_avx2) 1575 || (sizeof(__v0) == 16 && __have_avx 1576 && is_floating_point_v<_Tp>) 1577 || (sizeof(__v0) == 32 && __have_avx512f)) 1578 { 1579 // The ISA can handle wider input registers, so concat and use two-arg 1580 // implementation. This reduces code duplication considerably. 1581 return __convert_x86<_To>(__concat(__v0, __v1), __concat(__v2, __v3)); 1582 } 1583 else //{{{2 1584 { 1585 // conversion using bit reinterpretation (or no conversion at all) 1586 // should all go through the concat branch above: 1587 static_assert( 1588 !(is_floating_point_v< 1589 _Tp> == is_floating_point_v<_Up> && sizeof(_Tp) == sizeof(_Up))); 1590 // handle all zero extension{{{2 1591 if constexpr (4 * _Np < _M && sizeof(_To) > 16) 1592 { 1593 constexpr size_t Min = 16 / sizeof(_Up); 1594 return __zero_extend( 1595 __convert_x86< 1596 __vector_type_t<_Up, (Min > 4 * _Np) ? Min : 4 * _Np>>( 1597 __v0, __v1, __v2, __v3)); 1598 } 1599 else if constexpr (__i64_to_i16) //{{{2 1600 { 1601 if constexpr (__x_to_x && __have_sse4_1) 1602 { 1603 return __intrin_bitcast<_To>(_mm_shuffle_epi8( 1604 _mm_blend_epi16( 1605 _mm_blend_epi16(__i0, _mm_slli_si128(__i1, 2), 0x22), 1606 _mm_blend_epi16(_mm_slli_si128(__i2, 4), 1607 _mm_slli_si128(__i3, 6), 0x88), 1608 0xcc), 1609 _mm_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 1610 14, 15))); 1611 } 1612 else if constexpr (__y_to_y && __have_avx2) 1613 { 1614 return __intrin_bitcast<_To>(_mm256_shuffle_epi8( 1615 __xzyw(_mm256_blend_epi16( 1616 __auto_bitcast( 1617 _mm256_shuffle_ps(__vector_bitcast<float>(__v0), 1618 __vector_bitcast<float>(__v2), 1619 0x88)), // 0.1. 8.9. 2.3. A.B. 1620 __to_intrin(__vector_bitcast<int>(_mm256_shuffle_ps( 1621 __vector_bitcast<float>(__v1), 1622 __vector_bitcast<float>(__v3), 0x88)) 1623 << 16), // .4.5 .C.D .6.7 .E.F 1624 0xaa) // 0415 8C9D 2637 AEBF 1625 ), // 0415 2637 8C9D AEBF 1626 _mm256_setr_epi8(0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 1627 14, 15, 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 1628 10, 11, 14, 15))); 1629 /* 1630 auto __a = _mm256_unpacklo_epi16(__v0, __v1); // 04.. .... 26.. 1631 .... auto __b = _mm256_unpackhi_epi16(__v0, __v1); // 15.. 1632 .... 37.. .... auto __c = _mm256_unpacklo_epi16(__v2, __v3); // 1633 8C.. .... AE.. .... auto __d = _mm256_unpackhi_epi16(__v2, 1634 __v3); 1635 // 9D.. .... BF.. .... auto __e = _mm256_unpacklo_epi16(__a, 1636 __b); 1637 // 0145 .... 2367 .... auto __f = _mm256_unpacklo_epi16(__c, 1638 __d); 1639 // 89CD .... ABEF .... auto __g = _mm256_unpacklo_epi64(__e, 1640 __f); 1641 // 0145 89CD 2367 ABEF return __concat( 1642 _mm_unpacklo_epi32(__lo128(__g), __hi128(__g)), 1643 _mm_unpackhi_epi32(__lo128(__g), __hi128(__g))); // 0123 1644 4567 89AB CDEF 1645 */ 1646 } // else use fallback 1647 } 1648 else if constexpr (__i64_to_i8) //{{{2 1649 { 1650 if constexpr (__x_to_x) 1651 { 1652 // TODO: use fallback for now 1653 } 1654 else if constexpr (__y_to_x) 1655 { 1656 auto __a 1657 = _mm256_srli_epi32(_mm256_slli_epi32(__i0, 24), 24) 1658 | _mm256_srli_epi32(_mm256_slli_epi32(__i1, 24), 16) 1659 | _mm256_srli_epi32(_mm256_slli_epi32(__i2, 24), 8) 1660 | _mm256_slli_epi32( 1661 __i3, 24); // 048C .... 159D .... 26AE .... 37BF .... 1662 /*return _mm_shuffle_epi8( 1663 _mm_blend_epi32(__lo128(__a) << 32, __hi128(__a), 0x5), 1664 _mm_setr_epi8(4, 12, 0, 8, 5, 13, 1, 9, 6, 14, 2, 10, 7, 15, 1665 3, 11));*/ 1666 auto __b = _mm256_unpackhi_epi64( 1667 __a, __a); // 159D .... 159D .... 37BF .... 37BF .... 1668 auto __c = _mm256_unpacklo_epi8( 1669 __a, __b); // 0145 89CD .... .... 2367 ABEF .... .... 1670 return __intrin_bitcast<_To>( 1671 _mm_unpacklo_epi16(__lo128(__c), 1672 __hi128(__c))); // 0123 4567 89AB CDEF 1673 } 1674 } 1675 else if constexpr (__i32_to_i8) //{{{2 1676 { 1677 if constexpr (__x_to_x) 1678 { 1679 if constexpr (__have_ssse3) 1680 { 1681 const auto __x0 = __vector_bitcast<_UInt>(__v0) & 0xff; 1682 const auto __x1 = (__vector_bitcast<_UInt>(__v1) & 0xff) 1683 << 8; 1684 const auto __x2 = (__vector_bitcast<_UInt>(__v2) & 0xff) 1685 << 16; 1686 const auto __x3 = __vector_bitcast<_UInt>(__v3) << 24; 1687 return __intrin_bitcast<_To>( 1688 _mm_shuffle_epi8(__to_intrin(__x0 | __x1 | __x2 | __x3), 1689 _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 1690 2, 6, 10, 14, 3, 7, 11, 1691 15))); 1692 } 1693 else 1694 { 1695 auto __a 1696 = _mm_unpacklo_epi8(__i0, __i2); // 08.. .... 19.. .... 1697 auto __b 1698 = _mm_unpackhi_epi8(__i0, __i2); // 2A.. .... 3B.. .... 1699 auto __c 1700 = _mm_unpacklo_epi8(__i1, __i3); // 4C.. .... 5D.. .... 1701 auto __d 1702 = _mm_unpackhi_epi8(__i1, __i3); // 6E.. .... 7F.. .... 1703 auto __e 1704 = _mm_unpacklo_epi8(__a, __c); // 048C .... .... .... 1705 auto __f 1706 = _mm_unpackhi_epi8(__a, __c); // 159D .... .... .... 1707 auto __g 1708 = _mm_unpacklo_epi8(__b, __d); // 26AE .... .... .... 1709 auto __h 1710 = _mm_unpackhi_epi8(__b, __d); // 37BF .... .... .... 1711 return __intrin_bitcast<_To>(_mm_unpacklo_epi8( 1712 _mm_unpacklo_epi8(__e, __g), // 0246 8ACE .... .... 1713 _mm_unpacklo_epi8(__f, __h) // 1357 9BDF .... .... 1714 )); // 0123 4567 89AB CDEF 1715 } 1716 } 1717 else if constexpr (__y_to_y) 1718 { 1719 const auto __a = _mm256_shuffle_epi8( 1720 __to_intrin((__vector_bitcast<_UShort>(_mm256_blend_epi16( 1721 __i0, _mm256_slli_epi32(__i1, 16), 0xAA)) 1722 & 0xff) 1723 | (__vector_bitcast<_UShort>(_mm256_blend_epi16( 1724 __i2, _mm256_slli_epi32(__i3, 16), 0xAA)) 1725 << 8)), 1726 _mm256_setr_epi8(0, 4, 8, 12, 2, 6, 10, 14, 1, 5, 9, 13, 3, 7, 1727 11, 15, 0, 4, 8, 12, 2, 6, 10, 14, 1, 5, 9, 1728 13, 3, 7, 11, 15)); 1729 return __intrin_bitcast<_To>(_mm256_permutevar8x32_epi32( 1730 __a, _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7))); 1731 } 1732 } 1733 else if constexpr (__i64_to_f32) //{{{2 1734 { 1735 // this branch is only relevant with AVX and w/o AVX2 (i.e. no ymm 1736 // integers) 1737 if constexpr (__x_to_y) 1738 { 1739 return __make_wrapper<float>(__v0[0], __v0[1], __v1[0], __v1[1], 1740 __v2[0], __v2[1], __v3[0], 1741 __v3[1]); 1742 1743 const auto __a = _mm_unpacklo_epi32(__i0, __i1); // acAC 1744 const auto __b = _mm_unpackhi_epi32(__i0, __i1); // bdBD 1745 const auto __c = _mm_unpacklo_epi32(__i2, __i3); // egEG 1746 const auto __d = _mm_unpackhi_epi32(__i2, __i3); // fhFH 1747 const auto __lo32a = _mm_unpacklo_epi32(__a, __b); // abcd 1748 const auto __lo32b = _mm_unpacklo_epi32(__c, __d); // efgh 1749 const auto __hi32 = __vector_bitcast< 1750 conditional_t<is_signed_v<_Tp>, int, _UInt>>( 1751 __concat(_mm_unpackhi_epi32(__a, __b), 1752 _mm_unpackhi_epi32(__c, __d))); // ABCD EFGH 1753 const auto __hi 1754 = 0x100000000LL 1755 * __convert_x86<__vector_type_t<float, 8>>(__hi32); 1756 const auto __mid 1757 = 0x10000 1758 * _mm256_cvtepi32_ps(__concat(_mm_srli_epi32(__lo32a, 16), 1759 _mm_srli_epi32(__lo32b, 16))); 1760 const auto __lo = _mm256_cvtepi32_ps( 1761 __concat(_mm_set1_epi32(0x0000ffffu) & __lo32a, 1762 _mm_set1_epi32(0x0000ffffu) & __lo32b)); 1763 return (__hi + __mid) + __lo; 1764 } 1765 } 1766 else if constexpr (__f64_to_ibw) //{{{2 1767 { 1768 return __convert_x86<_To>( 1769 __convert_x86<__vector_type_t<int, _Np * 2>>(__v0, __v1), 1770 __convert_x86<__vector_type_t<int, _Np * 2>>(__v2, __v3)); 1771 } 1772 else if constexpr (__f32_to_ibw) //{{{2 1773 { 1774 return __convert_x86<_To>( 1775 __convert_x86<__vector_type_t<int, _Np>>(__v0), 1776 __convert_x86<__vector_type_t<int, _Np>>(__v1), 1777 __convert_x86<__vector_type_t<int, _Np>>(__v2), 1778 __convert_x86<__vector_type_t<int, _Np>>(__v3)); 1779 } //}}} 1780 1781 // fallback: {{{2 1782 if constexpr (sizeof(_To) >= 32) 1783 // if _To is ymm or zmm, then _SimdWrapper<_Up, _M / 2> is xmm or ymm 1784 return __concat(__convert_x86<__vector_type_t<_Up, _M / 2>>(__v0, 1785 __v1), 1786 __convert_x86<__vector_type_t<_Up, _M / 2>>(__v2, 1787 __v3)); 1788 else if constexpr (sizeof(_To) == 16) 1789 { 1790 const auto __lo = __to_intrin(__convert_x86<_To>(__v0, __v1)); 1791 const auto __hi = __to_intrin(__convert_x86<_To>(__v2, __v3)); 1792 if constexpr (sizeof(_Up) * _Np * 2 == 8) 1793 { 1794 if constexpr (is_floating_point_v<_Up>) 1795 return __auto_bitcast(_mm_unpacklo_pd(__lo, __hi)); 1796 else 1797 return __intrin_bitcast<_To>(_mm_unpacklo_epi64(__lo, __hi)); 1798 } 1799 else if constexpr (sizeof(_Up) * _Np * 2 == 4) 1800 { 1801 if constexpr (is_floating_point_v<_Up>) 1802 return __auto_bitcast(_mm_unpacklo_ps(__lo, __hi)); 1803 else 1804 return __intrin_bitcast<_To>(_mm_unpacklo_epi32(__lo, __hi)); 1805 } 1806 else 1807 __assert_unreachable<_Tp>(); 1808 } 1809 else 1810 return __vector_convert<_To>(__v0, __v1, __v2, __v3, 1811 make_index_sequence<_Np>()); 1812 //}}}2 1813 } 1814 } 1815 1816 //}}} 1817 // 8-arg __convert_x86 {{{1 1818 template <typename _To, typename _V, typename _Traits> 1819 _GLIBCXX_SIMD_INTRINSIC _To 1820 __convert_x86(_V __v0, _V __v1, _V __v2, _V __v3, _V __v4, _V __v5, _V __v6, 1821 _V __v7) 1822 { 1823 static_assert(__is_vector_type_v<_V>); 1824 using _Tp = typename _Traits::value_type; 1825 constexpr size_t _Np = _Traits::_S_full_size; 1826 [[maybe_unused]] const auto __i0 = __to_intrin(__v0); 1827 [[maybe_unused]] const auto __i1 = __to_intrin(__v1); 1828 [[maybe_unused]] const auto __i2 = __to_intrin(__v2); 1829 [[maybe_unused]] const auto __i3 = __to_intrin(__v3); 1830 [[maybe_unused]] const auto __i4 = __to_intrin(__v4); 1831 [[maybe_unused]] const auto __i5 = __to_intrin(__v5); 1832 [[maybe_unused]] const auto __i6 = __to_intrin(__v6); 1833 [[maybe_unused]] const auto __i7 = __to_intrin(__v7); 1834 using _Up = typename _VectorTraits<_To>::value_type; 1835 constexpr size_t _M = _VectorTraits<_To>::_S_full_size; 1836 1837 static_assert(8 * _Np <= _M, 1838 "__v4-__v7 would be discarded; use the four/two/one-argument " 1839 "__convert_x86 overload instead"); 1840 1841 // [xyz]_to_[xyz] {{{2 1842 [[maybe_unused]] constexpr bool __x_to_x 1843 = sizeof(__v0) <= 16 && sizeof(_To) <= 16; 1844 [[maybe_unused]] constexpr bool __x_to_y 1845 = sizeof(__v0) <= 16 && sizeof(_To) == 32; 1846 [[maybe_unused]] constexpr bool __x_to_z 1847 = sizeof(__v0) <= 16 && sizeof(_To) == 64; 1848 [[maybe_unused]] constexpr bool __y_to_x 1849 = sizeof(__v0) == 32 && sizeof(_To) <= 16; 1850 [[maybe_unused]] constexpr bool __y_to_y 1851 = sizeof(__v0) == 32 && sizeof(_To) == 32; 1852 [[maybe_unused]] constexpr bool __y_to_z 1853 = sizeof(__v0) == 32 && sizeof(_To) == 64; 1854 [[maybe_unused]] constexpr bool __z_to_x 1855 = sizeof(__v0) == 64 && sizeof(_To) <= 16; 1856 [[maybe_unused]] constexpr bool __z_to_y 1857 = sizeof(__v0) == 64 && sizeof(_To) == 32; 1858 [[maybe_unused]] constexpr bool __z_to_z 1859 = sizeof(__v0) == 64 && sizeof(_To) == 64; 1860 1861 // [if]X_to_i8 {{{2 1862 [[maybe_unused]] constexpr bool __i_to_i 1863 = is_integral_v<_Up> && is_integral_v<_Tp>; 1864 [[maybe_unused]] constexpr bool __i64_to_i8 1865 = __i_to_i && sizeof(_Tp) == 8 && sizeof(_Up) == 1; 1866 [[maybe_unused]] constexpr bool __f64_to_i8 1867 = is_integral_v<_Up> && sizeof(_Up) == 1 1868 && is_floating_point_v<_Tp> && sizeof(_Tp) == 8; 1869 1870 if constexpr (__i_to_i) // assert ISA {{{2 1871 { 1872 static_assert(__x_to_x || __have_avx2, 1873 "integral conversions with ymm registers require AVX2"); 1874 static_assert(__have_avx512bw 1875 || ((sizeof(_Tp) >= 4 || sizeof(__v0) < 64) 1876 && (sizeof(_Up) >= 4 || sizeof(_To) < 64)), 1877 "8/16-bit integers in zmm registers require AVX512BW"); 1878 static_assert((sizeof(__v0) < 64 && sizeof(_To) < 64) || __have_avx512f, 1879 "integral conversions with ymm registers require AVX2"); 1880 } 1881 // concat => use 4-arg __convert_x86 {{{2 1882 if constexpr (sizeof(__v0) < 16 || (sizeof(__v0) == 16 && __have_avx2) 1883 || (sizeof(__v0) == 16 && __have_avx 1884 && is_floating_point_v<_Tp>) 1885 || (sizeof(__v0) == 32 && __have_avx512f)) 1886 { 1887 // The ISA can handle wider input registers, so concat and use two-arg 1888 // implementation. This reduces code duplication considerably. 1889 return __convert_x86<_To>(__concat(__v0, __v1), __concat(__v2, __v3), 1890 __concat(__v4, __v5), __concat(__v6, __v7)); 1891 } 1892 else //{{{2 1893 { 1894 // conversion using bit reinterpretation (or no conversion at all) 1895 // should all go through the concat branch above: 1896 static_assert( 1897 !(is_floating_point_v< 1898 _Tp> == is_floating_point_v<_Up> && sizeof(_Tp) == sizeof(_Up))); 1899 static_assert(!(8 * _Np < _M && sizeof(_To) > 16), 1900 "zero extension should be impossible"); 1901 if constexpr (__i64_to_i8) //{{{2 1902 { 1903 if constexpr (__x_to_x && __have_ssse3) 1904 { 1905 // unsure whether this is better than the variant below 1906 return __intrin_bitcast<_To>(_mm_shuffle_epi8( 1907 __to_intrin( 1908 (((__v0 & 0xff) | ((__v1 & 0xff) << 8)) 1909 | (((__v2 & 0xff) << 16) | ((__v3 & 0xff) << 24))) 1910 | ((((__v4 & 0xff) << 32) | ((__v5 & 0xff) << 40)) 1911 | (((__v6 & 0xff) << 48) | (__v7 << 56)))), 1912 _mm_setr_epi8(0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 1913 7, 15))); 1914 } 1915 else if constexpr (__x_to_x) 1916 { 1917 const auto __a = _mm_unpacklo_epi8(__i0, __i1); // ac 1918 const auto __b = _mm_unpackhi_epi8(__i0, __i1); // bd 1919 const auto __c = _mm_unpacklo_epi8(__i2, __i3); // eg 1920 const auto __d = _mm_unpackhi_epi8(__i2, __i3); // fh 1921 const auto __e = _mm_unpacklo_epi8(__i4, __i5); // ik 1922 const auto __f = _mm_unpackhi_epi8(__i4, __i5); // jl 1923 const auto __g = _mm_unpacklo_epi8(__i6, __i7); // mo 1924 const auto __h = _mm_unpackhi_epi8(__i6, __i7); // np 1925 return __intrin_bitcast<_To>(_mm_unpacklo_epi64( 1926 _mm_unpacklo_epi32(_mm_unpacklo_epi8(__a, __b), // abcd 1927 _mm_unpacklo_epi8(__c, __d)), // efgh 1928 _mm_unpacklo_epi32(_mm_unpacklo_epi8(__e, __f), // ijkl 1929 _mm_unpacklo_epi8(__g, __h)) // mnop 1930 )); 1931 } 1932 else if constexpr (__y_to_y) 1933 { 1934 auto __a = // 048C GKOS 159D HLPT 26AE IMQU 37BF JNRV 1935 __to_intrin( 1936 (((__v0 & 0xff) | ((__v1 & 0xff) << 8)) 1937 | (((__v2 & 0xff) << 16) | ((__v3 & 0xff) << 24))) 1938 | ((((__v4 & 0xff) << 32) | ((__v5 & 0xff) << 40)) 1939 | (((__v6 & 0xff) << 48) | ((__v7 << 56))))); 1940 /* 1941 auto __b = _mm256_unpackhi_epi64(__a, __a); // 159D HLPT 159D 1942 HLPT 37BF JNRV 37BF JNRV auto __c = _mm256_unpacklo_epi8(__a, 1943 __b); // 0145 89CD GHKL OPST 2367 ABEF IJMN QRUV auto __d = 1944 __xzyw(__c); // 0145 89CD 2367 ABEF GHKL OPST IJMN QRUV return 1945 _mm256_shuffle_epi8( 1946 __d, _mm256_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 1947 13, 6, 7, 14, 15, 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 1948 14, 15)); 1949 */ 1950 auto __b = _mm256_shuffle_epi8( // 0145 89CD GHKL OPST 2367 ABEF 1951 // IJMN QRUV 1952 __a, _mm256_setr_epi8(0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 1953 6, 14, 7, 15, 0, 8, 1, 9, 2, 10, 3, 11, 1954 4, 12, 5, 13, 6, 14, 7, 15)); 1955 auto __c 1956 = __xzyw(__b); // 0145 89CD 2367 ABEF GHKL OPST IJMN QRUV 1957 return __intrin_bitcast<_To>(_mm256_shuffle_epi8( 1958 __c, _mm256_setr_epi8(0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 1959 6, 7, 14, 15, 0, 1, 8, 9, 2, 3, 10, 11, 1960 4, 5, 12, 13, 6, 7, 14, 15))); 1961 } 1962 else if constexpr (__z_to_z) 1963 { 1964 return __concat( 1965 __convert_x86<__vector_type_t<_Up, _M / 2>>(__v0, __v1, __v2, 1966 __v3), 1967 __convert_x86<__vector_type_t<_Up, _M / 2>>(__v4, __v5, __v6, 1968 __v7)); 1969 } 1970 } 1971 else if constexpr (__f64_to_i8) //{{{2 1972 { 1973 return __convert_x86<_To>( 1974 __convert_x86<__vector_type_t<int, _Np * 2>>(__v0, __v1), 1975 __convert_x86<__vector_type_t<int, _Np * 2>>(__v2, __v3), 1976 __convert_x86<__vector_type_t<int, _Np * 2>>(__v4, __v5), 1977 __convert_x86<__vector_type_t<int, _Np * 2>>(__v6, __v7)); 1978 } 1979 else // unreachable {{{2 1980 __assert_unreachable<_Tp>(); 1981 //}}} 1982 1983 // fallback: {{{2 1984 if constexpr (sizeof(_To) >= 32) 1985 // if _To is ymm or zmm, then _SimdWrapper<_Up, _M / 2> is xmm or ymm 1986 return __concat( 1987 __convert_x86<__vector_type_t<_Up, _M / 2>>(__v0, __v1, __v2, __v3), 1988 __convert_x86<__vector_type_t<_Up, _M / 2>>(__v4, __v5, __v6, 1989 __v7)); 1990 else if constexpr (sizeof(_To) == 16) 1991 { 1992 const auto __lo 1993 = __to_intrin(__convert_x86<_To>(__v0, __v1, __v2, __v3)); 1994 const auto __hi 1995 = __to_intrin(__convert_x86<_To>(__v4, __v5, __v6, __v7)); 1996 static_assert(sizeof(_Up) == 1 && _Np == 2); 1997 return __intrin_bitcast<_To>(_mm_unpacklo_epi64(__lo, __hi)); 1998 } 1999 else 2000 { 2001 __assert_unreachable<_Tp>(); 2002 // return __vector_convert<_To>(__v0, __v1, __v2, __v3, __v4, __v5, 2003 // __v6, __v7, 2004 // make_index_sequence<_Np>()); 2005 } //}}}2 2006 } 2007 } 2008 2009 //}}} 2010 // 16-arg __convert_x86 {{{1 2011 template <typename _To, typename _V, typename _Traits> 2012 _GLIBCXX_SIMD_INTRINSIC _To 2013 __convert_x86(_V __v0, _V __v1, _V __v2, _V __v3, _V __v4, _V __v5, _V __v6, 2014 _V __v7, _V __v8, _V __v9, _V __v10, _V __v11, _V __v12, 2015 _V __v13, _V __v14, _V __v15) 2016 { 2017 // concat => use 8-arg __convert_x86 2018 return __convert_x86<_To>(__concat(__v0, __v1), __concat(__v2, __v3), 2019 __concat(__v4, __v5), __concat(__v6, __v7), 2020 __concat(__v8, __v9), __concat(__v10, __v11), 2021 __concat(__v12, __v13), __concat(__v14, __v15)); 2022 } 2023 2024 //}}} 2025 2026 #endif // __cplusplus >= 201703L 2027 #endif // _GLIBCXX_EXPERIMENTAL_SIMD_X86_CONVERSIONS_H 2028 2029 // vim: foldmethod=marker 2030