1//===-- X86InstrAVX512.td - AVX512 Instruction Set ---------*- tablegen -*-===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file describes the X86 AVX512 instruction set, defining the 11// instructions, and properties of the instructions which are needed for code 12// generation, machine code emission, and analysis. 13// 14//===----------------------------------------------------------------------===// 15 16// Group template arguments that can be derived from the vector type (EltNum x 17// EltVT). These are things like the register class for the writemask, etc. 18// The idea is to pass one of these as the template argument rather than the 19// individual arguments. 20// The template is also used for scalar types, in this case numelts is 1. 21class X86VectorVTInfo<int numelts, ValueType eltvt, RegisterClass rc, 22 string suffix = ""> { 23 RegisterClass RC = rc; 24 ValueType EltVT = eltvt; 25 int NumElts = numelts; 26 27 // Corresponding mask register class. 28 RegisterClass KRC = !cast<RegisterClass>("VK" # NumElts); 29 30 // Corresponding write-mask register class. 31 RegisterClass KRCWM = !cast<RegisterClass>("VK" # NumElts # "WM"); 32 33 // The GPR register class that can hold the write mask. Use GR8 for fewer 34 // than 8 elements. Use shift-right and equal to work around the lack of 35 // !lt in tablegen. 36 RegisterClass MRC = 37 !cast<RegisterClass>("GR" # 38 !if (!eq (!srl(NumElts, 3), 0), 8, NumElts)); 39 40 // Suffix used in the instruction mnemonic. 41 string Suffix = suffix; 42 43 // VTName is a string name for vector VT. For vector types it will be 44 // v # NumElts # EltVT, so for vector of 8 elements of i32 it will be v8i32 45 // It is a little bit complex for scalar types, where NumElts = 1. 46 // In this case we build v4f32 or v2f64 47 string VTName = "v" # !if (!eq (NumElts, 1), 48 !if (!eq (EltVT.Size, 32), 4, 49 !if (!eq (EltVT.Size, 64), 2, NumElts)), NumElts) # EltVT; 50 51 // The vector VT. 52 ValueType VT = !cast<ValueType>(VTName); 53 54 string EltTypeName = !cast<string>(EltVT); 55 // Size of the element type in bits, e.g. 32 for v16i32. 56 string EltSizeName = !subst("i", "", !subst("f", "", EltTypeName)); 57 int EltSize = EltVT.Size; 58 59 // "i" for integer types and "f" for floating-point types 60 string TypeVariantName = !subst(EltSizeName, "", EltTypeName); 61 62 // Size of RC in bits, e.g. 512 for VR512. 63 int Size = VT.Size; 64 65 // The corresponding memory operand, e.g. i512mem for VR512. 66 X86MemOperand MemOp = !cast<X86MemOperand>(TypeVariantName # Size # "mem"); 67 X86MemOperand ScalarMemOp = !cast<X86MemOperand>(EltVT # "mem"); 68 69 // Load patterns 70 // Note: For 128/256-bit integer VT we choose loadv2i64/loadv4i64 71 // due to load promotion during legalization 72 PatFrag LdFrag = !cast<PatFrag>("load" # 73 !if (!eq (TypeVariantName, "i"), 74 !if (!eq (Size, 128), "v2i64", 75 !if (!eq (Size, 256), "v4i64", 76 VTName)), VTName)); 77 78 PatFrag AlignedLdFrag = !cast<PatFrag>("alignedload" # 79 !if (!eq (TypeVariantName, "i"), 80 !if (!eq (Size, 128), "v2i64", 81 !if (!eq (Size, 256), "v4i64", 82 !if (!eq (Size, 512), 83 !if (!eq (EltSize, 64), "v8i64", "v16i32"), 84 VTName))), VTName)); 85 86 PatFrag ScalarLdFrag = !cast<PatFrag>("load" # EltVT); 87 88 // The corresponding float type, e.g. v16f32 for v16i32 89 // Note: For EltSize < 32, FloatVT is illegal and TableGen 90 // fails to compile, so we choose FloatVT = VT 91 ValueType FloatVT = !cast<ValueType>( 92 !if (!eq (!srl(EltSize,5),0), 93 VTName, 94 !if (!eq(TypeVariantName, "i"), 95 "v" # NumElts # "f" # EltSize, 96 VTName))); 97 98 // The string to specify embedded broadcast in assembly. 99 string BroadcastStr = "{1to" # NumElts # "}"; 100 101 // 8-bit compressed displacement tuple/subvector format. This is only 102 // defined for NumElts <= 8. 103 CD8VForm CD8TupleForm = !if (!eq (!srl(NumElts, 4), 0), 104 !cast<CD8VForm>("CD8VT" # NumElts), ?); 105 106 SubRegIndex SubRegIdx = !if (!eq (Size, 128), sub_xmm, 107 !if (!eq (Size, 256), sub_ymm, ?)); 108 109 Domain ExeDomain = !if (!eq (EltTypeName, "f32"), SSEPackedSingle, 110 !if (!eq (EltTypeName, "f64"), SSEPackedDouble, 111 SSEPackedInt)); 112 113 RegisterClass FRC = !if (!eq (EltTypeName, "f32"), FR32X, FR64X); 114 115 // A vector type of the same width with element type i32. This is used to 116 // create the canonical constant zero node ImmAllZerosV. 117 ValueType i32VT = !cast<ValueType>("v" # !srl(Size, 5) # "i32"); 118 dag ImmAllZerosV = (VT (bitconvert (i32VT immAllZerosV))); 119 120 string ZSuffix = !if (!eq (Size, 128), "Z128", 121 !if (!eq (Size, 256), "Z256", "Z")); 122} 123 124def v64i8_info : X86VectorVTInfo<64, i8, VR512, "b">; 125def v32i16_info : X86VectorVTInfo<32, i16, VR512, "w">; 126def v16i32_info : X86VectorVTInfo<16, i32, VR512, "d">; 127def v8i64_info : X86VectorVTInfo<8, i64, VR512, "q">; 128def v16f32_info : X86VectorVTInfo<16, f32, VR512, "ps">; 129def v8f64_info : X86VectorVTInfo<8, f64, VR512, "pd">; 130 131// "x" in v32i8x_info means RC = VR256X 132def v32i8x_info : X86VectorVTInfo<32, i8, VR256X, "b">; 133def v16i16x_info : X86VectorVTInfo<16, i16, VR256X, "w">; 134def v8i32x_info : X86VectorVTInfo<8, i32, VR256X, "d">; 135def v4i64x_info : X86VectorVTInfo<4, i64, VR256X, "q">; 136def v8f32x_info : X86VectorVTInfo<8, f32, VR256X, "ps">; 137def v4f64x_info : X86VectorVTInfo<4, f64, VR256X, "pd">; 138 139def v16i8x_info : X86VectorVTInfo<16, i8, VR128X, "b">; 140def v8i16x_info : X86VectorVTInfo<8, i16, VR128X, "w">; 141def v4i32x_info : X86VectorVTInfo<4, i32, VR128X, "d">; 142def v2i64x_info : X86VectorVTInfo<2, i64, VR128X, "q">; 143def v4f32x_info : X86VectorVTInfo<4, f32, VR128X, "ps">; 144def v2f64x_info : X86VectorVTInfo<2, f64, VR128X, "pd">; 145 146// We map scalar types to the smallest (128-bit) vector type 147// with the appropriate element type. This allows to use the same masking logic. 148def f32x_info : X86VectorVTInfo<1, f32, VR128X, "ss">; 149def f64x_info : X86VectorVTInfo<1, f64, VR128X, "sd">; 150 151class AVX512VLVectorVTInfo<X86VectorVTInfo i512, X86VectorVTInfo i256, 152 X86VectorVTInfo i128> { 153 X86VectorVTInfo info512 = i512; 154 X86VectorVTInfo info256 = i256; 155 X86VectorVTInfo info128 = i128; 156} 157 158def avx512vl_i8_info : AVX512VLVectorVTInfo<v64i8_info, v32i8x_info, 159 v16i8x_info>; 160def avx512vl_i16_info : AVX512VLVectorVTInfo<v32i16_info, v16i16x_info, 161 v8i16x_info>; 162def avx512vl_i32_info : AVX512VLVectorVTInfo<v16i32_info, v8i32x_info, 163 v4i32x_info>; 164def avx512vl_i64_info : AVX512VLVectorVTInfo<v8i64_info, v4i64x_info, 165 v2i64x_info>; 166def avx512vl_f32_info : AVX512VLVectorVTInfo<v16f32_info, v8f32x_info, 167 v4f32x_info>; 168def avx512vl_f64_info : AVX512VLVectorVTInfo<v8f64_info, v4f64x_info, 169 v2f64x_info>; 170 171// This multiclass generates the masking variants from the non-masking 172// variant. It only provides the assembly pieces for the masking variants. 173// It assumes custom ISel patterns for masking which can be provided as 174// template arguments. 175multiclass AVX512_maskable_custom<bits<8> O, Format F, 176 dag Outs, 177 dag Ins, dag MaskingIns, dag ZeroMaskingIns, 178 string OpcodeStr, 179 string AttSrcAsm, string IntelSrcAsm, 180 list<dag> Pattern, 181 list<dag> MaskingPattern, 182 list<dag> ZeroMaskingPattern, 183 string MaskingConstraint = "", 184 InstrItinClass itin = NoItinerary, 185 bit IsCommutable = 0> { 186 let isCommutable = IsCommutable in 187 def NAME: AVX512<O, F, Outs, Ins, 188 OpcodeStr#"\t{"#AttSrcAsm#", $dst|"# 189 "$dst , "#IntelSrcAsm#"}", 190 Pattern, itin>; 191 192 // Prefer over VMOV*rrk Pat<> 193 let AddedComplexity = 20 in 194 def NAME#k: AVX512<O, F, Outs, MaskingIns, 195 OpcodeStr#"\t{"#AttSrcAsm#", $dst {${mask}}|"# 196 "$dst {${mask}}, "#IntelSrcAsm#"}", 197 MaskingPattern, itin>, 198 EVEX_K { 199 // In case of the 3src subclass this is overridden with a let. 200 string Constraints = MaskingConstraint; 201 } 202 let AddedComplexity = 30 in // Prefer over VMOV*rrkz Pat<> 203 def NAME#kz: AVX512<O, F, Outs, ZeroMaskingIns, 204 OpcodeStr#"\t{"#AttSrcAsm#", $dst {${mask}} {z}|"# 205 "$dst {${mask}} {z}, "#IntelSrcAsm#"}", 206 ZeroMaskingPattern, 207 itin>, 208 EVEX_KZ; 209} 210 211 212// Common base class of AVX512_maskable and AVX512_maskable_3src. 213multiclass AVX512_maskable_common<bits<8> O, Format F, X86VectorVTInfo _, 214 dag Outs, 215 dag Ins, dag MaskingIns, dag ZeroMaskingIns, 216 string OpcodeStr, 217 string AttSrcAsm, string IntelSrcAsm, 218 dag RHS, dag MaskingRHS, 219 SDNode Select = vselect, 220 string MaskingConstraint = "", 221 InstrItinClass itin = NoItinerary, 222 bit IsCommutable = 0> : 223 AVX512_maskable_custom<O, F, Outs, Ins, MaskingIns, ZeroMaskingIns, OpcodeStr, 224 AttSrcAsm, IntelSrcAsm, 225 [(set _.RC:$dst, RHS)], 226 [(set _.RC:$dst, MaskingRHS)], 227 [(set _.RC:$dst, 228 (Select _.KRCWM:$mask, RHS, _.ImmAllZerosV))], 229 MaskingConstraint, NoItinerary, IsCommutable>; 230 231// This multiclass generates the unconditional/non-masking, the masking and 232// the zero-masking variant of the vector instruction. In the masking case, the 233// perserved vector elements come from a new dummy input operand tied to $dst. 234multiclass AVX512_maskable<bits<8> O, Format F, X86VectorVTInfo _, 235 dag Outs, dag Ins, string OpcodeStr, 236 string AttSrcAsm, string IntelSrcAsm, 237 dag RHS, 238 InstrItinClass itin = NoItinerary, 239 bit IsCommutable = 0> : 240 AVX512_maskable_common<O, F, _, Outs, Ins, 241 !con((ins _.RC:$src0, _.KRCWM:$mask), Ins), 242 !con((ins _.KRCWM:$mask), Ins), 243 OpcodeStr, AttSrcAsm, IntelSrcAsm, RHS, 244 (vselect _.KRCWM:$mask, RHS, _.RC:$src0), vselect, 245 "$src0 = $dst", itin, IsCommutable>; 246 247// This multiclass generates the unconditional/non-masking, the masking and 248// the zero-masking variant of the scalar instruction. 249multiclass AVX512_maskable_scalar<bits<8> O, Format F, X86VectorVTInfo _, 250 dag Outs, dag Ins, string OpcodeStr, 251 string AttSrcAsm, string IntelSrcAsm, 252 dag RHS, 253 InstrItinClass itin = NoItinerary, 254 bit IsCommutable = 0> : 255 AVX512_maskable_common<O, F, _, Outs, Ins, 256 !con((ins _.RC:$src0, _.KRCWM:$mask), Ins), 257 !con((ins _.KRCWM:$mask), Ins), 258 OpcodeStr, AttSrcAsm, IntelSrcAsm, RHS, 259 (X86select _.KRCWM:$mask, RHS, _.RC:$src0), X86select, 260 "$src0 = $dst", itin, IsCommutable>; 261 262// Similar to AVX512_maskable but in this case one of the source operands 263// ($src1) is already tied to $dst so we just use that for the preserved 264// vector elements. NOTE that the NonTiedIns (the ins dag) should exclude 265// $src1. 266multiclass AVX512_maskable_3src<bits<8> O, Format F, X86VectorVTInfo _, 267 dag Outs, dag NonTiedIns, string OpcodeStr, 268 string AttSrcAsm, string IntelSrcAsm, 269 dag RHS> : 270 AVX512_maskable_common<O, F, _, Outs, 271 !con((ins _.RC:$src1), NonTiedIns), 272 !con((ins _.RC:$src1, _.KRCWM:$mask), NonTiedIns), 273 !con((ins _.RC:$src1, _.KRCWM:$mask), NonTiedIns), 274 OpcodeStr, AttSrcAsm, IntelSrcAsm, RHS, 275 (vselect _.KRCWM:$mask, RHS, _.RC:$src1)>; 276 277multiclass AVX512_maskable_3src_scalar<bits<8> O, Format F, X86VectorVTInfo _, 278 dag Outs, dag NonTiedIns, string OpcodeStr, 279 string AttSrcAsm, string IntelSrcAsm, 280 dag RHS> : 281 AVX512_maskable_common<O, F, _, Outs, 282 !con((ins _.RC:$src1), NonTiedIns), 283 !con((ins _.RC:$src1, _.KRCWM:$mask), NonTiedIns), 284 !con((ins _.RC:$src1, _.KRCWM:$mask), NonTiedIns), 285 OpcodeStr, AttSrcAsm, IntelSrcAsm, RHS, 286 (X86select _.KRCWM:$mask, RHS, _.RC:$src1)>; 287 288multiclass AVX512_maskable_in_asm<bits<8> O, Format F, X86VectorVTInfo _, 289 dag Outs, dag Ins, 290 string OpcodeStr, 291 string AttSrcAsm, string IntelSrcAsm, 292 list<dag> Pattern> : 293 AVX512_maskable_custom<O, F, Outs, Ins, 294 !con((ins _.RC:$src0, _.KRCWM:$mask), Ins), 295 !con((ins _.KRCWM:$mask), Ins), 296 OpcodeStr, AttSrcAsm, IntelSrcAsm, Pattern, [], [], 297 "$src0 = $dst">; 298 299 300// Instruction with mask that puts result in mask register, 301// like "compare" and "vptest" 302multiclass AVX512_maskable_custom_cmp<bits<8> O, Format F, 303 dag Outs, 304 dag Ins, dag MaskingIns, 305 string OpcodeStr, 306 string AttSrcAsm, string IntelSrcAsm, 307 list<dag> Pattern, 308 list<dag> MaskingPattern, 309 string Round = "", 310 InstrItinClass itin = NoItinerary> { 311 def NAME: AVX512<O, F, Outs, Ins, 312 OpcodeStr#"\t{"#AttSrcAsm#", $dst "#Round#"|"# 313 "$dst "#Round#", "#IntelSrcAsm#"}", 314 Pattern, itin>; 315 316 def NAME#k: AVX512<O, F, Outs, MaskingIns, 317 OpcodeStr#"\t{"#Round#AttSrcAsm#", $dst {${mask}}|"# 318 "$dst {${mask}}, "#IntelSrcAsm#Round#"}", 319 MaskingPattern, itin>, EVEX_K; 320} 321 322multiclass AVX512_maskable_common_cmp<bits<8> O, Format F, X86VectorVTInfo _, 323 dag Outs, 324 dag Ins, dag MaskingIns, 325 string OpcodeStr, 326 string AttSrcAsm, string IntelSrcAsm, 327 dag RHS, dag MaskingRHS, 328 string Round = "", 329 InstrItinClass itin = NoItinerary> : 330 AVX512_maskable_custom_cmp<O, F, Outs, Ins, MaskingIns, OpcodeStr, 331 AttSrcAsm, IntelSrcAsm, 332 [(set _.KRC:$dst, RHS)], 333 [(set _.KRC:$dst, MaskingRHS)], 334 Round, NoItinerary>; 335 336multiclass AVX512_maskable_cmp<bits<8> O, Format F, X86VectorVTInfo _, 337 dag Outs, dag Ins, string OpcodeStr, 338 string AttSrcAsm, string IntelSrcAsm, 339 dag RHS, string Round = "", 340 InstrItinClass itin = NoItinerary> : 341 AVX512_maskable_common_cmp<O, F, _, Outs, Ins, 342 !con((ins _.KRCWM:$mask), Ins), 343 OpcodeStr, AttSrcAsm, IntelSrcAsm, RHS, 344 (and _.KRCWM:$mask, RHS), 345 Round, itin>; 346 347multiclass AVX512_maskable_cmp_alt<bits<8> O, Format F, X86VectorVTInfo _, 348 dag Outs, dag Ins, string OpcodeStr, 349 string AttSrcAsm, string IntelSrcAsm> : 350 AVX512_maskable_custom_cmp<O, F, Outs, 351 Ins, !con((ins _.KRCWM:$mask),Ins), OpcodeStr, 352 AttSrcAsm, IntelSrcAsm, 353 [],[],"", NoItinerary>; 354 355// Bitcasts between 512-bit vector types. Return the original type since 356// no instruction is needed for the conversion 357let Predicates = [HasAVX512] in { 358 def : Pat<(v8f64 (bitconvert (v8i64 VR512:$src))), (v8f64 VR512:$src)>; 359 def : Pat<(v8f64 (bitconvert (v16i32 VR512:$src))), (v8f64 VR512:$src)>; 360 def : Pat<(v8f64 (bitconvert (v32i16 VR512:$src))), (v8f64 VR512:$src)>; 361 def : Pat<(v8f64 (bitconvert (v64i8 VR512:$src))), (v8f64 VR512:$src)>; 362 def : Pat<(v8f64 (bitconvert (v16f32 VR512:$src))), (v8f64 VR512:$src)>; 363 def : Pat<(v16f32 (bitconvert (v8i64 VR512:$src))), (v16f32 VR512:$src)>; 364 def : Pat<(v16f32 (bitconvert (v16i32 VR512:$src))), (v16f32 VR512:$src)>; 365 def : Pat<(v16f32 (bitconvert (v32i16 VR512:$src))), (v16f32 VR512:$src)>; 366 def : Pat<(v16f32 (bitconvert (v64i8 VR512:$src))), (v16f32 VR512:$src)>; 367 def : Pat<(v16f32 (bitconvert (v8f64 VR512:$src))), (v16f32 VR512:$src)>; 368 def : Pat<(v8i64 (bitconvert (v16i32 VR512:$src))), (v8i64 VR512:$src)>; 369 def : Pat<(v8i64 (bitconvert (v32i16 VR512:$src))), (v8i64 VR512:$src)>; 370 def : Pat<(v8i64 (bitconvert (v64i8 VR512:$src))), (v8i64 VR512:$src)>; 371 def : Pat<(v8i64 (bitconvert (v8f64 VR512:$src))), (v8i64 VR512:$src)>; 372 def : Pat<(v8i64 (bitconvert (v16f32 VR512:$src))), (v8i64 VR512:$src)>; 373 def : Pat<(v16i32 (bitconvert (v8i64 VR512:$src))), (v16i32 VR512:$src)>; 374 def : Pat<(v16i32 (bitconvert (v16f32 VR512:$src))), (v16i32 VR512:$src)>; 375 def : Pat<(v16i32 (bitconvert (v32i16 VR512:$src))), (v16i32 VR512:$src)>; 376 def : Pat<(v16i32 (bitconvert (v64i8 VR512:$src))), (v16i32 VR512:$src)>; 377 def : Pat<(v16i32 (bitconvert (v8f64 VR512:$src))), (v16i32 VR512:$src)>; 378 def : Pat<(v32i16 (bitconvert (v8i64 VR512:$src))), (v32i16 VR512:$src)>; 379 def : Pat<(v32i16 (bitconvert (v16i32 VR512:$src))), (v32i16 VR512:$src)>; 380 def : Pat<(v32i16 (bitconvert (v64i8 VR512:$src))), (v32i16 VR512:$src)>; 381 def : Pat<(v32i16 (bitconvert (v8f64 VR512:$src))), (v32i16 VR512:$src)>; 382 def : Pat<(v32i16 (bitconvert (v16f32 VR512:$src))), (v32i16 VR512:$src)>; 383 def : Pat<(v32i16 (bitconvert (v16f32 VR512:$src))), (v32i16 VR512:$src)>; 384 def : Pat<(v64i8 (bitconvert (v8i64 VR512:$src))), (v64i8 VR512:$src)>; 385 def : Pat<(v64i8 (bitconvert (v16i32 VR512:$src))), (v64i8 VR512:$src)>; 386 def : Pat<(v64i8 (bitconvert (v32i16 VR512:$src))), (v64i8 VR512:$src)>; 387 def : Pat<(v64i8 (bitconvert (v8f64 VR512:$src))), (v64i8 VR512:$src)>; 388 def : Pat<(v64i8 (bitconvert (v16f32 VR512:$src))), (v64i8 VR512:$src)>; 389 390 def : Pat<(v2i64 (bitconvert (v4i32 VR128X:$src))), (v2i64 VR128X:$src)>; 391 def : Pat<(v2i64 (bitconvert (v8i16 VR128X:$src))), (v2i64 VR128X:$src)>; 392 def : Pat<(v2i64 (bitconvert (v16i8 VR128X:$src))), (v2i64 VR128X:$src)>; 393 def : Pat<(v2i64 (bitconvert (v2f64 VR128X:$src))), (v2i64 VR128X:$src)>; 394 def : Pat<(v2i64 (bitconvert (v4f32 VR128X:$src))), (v2i64 VR128X:$src)>; 395 def : Pat<(v4i32 (bitconvert (v2i64 VR128X:$src))), (v4i32 VR128X:$src)>; 396 def : Pat<(v4i32 (bitconvert (v8i16 VR128X:$src))), (v4i32 VR128X:$src)>; 397 def : Pat<(v4i32 (bitconvert (v16i8 VR128X:$src))), (v4i32 VR128X:$src)>; 398 def : Pat<(v4i32 (bitconvert (v2f64 VR128X:$src))), (v4i32 VR128X:$src)>; 399 def : Pat<(v4i32 (bitconvert (v4f32 VR128X:$src))), (v4i32 VR128X:$src)>; 400 def : Pat<(v8i16 (bitconvert (v2i64 VR128X:$src))), (v8i16 VR128X:$src)>; 401 def : Pat<(v8i16 (bitconvert (v4i32 VR128X:$src))), (v8i16 VR128X:$src)>; 402 def : Pat<(v8i16 (bitconvert (v16i8 VR128X:$src))), (v8i16 VR128X:$src)>; 403 def : Pat<(v8i16 (bitconvert (v2f64 VR128X:$src))), (v8i16 VR128X:$src)>; 404 def : Pat<(v8i16 (bitconvert (v4f32 VR128X:$src))), (v8i16 VR128X:$src)>; 405 def : Pat<(v16i8 (bitconvert (v2i64 VR128X:$src))), (v16i8 VR128X:$src)>; 406 def : Pat<(v16i8 (bitconvert (v4i32 VR128X:$src))), (v16i8 VR128X:$src)>; 407 def : Pat<(v16i8 (bitconvert (v8i16 VR128X:$src))), (v16i8 VR128X:$src)>; 408 def : Pat<(v16i8 (bitconvert (v2f64 VR128X:$src))), (v16i8 VR128X:$src)>; 409 def : Pat<(v16i8 (bitconvert (v4f32 VR128X:$src))), (v16i8 VR128X:$src)>; 410 def : Pat<(v4f32 (bitconvert (v2i64 VR128X:$src))), (v4f32 VR128X:$src)>; 411 def : Pat<(v4f32 (bitconvert (v4i32 VR128X:$src))), (v4f32 VR128X:$src)>; 412 def : Pat<(v4f32 (bitconvert (v8i16 VR128X:$src))), (v4f32 VR128X:$src)>; 413 def : Pat<(v4f32 (bitconvert (v16i8 VR128X:$src))), (v4f32 VR128X:$src)>; 414 def : Pat<(v4f32 (bitconvert (v2f64 VR128X:$src))), (v4f32 VR128X:$src)>; 415 def : Pat<(v2f64 (bitconvert (v2i64 VR128X:$src))), (v2f64 VR128X:$src)>; 416 def : Pat<(v2f64 (bitconvert (v4i32 VR128X:$src))), (v2f64 VR128X:$src)>; 417 def : Pat<(v2f64 (bitconvert (v8i16 VR128X:$src))), (v2f64 VR128X:$src)>; 418 def : Pat<(v2f64 (bitconvert (v16i8 VR128X:$src))), (v2f64 VR128X:$src)>; 419 def : Pat<(v2f64 (bitconvert (v4f32 VR128X:$src))), (v2f64 VR128X:$src)>; 420 421// Bitcasts between 256-bit vector types. Return the original type since 422// no instruction is needed for the conversion 423 def : Pat<(v4f64 (bitconvert (v8f32 VR256X:$src))), (v4f64 VR256X:$src)>; 424 def : Pat<(v4f64 (bitconvert (v8i32 VR256X:$src))), (v4f64 VR256X:$src)>; 425 def : Pat<(v4f64 (bitconvert (v4i64 VR256X:$src))), (v4f64 VR256X:$src)>; 426 def : Pat<(v4f64 (bitconvert (v16i16 VR256X:$src))), (v4f64 VR256X:$src)>; 427 def : Pat<(v4f64 (bitconvert (v32i8 VR256X:$src))), (v4f64 VR256X:$src)>; 428 def : Pat<(v8f32 (bitconvert (v8i32 VR256X:$src))), (v8f32 VR256X:$src)>; 429 def : Pat<(v8f32 (bitconvert (v4i64 VR256X:$src))), (v8f32 VR256X:$src)>; 430 def : Pat<(v8f32 (bitconvert (v4f64 VR256X:$src))), (v8f32 VR256X:$src)>; 431 def : Pat<(v8f32 (bitconvert (v32i8 VR256X:$src))), (v8f32 VR256X:$src)>; 432 def : Pat<(v8f32 (bitconvert (v16i16 VR256X:$src))), (v8f32 VR256X:$src)>; 433 def : Pat<(v4i64 (bitconvert (v8f32 VR256X:$src))), (v4i64 VR256X:$src)>; 434 def : Pat<(v4i64 (bitconvert (v8i32 VR256X:$src))), (v4i64 VR256X:$src)>; 435 def : Pat<(v4i64 (bitconvert (v4f64 VR256X:$src))), (v4i64 VR256X:$src)>; 436 def : Pat<(v4i64 (bitconvert (v32i8 VR256X:$src))), (v4i64 VR256X:$src)>; 437 def : Pat<(v4i64 (bitconvert (v16i16 VR256X:$src))), (v4i64 VR256X:$src)>; 438 def : Pat<(v32i8 (bitconvert (v4f64 VR256X:$src))), (v32i8 VR256X:$src)>; 439 def : Pat<(v32i8 (bitconvert (v4i64 VR256X:$src))), (v32i8 VR256X:$src)>; 440 def : Pat<(v32i8 (bitconvert (v8f32 VR256X:$src))), (v32i8 VR256X:$src)>; 441 def : Pat<(v32i8 (bitconvert (v8i32 VR256X:$src))), (v32i8 VR256X:$src)>; 442 def : Pat<(v32i8 (bitconvert (v16i16 VR256X:$src))), (v32i8 VR256X:$src)>; 443 def : Pat<(v8i32 (bitconvert (v32i8 VR256X:$src))), (v8i32 VR256X:$src)>; 444 def : Pat<(v8i32 (bitconvert (v16i16 VR256X:$src))), (v8i32 VR256X:$src)>; 445 def : Pat<(v8i32 (bitconvert (v8f32 VR256X:$src))), (v8i32 VR256X:$src)>; 446 def : Pat<(v8i32 (bitconvert (v4i64 VR256X:$src))), (v8i32 VR256X:$src)>; 447 def : Pat<(v8i32 (bitconvert (v4f64 VR256X:$src))), (v8i32 VR256X:$src)>; 448 def : Pat<(v16i16 (bitconvert (v8f32 VR256X:$src))), (v16i16 VR256X:$src)>; 449 def : Pat<(v16i16 (bitconvert (v8i32 VR256X:$src))), (v16i16 VR256X:$src)>; 450 def : Pat<(v16i16 (bitconvert (v4i64 VR256X:$src))), (v16i16 VR256X:$src)>; 451 def : Pat<(v16i16 (bitconvert (v4f64 VR256X:$src))), (v16i16 VR256X:$src)>; 452 def : Pat<(v16i16 (bitconvert (v32i8 VR256X:$src))), (v16i16 VR256X:$src)>; 453} 454 455// 456// AVX-512: VPXOR instruction writes zero to its upper part, it's safe build zeros. 457// 458 459let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1, 460 isPseudo = 1, Predicates = [HasAVX512] in { 461def AVX512_512_SET0 : I<0, Pseudo, (outs VR512:$dst), (ins), "", 462 [(set VR512:$dst, (v16f32 immAllZerosV))]>; 463} 464 465let Predicates = [HasAVX512] in { 466def : Pat<(v8i64 immAllZerosV), (AVX512_512_SET0)>; 467def : Pat<(v16i32 immAllZerosV), (AVX512_512_SET0)>; 468def : Pat<(v8f64 immAllZerosV), (AVX512_512_SET0)>; 469} 470 471//===----------------------------------------------------------------------===// 472// AVX-512 - VECTOR INSERT 473// 474 475multiclass vinsert_for_size_no_alt<int Opcode, 476 X86VectorVTInfo From, X86VectorVTInfo To, 477 PatFrag vinsert_insert, 478 SDNodeXForm INSERT_get_vinsert_imm> { 479 let hasSideEffects = 0, ExeDomain = To.ExeDomain in { 480 def rr : AVX512AIi8<Opcode, MRMSrcReg, (outs VR512:$dst), 481 (ins VR512:$src1, From.RC:$src2, u8imm:$src3), 482 "vinsert" # From.EltTypeName # "x" # From.NumElts # 483 "\t{$src3, $src2, $src1, $dst|" 484 "$dst, $src1, $src2, $src3}", 485 [(set To.RC:$dst, (vinsert_insert:$src3 (To.VT VR512:$src1), 486 (From.VT From.RC:$src2), 487 (iPTR imm)))]>, 488 EVEX_4V, EVEX_V512; 489 490 let mayLoad = 1 in 491 def rm : AVX512AIi8<Opcode, MRMSrcMem, (outs VR512:$dst), 492 (ins VR512:$src1, From.MemOp:$src2, u8imm:$src3), 493 "vinsert" # From.EltTypeName # "x" # From.NumElts # 494 "\t{$src3, $src2, $src1, $dst|" 495 "$dst, $src1, $src2, $src3}", 496 []>, 497 EVEX_4V, EVEX_V512, EVEX_CD8<From.EltSize, From.CD8TupleForm>; 498 } 499} 500 501multiclass vinsert_for_size<int Opcode, 502 X86VectorVTInfo From, X86VectorVTInfo To, 503 X86VectorVTInfo AltFrom, X86VectorVTInfo AltTo, 504 PatFrag vinsert_insert, 505 SDNodeXForm INSERT_get_vinsert_imm> : 506 vinsert_for_size_no_alt<Opcode, From, To, 507 vinsert_insert, INSERT_get_vinsert_imm> { 508 // Codegen pattern with the alternative types, e.g. v2i64 -> v8i64 for 509 // vinserti32x4. Only add this if 64x2 and friends are not supported 510 // natively via AVX512DQ. 511 let Predicates = [NoDQI] in 512 def : Pat<(vinsert_insert:$ins 513 (AltTo.VT VR512:$src1), (AltFrom.VT From.RC:$src2), (iPTR imm)), 514 (AltTo.VT (!cast<Instruction>(NAME # From.EltSize # "x4rr") 515 VR512:$src1, From.RC:$src2, 516 (INSERT_get_vinsert_imm VR512:$ins)))>; 517} 518 519multiclass vinsert_for_type<ValueType EltVT32, int Opcode128, 520 ValueType EltVT64, int Opcode256> { 521 defm NAME # "32x4" : vinsert_for_size<Opcode128, 522 X86VectorVTInfo< 4, EltVT32, VR128X>, 523 X86VectorVTInfo<16, EltVT32, VR512>, 524 X86VectorVTInfo< 2, EltVT64, VR128X>, 525 X86VectorVTInfo< 8, EltVT64, VR512>, 526 vinsert128_insert, 527 INSERT_get_vinsert128_imm>; 528 let Predicates = [HasDQI] in 529 defm NAME # "64x2" : vinsert_for_size_no_alt<Opcode128, 530 X86VectorVTInfo< 2, EltVT64, VR128X>, 531 X86VectorVTInfo< 8, EltVT64, VR512>, 532 vinsert128_insert, 533 INSERT_get_vinsert128_imm>, VEX_W; 534 defm NAME # "64x4" : vinsert_for_size<Opcode256, 535 X86VectorVTInfo< 4, EltVT64, VR256X>, 536 X86VectorVTInfo< 8, EltVT64, VR512>, 537 X86VectorVTInfo< 8, EltVT32, VR256>, 538 X86VectorVTInfo<16, EltVT32, VR512>, 539 vinsert256_insert, 540 INSERT_get_vinsert256_imm>, VEX_W; 541 let Predicates = [HasDQI] in 542 defm NAME # "32x8" : vinsert_for_size_no_alt<Opcode256, 543 X86VectorVTInfo< 8, EltVT32, VR256X>, 544 X86VectorVTInfo<16, EltVT32, VR512>, 545 vinsert256_insert, 546 INSERT_get_vinsert256_imm>; 547} 548 549defm VINSERTF : vinsert_for_type<f32, 0x18, f64, 0x1a>; 550defm VINSERTI : vinsert_for_type<i32, 0x38, i64, 0x3a>; 551 552// vinsertps - insert f32 to XMM 553def VINSERTPSzrr : AVX512AIi8<0x21, MRMSrcReg, (outs VR128X:$dst), 554 (ins VR128X:$src1, VR128X:$src2, u8imm:$src3), 555 "vinsertps\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", 556 [(set VR128X:$dst, (X86insertps VR128X:$src1, VR128X:$src2, imm:$src3))]>, 557 EVEX_4V; 558def VINSERTPSzrm: AVX512AIi8<0x21, MRMSrcMem, (outs VR128X:$dst), 559 (ins VR128X:$src1, f32mem:$src2, u8imm:$src3), 560 "vinsertps\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", 561 [(set VR128X:$dst, (X86insertps VR128X:$src1, 562 (v4f32 (scalar_to_vector (loadf32 addr:$src2))), 563 imm:$src3))]>, EVEX_4V, EVEX_CD8<32, CD8VT1>; 564 565//===----------------------------------------------------------------------===// 566// AVX-512 VECTOR EXTRACT 567//--- 568 569multiclass vextract_for_size<int Opcode, 570 X86VectorVTInfo From, X86VectorVTInfo To, 571 X86VectorVTInfo AltFrom, X86VectorVTInfo AltTo, 572 PatFrag vextract_extract, 573 SDNodeXForm EXTRACT_get_vextract_imm> { 574 let hasSideEffects = 0, ExeDomain = To.ExeDomain in { 575 defm rr : AVX512_maskable_in_asm<Opcode, MRMDestReg, To, (outs To.RC:$dst), 576 (ins VR512:$src1, u8imm:$idx), 577 "vextract" # To.EltTypeName # "x4", 578 "$idx, $src1", "$src1, $idx", 579 [(set To.RC:$dst, (vextract_extract:$idx (From.VT VR512:$src1), 580 (iPTR imm)))]>, 581 AVX512AIi8Base, EVEX, EVEX_V512; 582 let mayStore = 1 in 583 def rm : AVX512AIi8<Opcode, MRMDestMem, (outs), 584 (ins To.MemOp:$dst, VR512:$src1, u8imm:$src2), 585 "vextract" # To.EltTypeName # "x4\t{$src2, $src1, $dst|" 586 "$dst, $src1, $src2}", 587 []>, EVEX, EVEX_V512, EVEX_CD8<To.EltSize, CD8VT4>; 588 } 589 590 // Codegen pattern with the alternative types, e.g. v8i64 -> v2i64 for 591 // vextracti32x4 592 def : Pat<(vextract_extract:$ext (AltFrom.VT VR512:$src1), (iPTR imm)), 593 (AltTo.VT (!cast<Instruction>(NAME # To.EltSize # "x4rr") 594 VR512:$src1, 595 (EXTRACT_get_vextract_imm To.RC:$ext)))>; 596 597 // A 128/256-bit subvector extract from the first 512-bit vector position is 598 // a subregister copy that needs no instruction. 599 def : Pat<(To.VT (extract_subvector (From.VT VR512:$src), (iPTR 0))), 600 (To.VT 601 (EXTRACT_SUBREG (From.VT VR512:$src), To.SubRegIdx))>; 602 603 // And for the alternative types. 604 def : Pat<(AltTo.VT (extract_subvector (AltFrom.VT VR512:$src), (iPTR 0))), 605 (AltTo.VT 606 (EXTRACT_SUBREG (AltFrom.VT VR512:$src), AltTo.SubRegIdx))>; 607 608 // Intrinsic call with masking. 609 def : Pat<(!cast<Intrinsic>("int_x86_avx512_mask_vextract" # To.EltTypeName # 610 "x4_512") 611 VR512:$src1, (iPTR imm:$idx), To.RC:$src0, GR8:$mask), 612 (!cast<Instruction>(NAME # To.EltSize # "x4rrk") To.RC:$src0, 613 (v4i1 (COPY_TO_REGCLASS GR8:$mask, VK4WM)), 614 VR512:$src1, imm:$idx)>; 615 616 // Intrinsic call with zero-masking. 617 def : Pat<(!cast<Intrinsic>("int_x86_avx512_mask_vextract" # To.EltTypeName # 618 "x4_512") 619 VR512:$src1, (iPTR imm:$idx), To.ImmAllZerosV, GR8:$mask), 620 (!cast<Instruction>(NAME # To.EltSize # "x4rrkz") 621 (v4i1 (COPY_TO_REGCLASS GR8:$mask, VK4WM)), 622 VR512:$src1, imm:$idx)>; 623 624 // Intrinsic call without masking. 625 def : Pat<(!cast<Intrinsic>("int_x86_avx512_mask_vextract" # To.EltTypeName # 626 "x4_512") 627 VR512:$src1, (iPTR imm:$idx), To.ImmAllZerosV, (i8 -1)), 628 (!cast<Instruction>(NAME # To.EltSize # "x4rr") 629 VR512:$src1, imm:$idx)>; 630} 631 632multiclass vextract_for_type<ValueType EltVT32, int Opcode32, 633 ValueType EltVT64, int Opcode64> { 634 defm NAME # "32x4" : vextract_for_size<Opcode32, 635 X86VectorVTInfo<16, EltVT32, VR512>, 636 X86VectorVTInfo< 4, EltVT32, VR128X>, 637 X86VectorVTInfo< 8, EltVT64, VR512>, 638 X86VectorVTInfo< 2, EltVT64, VR128X>, 639 vextract128_extract, 640 EXTRACT_get_vextract128_imm>; 641 defm NAME # "64x4" : vextract_for_size<Opcode64, 642 X86VectorVTInfo< 8, EltVT64, VR512>, 643 X86VectorVTInfo< 4, EltVT64, VR256X>, 644 X86VectorVTInfo<16, EltVT32, VR512>, 645 X86VectorVTInfo< 8, EltVT32, VR256>, 646 vextract256_extract, 647 EXTRACT_get_vextract256_imm>, VEX_W; 648} 649 650defm VEXTRACTF : vextract_for_type<f32, 0x19, f64, 0x1b>; 651defm VEXTRACTI : vextract_for_type<i32, 0x39, i64, 0x3b>; 652 653// A 128-bit subvector insert to the first 512-bit vector position 654// is a subregister copy that needs no instruction. 655def : Pat<(insert_subvector undef, (v2i64 VR128X:$src), (iPTR 0)), 656 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), 657 (INSERT_SUBREG (v4i64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm), 658 sub_ymm)>; 659def : Pat<(insert_subvector undef, (v2f64 VR128X:$src), (iPTR 0)), 660 (INSERT_SUBREG (v8f64 (IMPLICIT_DEF)), 661 (INSERT_SUBREG (v4f64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm), 662 sub_ymm)>; 663def : Pat<(insert_subvector undef, (v4i32 VR128X:$src), (iPTR 0)), 664 (INSERT_SUBREG (v16i32 (IMPLICIT_DEF)), 665 (INSERT_SUBREG (v8i32 (IMPLICIT_DEF)), VR128X:$src, sub_xmm), 666 sub_ymm)>; 667def : Pat<(insert_subvector undef, (v4f32 VR128X:$src), (iPTR 0)), 668 (INSERT_SUBREG (v16f32 (IMPLICIT_DEF)), 669 (INSERT_SUBREG (v8f32 (IMPLICIT_DEF)), VR128X:$src, sub_xmm), 670 sub_ymm)>; 671 672def : Pat<(insert_subvector undef, (v4i64 VR256X:$src), (iPTR 0)), 673 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src, sub_ymm)>; 674def : Pat<(insert_subvector undef, (v4f64 VR256X:$src), (iPTR 0)), 675 (INSERT_SUBREG (v8f64 (IMPLICIT_DEF)), VR256X:$src, sub_ymm)>; 676def : Pat<(insert_subvector undef, (v8i32 VR256X:$src), (iPTR 0)), 677 (INSERT_SUBREG (v16i32 (IMPLICIT_DEF)), VR256X:$src, sub_ymm)>; 678def : Pat<(insert_subvector undef, (v8f32 VR256X:$src), (iPTR 0)), 679 (INSERT_SUBREG (v16f32 (IMPLICIT_DEF)), VR256X:$src, sub_ymm)>; 680 681// vextractps - extract 32 bits from XMM 682def VEXTRACTPSzrr : AVX512AIi8<0x17, MRMDestReg, (outs GR32:$dst), 683 (ins VR128X:$src1, u8imm:$src2), 684 "vextractps\t{$src2, $src1, $dst|$dst, $src1, $src2}", 685 [(set GR32:$dst, (extractelt (bc_v4i32 (v4f32 VR128X:$src1)), imm:$src2))]>, 686 EVEX; 687 688def VEXTRACTPSzmr : AVX512AIi8<0x17, MRMDestMem, (outs), 689 (ins f32mem:$dst, VR128X:$src1, u8imm:$src2), 690 "vextractps\t{$src2, $src1, $dst|$dst, $src1, $src2}", 691 [(store (extractelt (bc_v4i32 (v4f32 VR128X:$src1)), imm:$src2), 692 addr:$dst)]>, EVEX, EVEX_CD8<32, CD8VT1>; 693 694//===---------------------------------------------------------------------===// 695// AVX-512 BROADCAST 696//--- 697multiclass avx512_fp_broadcast<bits<8> opc, SDNode OpNode, RegisterClass SrcRC, 698 ValueType svt, X86VectorVTInfo _> { 699 defm r : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), 700 (ins SrcRC:$src), "vbroadcast"## !subst("p", "s", _.Suffix), 701 "$src", "$src", (_.VT (OpNode (svt SrcRC:$src)))>, 702 T8PD, EVEX; 703 704 let mayLoad = 1 in { 705 defm m : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), 706 (ins _.ScalarMemOp:$src), 707 "vbroadcast"##!subst("p", "s", _.Suffix), "$src", "$src", 708 (_.VT (OpNode (_.ScalarLdFrag addr:$src)))>, 709 T8PD, EVEX; 710 } 711} 712 713multiclass avx512_fp_broadcast_vl<bits<8> opc, SDNode OpNode, 714 AVX512VLVectorVTInfo _> { 715 defm Z : avx512_fp_broadcast<opc, OpNode, VR128X, _.info128.VT, _.info512>, 716 EVEX_V512; 717 718 let Predicates = [HasVLX] in { 719 defm Z256 : avx512_fp_broadcast<opc, OpNode, VR128X, _.info128.VT, _.info256>, 720 EVEX_V256; 721 } 722} 723 724let ExeDomain = SSEPackedSingle in { 725 defm VBROADCASTSS : avx512_fp_broadcast_vl<0x18, X86VBroadcast, 726 avx512vl_f32_info>, EVEX_CD8<32, CD8VT1>; 727 let Predicates = [HasVLX] in { 728 defm VBROADCASTSSZ128 : avx512_fp_broadcast<0x18, X86VBroadcast, VR128X, 729 v4f32, v4f32x_info>, EVEX_V128, 730 EVEX_CD8<32, CD8VT1>; 731 } 732} 733 734let ExeDomain = SSEPackedDouble in { 735 defm VBROADCASTSD : avx512_fp_broadcast_vl<0x19, X86VBroadcast, 736 avx512vl_f64_info>, VEX_W, EVEX_CD8<64, CD8VT1>; 737} 738 739// avx512_broadcast_pat introduces patterns for broadcast with a scalar argument. 740// Later, we can canonize broadcast instructions before ISel phase and 741// eliminate additional patterns on ISel. 742// SrcRC_v and SrcRC_s are RegisterClasses for vector and scalar 743// representations of source 744multiclass avx512_broadcast_pat<string InstName, SDNode OpNode, 745 X86VectorVTInfo _, RegisterClass SrcRC_v, 746 RegisterClass SrcRC_s> { 747 def : Pat<(_.VT (OpNode (_.EltVT SrcRC_s:$src))), 748 (!cast<Instruction>(InstName##"r") 749 (COPY_TO_REGCLASS SrcRC_s:$src, SrcRC_v))>; 750 751 let AddedComplexity = 30 in { 752 def : Pat<(_.VT (vselect _.KRCWM:$mask, 753 (OpNode (_.EltVT SrcRC_s:$src)), _.RC:$src0)), 754 (!cast<Instruction>(InstName##"rk") _.RC:$src0, _.KRCWM:$mask, 755 (COPY_TO_REGCLASS SrcRC_s:$src, SrcRC_v))>; 756 757 def : Pat<(_.VT(vselect _.KRCWM:$mask, 758 (OpNode (_.EltVT SrcRC_s:$src)), _.ImmAllZerosV)), 759 (!cast<Instruction>(InstName##"rkz") _.KRCWM:$mask, 760 (COPY_TO_REGCLASS SrcRC_s:$src, SrcRC_v))>; 761 } 762} 763 764defm : avx512_broadcast_pat<"VBROADCASTSSZ", X86VBroadcast, v16f32_info, 765 VR128X, FR32X>; 766defm : avx512_broadcast_pat<"VBROADCASTSDZ", X86VBroadcast, v8f64_info, 767 VR128X, FR64X>; 768 769let Predicates = [HasVLX] in { 770 defm : avx512_broadcast_pat<"VBROADCASTSSZ256", X86VBroadcast, 771 v8f32x_info, VR128X, FR32X>; 772 defm : avx512_broadcast_pat<"VBROADCASTSSZ128", X86VBroadcast, 773 v4f32x_info, VR128X, FR32X>; 774 defm : avx512_broadcast_pat<"VBROADCASTSDZ256", X86VBroadcast, 775 v4f64x_info, VR128X, FR64X>; 776} 777 778def : Pat<(v16f32 (X86VBroadcast (loadf32 addr:$src))), 779 (VBROADCASTSSZm addr:$src)>; 780def : Pat<(v8f64 (X86VBroadcast (loadf64 addr:$src))), 781 (VBROADCASTSDZm addr:$src)>; 782 783def : Pat<(int_x86_avx512_vbroadcast_ss_512 addr:$src), 784 (VBROADCASTSSZm addr:$src)>; 785def : Pat<(int_x86_avx512_vbroadcast_sd_512 addr:$src), 786 (VBROADCASTSDZm addr:$src)>; 787 788multiclass avx512_int_broadcast_reg<bits<8> opc, X86VectorVTInfo _, 789 RegisterClass SrcRC> { 790 defm r : AVX512_maskable_in_asm<opc, MRMSrcReg, _, (outs _.RC:$dst), 791 (ins SrcRC:$src), "vpbroadcast"##_.Suffix, 792 "$src", "$src", []>, T8PD, EVEX; 793} 794 795multiclass avx512_int_broadcast_reg_vl<bits<8> opc, AVX512VLVectorVTInfo _, 796 RegisterClass SrcRC, Predicate prd> { 797 let Predicates = [prd] in 798 defm Z : avx512_int_broadcast_reg<opc, _.info512, SrcRC>, EVEX_V512; 799 let Predicates = [prd, HasVLX] in { 800 defm Z256 : avx512_int_broadcast_reg<opc, _.info256, SrcRC>, EVEX_V256; 801 defm Z128 : avx512_int_broadcast_reg<opc, _.info128, SrcRC>, EVEX_V128; 802 } 803} 804 805defm VPBROADCASTBr : avx512_int_broadcast_reg_vl<0x7A, avx512vl_i8_info, GR32, 806 HasBWI>; 807defm VPBROADCASTWr : avx512_int_broadcast_reg_vl<0x7B, avx512vl_i16_info, GR32, 808 HasBWI>; 809defm VPBROADCASTDr : avx512_int_broadcast_reg_vl<0x7C, avx512vl_i32_info, GR32, 810 HasAVX512>; 811defm VPBROADCASTQr : avx512_int_broadcast_reg_vl<0x7C, avx512vl_i64_info, GR64, 812 HasAVX512>, VEX_W; 813 814def : Pat <(v16i32 (X86vzext VK16WM:$mask)), 815 (VPBROADCASTDrZrkz VK16WM:$mask, (i32 (MOV32ri 0x1)))>; 816 817def : Pat <(v8i64 (X86vzext VK8WM:$mask)), 818 (VPBROADCASTQrZrkz VK8WM:$mask, (i64 (MOV64ri 0x1)))>; 819 820def : Pat<(v16i32 (X86VBroadcast (i32 GR32:$src))), 821 (VPBROADCASTDrZr GR32:$src)>; 822def : Pat<(v8i64 (X86VBroadcast (i64 GR64:$src))), 823 (VPBROADCASTQrZr GR64:$src)>; 824 825def : Pat<(v16i32 (int_x86_avx512_pbroadcastd_i32_512 (i32 GR32:$src))), 826 (VPBROADCASTDrZr GR32:$src)>; 827def : Pat<(v8i64 (int_x86_avx512_pbroadcastq_i64_512 (i64 GR64:$src))), 828 (VPBROADCASTQrZr GR64:$src)>; 829 830def : Pat<(v16i32 (int_x86_avx512_mask_pbroadcast_d_gpr_512 (i32 GR32:$src), 831 (v16i32 immAllZerosV), (i16 GR16:$mask))), 832 (VPBROADCASTDrZrkz (COPY_TO_REGCLASS GR16:$mask, VK16WM), GR32:$src)>; 833def : Pat<(v8i64 (int_x86_avx512_mask_pbroadcast_q_gpr_512 (i64 GR64:$src), 834 (bc_v8i64 (v16i32 immAllZerosV)), (i8 GR8:$mask))), 835 (VPBROADCASTQrZrkz (COPY_TO_REGCLASS GR8:$mask, VK8WM), GR64:$src)>; 836 837multiclass avx512_int_broadcast_rm<bits<8> opc, string OpcodeStr, 838 X86MemOperand x86memop, PatFrag ld_frag, 839 RegisterClass DstRC, ValueType OpVT, ValueType SrcVT, 840 RegisterClass KRC> { 841 def rr : AVX5128I<opc, MRMSrcReg, (outs DstRC:$dst), (ins VR128X:$src), 842 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 843 [(set DstRC:$dst, 844 (OpVT (X86VBroadcast (SrcVT VR128X:$src))))]>, EVEX; 845 def rrk : AVX5128I<opc, MRMSrcReg, (outs DstRC:$dst), (ins KRC:$mask, 846 VR128X:$src), 847 !strconcat(OpcodeStr, 848 "\t{$src, ${dst} {${mask}} |${dst} {${mask}}, $src}"), 849 []>, EVEX, EVEX_K; 850 def rrkz : AVX5128I<opc, MRMSrcReg, (outs DstRC:$dst), (ins KRC:$mask, 851 VR128X:$src), 852 !strconcat(OpcodeStr, 853 "\t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}"), 854 []>, EVEX, EVEX_KZ; 855 let mayLoad = 1 in { 856 def rm : AVX5128I<opc, MRMSrcMem, (outs DstRC:$dst), (ins x86memop:$src), 857 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 858 [(set DstRC:$dst, 859 (OpVT (X86VBroadcast (ld_frag addr:$src))))]>, EVEX; 860 def rmk : AVX5128I<opc, MRMSrcMem, (outs DstRC:$dst), (ins KRC:$mask, 861 x86memop:$src), 862 !strconcat(OpcodeStr, 863 "\t{$src, ${dst} {${mask}}|${dst} {${mask}} , $src}"), 864 []>, EVEX, EVEX_K; 865 def rmkz : AVX5128I<opc, MRMSrcMem, (outs DstRC:$dst), (ins KRC:$mask, 866 x86memop:$src), 867 !strconcat(OpcodeStr, 868 "\t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}"), 869 [(set DstRC:$dst, (OpVT (vselect KRC:$mask, 870 (X86VBroadcast (ld_frag addr:$src)), 871 (OpVT (bitconvert (v16i32 immAllZerosV))))))]>, EVEX, EVEX_KZ; 872 } 873} 874 875defm VPBROADCASTDZ : avx512_int_broadcast_rm<0x58, "vpbroadcastd", i32mem, 876 loadi32, VR512, v16i32, v4i32, VK16WM>, 877 EVEX_V512, EVEX_CD8<32, CD8VT1>; 878defm VPBROADCASTQZ : avx512_int_broadcast_rm<0x59, "vpbroadcastq", i64mem, 879 loadi64, VR512, v8i64, v2i64, VK8WM>, EVEX_V512, VEX_W, 880 EVEX_CD8<64, CD8VT1>; 881 882multiclass avx512_subvec_broadcast_rm<bits<8> opc, string OpcodeStr, 883 X86VectorVTInfo _Dst, X86VectorVTInfo _Src> { 884 let mayLoad = 1 in { 885 def rm : AVX5128I<opc, MRMSrcMem, (outs _Dst.RC:$dst), (ins _Src.MemOp:$src), 886 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 887 [(set _Dst.RC:$dst, 888 (_Dst.VT (X86SubVBroadcast 889 (_Src.VT (bitconvert (_Src.LdFrag addr:$src))))))]>, EVEX; 890 def rmk : AVX5128I<opc, MRMSrcMem, (outs _Dst.RC:$dst), (ins _Dst.KRCWM:$mask, 891 _Src.MemOp:$src), 892 !strconcat(OpcodeStr, 893 "\t{$src, ${dst} {${mask}}|${dst} {${mask}}, $src}"), 894 []>, EVEX, EVEX_K; 895 def rmkz : AVX5128I<opc, MRMSrcMem, (outs _Dst.RC:$dst), (ins _Dst.KRCWM:$mask, 896 _Src.MemOp:$src), 897 !strconcat(OpcodeStr, 898 "\t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}"), 899 []>, EVEX, EVEX_KZ; 900 } 901} 902 903defm VBROADCASTI32X4 : avx512_subvec_broadcast_rm<0x5a, "vbroadcasti32x4", 904 v16i32_info, v4i32x_info>, 905 EVEX_V512, EVEX_CD8<32, CD8VT4>; 906defm VBROADCASTF32X4 : avx512_subvec_broadcast_rm<0x1a, "vbroadcastf32x4", 907 v16f32_info, v4f32x_info>, 908 EVEX_V512, EVEX_CD8<32, CD8VT4>; 909defm VBROADCASTI64X4 : avx512_subvec_broadcast_rm<0x5b, "vbroadcasti64x4", 910 v8i64_info, v4i64x_info>, VEX_W, 911 EVEX_V512, EVEX_CD8<64, CD8VT4>; 912defm VBROADCASTF64X4 : avx512_subvec_broadcast_rm<0x1b, "vbroadcastf64x4", 913 v8f64_info, v4f64x_info>, VEX_W, 914 EVEX_V512, EVEX_CD8<64, CD8VT4>; 915 916let Predicates = [HasVLX] in { 917defm VBROADCASTI32X4Z256 : avx512_subvec_broadcast_rm<0x5a, "vbroadcasti32x4", 918 v8i32x_info, v4i32x_info>, 919 EVEX_V256, EVEX_CD8<32, CD8VT4>; 920defm VBROADCASTF32X4Z256 : avx512_subvec_broadcast_rm<0x1a, "vbroadcastf32x4", 921 v8f32x_info, v4f32x_info>, 922 EVEX_V256, EVEX_CD8<32, CD8VT4>; 923} 924let Predicates = [HasVLX, HasDQI] in { 925defm VBROADCASTI64X2Z128 : avx512_subvec_broadcast_rm<0x5a, "vbroadcasti64x2", 926 v4i64x_info, v2i64x_info>, VEX_W, 927 EVEX_V256, EVEX_CD8<64, CD8VT2>; 928defm VBROADCASTF64X2Z128 : avx512_subvec_broadcast_rm<0x1a, "vbroadcastf64x2", 929 v4f64x_info, v2f64x_info>, VEX_W, 930 EVEX_V256, EVEX_CD8<64, CD8VT2>; 931} 932let Predicates = [HasDQI] in { 933defm VBROADCASTI64X2 : avx512_subvec_broadcast_rm<0x5a, "vbroadcasti64x2", 934 v8i64_info, v2i64x_info>, VEX_W, 935 EVEX_V512, EVEX_CD8<64, CD8VT2>; 936defm VBROADCASTI32X8 : avx512_subvec_broadcast_rm<0x5b, "vbroadcasti32x8", 937 v16i32_info, v8i32x_info>, 938 EVEX_V512, EVEX_CD8<32, CD8VT8>; 939defm VBROADCASTF64X2 : avx512_subvec_broadcast_rm<0x1a, "vbroadcastf64x2", 940 v8f64_info, v2f64x_info>, VEX_W, 941 EVEX_V512, EVEX_CD8<64, CD8VT2>; 942defm VBROADCASTF32X8 : avx512_subvec_broadcast_rm<0x1b, "vbroadcastf32x8", 943 v16f32_info, v8f32x_info>, 944 EVEX_V512, EVEX_CD8<32, CD8VT8>; 945} 946 947def : Pat<(v16i32 (int_x86_avx512_pbroadcastd_512 (v4i32 VR128X:$src))), 948 (VPBROADCASTDZrr VR128X:$src)>; 949def : Pat<(v8i64 (int_x86_avx512_pbroadcastq_512 (v2i64 VR128X:$src))), 950 (VPBROADCASTQZrr VR128X:$src)>; 951 952def : Pat<(v16f32 (X86VBroadcast (v16f32 VR512:$src))), 953 (VBROADCASTSSZr (EXTRACT_SUBREG (v16f32 VR512:$src), sub_xmm))>; 954def : Pat<(v16f32 (X86VBroadcast (v8f32 VR256X:$src))), 955 (VBROADCASTSSZr (EXTRACT_SUBREG (v8f32 VR256X:$src), sub_xmm))>; 956 957def : Pat<(v8f64 (X86VBroadcast (v8f64 VR512:$src))), 958 (VBROADCASTSDZr (EXTRACT_SUBREG (v8f64 VR512:$src), sub_xmm))>; 959def : Pat<(v8f64 (X86VBroadcast (v4f64 VR256X:$src))), 960 (VBROADCASTSDZr (EXTRACT_SUBREG (v4f64 VR256X:$src), sub_xmm))>; 961 962def : Pat<(v16i32 (X86VBroadcast (v16i32 VR512:$src))), 963 (VPBROADCASTDZrr (EXTRACT_SUBREG (v16i32 VR512:$src), sub_xmm))>; 964def : Pat<(v16i32 (X86VBroadcast (v8i32 VR256X:$src))), 965 (VPBROADCASTDZrr (EXTRACT_SUBREG (v8i32 VR256X:$src), sub_xmm))>; 966 967def : Pat<(v8i64 (X86VBroadcast (v8i64 VR512:$src))), 968 (VPBROADCASTQZrr (EXTRACT_SUBREG (v8i64 VR512:$src), sub_xmm))>; 969def : Pat<(v8i64 (X86VBroadcast (v4i64 VR256X:$src))), 970 (VPBROADCASTQZrr (EXTRACT_SUBREG (v4i64 VR256X:$src), sub_xmm))>; 971 972def : Pat<(v16f32 (int_x86_avx512_vbroadcast_ss_ps_512 (v4f32 VR128X:$src))), 973 (VBROADCASTSSZr VR128X:$src)>; 974def : Pat<(v8f64 (int_x86_avx512_vbroadcast_sd_pd_512 (v2f64 VR128X:$src))), 975 (VBROADCASTSDZr VR128X:$src)>; 976 977// Provide fallback in case the load node that is used in the patterns above 978// is used by additional users, which prevents the pattern selection. 979def : Pat<(v16f32 (X86VBroadcast FR32X:$src)), 980 (VBROADCASTSSZr (COPY_TO_REGCLASS FR32X:$src, VR128X))>; 981def : Pat<(v8f64 (X86VBroadcast FR64X:$src)), 982 (VBROADCASTSDZr (COPY_TO_REGCLASS FR64X:$src, VR128X))>; 983 984 985//===----------------------------------------------------------------------===// 986// AVX-512 BROADCAST MASK TO VECTOR REGISTER 987//--- 988 989multiclass avx512_mask_broadcast<bits<8> opc, string OpcodeStr, 990 RegisterClass KRC> { 991let Predicates = [HasCDI] in 992def Zrr : AVX512XS8I<opc, MRMSrcReg, (outs VR512:$dst), (ins KRC:$src), 993 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 994 []>, EVEX, EVEX_V512; 995 996let Predicates = [HasCDI, HasVLX] in { 997def Z128rr : AVX512XS8I<opc, MRMSrcReg, (outs VR128:$dst), (ins KRC:$src), 998 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 999 []>, EVEX, EVEX_V128; 1000def Z256rr : AVX512XS8I<opc, MRMSrcReg, (outs VR256:$dst), (ins KRC:$src), 1001 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 1002 []>, EVEX, EVEX_V256; 1003} 1004} 1005 1006let Predicates = [HasCDI] in { 1007defm VPBROADCASTMW2D : avx512_mask_broadcast<0x3A, "vpbroadcastmw2d", 1008 VK16>; 1009defm VPBROADCASTMB2Q : avx512_mask_broadcast<0x2A, "vpbroadcastmb2q", 1010 VK8>, VEX_W; 1011} 1012 1013//===----------------------------------------------------------------------===// 1014// AVX-512 - VPERM 1015// 1016// -- immediate form -- 1017multiclass avx512_perm_imm<bits<8> opc, string OpcodeStr, SDNode OpNode, 1018 X86VectorVTInfo _> { 1019 let ExeDomain = _.ExeDomain in { 1020 def ri : AVX512AIi8<opc, MRMSrcReg, (outs _.RC:$dst), 1021 (ins _.RC:$src1, u8imm:$src2), 1022 !strconcat(OpcodeStr, 1023 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 1024 [(set _.RC:$dst, 1025 (_.VT (OpNode _.RC:$src1, (i8 imm:$src2))))]>, 1026 EVEX; 1027 def mi : AVX512AIi8<opc, MRMSrcMem, (outs _.RC:$dst), 1028 (ins _.MemOp:$src1, u8imm:$src2), 1029 !strconcat(OpcodeStr, 1030 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 1031 [(set _.RC:$dst, 1032 (_.VT (OpNode (_.LdFrag addr:$src1), 1033 (i8 imm:$src2))))]>, 1034 EVEX, EVEX_CD8<_.EltSize, CD8VF>; 1035} 1036} 1037 1038multiclass avx512_permil<bits<8> OpcImm, bits<8> OpcVar, X86VectorVTInfo _, 1039 X86VectorVTInfo Ctrl> : 1040 avx512_perm_imm<OpcImm, "vpermil" # _.Suffix, X86VPermilpi, _> { 1041 let ExeDomain = _.ExeDomain in { 1042 def rr : AVX5128I<OpcVar, MRMSrcReg, (outs _.RC:$dst), 1043 (ins _.RC:$src1, _.RC:$src2), 1044 !strconcat("vpermil" # _.Suffix, 1045 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 1046 [(set _.RC:$dst, 1047 (_.VT (X86VPermilpv _.RC:$src1, 1048 (Ctrl.VT Ctrl.RC:$src2))))]>, 1049 EVEX_4V; 1050 def rm : AVX5128I<OpcVar, MRMSrcMem, (outs _.RC:$dst), 1051 (ins _.RC:$src1, Ctrl.MemOp:$src2), 1052 !strconcat("vpermil" # _.Suffix, 1053 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 1054 [(set _.RC:$dst, 1055 (_.VT (X86VPermilpv _.RC:$src1, 1056 (Ctrl.VT (Ctrl.LdFrag addr:$src2)))))]>, 1057 EVEX_4V; 1058 } 1059} 1060defm VPERMILPSZ : avx512_permil<0x04, 0x0C, v16f32_info, v16i32_info>, 1061 EVEX_V512; 1062defm VPERMILPDZ : avx512_permil<0x05, 0x0D, v8f64_info, v8i64_info>, 1063 EVEX_V512, VEX_W; 1064 1065def : Pat<(v16i32 (X86VPermilpi VR512:$src1, (i8 imm:$imm))), 1066 (VPERMILPSZri VR512:$src1, imm:$imm)>; 1067def : Pat<(v8i64 (X86VPermilpi VR512:$src1, (i8 imm:$imm))), 1068 (VPERMILPDZri VR512:$src1, imm:$imm)>; 1069 1070// -- VPERM2I - 3 source operands form -- 1071multiclass avx512_perm_3src<bits<8> opc, string OpcodeStr, 1072 SDNode OpNode, X86VectorVTInfo _> { 1073let Constraints = "$src1 = $dst" in { 1074 defm rr: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst), 1075 (ins _.RC:$src2, _.RC:$src3), 1076 OpcodeStr, "$src3, $src2", "$src2, $src3", 1077 (_.VT (OpNode _.RC:$src1, _.RC:$src2, _.RC:$src3))>, EVEX_4V, 1078 AVX5128IBase; 1079 1080 let mayLoad = 1 in 1081 defm rm: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst), 1082 (ins _.RC:$src2, _.MemOp:$src3), 1083 OpcodeStr, "$src3, $src2", "$src2, $src3", 1084 (_.VT (OpNode _.RC:$src1, _.RC:$src2, 1085 (_.VT (bitconvert (_.LdFrag addr:$src3)))))>, 1086 EVEX_4V, AVX5128IBase; 1087 } 1088} 1089multiclass avx512_perm_3src_mb<bits<8> opc, string OpcodeStr, 1090 SDNode OpNode, X86VectorVTInfo _> { 1091 let mayLoad = 1, Constraints = "$src1 = $dst" in 1092 defm rmb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst), 1093 (ins _.RC:$src2, _.ScalarMemOp:$src3), 1094 OpcodeStr, !strconcat("${src3}", _.BroadcastStr,", $src2"), 1095 !strconcat("$src2, ${src3}", _.BroadcastStr ), 1096 (_.VT (OpNode _.RC:$src1, 1097 _.RC:$src2,(_.VT (X86VBroadcast (_.ScalarLdFrag addr:$src3)))))>, 1098 AVX5128IBase, EVEX_4V, EVEX_B; 1099} 1100 1101multiclass avx512_perm_3src_sizes<bits<8> opc, string OpcodeStr, 1102 SDNode OpNode, AVX512VLVectorVTInfo VTInfo> { 1103 let Predicates = [HasAVX512] in 1104 defm NAME: avx512_perm_3src<opc, OpcodeStr, OpNode, VTInfo.info512>, 1105 avx512_perm_3src_mb<opc, OpcodeStr, OpNode, VTInfo.info512>, EVEX_V512; 1106 let Predicates = [HasVLX] in { 1107 defm NAME#128: avx512_perm_3src<opc, OpcodeStr, OpNode, VTInfo.info128>, 1108 avx512_perm_3src_mb<opc, OpcodeStr, OpNode, VTInfo.info128>, 1109 EVEX_V128; 1110 defm NAME#256: avx512_perm_3src<opc, OpcodeStr, OpNode, VTInfo.info256>, 1111 avx512_perm_3src_mb<opc, OpcodeStr, OpNode, VTInfo.info256>, 1112 EVEX_V256; 1113 } 1114} 1115multiclass avx512_perm_3src_sizes_w<bits<8> opc, string OpcodeStr, 1116 SDNode OpNode, AVX512VLVectorVTInfo VTInfo> { 1117 let Predicates = [HasBWI] in 1118 defm NAME: avx512_perm_3src<opc, OpcodeStr, OpNode, VTInfo.info512>, 1119 avx512_perm_3src_mb<opc, OpcodeStr, OpNode, VTInfo.info512>, 1120 EVEX_V512; 1121 let Predicates = [HasBWI, HasVLX] in { 1122 defm NAME#128: avx512_perm_3src<opc, OpcodeStr, OpNode, VTInfo.info128>, 1123 avx512_perm_3src_mb<opc, OpcodeStr, OpNode, VTInfo.info128>, 1124 EVEX_V128; 1125 defm NAME#256: avx512_perm_3src<opc, OpcodeStr, OpNode, VTInfo.info256>, 1126 avx512_perm_3src_mb<opc, OpcodeStr, OpNode, VTInfo.info256>, 1127 EVEX_V256; 1128 } 1129} 1130defm VPERMI2D : avx512_perm_3src_sizes<0x76, "vpermi2d", X86VPermiv3, 1131 avx512vl_i32_info>, EVEX_CD8<32, CD8VF>; 1132defm VPERMI2Q : avx512_perm_3src_sizes<0x76, "vpermi2q", X86VPermiv3, 1133 avx512vl_i64_info>, VEX_W, EVEX_CD8<64, CD8VF>; 1134defm VPERMI2PS : avx512_perm_3src_sizes<0x77, "vpermi2ps", X86VPermiv3, 1135 avx512vl_f32_info>, EVEX_CD8<32, CD8VF>; 1136defm VPERMI2PD : avx512_perm_3src_sizes<0x77, "vpermi2pd", X86VPermiv3, 1137 avx512vl_f64_info>, VEX_W, EVEX_CD8<64, CD8VF>; 1138 1139defm VPERMT2D : avx512_perm_3src_sizes<0x7E, "vpermt2d", X86VPermv3, 1140 avx512vl_i32_info>, EVEX_CD8<32, CD8VF>; 1141defm VPERMT2Q : avx512_perm_3src_sizes<0x7E, "vpermt2q", X86VPermv3, 1142 avx512vl_i64_info>, VEX_W, EVEX_CD8<64, CD8VF>; 1143defm VPERMT2PS : avx512_perm_3src_sizes<0x7F, "vpermt2ps", X86VPermv3, 1144 avx512vl_f32_info>, EVEX_CD8<32, CD8VF>; 1145defm VPERMT2PD : avx512_perm_3src_sizes<0x7F, "vpermt2pd", X86VPermv3, 1146 avx512vl_f64_info>, VEX_W, EVEX_CD8<64, CD8VF>; 1147 1148defm VPERMT2W : avx512_perm_3src_sizes_w<0x7D, "vpermt2w", X86VPermv3, 1149 avx512vl_i16_info>, VEX_W, EVEX_CD8<16, CD8VF>; 1150defm VPERMI2W : avx512_perm_3src_sizes_w<0x75, "vpermi2w", X86VPermiv3, 1151 avx512vl_i16_info>, VEX_W, EVEX_CD8<16, CD8VF>; 1152 1153//===----------------------------------------------------------------------===// 1154// AVX-512 - BLEND using mask 1155// 1156multiclass avx512_blendmask<bits<8> opc, string OpcodeStr, X86VectorVTInfo _> { 1157 let ExeDomain = _.ExeDomain in { 1158 def rr : AVX5128I<opc, MRMSrcReg, (outs _.RC:$dst), 1159 (ins _.RC:$src1, _.RC:$src2), 1160 !strconcat(OpcodeStr, 1161 "\t{$src2, $src1, ${dst} |${dst}, $src1, $src2}"), 1162 []>, EVEX_4V; 1163 def rrk : AVX5128I<opc, MRMSrcReg, (outs _.RC:$dst), 1164 (ins _.KRCWM:$mask, _.RC:$src1, _.RC:$src2), 1165 !strconcat(OpcodeStr, 1166 "\t{$src2, $src1, ${dst} {${mask}}|${dst} {${mask}}, $src1, $src2}"), 1167 [(set _.RC:$dst, (X86select _.KRCWM:$mask, (_.VT _.RC:$src1), 1168 (_.VT _.RC:$src2)))]>, EVEX_4V, EVEX_K; 1169 def rrkz : AVX5128I<opc, MRMSrcReg, (outs _.RC:$dst), 1170 (ins _.KRCWM:$mask, _.RC:$src1, _.RC:$src2), 1171 !strconcat(OpcodeStr, 1172 "\t{$src2, $src1, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src1, $src2}"), 1173 []>, EVEX_4V, EVEX_KZ; 1174 let mayLoad = 1 in { 1175 def rm : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst), 1176 (ins _.RC:$src1, _.MemOp:$src2), 1177 !strconcat(OpcodeStr, 1178 "\t{$src2, $src1, ${dst} |${dst}, $src1, $src2}"), 1179 []>, EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>; 1180 def rmk : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst), 1181 (ins _.KRCWM:$mask, _.RC:$src1, _.MemOp:$src2), 1182 !strconcat(OpcodeStr, 1183 "\t{$src2, $src1, ${dst} {${mask}}|${dst} {${mask}}, $src1, $src2}"), 1184 [(set _.RC:$dst, (X86select _.KRCWM:$mask, (_.VT _.RC:$src1), 1185 (_.VT (bitconvert (_.LdFrag addr:$src2)))))]>, 1186 EVEX_4V, EVEX_K, EVEX_CD8<_.EltSize, CD8VF>; 1187 def rmkz : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst), 1188 (ins _.KRCWM:$mask, _.RC:$src1, _.MemOp:$src2), 1189 !strconcat(OpcodeStr, 1190 "\t{$src2, $src1, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src1, $src2}"), 1191 []>, EVEX_4V, EVEX_KZ, EVEX_CD8<_.EltSize, CD8VF>; 1192 } 1193 } 1194} 1195multiclass avx512_blendmask_rmb<bits<8> opc, string OpcodeStr, X86VectorVTInfo _> { 1196 1197 def rmbk : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst), 1198 (ins _.KRCWM:$mask, _.RC:$src1, _.ScalarMemOp:$src2), 1199 !strconcat(OpcodeStr, 1200 "\t{${src2}", _.BroadcastStr, ", $src1, $dst {${mask}}|", 1201 "$dst {${mask}}, $src1, ${src2}", _.BroadcastStr, "}"), 1202 [(set _.RC:$dst,(X86select _.KRCWM:$mask, (_.VT _.RC:$src1), 1203 (X86VBroadcast (_.ScalarLdFrag addr:$src2))))]>, 1204 EVEX_4V, EVEX_K, EVEX_B, EVEX_CD8<_.EltSize, CD8VF>; 1205 1206 def rmb : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst), 1207 (ins _.RC:$src1, _.ScalarMemOp:$src2), 1208 !strconcat(OpcodeStr, 1209 "\t{${src2}", _.BroadcastStr, ", $src1, $dst|", 1210 "$dst, $src1, ${src2}", _.BroadcastStr, "}"), 1211 []>, EVEX_4V, EVEX_B, EVEX_CD8<_.EltSize, CD8VF>; 1212 1213} 1214 1215multiclass blendmask_dq <bits<8> opc, string OpcodeStr, 1216 AVX512VLVectorVTInfo VTInfo> { 1217 defm Z : avx512_blendmask <opc, OpcodeStr, VTInfo.info512>, 1218 avx512_blendmask_rmb <opc, OpcodeStr, VTInfo.info512>, EVEX_V512; 1219 1220 let Predicates = [HasVLX] in { 1221 defm Z256 : avx512_blendmask<opc, OpcodeStr, VTInfo.info256>, 1222 avx512_blendmask_rmb <opc, OpcodeStr, VTInfo.info256>, EVEX_V256; 1223 defm Z128 : avx512_blendmask<opc, OpcodeStr, VTInfo.info128>, 1224 avx512_blendmask_rmb <opc, OpcodeStr, VTInfo.info128>, EVEX_V128; 1225 } 1226} 1227 1228multiclass blendmask_bw <bits<8> opc, string OpcodeStr, 1229 AVX512VLVectorVTInfo VTInfo> { 1230 let Predicates = [HasBWI] in 1231 defm Z : avx512_blendmask <opc, OpcodeStr, VTInfo.info512>, EVEX_V512; 1232 1233 let Predicates = [HasBWI, HasVLX] in { 1234 defm Z256 : avx512_blendmask <opc, OpcodeStr, VTInfo.info256>, EVEX_V256; 1235 defm Z128 : avx512_blendmask <opc, OpcodeStr, VTInfo.info128>, EVEX_V128; 1236 } 1237} 1238 1239 1240defm VBLENDMPS : blendmask_dq <0x65, "vblendmps", avx512vl_f32_info>; 1241defm VBLENDMPD : blendmask_dq <0x65, "vblendmpd", avx512vl_f64_info>, VEX_W; 1242defm VPBLENDMD : blendmask_dq <0x64, "vpblendmd", avx512vl_i32_info>; 1243defm VPBLENDMQ : blendmask_dq <0x64, "vpblendmq", avx512vl_i64_info>, VEX_W; 1244defm VPBLENDMB : blendmask_bw <0x66, "vpblendmb", avx512vl_i8_info>; 1245defm VPBLENDMW : blendmask_bw <0x66, "vpblendmw", avx512vl_i16_info>, VEX_W; 1246 1247 1248let Predicates = [HasAVX512] in { 1249def : Pat<(v8f32 (vselect (v8i1 VK8WM:$mask), (v8f32 VR256X:$src1), 1250 (v8f32 VR256X:$src2))), 1251 (EXTRACT_SUBREG 1252 (v16f32 (VBLENDMPSZrrk (COPY_TO_REGCLASS VK8WM:$mask, VK16WM), 1253 (v16f32 (SUBREG_TO_REG (i32 0), VR256X:$src2, sub_ymm)), 1254 (v16f32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)))), sub_ymm)>; 1255 1256def : Pat<(v8i32 (vselect (v8i1 VK8WM:$mask), (v8i32 VR256X:$src1), 1257 (v8i32 VR256X:$src2))), 1258 (EXTRACT_SUBREG 1259 (v16i32 (VPBLENDMDZrrk (COPY_TO_REGCLASS VK8WM:$mask, VK16WM), 1260 (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src2, sub_ymm)), 1261 (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)))), sub_ymm)>; 1262} 1263//===----------------------------------------------------------------------===// 1264// Compare Instructions 1265//===----------------------------------------------------------------------===// 1266 1267// avx512_cmp_scalar - AVX512 CMPSS and CMPSD 1268multiclass avx512_cmp_scalar<RegisterClass RC, X86MemOperand x86memop, 1269 SDNode OpNode, ValueType VT, 1270 PatFrag ld_frag, string Suffix> { 1271 def rr : AVX512Ii8<0xC2, MRMSrcReg, 1272 (outs VK1:$dst), (ins RC:$src1, RC:$src2, AVXCC:$cc), 1273 !strconcat("vcmp${cc}", Suffix, 1274 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 1275 [(set VK1:$dst, (OpNode (VT RC:$src1), RC:$src2, imm:$cc))], 1276 IIC_SSE_ALU_F32S_RR>, EVEX_4V; 1277 def rm : AVX512Ii8<0xC2, MRMSrcMem, 1278 (outs VK1:$dst), (ins RC:$src1, x86memop:$src2, AVXCC:$cc), 1279 !strconcat("vcmp${cc}", Suffix, 1280 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 1281 [(set VK1:$dst, (OpNode (VT RC:$src1), 1282 (ld_frag addr:$src2), imm:$cc))], IIC_SSE_ALU_F32P_RM>, EVEX_4V; 1283 let isAsmParserOnly = 1, hasSideEffects = 0 in { 1284 def rri_alt : AVX512Ii8<0xC2, MRMSrcReg, 1285 (outs VK1:$dst), (ins RC:$src1, RC:$src2, u8imm:$cc), 1286 !strconcat("vcmp", Suffix, 1287 "\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}"), 1288 [], IIC_SSE_ALU_F32S_RR>, EVEX_4V; 1289 let mayLoad = 1 in 1290 def rmi_alt : AVX512Ii8<0xC2, MRMSrcMem, 1291 (outs VK1:$dst), (ins RC:$src1, x86memop:$src2, u8imm:$cc), 1292 !strconcat("vcmp", Suffix, 1293 "\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}"), 1294 [], IIC_SSE_ALU_F32P_RM>, EVEX_4V; 1295 } 1296} 1297 1298let Predicates = [HasAVX512] in { 1299defm VCMPSSZ : avx512_cmp_scalar<FR32X, f32mem, X86cmpms, f32, loadf32, "ss">, 1300 XS; 1301defm VCMPSDZ : avx512_cmp_scalar<FR64X, f64mem, X86cmpms, f64, loadf64, "sd">, 1302 XD, VEX_W; 1303} 1304 1305multiclass avx512_icmp_packed<bits<8> opc, string OpcodeStr, SDNode OpNode, 1306 X86VectorVTInfo _> { 1307 def rr : AVX512BI<opc, MRMSrcReg, 1308 (outs _.KRC:$dst), (ins _.RC:$src1, _.RC:$src2), 1309 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 1310 [(set _.KRC:$dst, (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2)))], 1311 IIC_SSE_ALU_F32P_RR>, EVEX_4V; 1312 let mayLoad = 1 in 1313 def rm : AVX512BI<opc, MRMSrcMem, 1314 (outs _.KRC:$dst), (ins _.RC:$src1, _.MemOp:$src2), 1315 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 1316 [(set _.KRC:$dst, (OpNode (_.VT _.RC:$src1), 1317 (_.VT (bitconvert (_.LdFrag addr:$src2)))))], 1318 IIC_SSE_ALU_F32P_RM>, EVEX_4V; 1319 def rrk : AVX512BI<opc, MRMSrcReg, 1320 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, _.RC:$src2), 1321 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst {${mask}}|", 1322 "$dst {${mask}}, $src1, $src2}"), 1323 [(set _.KRC:$dst, (and _.KRCWM:$mask, 1324 (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2))))], 1325 IIC_SSE_ALU_F32P_RR>, EVEX_4V, EVEX_K; 1326 let mayLoad = 1 in 1327 def rmk : AVX512BI<opc, MRMSrcMem, 1328 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, _.MemOp:$src2), 1329 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst {${mask}}|", 1330 "$dst {${mask}}, $src1, $src2}"), 1331 [(set _.KRC:$dst, (and _.KRCWM:$mask, 1332 (OpNode (_.VT _.RC:$src1), 1333 (_.VT (bitconvert 1334 (_.LdFrag addr:$src2))))))], 1335 IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_K; 1336} 1337 1338multiclass avx512_icmp_packed_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode, 1339 X86VectorVTInfo _> : 1340 avx512_icmp_packed<opc, OpcodeStr, OpNode, _> { 1341 let mayLoad = 1 in { 1342 def rmb : AVX512BI<opc, MRMSrcMem, 1343 (outs _.KRC:$dst), (ins _.RC:$src1, _.ScalarMemOp:$src2), 1344 !strconcat(OpcodeStr, "\t{${src2}", _.BroadcastStr, ", $src1, $dst", 1345 "|$dst, $src1, ${src2}", _.BroadcastStr, "}"), 1346 [(set _.KRC:$dst, (OpNode (_.VT _.RC:$src1), 1347 (X86VBroadcast (_.ScalarLdFrag addr:$src2))))], 1348 IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_B; 1349 def rmbk : AVX512BI<opc, MRMSrcMem, 1350 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, 1351 _.ScalarMemOp:$src2), 1352 !strconcat(OpcodeStr, 1353 "\t{${src2}", _.BroadcastStr, ", $src1, $dst {${mask}}|", 1354 "$dst {${mask}}, $src1, ${src2}", _.BroadcastStr, "}"), 1355 [(set _.KRC:$dst, (and _.KRCWM:$mask, 1356 (OpNode (_.VT _.RC:$src1), 1357 (X86VBroadcast 1358 (_.ScalarLdFrag addr:$src2)))))], 1359 IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_K, EVEX_B; 1360 } 1361} 1362 1363multiclass avx512_icmp_packed_vl<bits<8> opc, string OpcodeStr, SDNode OpNode, 1364 AVX512VLVectorVTInfo VTInfo, Predicate prd> { 1365 let Predicates = [prd] in 1366 defm Z : avx512_icmp_packed<opc, OpcodeStr, OpNode, VTInfo.info512>, 1367 EVEX_V512; 1368 1369 let Predicates = [prd, HasVLX] in { 1370 defm Z256 : avx512_icmp_packed<opc, OpcodeStr, OpNode, VTInfo.info256>, 1371 EVEX_V256; 1372 defm Z128 : avx512_icmp_packed<opc, OpcodeStr, OpNode, VTInfo.info128>, 1373 EVEX_V128; 1374 } 1375} 1376 1377multiclass avx512_icmp_packed_rmb_vl<bits<8> opc, string OpcodeStr, 1378 SDNode OpNode, AVX512VLVectorVTInfo VTInfo, 1379 Predicate prd> { 1380 let Predicates = [prd] in 1381 defm Z : avx512_icmp_packed_rmb<opc, OpcodeStr, OpNode, VTInfo.info512>, 1382 EVEX_V512; 1383 1384 let Predicates = [prd, HasVLX] in { 1385 defm Z256 : avx512_icmp_packed_rmb<opc, OpcodeStr, OpNode, VTInfo.info256>, 1386 EVEX_V256; 1387 defm Z128 : avx512_icmp_packed_rmb<opc, OpcodeStr, OpNode, VTInfo.info128>, 1388 EVEX_V128; 1389 } 1390} 1391 1392defm VPCMPEQB : avx512_icmp_packed_vl<0x74, "vpcmpeqb", X86pcmpeqm, 1393 avx512vl_i8_info, HasBWI>, 1394 EVEX_CD8<8, CD8VF>; 1395 1396defm VPCMPEQW : avx512_icmp_packed_vl<0x75, "vpcmpeqw", X86pcmpeqm, 1397 avx512vl_i16_info, HasBWI>, 1398 EVEX_CD8<16, CD8VF>; 1399 1400defm VPCMPEQD : avx512_icmp_packed_rmb_vl<0x76, "vpcmpeqd", X86pcmpeqm, 1401 avx512vl_i32_info, HasAVX512>, 1402 EVEX_CD8<32, CD8VF>; 1403 1404defm VPCMPEQQ : avx512_icmp_packed_rmb_vl<0x29, "vpcmpeqq", X86pcmpeqm, 1405 avx512vl_i64_info, HasAVX512>, 1406 T8PD, VEX_W, EVEX_CD8<64, CD8VF>; 1407 1408defm VPCMPGTB : avx512_icmp_packed_vl<0x64, "vpcmpgtb", X86pcmpgtm, 1409 avx512vl_i8_info, HasBWI>, 1410 EVEX_CD8<8, CD8VF>; 1411 1412defm VPCMPGTW : avx512_icmp_packed_vl<0x65, "vpcmpgtw", X86pcmpgtm, 1413 avx512vl_i16_info, HasBWI>, 1414 EVEX_CD8<16, CD8VF>; 1415 1416defm VPCMPGTD : avx512_icmp_packed_rmb_vl<0x66, "vpcmpgtd", X86pcmpgtm, 1417 avx512vl_i32_info, HasAVX512>, 1418 EVEX_CD8<32, CD8VF>; 1419 1420defm VPCMPGTQ : avx512_icmp_packed_rmb_vl<0x37, "vpcmpgtq", X86pcmpgtm, 1421 avx512vl_i64_info, HasAVX512>, 1422 T8PD, VEX_W, EVEX_CD8<64, CD8VF>; 1423 1424def : Pat<(v8i1 (X86pcmpgtm (v8i32 VR256X:$src1), (v8i32 VR256X:$src2))), 1425 (COPY_TO_REGCLASS (VPCMPGTDZrr 1426 (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)), 1427 (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src2, sub_ymm))), VK8)>; 1428 1429def : Pat<(v8i1 (X86pcmpeqm (v8i32 VR256X:$src1), (v8i32 VR256X:$src2))), 1430 (COPY_TO_REGCLASS (VPCMPEQDZrr 1431 (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)), 1432 (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src2, sub_ymm))), VK8)>; 1433 1434multiclass avx512_icmp_cc<bits<8> opc, string Suffix, SDNode OpNode, 1435 X86VectorVTInfo _> { 1436 def rri : AVX512AIi8<opc, MRMSrcReg, 1437 (outs _.KRC:$dst), (ins _.RC:$src1, _.RC:$src2, AVX512ICC:$cc), 1438 !strconcat("vpcmp${cc}", Suffix, 1439 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 1440 [(set _.KRC:$dst, (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2), 1441 imm:$cc))], 1442 IIC_SSE_ALU_F32P_RR>, EVEX_4V; 1443 let mayLoad = 1 in 1444 def rmi : AVX512AIi8<opc, MRMSrcMem, 1445 (outs _.KRC:$dst), (ins _.RC:$src1, _.MemOp:$src2, AVX512ICC:$cc), 1446 !strconcat("vpcmp${cc}", Suffix, 1447 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 1448 [(set _.KRC:$dst, (OpNode (_.VT _.RC:$src1), 1449 (_.VT (bitconvert (_.LdFrag addr:$src2))), 1450 imm:$cc))], 1451 IIC_SSE_ALU_F32P_RM>, EVEX_4V; 1452 def rrik : AVX512AIi8<opc, MRMSrcReg, 1453 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, _.RC:$src2, 1454 AVX512ICC:$cc), 1455 !strconcat("vpcmp${cc}", Suffix, 1456 "\t{$src2, $src1, $dst {${mask}}|", 1457 "$dst {${mask}}, $src1, $src2}"), 1458 [(set _.KRC:$dst, (and _.KRCWM:$mask, 1459 (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2), 1460 imm:$cc)))], 1461 IIC_SSE_ALU_F32P_RR>, EVEX_4V, EVEX_K; 1462 let mayLoad = 1 in 1463 def rmik : AVX512AIi8<opc, MRMSrcMem, 1464 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, _.MemOp:$src2, 1465 AVX512ICC:$cc), 1466 !strconcat("vpcmp${cc}", Suffix, 1467 "\t{$src2, $src1, $dst {${mask}}|", 1468 "$dst {${mask}}, $src1, $src2}"), 1469 [(set _.KRC:$dst, (and _.KRCWM:$mask, 1470 (OpNode (_.VT _.RC:$src1), 1471 (_.VT (bitconvert (_.LdFrag addr:$src2))), 1472 imm:$cc)))], 1473 IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_K; 1474 1475 // Accept explicit immediate argument form instead of comparison code. 1476 let isAsmParserOnly = 1, hasSideEffects = 0 in { 1477 def rri_alt : AVX512AIi8<opc, MRMSrcReg, 1478 (outs _.KRC:$dst), (ins _.RC:$src1, _.RC:$src2, u8imm:$cc), 1479 !strconcat("vpcmp", Suffix, "\t{$cc, $src2, $src1, $dst|", 1480 "$dst, $src1, $src2, $cc}"), 1481 [], IIC_SSE_ALU_F32P_RR>, EVEX_4V; 1482 let mayLoad = 1 in 1483 def rmi_alt : AVX512AIi8<opc, MRMSrcMem, 1484 (outs _.KRC:$dst), (ins _.RC:$src1, _.MemOp:$src2, u8imm:$cc), 1485 !strconcat("vpcmp", Suffix, "\t{$cc, $src2, $src1, $dst|", 1486 "$dst, $src1, $src2, $cc}"), 1487 [], IIC_SSE_ALU_F32P_RM>, EVEX_4V; 1488 def rrik_alt : AVX512AIi8<opc, MRMSrcReg, 1489 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, _.RC:$src2, 1490 u8imm:$cc), 1491 !strconcat("vpcmp", Suffix, 1492 "\t{$cc, $src2, $src1, $dst {${mask}}|", 1493 "$dst {${mask}}, $src1, $src2, $cc}"), 1494 [], IIC_SSE_ALU_F32P_RR>, EVEX_4V, EVEX_K; 1495 let mayLoad = 1 in 1496 def rmik_alt : AVX512AIi8<opc, MRMSrcMem, 1497 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, _.MemOp:$src2, 1498 u8imm:$cc), 1499 !strconcat("vpcmp", Suffix, 1500 "\t{$cc, $src2, $src1, $dst {${mask}}|", 1501 "$dst {${mask}}, $src1, $src2, $cc}"), 1502 [], IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_K; 1503 } 1504} 1505 1506multiclass avx512_icmp_cc_rmb<bits<8> opc, string Suffix, SDNode OpNode, 1507 X86VectorVTInfo _> : 1508 avx512_icmp_cc<opc, Suffix, OpNode, _> { 1509 def rmib : AVX512AIi8<opc, MRMSrcMem, 1510 (outs _.KRC:$dst), (ins _.RC:$src1, _.ScalarMemOp:$src2, 1511 AVX512ICC:$cc), 1512 !strconcat("vpcmp${cc}", Suffix, 1513 "\t{${src2}", _.BroadcastStr, ", $src1, $dst|", 1514 "$dst, $src1, ${src2}", _.BroadcastStr, "}"), 1515 [(set _.KRC:$dst, (OpNode (_.VT _.RC:$src1), 1516 (X86VBroadcast (_.ScalarLdFrag addr:$src2)), 1517 imm:$cc))], 1518 IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_B; 1519 def rmibk : AVX512AIi8<opc, MRMSrcMem, 1520 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, 1521 _.ScalarMemOp:$src2, AVX512ICC:$cc), 1522 !strconcat("vpcmp${cc}", Suffix, 1523 "\t{${src2}", _.BroadcastStr, ", $src1, $dst {${mask}}|", 1524 "$dst {${mask}}, $src1, ${src2}", _.BroadcastStr, "}"), 1525 [(set _.KRC:$dst, (and _.KRCWM:$mask, 1526 (OpNode (_.VT _.RC:$src1), 1527 (X86VBroadcast (_.ScalarLdFrag addr:$src2)), 1528 imm:$cc)))], 1529 IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_K, EVEX_B; 1530 1531 // Accept explicit immediate argument form instead of comparison code. 1532 let isAsmParserOnly = 1, hasSideEffects = 0, mayLoad = 1 in { 1533 def rmib_alt : AVX512AIi8<opc, MRMSrcMem, 1534 (outs _.KRC:$dst), (ins _.RC:$src1, _.ScalarMemOp:$src2, 1535 u8imm:$cc), 1536 !strconcat("vpcmp", Suffix, 1537 "\t{$cc, ${src2}", _.BroadcastStr, ", $src1, $dst|", 1538 "$dst, $src1, ${src2}", _.BroadcastStr, ", $cc}"), 1539 [], IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_B; 1540 def rmibk_alt : AVX512AIi8<opc, MRMSrcMem, 1541 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, 1542 _.ScalarMemOp:$src2, u8imm:$cc), 1543 !strconcat("vpcmp", Suffix, 1544 "\t{$cc, ${src2}", _.BroadcastStr, ", $src1, $dst {${mask}}|", 1545 "$dst {${mask}}, $src1, ${src2}", _.BroadcastStr, ", $cc}"), 1546 [], IIC_SSE_ALU_F32P_RM>, EVEX_4V, EVEX_K, EVEX_B; 1547 } 1548} 1549 1550multiclass avx512_icmp_cc_vl<bits<8> opc, string Suffix, SDNode OpNode, 1551 AVX512VLVectorVTInfo VTInfo, Predicate prd> { 1552 let Predicates = [prd] in 1553 defm Z : avx512_icmp_cc<opc, Suffix, OpNode, VTInfo.info512>, EVEX_V512; 1554 1555 let Predicates = [prd, HasVLX] in { 1556 defm Z256 : avx512_icmp_cc<opc, Suffix, OpNode, VTInfo.info256>, EVEX_V256; 1557 defm Z128 : avx512_icmp_cc<opc, Suffix, OpNode, VTInfo.info128>, EVEX_V128; 1558 } 1559} 1560 1561multiclass avx512_icmp_cc_rmb_vl<bits<8> opc, string Suffix, SDNode OpNode, 1562 AVX512VLVectorVTInfo VTInfo, Predicate prd> { 1563 let Predicates = [prd] in 1564 defm Z : avx512_icmp_cc_rmb<opc, Suffix, OpNode, VTInfo.info512>, 1565 EVEX_V512; 1566 1567 let Predicates = [prd, HasVLX] in { 1568 defm Z256 : avx512_icmp_cc_rmb<opc, Suffix, OpNode, VTInfo.info256>, 1569 EVEX_V256; 1570 defm Z128 : avx512_icmp_cc_rmb<opc, Suffix, OpNode, VTInfo.info128>, 1571 EVEX_V128; 1572 } 1573} 1574 1575defm VPCMPB : avx512_icmp_cc_vl<0x3F, "b", X86cmpm, avx512vl_i8_info, 1576 HasBWI>, EVEX_CD8<8, CD8VF>; 1577defm VPCMPUB : avx512_icmp_cc_vl<0x3E, "ub", X86cmpmu, avx512vl_i8_info, 1578 HasBWI>, EVEX_CD8<8, CD8VF>; 1579 1580defm VPCMPW : avx512_icmp_cc_vl<0x3F, "w", X86cmpm, avx512vl_i16_info, 1581 HasBWI>, VEX_W, EVEX_CD8<16, CD8VF>; 1582defm VPCMPUW : avx512_icmp_cc_vl<0x3E, "uw", X86cmpmu, avx512vl_i16_info, 1583 HasBWI>, VEX_W, EVEX_CD8<16, CD8VF>; 1584 1585defm VPCMPD : avx512_icmp_cc_rmb_vl<0x1F, "d", X86cmpm, avx512vl_i32_info, 1586 HasAVX512>, EVEX_CD8<32, CD8VF>; 1587defm VPCMPUD : avx512_icmp_cc_rmb_vl<0x1E, "ud", X86cmpmu, avx512vl_i32_info, 1588 HasAVX512>, EVEX_CD8<32, CD8VF>; 1589 1590defm VPCMPQ : avx512_icmp_cc_rmb_vl<0x1F, "q", X86cmpm, avx512vl_i64_info, 1591 HasAVX512>, VEX_W, EVEX_CD8<64, CD8VF>; 1592defm VPCMPUQ : avx512_icmp_cc_rmb_vl<0x1E, "uq", X86cmpmu, avx512vl_i64_info, 1593 HasAVX512>, VEX_W, EVEX_CD8<64, CD8VF>; 1594 1595multiclass avx512_vcmp_common<X86VectorVTInfo _> { 1596 1597 defm rri : AVX512_maskable_cmp<0xC2, MRMSrcReg, _, 1598 (outs _.KRC:$dst), (ins _.RC:$src1, _.RC:$src2,AVXCC:$cc), 1599 "vcmp${cc}"#_.Suffix, 1600 "$src2, $src1", "$src1, $src2", 1601 (X86cmpm (_.VT _.RC:$src1), 1602 (_.VT _.RC:$src2), 1603 imm:$cc)>; 1604 1605 let mayLoad = 1 in { 1606 defm rmi : AVX512_maskable_cmp<0xC2, MRMSrcMem, _, 1607 (outs _.KRC:$dst),(ins _.RC:$src1, _.MemOp:$src2, AVXCC:$cc), 1608 "vcmp${cc}"#_.Suffix, 1609 "$src2, $src1", "$src1, $src2", 1610 (X86cmpm (_.VT _.RC:$src1), 1611 (_.VT (bitconvert (_.LdFrag addr:$src2))), 1612 imm:$cc)>; 1613 1614 defm rmbi : AVX512_maskable_cmp<0xC2, MRMSrcMem, _, 1615 (outs _.KRC:$dst), 1616 (ins _.RC:$src1, _.ScalarMemOp:$src2, AVXCC:$cc), 1617 "vcmp${cc}"#_.Suffix, 1618 "${src2}"##_.BroadcastStr##", $src1", 1619 "$src1, ${src2}"##_.BroadcastStr, 1620 (X86cmpm (_.VT _.RC:$src1), 1621 (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src2))), 1622 imm:$cc)>,EVEX_B; 1623 } 1624 // Accept explicit immediate argument form instead of comparison code. 1625 let isAsmParserOnly = 1, hasSideEffects = 0 in { 1626 defm rri_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcReg, _, 1627 (outs _.KRC:$dst), 1628 (ins _.RC:$src1, _.RC:$src2, u8imm:$cc), 1629 "vcmp"#_.Suffix, 1630 "$cc, $src2, $src1", "$src1, $src2, $cc">; 1631 1632 let mayLoad = 1 in { 1633 defm rmi_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcMem, _, 1634 (outs _.KRC:$dst), 1635 (ins _.RC:$src1, _.MemOp:$src2, u8imm:$cc), 1636 "vcmp"#_.Suffix, 1637 "$cc, $src2, $src1", "$src1, $src2, $cc">; 1638 1639 defm rmbi_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcMem, _, 1640 (outs _.KRC:$dst), 1641 (ins _.RC:$src1, _.ScalarMemOp:$src2, u8imm:$cc), 1642 "vcmp"#_.Suffix, 1643 "$cc, ${src2}"##_.BroadcastStr##", $src1", 1644 "$src1, ${src2}"##_.BroadcastStr##", $cc">,EVEX_B; 1645 } 1646 } 1647} 1648 1649multiclass avx512_vcmp_sae<X86VectorVTInfo _> { 1650 // comparison code form (VCMP[EQ/LT/LE/...] 1651 defm rrib : AVX512_maskable_cmp<0xC2, MRMSrcReg, _, 1652 (outs _.KRC:$dst),(ins _.RC:$src1, _.RC:$src2, AVXCC:$cc), 1653 "vcmp${cc}"#_.Suffix, 1654 "{sae}, $src2, $src1", "$src1, $src2,{sae}", 1655 (X86cmpmRnd (_.VT _.RC:$src1), 1656 (_.VT _.RC:$src2), 1657 imm:$cc, 1658 (i32 FROUND_NO_EXC))>, EVEX_B; 1659 1660 let isAsmParserOnly = 1, hasSideEffects = 0 in { 1661 defm rrib_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcReg, _, 1662 (outs _.KRC:$dst), 1663 (ins _.RC:$src1, _.RC:$src2, u8imm:$cc), 1664 "vcmp"#_.Suffix, 1665 "$cc,{sae}, $src2, $src1", 1666 "$src1, $src2,{sae}, $cc">, EVEX_B; 1667 } 1668} 1669 1670multiclass avx512_vcmp<AVX512VLVectorVTInfo _> { 1671 let Predicates = [HasAVX512] in { 1672 defm Z : avx512_vcmp_common<_.info512>, 1673 avx512_vcmp_sae<_.info512>, EVEX_V512; 1674 1675 } 1676 let Predicates = [HasAVX512,HasVLX] in { 1677 defm Z128 : avx512_vcmp_common<_.info128>, EVEX_V128; 1678 defm Z256 : avx512_vcmp_common<_.info256>, EVEX_V256; 1679 } 1680} 1681 1682defm VCMPPD : avx512_vcmp<avx512vl_f64_info>, 1683 AVX512PDIi8Base, EVEX_4V, EVEX_CD8<64, CD8VF>, VEX_W; 1684defm VCMPPS : avx512_vcmp<avx512vl_f32_info>, 1685 AVX512PSIi8Base, EVEX_4V, EVEX_CD8<32, CD8VF>; 1686 1687def : Pat<(v8i1 (X86cmpm (v8f32 VR256X:$src1), (v8f32 VR256X:$src2), imm:$cc)), 1688 (COPY_TO_REGCLASS (VCMPPSZrri 1689 (v16f32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)), 1690 (v16f32 (SUBREG_TO_REG (i32 0), VR256X:$src2, sub_ymm)), 1691 imm:$cc), VK8)>; 1692def : Pat<(v8i1 (X86cmpm (v8i32 VR256X:$src1), (v8i32 VR256X:$src2), imm:$cc)), 1693 (COPY_TO_REGCLASS (VPCMPDZrri 1694 (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)), 1695 (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src2, sub_ymm)), 1696 imm:$cc), VK8)>; 1697def : Pat<(v8i1 (X86cmpmu (v8i32 VR256X:$src1), (v8i32 VR256X:$src2), imm:$cc)), 1698 (COPY_TO_REGCLASS (VPCMPUDZrri 1699 (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)), 1700 (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src2, sub_ymm)), 1701 imm:$cc), VK8)>; 1702 1703//----------------------------------------------------------------- 1704// Mask register copy, including 1705// - copy between mask registers 1706// - load/store mask registers 1707// - copy from GPR to mask register and vice versa 1708// 1709multiclass avx512_mask_mov<bits<8> opc_kk, bits<8> opc_km, bits<8> opc_mk, 1710 string OpcodeStr, RegisterClass KRC, 1711 ValueType vvt, X86MemOperand x86memop> { 1712 let hasSideEffects = 0 in { 1713 def kk : I<opc_kk, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src), 1714 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), []>; 1715 let mayLoad = 1 in 1716 def km : I<opc_km, MRMSrcMem, (outs KRC:$dst), (ins x86memop:$src), 1717 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 1718 [(set KRC:$dst, (vvt (load addr:$src)))]>; 1719 let mayStore = 1 in 1720 def mk : I<opc_mk, MRMDestMem, (outs), (ins x86memop:$dst, KRC:$src), 1721 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 1722 [(store KRC:$src, addr:$dst)]>; 1723 } 1724} 1725 1726multiclass avx512_mask_mov_gpr<bits<8> opc_kr, bits<8> opc_rk, 1727 string OpcodeStr, 1728 RegisterClass KRC, RegisterClass GRC> { 1729 let hasSideEffects = 0 in { 1730 def kr : I<opc_kr, MRMSrcReg, (outs KRC:$dst), (ins GRC:$src), 1731 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), []>; 1732 def rk : I<opc_rk, MRMSrcReg, (outs GRC:$dst), (ins KRC:$src), 1733 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), []>; 1734 } 1735} 1736 1737let Predicates = [HasDQI] in 1738 defm KMOVB : avx512_mask_mov<0x90, 0x90, 0x91, "kmovb", VK8, v8i1, i8mem>, 1739 avx512_mask_mov_gpr<0x92, 0x93, "kmovb", VK8, GR32>, 1740 VEX, PD; 1741 1742let Predicates = [HasAVX512] in 1743 defm KMOVW : avx512_mask_mov<0x90, 0x90, 0x91, "kmovw", VK16, v16i1, i16mem>, 1744 avx512_mask_mov_gpr<0x92, 0x93, "kmovw", VK16, GR32>, 1745 VEX, PS; 1746 1747let Predicates = [HasBWI] in { 1748 defm KMOVD : avx512_mask_mov<0x90, 0x90, 0x91, "kmovd", VK32, v32i1,i32mem>, 1749 VEX, PD, VEX_W; 1750 defm KMOVD : avx512_mask_mov_gpr<0x92, 0x93, "kmovd", VK32, GR32>, 1751 VEX, XD; 1752} 1753 1754let Predicates = [HasBWI] in { 1755 defm KMOVQ : avx512_mask_mov<0x90, 0x90, 0x91, "kmovq", VK64, v64i1, i64mem>, 1756 VEX, PS, VEX_W; 1757 defm KMOVQ : avx512_mask_mov_gpr<0x92, 0x93, "kmovq", VK64, GR64>, 1758 VEX, XD, VEX_W; 1759} 1760 1761// GR from/to mask register 1762let Predicates = [HasDQI] in { 1763 def : Pat<(v8i1 (bitconvert (i8 GR8:$src))), 1764 (KMOVBkr (SUBREG_TO_REG (i32 0), GR8:$src, sub_8bit))>; 1765 def : Pat<(i8 (bitconvert (v8i1 VK8:$src))), 1766 (EXTRACT_SUBREG (KMOVBrk VK8:$src), sub_8bit)>; 1767} 1768let Predicates = [HasAVX512] in { 1769 def : Pat<(v16i1 (bitconvert (i16 GR16:$src))), 1770 (KMOVWkr (SUBREG_TO_REG (i32 0), GR16:$src, sub_16bit))>; 1771 def : Pat<(i16 (bitconvert (v16i1 VK16:$src))), 1772 (EXTRACT_SUBREG (KMOVWrk VK16:$src), sub_16bit)>; 1773} 1774let Predicates = [HasBWI] in { 1775 def : Pat<(v32i1 (bitconvert (i32 GR32:$src))), (KMOVDkr GR32:$src)>; 1776 def : Pat<(i32 (bitconvert (v32i1 VK32:$src))), (KMOVDrk VK32:$src)>; 1777} 1778let Predicates = [HasBWI] in { 1779 def : Pat<(v64i1 (bitconvert (i64 GR64:$src))), (KMOVQkr GR64:$src)>; 1780 def : Pat<(i64 (bitconvert (v64i1 VK64:$src))), (KMOVQrk VK64:$src)>; 1781} 1782 1783// Load/store kreg 1784let Predicates = [HasDQI] in { 1785 def : Pat<(store (i8 (bitconvert (v8i1 VK8:$src))), addr:$dst), 1786 (KMOVBmk addr:$dst, VK8:$src)>; 1787 def : Pat<(v8i1 (bitconvert (i8 (load addr:$src)))), 1788 (KMOVBkm addr:$src)>; 1789} 1790let Predicates = [HasAVX512, NoDQI] in { 1791 def : Pat<(store (i8 (bitconvert (v8i1 VK8:$src))), addr:$dst), 1792 (KMOVWmk addr:$dst, (COPY_TO_REGCLASS VK8:$src, VK16))>; 1793 def : Pat<(v8i1 (bitconvert (i8 (load addr:$src)))), 1794 (COPY_TO_REGCLASS (KMOVWkm addr:$src), VK8)>; 1795} 1796let Predicates = [HasAVX512] in { 1797 def : Pat<(store (i16 (bitconvert (v16i1 VK16:$src))), addr:$dst), 1798 (KMOVWmk addr:$dst, VK16:$src)>; 1799 def : Pat<(i1 (load addr:$src)), 1800 (COPY_TO_REGCLASS (AND16ri (i16 (SUBREG_TO_REG (i32 0), 1801 (MOV8rm addr:$src), sub_8bit)), 1802 (i16 1)), VK1)>; 1803 def : Pat<(v16i1 (bitconvert (i16 (load addr:$src)))), 1804 (KMOVWkm addr:$src)>; 1805} 1806let Predicates = [HasBWI] in { 1807 def : Pat<(store (i32 (bitconvert (v32i1 VK32:$src))), addr:$dst), 1808 (KMOVDmk addr:$dst, VK32:$src)>; 1809 def : Pat<(v32i1 (bitconvert (i32 (load addr:$src)))), 1810 (KMOVDkm addr:$src)>; 1811} 1812let Predicates = [HasBWI] in { 1813 def : Pat<(store (i64 (bitconvert (v64i1 VK64:$src))), addr:$dst), 1814 (KMOVQmk addr:$dst, VK64:$src)>; 1815 def : Pat<(v64i1 (bitconvert (i64 (load addr:$src)))), 1816 (KMOVQkm addr:$src)>; 1817} 1818 1819let Predicates = [HasAVX512] in { 1820 def : Pat<(i1 (trunc (i64 GR64:$src))), 1821 (COPY_TO_REGCLASS (KMOVWkr (AND32ri (EXTRACT_SUBREG $src, sub_32bit), 1822 (i32 1))), VK1)>; 1823 1824 def : Pat<(i1 (trunc (i32 GR32:$src))), 1825 (COPY_TO_REGCLASS (KMOVWkr (AND32ri $src, (i32 1))), VK1)>; 1826 1827 def : Pat<(i1 (trunc (i8 GR8:$src))), 1828 (COPY_TO_REGCLASS 1829 (KMOVWkr (AND32ri (SUBREG_TO_REG (i32 0), GR8:$src, sub_8bit), (i32 1))), 1830 VK1)>; 1831 def : Pat<(i1 (trunc (i16 GR16:$src))), 1832 (COPY_TO_REGCLASS 1833 (KMOVWkr (AND32ri (SUBREG_TO_REG (i32 0), $src, sub_16bit), (i32 1))), 1834 VK1)>; 1835 1836 def : Pat<(i32 (zext VK1:$src)), 1837 (AND32ri (KMOVWrk (COPY_TO_REGCLASS VK1:$src, VK16)), (i32 1))>; 1838 def : Pat<(i32 (anyext VK1:$src)), 1839 (KMOVWrk (COPY_TO_REGCLASS VK1:$src, VK16))>; 1840 def : Pat<(i8 (zext VK1:$src)), 1841 (EXTRACT_SUBREG 1842 (AND32ri (KMOVWrk 1843 (COPY_TO_REGCLASS VK1:$src, VK16)), (i32 1)), sub_8bit)>; 1844 def : Pat<(i64 (zext VK1:$src)), 1845 (AND64ri8 (SUBREG_TO_REG (i64 0), 1846 (KMOVWrk (COPY_TO_REGCLASS VK1:$src, VK16)), sub_32bit), (i64 1))>; 1847 def : Pat<(i16 (zext VK1:$src)), 1848 (EXTRACT_SUBREG 1849 (AND32ri (KMOVWrk (COPY_TO_REGCLASS VK1:$src, VK16)), (i32 1)), 1850 sub_16bit)>; 1851 def : Pat<(v16i1 (scalar_to_vector VK1:$src)), 1852 (COPY_TO_REGCLASS VK1:$src, VK16)>; 1853 def : Pat<(v8i1 (scalar_to_vector VK1:$src)), 1854 (COPY_TO_REGCLASS VK1:$src, VK8)>; 1855} 1856let Predicates = [HasBWI] in { 1857 def : Pat<(v32i1 (scalar_to_vector VK1:$src)), 1858 (COPY_TO_REGCLASS VK1:$src, VK32)>; 1859 def : Pat<(v64i1 (scalar_to_vector VK1:$src)), 1860 (COPY_TO_REGCLASS VK1:$src, VK64)>; 1861} 1862 1863 1864// With AVX-512 only, 8-bit mask is promoted to 16-bit mask. 1865let Predicates = [HasAVX512, NoDQI] in { 1866 // GR from/to 8-bit mask without native support 1867 def : Pat<(v8i1 (bitconvert (i8 GR8:$src))), 1868 (COPY_TO_REGCLASS 1869 (KMOVWkr (MOVZX32rr8 GR8 :$src)), VK8)>; 1870 def : Pat<(i8 (bitconvert (v8i1 VK8:$src))), 1871 (EXTRACT_SUBREG 1872 (KMOVWrk (COPY_TO_REGCLASS VK8:$src, VK16)), 1873 sub_8bit)>; 1874} 1875 1876let Predicates = [HasAVX512] in { 1877 def : Pat<(i1 (X86Vextract VK16:$src, (iPTR 0))), 1878 (COPY_TO_REGCLASS VK16:$src, VK1)>; 1879 def : Pat<(i1 (X86Vextract VK8:$src, (iPTR 0))), 1880 (COPY_TO_REGCLASS VK8:$src, VK1)>; 1881} 1882let Predicates = [HasBWI] in { 1883 def : Pat<(i1 (X86Vextract VK32:$src, (iPTR 0))), 1884 (COPY_TO_REGCLASS VK32:$src, VK1)>; 1885 def : Pat<(i1 (X86Vextract VK64:$src, (iPTR 0))), 1886 (COPY_TO_REGCLASS VK64:$src, VK1)>; 1887} 1888 1889// Mask unary operation 1890// - KNOT 1891multiclass avx512_mask_unop<bits<8> opc, string OpcodeStr, 1892 RegisterClass KRC, SDPatternOperator OpNode, 1893 Predicate prd> { 1894 let Predicates = [prd] in 1895 def rr : I<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src), 1896 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 1897 [(set KRC:$dst, (OpNode KRC:$src))]>; 1898} 1899 1900multiclass avx512_mask_unop_all<bits<8> opc, string OpcodeStr, 1901 SDPatternOperator OpNode> { 1902 defm B : avx512_mask_unop<opc, !strconcat(OpcodeStr, "b"), VK8, OpNode, 1903 HasDQI>, VEX, PD; 1904 defm W : avx512_mask_unop<opc, !strconcat(OpcodeStr, "w"), VK16, OpNode, 1905 HasAVX512>, VEX, PS; 1906 defm D : avx512_mask_unop<opc, !strconcat(OpcodeStr, "d"), VK32, OpNode, 1907 HasBWI>, VEX, PD, VEX_W; 1908 defm Q : avx512_mask_unop<opc, !strconcat(OpcodeStr, "q"), VK64, OpNode, 1909 HasBWI>, VEX, PS, VEX_W; 1910} 1911 1912defm KNOT : avx512_mask_unop_all<0x44, "knot", not>; 1913 1914multiclass avx512_mask_unop_int<string IntName, string InstName> { 1915 let Predicates = [HasAVX512] in 1916 def : Pat<(!cast<Intrinsic>("int_x86_avx512_"##IntName##"_w") 1917 (i16 GR16:$src)), 1918 (COPY_TO_REGCLASS (!cast<Instruction>(InstName##"Wrr") 1919 (v16i1 (COPY_TO_REGCLASS GR16:$src, VK16))), GR16)>; 1920} 1921defm : avx512_mask_unop_int<"knot", "KNOT">; 1922 1923let Predicates = [HasDQI] in 1924def : Pat<(xor VK8:$src1, (v8i1 immAllOnesV)), (KNOTBrr VK8:$src1)>; 1925let Predicates = [HasAVX512] in 1926def : Pat<(xor VK16:$src1, (v16i1 immAllOnesV)), (KNOTWrr VK16:$src1)>; 1927let Predicates = [HasBWI] in 1928def : Pat<(xor VK32:$src1, (v32i1 immAllOnesV)), (KNOTDrr VK32:$src1)>; 1929let Predicates = [HasBWI] in 1930def : Pat<(xor VK64:$src1, (v64i1 immAllOnesV)), (KNOTQrr VK64:$src1)>; 1931 1932// KNL does not support KMOVB, 8-bit mask is promoted to 16-bit 1933let Predicates = [HasAVX512, NoDQI] in { 1934def : Pat<(xor VK8:$src1, (v8i1 immAllOnesV)), 1935 (COPY_TO_REGCLASS (KNOTWrr (COPY_TO_REGCLASS VK8:$src1, VK16)), VK8)>; 1936def : Pat<(not VK8:$src), 1937 (COPY_TO_REGCLASS 1938 (KNOTWrr (COPY_TO_REGCLASS VK8:$src, VK16)), VK8)>; 1939} 1940def : Pat<(xor VK4:$src1, (v4i1 immAllOnesV)), 1941 (COPY_TO_REGCLASS (KNOTWrr (COPY_TO_REGCLASS VK4:$src1, VK16)), VK4)>; 1942def : Pat<(xor VK2:$src1, (v2i1 immAllOnesV)), 1943 (COPY_TO_REGCLASS (KNOTWrr (COPY_TO_REGCLASS VK2:$src1, VK16)), VK2)>; 1944 1945// Mask binary operation 1946// - KAND, KANDN, KOR, KXNOR, KXOR 1947multiclass avx512_mask_binop<bits<8> opc, string OpcodeStr, 1948 RegisterClass KRC, SDPatternOperator OpNode, 1949 Predicate prd, bit IsCommutable> { 1950 let Predicates = [prd], isCommutable = IsCommutable in 1951 def rr : I<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src1, KRC:$src2), 1952 !strconcat(OpcodeStr, 1953 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 1954 [(set KRC:$dst, (OpNode KRC:$src1, KRC:$src2))]>; 1955} 1956 1957multiclass avx512_mask_binop_all<bits<8> opc, string OpcodeStr, 1958 SDPatternOperator OpNode, bit IsCommutable> { 1959 defm B : avx512_mask_binop<opc, !strconcat(OpcodeStr, "b"), VK8, OpNode, 1960 HasDQI, IsCommutable>, VEX_4V, VEX_L, PD; 1961 defm W : avx512_mask_binop<opc, !strconcat(OpcodeStr, "w"), VK16, OpNode, 1962 HasAVX512, IsCommutable>, VEX_4V, VEX_L, PS; 1963 defm D : avx512_mask_binop<opc, !strconcat(OpcodeStr, "d"), VK32, OpNode, 1964 HasBWI, IsCommutable>, VEX_4V, VEX_L, VEX_W, PD; 1965 defm Q : avx512_mask_binop<opc, !strconcat(OpcodeStr, "q"), VK64, OpNode, 1966 HasBWI, IsCommutable>, VEX_4V, VEX_L, VEX_W, PS; 1967} 1968 1969def andn : PatFrag<(ops node:$i0, node:$i1), (and (not node:$i0), node:$i1)>; 1970def xnor : PatFrag<(ops node:$i0, node:$i1), (not (xor node:$i0, node:$i1))>; 1971 1972defm KAND : avx512_mask_binop_all<0x41, "kand", and, 1>; 1973defm KOR : avx512_mask_binop_all<0x45, "kor", or, 1>; 1974defm KXNOR : avx512_mask_binop_all<0x46, "kxnor", xnor, 1>; 1975defm KXOR : avx512_mask_binop_all<0x47, "kxor", xor, 1>; 1976defm KANDN : avx512_mask_binop_all<0x42, "kandn", andn, 0>; 1977 1978multiclass avx512_mask_binop_int<string IntName, string InstName> { 1979 let Predicates = [HasAVX512] in 1980 def : Pat<(!cast<Intrinsic>("int_x86_avx512_"##IntName##"_w") 1981 (i16 GR16:$src1), (i16 GR16:$src2)), 1982 (COPY_TO_REGCLASS (!cast<Instruction>(InstName##"Wrr") 1983 (v16i1 (COPY_TO_REGCLASS GR16:$src1, VK16)), 1984 (v16i1 (COPY_TO_REGCLASS GR16:$src2, VK16))), GR16)>; 1985} 1986 1987defm : avx512_mask_binop_int<"kand", "KAND">; 1988defm : avx512_mask_binop_int<"kandn", "KANDN">; 1989defm : avx512_mask_binop_int<"kor", "KOR">; 1990defm : avx512_mask_binop_int<"kxnor", "KXNOR">; 1991defm : avx512_mask_binop_int<"kxor", "KXOR">; 1992 1993multiclass avx512_binop_pat<SDPatternOperator OpNode, Instruction Inst> { 1994 // With AVX512F, 8-bit mask is promoted to 16-bit mask, 1995 // for the DQI set, this type is legal and KxxxB instruction is used 1996 let Predicates = [NoDQI] in 1997 def : Pat<(OpNode VK8:$src1, VK8:$src2), 1998 (COPY_TO_REGCLASS 1999 (Inst (COPY_TO_REGCLASS VK8:$src1, VK16), 2000 (COPY_TO_REGCLASS VK8:$src2, VK16)), VK8)>; 2001 2002 // All types smaller than 8 bits require conversion anyway 2003 def : Pat<(OpNode VK1:$src1, VK1:$src2), 2004 (COPY_TO_REGCLASS (Inst 2005 (COPY_TO_REGCLASS VK1:$src1, VK16), 2006 (COPY_TO_REGCLASS VK1:$src2, VK16)), VK1)>; 2007 def : Pat<(OpNode VK2:$src1, VK2:$src2), 2008 (COPY_TO_REGCLASS (Inst 2009 (COPY_TO_REGCLASS VK2:$src1, VK16), 2010 (COPY_TO_REGCLASS VK2:$src2, VK16)), VK1)>; 2011 def : Pat<(OpNode VK4:$src1, VK4:$src2), 2012 (COPY_TO_REGCLASS (Inst 2013 (COPY_TO_REGCLASS VK4:$src1, VK16), 2014 (COPY_TO_REGCLASS VK4:$src2, VK16)), VK1)>; 2015} 2016 2017defm : avx512_binop_pat<and, KANDWrr>; 2018defm : avx512_binop_pat<andn, KANDNWrr>; 2019defm : avx512_binop_pat<or, KORWrr>; 2020defm : avx512_binop_pat<xnor, KXNORWrr>; 2021defm : avx512_binop_pat<xor, KXORWrr>; 2022 2023def : Pat<(xor (xor VK16:$src1, VK16:$src2), (v16i1 immAllOnesV)), 2024 (KXNORWrr VK16:$src1, VK16:$src2)>; 2025def : Pat<(xor (xor VK8:$src1, VK8:$src2), (v8i1 immAllOnesV)), 2026 (KXNORBrr VK8:$src1, VK8:$src2)>, Requires<[HasDQI]>; 2027def : Pat<(xor (xor VK32:$src1, VK32:$src2), (v32i1 immAllOnesV)), 2028 (KXNORDrr VK32:$src1, VK32:$src2)>, Requires<[HasBWI]>; 2029def : Pat<(xor (xor VK64:$src1, VK64:$src2), (v64i1 immAllOnesV)), 2030 (KXNORQrr VK64:$src1, VK64:$src2)>, Requires<[HasBWI]>; 2031 2032let Predicates = [NoDQI] in 2033def : Pat<(xor (xor VK8:$src1, VK8:$src2), (v8i1 immAllOnesV)), 2034 (COPY_TO_REGCLASS (KXNORWrr (COPY_TO_REGCLASS VK8:$src1, VK16), 2035 (COPY_TO_REGCLASS VK8:$src2, VK16)), VK8)>; 2036 2037def : Pat<(xor (xor VK4:$src1, VK4:$src2), (v4i1 immAllOnesV)), 2038 (COPY_TO_REGCLASS (KXNORWrr (COPY_TO_REGCLASS VK4:$src1, VK16), 2039 (COPY_TO_REGCLASS VK4:$src2, VK16)), VK4)>; 2040 2041def : Pat<(xor (xor VK2:$src1, VK2:$src2), (v2i1 immAllOnesV)), 2042 (COPY_TO_REGCLASS (KXNORWrr (COPY_TO_REGCLASS VK2:$src1, VK16), 2043 (COPY_TO_REGCLASS VK2:$src2, VK16)), VK2)>; 2044 2045def : Pat<(xor (xor VK1:$src1, VK1:$src2), (i1 1)), 2046 (COPY_TO_REGCLASS (KXNORWrr (COPY_TO_REGCLASS VK1:$src1, VK16), 2047 (COPY_TO_REGCLASS VK1:$src2, VK16)), VK1)>; 2048 2049// Mask unpacking 2050multiclass avx512_mask_unpck<bits<8> opc, string OpcodeStr, 2051 RegisterClass KRC> { 2052 let Predicates = [HasAVX512] in 2053 def rr : I<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src1, KRC:$src2), 2054 !strconcat(OpcodeStr, 2055 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>; 2056} 2057 2058multiclass avx512_mask_unpck_bw<bits<8> opc, string OpcodeStr> { 2059 defm BW : avx512_mask_unpck<opc, !strconcat(OpcodeStr, "bw"), VK16>, 2060 VEX_4V, VEX_L, PD; 2061} 2062 2063defm KUNPCK : avx512_mask_unpck_bw<0x4b, "kunpck">; 2064def : Pat<(v16i1 (concat_vectors (v8i1 VK8:$src1), (v8i1 VK8:$src2))), 2065 (KUNPCKBWrr (COPY_TO_REGCLASS VK8:$src2, VK16), 2066 (COPY_TO_REGCLASS VK8:$src1, VK16))>; 2067 2068 2069multiclass avx512_mask_unpck_int<string IntName, string InstName> { 2070 let Predicates = [HasAVX512] in 2071 def : Pat<(!cast<Intrinsic>("int_x86_avx512_"##IntName##"_bw") 2072 (i16 GR16:$src1), (i16 GR16:$src2)), 2073 (COPY_TO_REGCLASS (!cast<Instruction>(InstName##"BWrr") 2074 (v16i1 (COPY_TO_REGCLASS GR16:$src1, VK16)), 2075 (v16i1 (COPY_TO_REGCLASS GR16:$src2, VK16))), GR16)>; 2076} 2077defm : avx512_mask_unpck_int<"kunpck", "KUNPCK">; 2078 2079// Mask bit testing 2080multiclass avx512_mask_testop<bits<8> opc, string OpcodeStr, RegisterClass KRC, 2081 SDNode OpNode> { 2082 let Predicates = [HasAVX512], Defs = [EFLAGS] in 2083 def rr : I<opc, MRMSrcReg, (outs), (ins KRC:$src1, KRC:$src2), 2084 !strconcat(OpcodeStr, "\t{$src2, $src1|$src1, $src2}"), 2085 [(set EFLAGS, (OpNode KRC:$src1, KRC:$src2))]>; 2086} 2087 2088multiclass avx512_mask_testop_w<bits<8> opc, string OpcodeStr, SDNode OpNode> { 2089 defm W : avx512_mask_testop<opc, !strconcat(OpcodeStr, "w"), VK16, OpNode>, 2090 VEX, PS; 2091 let Predicates = [HasDQI] in 2092 defm B : avx512_mask_testop<opc, !strconcat(OpcodeStr, "b"), VK8, OpNode>, 2093 VEX, PD; 2094 let Predicates = [HasBWI] in { 2095 defm Q : avx512_mask_testop<opc, !strconcat(OpcodeStr, "q"), VK64, OpNode>, 2096 VEX, PS, VEX_W; 2097 defm D : avx512_mask_testop<opc, !strconcat(OpcodeStr, "d"), VK32, OpNode>, 2098 VEX, PD, VEX_W; 2099 } 2100} 2101 2102defm KORTEST : avx512_mask_testop_w<0x98, "kortest", X86kortest>; 2103 2104// Mask shift 2105multiclass avx512_mask_shiftop<bits<8> opc, string OpcodeStr, RegisterClass KRC, 2106 SDNode OpNode> { 2107 let Predicates = [HasAVX512] in 2108 def ri : Ii8<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src, u8imm:$imm), 2109 !strconcat(OpcodeStr, 2110 "\t{$imm, $src, $dst|$dst, $src, $imm}"), 2111 [(set KRC:$dst, (OpNode KRC:$src, (i8 imm:$imm)))]>; 2112} 2113 2114multiclass avx512_mask_shiftop_w<bits<8> opc1, bits<8> opc2, string OpcodeStr, 2115 SDNode OpNode> { 2116 defm W : avx512_mask_shiftop<opc1, !strconcat(OpcodeStr, "w"), VK16, OpNode>, 2117 VEX, TAPD, VEX_W; 2118 let Predicates = [HasDQI] in 2119 defm B : avx512_mask_shiftop<opc1, !strconcat(OpcodeStr, "b"), VK8, OpNode>, 2120 VEX, TAPD; 2121 let Predicates = [HasBWI] in { 2122 defm Q : avx512_mask_shiftop<opc2, !strconcat(OpcodeStr, "q"), VK64, OpNode>, 2123 VEX, TAPD, VEX_W; 2124 let Predicates = [HasDQI] in 2125 defm D : avx512_mask_shiftop<opc2, !strconcat(OpcodeStr, "d"), VK32, OpNode>, 2126 VEX, TAPD; 2127 } 2128} 2129 2130defm KSHIFTL : avx512_mask_shiftop_w<0x32, 0x33, "kshiftl", X86vshli>; 2131defm KSHIFTR : avx512_mask_shiftop_w<0x30, 0x31, "kshiftr", X86vsrli>; 2132 2133// Mask setting all 0s or 1s 2134multiclass avx512_mask_setop<RegisterClass KRC, ValueType VT, PatFrag Val> { 2135 let Predicates = [HasAVX512] in 2136 let isReMaterializable = 1, isAsCheapAsAMove = 1, isPseudo = 1 in 2137 def #NAME# : I<0, Pseudo, (outs KRC:$dst), (ins), "", 2138 [(set KRC:$dst, (VT Val))]>; 2139} 2140 2141multiclass avx512_mask_setop_w<PatFrag Val> { 2142 defm B : avx512_mask_setop<VK8, v8i1, Val>; 2143 defm W : avx512_mask_setop<VK16, v16i1, Val>; 2144 defm D : avx512_mask_setop<VK32, v32i1, Val>; 2145 defm Q : avx512_mask_setop<VK64, v64i1, Val>; 2146} 2147 2148defm KSET0 : avx512_mask_setop_w<immAllZerosV>; 2149defm KSET1 : avx512_mask_setop_w<immAllOnesV>; 2150 2151// With AVX-512 only, 8-bit mask is promoted to 16-bit mask. 2152let Predicates = [HasAVX512] in { 2153 def : Pat<(v8i1 immAllZerosV), (COPY_TO_REGCLASS (KSET0W), VK8)>; 2154 def : Pat<(v8i1 immAllOnesV), (COPY_TO_REGCLASS (KSET1W), VK8)>; 2155 def : Pat<(v4i1 immAllOnesV), (COPY_TO_REGCLASS (KSET1W), VK4)>; 2156 def : Pat<(v2i1 immAllOnesV), (COPY_TO_REGCLASS (KSET1W), VK2)>; 2157 def : Pat<(i1 0), (COPY_TO_REGCLASS (KSET0W), VK1)>; 2158 def : Pat<(i1 1), (COPY_TO_REGCLASS (KSHIFTRWri (KSET1W), (i8 15)), VK1)>; 2159 def : Pat<(i1 -1), (COPY_TO_REGCLASS (KSHIFTRWri (KSET1W), (i8 15)), VK1)>; 2160} 2161def : Pat<(v8i1 (extract_subvector (v16i1 VK16:$src), (iPTR 0))), 2162 (v8i1 (COPY_TO_REGCLASS VK16:$src, VK8))>; 2163 2164def : Pat<(v16i1 (insert_subvector undef, (v8i1 VK8:$src), (iPTR 0))), 2165 (v16i1 (COPY_TO_REGCLASS VK8:$src, VK16))>; 2166 2167def : Pat<(v8i1 (extract_subvector (v16i1 VK16:$src), (iPTR 8))), 2168 (v8i1 (COPY_TO_REGCLASS (KSHIFTRWri VK16:$src, (i8 8)), VK8))>; 2169 2170def : Pat<(v32i1 (extract_subvector (v64i1 VK64:$src), (iPTR 0))), 2171 (v32i1 (COPY_TO_REGCLASS VK64:$src, VK32))>; 2172 2173def : Pat<(v32i1 (extract_subvector (v64i1 VK64:$src), (iPTR 32))), 2174 (v32i1 (COPY_TO_REGCLASS (KSHIFTRQri VK64:$src, (i8 32)), VK32))>; 2175 2176let Predicates = [HasVLX] in { 2177 def : Pat<(v8i1 (insert_subvector undef, (v4i1 VK4:$src), (iPTR 0))), 2178 (v8i1 (COPY_TO_REGCLASS VK4:$src, VK8))>; 2179 def : Pat<(v8i1 (insert_subvector undef, (v2i1 VK2:$src), (iPTR 0))), 2180 (v8i1 (COPY_TO_REGCLASS VK2:$src, VK8))>; 2181 def : Pat<(v4i1 (insert_subvector undef, (v2i1 VK2:$src), (iPTR 0))), 2182 (v4i1 (COPY_TO_REGCLASS VK2:$src, VK4))>; 2183 def : Pat<(v4i1 (extract_subvector (v8i1 VK8:$src), (iPTR 0))), 2184 (v4i1 (COPY_TO_REGCLASS VK8:$src, VK4))>; 2185 def : Pat<(v2i1 (extract_subvector (v8i1 VK8:$src), (iPTR 0))), 2186 (v2i1 (COPY_TO_REGCLASS VK8:$src, VK2))>; 2187} 2188 2189def : Pat<(v8i1 (X86vshli VK8:$src, (i8 imm:$imm))), 2190 (v8i1 (COPY_TO_REGCLASS 2191 (KSHIFTLWri (COPY_TO_REGCLASS VK8:$src, VK16), 2192 (I8Imm $imm)), VK8))>, Requires<[HasAVX512, NoDQI]>; 2193 2194def : Pat<(v8i1 (X86vsrli VK8:$src, (i8 imm:$imm))), 2195 (v8i1 (COPY_TO_REGCLASS 2196 (KSHIFTRWri (COPY_TO_REGCLASS VK8:$src, VK16), 2197 (I8Imm $imm)), VK8))>, Requires<[HasAVX512, NoDQI]>; 2198 2199def : Pat<(v4i1 (X86vshli VK4:$src, (i8 imm:$imm))), 2200 (v4i1 (COPY_TO_REGCLASS 2201 (KSHIFTLWri (COPY_TO_REGCLASS VK4:$src, VK16), 2202 (I8Imm $imm)), VK4))>, Requires<[HasAVX512]>; 2203 2204def : Pat<(v4i1 (X86vsrli VK4:$src, (i8 imm:$imm))), 2205 (v4i1 (COPY_TO_REGCLASS 2206 (KSHIFTRWri (COPY_TO_REGCLASS VK4:$src, VK16), 2207 (I8Imm $imm)), VK4))>, Requires<[HasAVX512]>; 2208 2209//===----------------------------------------------------------------------===// 2210// AVX-512 - Aligned and unaligned load and store 2211// 2212 2213 2214multiclass avx512_load<bits<8> opc, string OpcodeStr, X86VectorVTInfo _, 2215 PatFrag ld_frag, PatFrag mload, 2216 bit IsReMaterializable = 1> { 2217 let hasSideEffects = 0 in { 2218 def rr : AVX512PI<opc, MRMSrcReg, (outs _.RC:$dst), (ins _.RC:$src), 2219 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), [], 2220 _.ExeDomain>, EVEX; 2221 def rrkz : AVX512PI<opc, MRMSrcReg, (outs _.RC:$dst), 2222 (ins _.KRCWM:$mask, _.RC:$src), 2223 !strconcat(OpcodeStr, "\t{$src, ${dst} {${mask}} {z}|", 2224 "${dst} {${mask}} {z}, $src}"), [], _.ExeDomain>, 2225 EVEX, EVEX_KZ; 2226 2227 let canFoldAsLoad = 1, isReMaterializable = IsReMaterializable, 2228 SchedRW = [WriteLoad] in 2229 def rm : AVX512PI<opc, MRMSrcMem, (outs _.RC:$dst), (ins _.MemOp:$src), 2230 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 2231 [(set _.RC:$dst, (_.VT (bitconvert (ld_frag addr:$src))))], 2232 _.ExeDomain>, EVEX; 2233 2234 let Constraints = "$src0 = $dst" in { 2235 def rrk : AVX512PI<opc, MRMSrcReg, (outs _.RC:$dst), 2236 (ins _.RC:$src0, _.KRCWM:$mask, _.RC:$src1), 2237 !strconcat(OpcodeStr, "\t{$src1, ${dst} {${mask}}|", 2238 "${dst} {${mask}}, $src1}"), 2239 [(set _.RC:$dst, (_.VT (vselect _.KRCWM:$mask, 2240 (_.VT _.RC:$src1), 2241 (_.VT _.RC:$src0))))], _.ExeDomain>, 2242 EVEX, EVEX_K; 2243 let mayLoad = 1, SchedRW = [WriteLoad] in 2244 def rmk : AVX512PI<opc, MRMSrcMem, (outs _.RC:$dst), 2245 (ins _.RC:$src0, _.KRCWM:$mask, _.MemOp:$src1), 2246 !strconcat(OpcodeStr, "\t{$src1, ${dst} {${mask}}|", 2247 "${dst} {${mask}}, $src1}"), 2248 [(set _.RC:$dst, (_.VT 2249 (vselect _.KRCWM:$mask, 2250 (_.VT (bitconvert (ld_frag addr:$src1))), 2251 (_.VT _.RC:$src0))))], _.ExeDomain>, EVEX, EVEX_K; 2252 } 2253 let mayLoad = 1, SchedRW = [WriteLoad] in 2254 def rmkz : AVX512PI<opc, MRMSrcMem, (outs _.RC:$dst), 2255 (ins _.KRCWM:$mask, _.MemOp:$src), 2256 OpcodeStr #"\t{$src, ${dst} {${mask}} {z}|"# 2257 "${dst} {${mask}} {z}, $src}", 2258 [(set _.RC:$dst, (_.VT (vselect _.KRCWM:$mask, 2259 (_.VT (bitconvert (ld_frag addr:$src))), _.ImmAllZerosV)))], 2260 _.ExeDomain>, EVEX, EVEX_KZ; 2261 } 2262 def : Pat<(_.VT (mload addr:$ptr, _.KRCWM:$mask, undef)), 2263 (!cast<Instruction>(NAME#_.ZSuffix##rmkz) _.KRCWM:$mask, addr:$ptr)>; 2264 2265 def : Pat<(_.VT (mload addr:$ptr, _.KRCWM:$mask, _.ImmAllZerosV)), 2266 (!cast<Instruction>(NAME#_.ZSuffix##rmkz) _.KRCWM:$mask, addr:$ptr)>; 2267 2268 def : Pat<(_.VT (mload addr:$ptr, _.KRCWM:$mask, (_.VT _.RC:$src0))), 2269 (!cast<Instruction>(NAME#_.ZSuffix##rmk) _.RC:$src0, 2270 _.KRCWM:$mask, addr:$ptr)>; 2271} 2272 2273multiclass avx512_alignedload_vl<bits<8> opc, string OpcodeStr, 2274 AVX512VLVectorVTInfo _, 2275 Predicate prd, 2276 bit IsReMaterializable = 1> { 2277 let Predicates = [prd] in 2278 defm Z : avx512_load<opc, OpcodeStr, _.info512, _.info512.AlignedLdFrag, 2279 masked_load_aligned512, IsReMaterializable>, EVEX_V512; 2280 2281 let Predicates = [prd, HasVLX] in { 2282 defm Z256 : avx512_load<opc, OpcodeStr, _.info256, _.info256.AlignedLdFrag, 2283 masked_load_aligned256, IsReMaterializable>, EVEX_V256; 2284 defm Z128 : avx512_load<opc, OpcodeStr, _.info128, _.info128.AlignedLdFrag, 2285 masked_load_aligned128, IsReMaterializable>, EVEX_V128; 2286 } 2287} 2288 2289multiclass avx512_load_vl<bits<8> opc, string OpcodeStr, 2290 AVX512VLVectorVTInfo _, 2291 Predicate prd, 2292 bit IsReMaterializable = 1> { 2293 let Predicates = [prd] in 2294 defm Z : avx512_load<opc, OpcodeStr, _.info512, _.info512.LdFrag, 2295 masked_load_unaligned, IsReMaterializable>, EVEX_V512; 2296 2297 let Predicates = [prd, HasVLX] in { 2298 defm Z256 : avx512_load<opc, OpcodeStr, _.info256, _.info256.LdFrag, 2299 masked_load_unaligned, IsReMaterializable>, EVEX_V256; 2300 defm Z128 : avx512_load<opc, OpcodeStr, _.info128, _.info128.LdFrag, 2301 masked_load_unaligned, IsReMaterializable>, EVEX_V128; 2302 } 2303} 2304 2305multiclass avx512_store<bits<8> opc, string OpcodeStr, X86VectorVTInfo _, 2306 PatFrag st_frag, PatFrag mstore> { 2307 let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0 in { 2308 def rr_alt : AVX512PI<opc, MRMDestReg, (outs _.RC:$dst), (ins _.RC:$src), 2309 OpcodeStr # "\t{$src, $dst|$dst, $src}", [], 2310 _.ExeDomain>, EVEX; 2311 let Constraints = "$src1 = $dst" in 2312 def rrk_alt : AVX512PI<opc, MRMDestReg, (outs _.RC:$dst), 2313 (ins _.RC:$src1, _.KRCWM:$mask, _.RC:$src2), 2314 OpcodeStr # 2315 "\t{$src2, ${dst} {${mask}}|${dst} {${mask}}, $src2}", 2316 [], _.ExeDomain>, EVEX, EVEX_K; 2317 def rrkz_alt : AVX512PI<opc, MRMDestReg, (outs _.RC:$dst), 2318 (ins _.KRCWM:$mask, _.RC:$src), 2319 OpcodeStr # 2320 "\t{$src, ${dst} {${mask}} {z}|" # 2321 "${dst} {${mask}} {z}, $src}", 2322 [], _.ExeDomain>, EVEX, EVEX_KZ; 2323 } 2324 let mayStore = 1 in { 2325 def mr : AVX512PI<opc, MRMDestMem, (outs), (ins _.MemOp:$dst, _.RC:$src), 2326 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 2327 [(st_frag (_.VT _.RC:$src), addr:$dst)], _.ExeDomain>, EVEX; 2328 def mrk : AVX512PI<opc, MRMDestMem, (outs), 2329 (ins _.MemOp:$dst, _.KRCWM:$mask, _.RC:$src), 2330 OpcodeStr # "\t{$src, ${dst} {${mask}}|${dst} {${mask}}, $src}", 2331 [], _.ExeDomain>, EVEX, EVEX_K; 2332 } 2333 2334 def: Pat<(mstore addr:$ptr, _.KRCWM:$mask, (_.VT _.RC:$src)), 2335 (!cast<Instruction>(NAME#_.ZSuffix##mrk) addr:$ptr, 2336 _.KRCWM:$mask, _.RC:$src)>; 2337} 2338 2339 2340multiclass avx512_store_vl< bits<8> opc, string OpcodeStr, 2341 AVX512VLVectorVTInfo _, Predicate prd> { 2342 let Predicates = [prd] in 2343 defm Z : avx512_store<opc, OpcodeStr, _.info512, store, 2344 masked_store_unaligned>, EVEX_V512; 2345 2346 let Predicates = [prd, HasVLX] in { 2347 defm Z256 : avx512_store<opc, OpcodeStr, _.info256, store, 2348 masked_store_unaligned>, EVEX_V256; 2349 defm Z128 : avx512_store<opc, OpcodeStr, _.info128, store, 2350 masked_store_unaligned>, EVEX_V128; 2351 } 2352} 2353 2354multiclass avx512_alignedstore_vl<bits<8> opc, string OpcodeStr, 2355 AVX512VLVectorVTInfo _, Predicate prd> { 2356 let Predicates = [prd] in 2357 defm Z : avx512_store<opc, OpcodeStr, _.info512, alignedstore512, 2358 masked_store_aligned512>, EVEX_V512; 2359 2360 let Predicates = [prd, HasVLX] in { 2361 defm Z256 : avx512_store<opc, OpcodeStr, _.info256, alignedstore256, 2362 masked_store_aligned256>, EVEX_V256; 2363 defm Z128 : avx512_store<opc, OpcodeStr, _.info128, alignedstore, 2364 masked_store_aligned128>, EVEX_V128; 2365 } 2366} 2367 2368defm VMOVAPS : avx512_alignedload_vl<0x28, "vmovaps", avx512vl_f32_info, 2369 HasAVX512>, 2370 avx512_alignedstore_vl<0x29, "vmovaps", avx512vl_f32_info, 2371 HasAVX512>, PS, EVEX_CD8<32, CD8VF>; 2372 2373defm VMOVAPD : avx512_alignedload_vl<0x28, "vmovapd", avx512vl_f64_info, 2374 HasAVX512>, 2375 avx512_alignedstore_vl<0x29, "vmovapd", avx512vl_f64_info, 2376 HasAVX512>, PD, VEX_W, EVEX_CD8<64, CD8VF>; 2377 2378defm VMOVUPS : avx512_load_vl<0x10, "vmovups", avx512vl_f32_info, HasAVX512>, 2379 avx512_store_vl<0x11, "vmovups", avx512vl_f32_info, HasAVX512>, 2380 PS, EVEX_CD8<32, CD8VF>; 2381 2382defm VMOVUPD : avx512_load_vl<0x10, "vmovupd", avx512vl_f64_info, HasAVX512, 0>, 2383 avx512_store_vl<0x11, "vmovupd", avx512vl_f64_info, HasAVX512>, 2384 PD, VEX_W, EVEX_CD8<64, CD8VF>; 2385 2386def: Pat<(v8f64 (int_x86_avx512_mask_loadu_pd_512 addr:$ptr, 2387 (bc_v8f64 (v16i32 immAllZerosV)), GR8:$mask)), 2388 (VMOVUPDZrmkz (v8i1 (COPY_TO_REGCLASS GR8:$mask, VK8WM)), addr:$ptr)>; 2389 2390def: Pat<(v16f32 (int_x86_avx512_mask_loadu_ps_512 addr:$ptr, 2391 (bc_v16f32 (v16i32 immAllZerosV)), GR16:$mask)), 2392 (VMOVUPSZrmkz (v16i1 (COPY_TO_REGCLASS GR16:$mask, VK16WM)), addr:$ptr)>; 2393 2394def: Pat<(v8f64 (int_x86_avx512_mask_load_pd_512 addr:$ptr, 2395 (bc_v8f64 (v16i32 immAllZerosV)), GR8:$mask)), 2396 (VMOVAPDZrmkz (v8i1 (COPY_TO_REGCLASS GR8:$mask, VK8WM)), addr:$ptr)>; 2397 2398def: Pat<(v16f32 (int_x86_avx512_mask_load_ps_512 addr:$ptr, 2399 (bc_v16f32 (v16i32 immAllZerosV)), GR16:$mask)), 2400 (VMOVAPSZrmkz (v16i1 (COPY_TO_REGCLASS GR16:$mask, VK16WM)), addr:$ptr)>; 2401 2402def: Pat<(v8f64 (int_x86_avx512_mask_load_pd_512 addr:$ptr, 2403 (bc_v8f64 (v16i32 immAllZerosV)), (i8 -1))), 2404 (VMOVAPDZrm addr:$ptr)>; 2405 2406def: Pat<(v16f32 (int_x86_avx512_mask_load_ps_512 addr:$ptr, 2407 (bc_v16f32 (v16i32 immAllZerosV)), (i16 -1))), 2408 (VMOVAPSZrm addr:$ptr)>; 2409 2410def: Pat<(int_x86_avx512_mask_storeu_ps_512 addr:$ptr, (v16f32 VR512:$src), 2411 GR16:$mask), 2412 (VMOVUPSZmrk addr:$ptr, (v16i1 (COPY_TO_REGCLASS GR16:$mask, VK16WM)), 2413 VR512:$src)>; 2414def: Pat<(int_x86_avx512_mask_storeu_pd_512 addr:$ptr, (v8f64 VR512:$src), 2415 GR8:$mask), 2416 (VMOVUPDZmrk addr:$ptr, (v8i1 (COPY_TO_REGCLASS GR8:$mask, VK8WM)), 2417 VR512:$src)>; 2418 2419def: Pat<(int_x86_avx512_mask_store_ps_512 addr:$ptr, (v16f32 VR512:$src), 2420 GR16:$mask), 2421 (VMOVAPSZmrk addr:$ptr, (v16i1 (COPY_TO_REGCLASS GR16:$mask, VK16WM)), 2422 VR512:$src)>; 2423def: Pat<(int_x86_avx512_mask_store_pd_512 addr:$ptr, (v8f64 VR512:$src), 2424 GR8:$mask), 2425 (VMOVAPDZmrk addr:$ptr, (v8i1 (COPY_TO_REGCLASS GR8:$mask, VK8WM)), 2426 VR512:$src)>; 2427 2428let Predicates = [HasAVX512, NoVLX] in { 2429def: Pat<(masked_store addr:$ptr, VK8WM:$mask, (v8f32 VR256:$src)), 2430 (VMOVUPSZmrk addr:$ptr, 2431 (v16i1 (COPY_TO_REGCLASS VK8WM:$mask, VK16WM)), 2432 (INSERT_SUBREG (v16f32 (IMPLICIT_DEF)), VR256:$src, sub_ymm))>; 2433 2434def: Pat<(v8f32 (masked_load addr:$ptr, VK8WM:$mask, undef)), 2435 (v8f32 (EXTRACT_SUBREG (v16f32 (VMOVUPSZrmkz 2436 (v16i1 (COPY_TO_REGCLASS VK8WM:$mask, VK16WM)), addr:$ptr)), sub_ymm))>; 2437 2438def: Pat<(v8f32 (masked_load addr:$ptr, VK8WM:$mask, (v8f32 VR256:$src0))), 2439 (v8f32 (EXTRACT_SUBREG (v16f32 (VMOVUPSZrmk 2440 (INSERT_SUBREG (v16f32 (IMPLICIT_DEF)), VR256:$src0, sub_ymm), 2441 (v16i1 (COPY_TO_REGCLASS VK8WM:$mask, VK16WM)), addr:$ptr)), sub_ymm))>; 2442} 2443 2444defm VMOVDQA32 : avx512_alignedload_vl<0x6F, "vmovdqa32", avx512vl_i32_info, 2445 HasAVX512>, 2446 avx512_alignedstore_vl<0x7F, "vmovdqa32", avx512vl_i32_info, 2447 HasAVX512>, PD, EVEX_CD8<32, CD8VF>; 2448 2449defm VMOVDQA64 : avx512_alignedload_vl<0x6F, "vmovdqa64", avx512vl_i64_info, 2450 HasAVX512>, 2451 avx512_alignedstore_vl<0x7F, "vmovdqa64", avx512vl_i64_info, 2452 HasAVX512>, PD, VEX_W, EVEX_CD8<64, CD8VF>; 2453 2454defm VMOVDQU8 : avx512_load_vl<0x6F, "vmovdqu8", avx512vl_i8_info, HasBWI>, 2455 avx512_store_vl<0x7F, "vmovdqu8", avx512vl_i8_info, 2456 HasBWI>, XD, EVEX_CD8<8, CD8VF>; 2457 2458defm VMOVDQU16 : avx512_load_vl<0x6F, "vmovdqu16", avx512vl_i16_info, HasBWI>, 2459 avx512_store_vl<0x7F, "vmovdqu16", avx512vl_i16_info, 2460 HasBWI>, XD, VEX_W, EVEX_CD8<16, CD8VF>; 2461 2462defm VMOVDQU32 : avx512_load_vl<0x6F, "vmovdqu32", avx512vl_i32_info, HasAVX512>, 2463 avx512_store_vl<0x7F, "vmovdqu32", avx512vl_i32_info, 2464 HasAVX512>, XS, EVEX_CD8<32, CD8VF>; 2465 2466defm VMOVDQU64 : avx512_load_vl<0x6F, "vmovdqu64", avx512vl_i64_info, HasAVX512>, 2467 avx512_store_vl<0x7F, "vmovdqu64", avx512vl_i64_info, 2468 HasAVX512>, XS, VEX_W, EVEX_CD8<64, CD8VF>; 2469 2470def: Pat<(v16i32 (int_x86_avx512_mask_loadu_d_512 addr:$ptr, 2471 (v16i32 immAllZerosV), GR16:$mask)), 2472 (VMOVDQU32Zrmkz (v16i1 (COPY_TO_REGCLASS GR16:$mask, VK16WM)), addr:$ptr)>; 2473 2474def: Pat<(v8i64 (int_x86_avx512_mask_loadu_q_512 addr:$ptr, 2475 (bc_v8i64 (v16i32 immAllZerosV)), GR8:$mask)), 2476 (VMOVDQU64Zrmkz (v8i1 (COPY_TO_REGCLASS GR8:$mask, VK8WM)), addr:$ptr)>; 2477 2478def: Pat<(int_x86_avx512_mask_storeu_d_512 addr:$ptr, (v16i32 VR512:$src), 2479 GR16:$mask), 2480 (VMOVDQU32Zmrk addr:$ptr, (v16i1 (COPY_TO_REGCLASS GR16:$mask, VK16WM)), 2481 VR512:$src)>; 2482def: Pat<(int_x86_avx512_mask_storeu_q_512 addr:$ptr, (v8i64 VR512:$src), 2483 GR8:$mask), 2484 (VMOVDQU64Zmrk addr:$ptr, (v8i1 (COPY_TO_REGCLASS GR8:$mask, VK8WM)), 2485 VR512:$src)>; 2486 2487let AddedComplexity = 20 in { 2488def : Pat<(v8i64 (vselect VK8WM:$mask, (v8i64 VR512:$src), 2489 (bc_v8i64 (v16i32 immAllZerosV)))), 2490 (VMOVDQU64Zrrkz VK8WM:$mask, VR512:$src)>; 2491 2492def : Pat<(v8i64 (vselect VK8WM:$mask, (bc_v8i64 (v16i32 immAllZerosV)), 2493 (v8i64 VR512:$src))), 2494 (VMOVDQU64Zrrkz (COPY_TO_REGCLASS (KNOTWrr (COPY_TO_REGCLASS VK8:$mask, VK16)), 2495 VK8), VR512:$src)>; 2496 2497def : Pat<(v16i32 (vselect VK16WM:$mask, (v16i32 VR512:$src), 2498 (v16i32 immAllZerosV))), 2499 (VMOVDQU32Zrrkz VK16WM:$mask, VR512:$src)>; 2500 2501def : Pat<(v16i32 (vselect VK16WM:$mask, (v16i32 immAllZerosV), 2502 (v16i32 VR512:$src))), 2503 (VMOVDQU32Zrrkz (KNOTWrr VK16WM:$mask), VR512:$src)>; 2504} 2505// NoVLX patterns 2506let Predicates = [HasAVX512, NoVLX] in { 2507def: Pat<(masked_store addr:$ptr, VK8WM:$mask, (v8i32 VR256:$src)), 2508 (VMOVDQU32Zmrk addr:$ptr, 2509 (v16i1 (COPY_TO_REGCLASS VK8WM:$mask, VK16WM)), 2510 (INSERT_SUBREG (v16i32 (IMPLICIT_DEF)), VR256:$src, sub_ymm))>; 2511 2512def: Pat<(v8i32 (masked_load addr:$ptr, VK8WM:$mask, undef)), 2513 (v8i32 (EXTRACT_SUBREG (v16i32 (VMOVDQU32Zrmkz 2514 (v16i1 (COPY_TO_REGCLASS VK8WM:$mask, VK16WM)), addr:$ptr)), sub_ymm))>; 2515} 2516 2517// Move Int Doubleword to Packed Double Int 2518// 2519def VMOVDI2PDIZrr : AVX512BI<0x6E, MRMSrcReg, (outs VR128X:$dst), (ins GR32:$src), 2520 "vmovd\t{$src, $dst|$dst, $src}", 2521 [(set VR128X:$dst, 2522 (v4i32 (scalar_to_vector GR32:$src)))], IIC_SSE_MOVDQ>, 2523 EVEX, VEX_LIG; 2524def VMOVDI2PDIZrm : AVX512BI<0x6E, MRMSrcMem, (outs VR128X:$dst), (ins i32mem:$src), 2525 "vmovd\t{$src, $dst|$dst, $src}", 2526 [(set VR128X:$dst, 2527 (v4i32 (scalar_to_vector (loadi32 addr:$src))))], 2528 IIC_SSE_MOVDQ>, EVEX, VEX_LIG, EVEX_CD8<32, CD8VT1>; 2529def VMOV64toPQIZrr : AVX512BI<0x6E, MRMSrcReg, (outs VR128X:$dst), (ins GR64:$src), 2530 "vmovq\t{$src, $dst|$dst, $src}", 2531 [(set VR128X:$dst, 2532 (v2i64 (scalar_to_vector GR64:$src)))], 2533 IIC_SSE_MOVDQ>, EVEX, VEX_W, VEX_LIG; 2534let isCodeGenOnly = 1 in { 2535def VMOV64toSDZrr : AVX512BI<0x6E, MRMSrcReg, (outs FR64:$dst), (ins GR64:$src), 2536 "vmovq\t{$src, $dst|$dst, $src}", 2537 [(set FR64:$dst, (bitconvert GR64:$src))], 2538 IIC_SSE_MOVDQ>, EVEX, VEX_W, Sched<[WriteMove]>; 2539def VMOVSDto64Zrr : AVX512BI<0x7E, MRMDestReg, (outs GR64:$dst), (ins FR64:$src), 2540 "vmovq\t{$src, $dst|$dst, $src}", 2541 [(set GR64:$dst, (bitconvert FR64:$src))], 2542 IIC_SSE_MOVDQ>, EVEX, VEX_W, Sched<[WriteMove]>; 2543} 2544def VMOVSDto64Zmr : AVX512BI<0x7E, MRMDestMem, (outs), (ins i64mem:$dst, FR64:$src), 2545 "vmovq\t{$src, $dst|$dst, $src}", 2546 [(store (i64 (bitconvert FR64:$src)), addr:$dst)], 2547 IIC_SSE_MOVDQ>, EVEX, VEX_W, Sched<[WriteStore]>, 2548 EVEX_CD8<64, CD8VT1>; 2549 2550// Move Int Doubleword to Single Scalar 2551// 2552let isCodeGenOnly = 1 in { 2553def VMOVDI2SSZrr : AVX512BI<0x6E, MRMSrcReg, (outs FR32X:$dst), (ins GR32:$src), 2554 "vmovd\t{$src, $dst|$dst, $src}", 2555 [(set FR32X:$dst, (bitconvert GR32:$src))], 2556 IIC_SSE_MOVDQ>, EVEX, VEX_LIG; 2557 2558def VMOVDI2SSZrm : AVX512BI<0x6E, MRMSrcMem, (outs FR32X:$dst), (ins i32mem:$src), 2559 "vmovd\t{$src, $dst|$dst, $src}", 2560 [(set FR32X:$dst, (bitconvert (loadi32 addr:$src)))], 2561 IIC_SSE_MOVDQ>, EVEX, VEX_LIG, EVEX_CD8<32, CD8VT1>; 2562} 2563 2564// Move doubleword from xmm register to r/m32 2565// 2566def VMOVPDI2DIZrr : AVX512BI<0x7E, MRMDestReg, (outs GR32:$dst), (ins VR128X:$src), 2567 "vmovd\t{$src, $dst|$dst, $src}", 2568 [(set GR32:$dst, (vector_extract (v4i32 VR128X:$src), 2569 (iPTR 0)))], IIC_SSE_MOVD_ToGP>, 2570 EVEX, VEX_LIG; 2571def VMOVPDI2DIZmr : AVX512BI<0x7E, MRMDestMem, (outs), 2572 (ins i32mem:$dst, VR128X:$src), 2573 "vmovd\t{$src, $dst|$dst, $src}", 2574 [(store (i32 (vector_extract (v4i32 VR128X:$src), 2575 (iPTR 0))), addr:$dst)], IIC_SSE_MOVDQ>, 2576 EVEX, VEX_LIG, EVEX_CD8<32, CD8VT1>; 2577 2578// Move quadword from xmm1 register to r/m64 2579// 2580def VMOVPQIto64Zrr : I<0x7E, MRMDestReg, (outs GR64:$dst), (ins VR128X:$src), 2581 "vmovq\t{$src, $dst|$dst, $src}", 2582 [(set GR64:$dst, (extractelt (v2i64 VR128X:$src), 2583 (iPTR 0)))], 2584 IIC_SSE_MOVD_ToGP>, PD, EVEX, VEX_LIG, VEX_W, 2585 Requires<[HasAVX512, In64BitMode]>; 2586 2587def VMOVPQIto64Zmr : I<0xD6, MRMDestMem, (outs), 2588 (ins i64mem:$dst, VR128X:$src), 2589 "vmovq\t{$src, $dst|$dst, $src}", 2590 [(store (extractelt (v2i64 VR128X:$src), (iPTR 0)), 2591 addr:$dst)], IIC_SSE_MOVDQ>, 2592 EVEX, PD, VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>, 2593 Sched<[WriteStore]>, Requires<[HasAVX512, In64BitMode]>; 2594 2595// Move Scalar Single to Double Int 2596// 2597let isCodeGenOnly = 1 in { 2598def VMOVSS2DIZrr : AVX512BI<0x7E, MRMDestReg, (outs GR32:$dst), 2599 (ins FR32X:$src), 2600 "vmovd\t{$src, $dst|$dst, $src}", 2601 [(set GR32:$dst, (bitconvert FR32X:$src))], 2602 IIC_SSE_MOVD_ToGP>, EVEX, VEX_LIG; 2603def VMOVSS2DIZmr : AVX512BI<0x7E, MRMDestMem, (outs), 2604 (ins i32mem:$dst, FR32X:$src), 2605 "vmovd\t{$src, $dst|$dst, $src}", 2606 [(store (i32 (bitconvert FR32X:$src)), addr:$dst)], 2607 IIC_SSE_MOVDQ>, EVEX, VEX_LIG, EVEX_CD8<32, CD8VT1>; 2608} 2609 2610// Move Quadword Int to Packed Quadword Int 2611// 2612def VMOVQI2PQIZrm : AVX512BI<0x6E, MRMSrcMem, (outs VR128X:$dst), 2613 (ins i64mem:$src), 2614 "vmovq\t{$src, $dst|$dst, $src}", 2615 [(set VR128X:$dst, 2616 (v2i64 (scalar_to_vector (loadi64 addr:$src))))]>, 2617 EVEX, VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>; 2618 2619//===----------------------------------------------------------------------===// 2620// AVX-512 MOVSS, MOVSD 2621//===----------------------------------------------------------------------===// 2622 2623multiclass avx512_move_scalar <string asm, RegisterClass RC, 2624 SDNode OpNode, ValueType vt, 2625 X86MemOperand x86memop, PatFrag mem_pat> { 2626 let hasSideEffects = 0 in { 2627 def rr : SI<0x10, MRMSrcReg, (outs VR128X:$dst), (ins VR128X:$src1, RC:$src2), 2628 !strconcat(asm, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 2629 [(set VR128X:$dst, (vt (OpNode VR128X:$src1, 2630 (scalar_to_vector RC:$src2))))], 2631 IIC_SSE_MOV_S_RR>, EVEX_4V, VEX_LIG; 2632 let Constraints = "$src1 = $dst" in 2633 def rrk : SI<0x10, MRMSrcReg, (outs VR128X:$dst), 2634 (ins VR128X:$src1, VK1WM:$mask, RC:$src2, RC:$src3), 2635 !strconcat(asm, 2636 "\t{$src3, $src2, $dst {${mask}}|$dst {${mask}}, $src2, $src3}"), 2637 [], IIC_SSE_MOV_S_RR>, EVEX_4V, VEX_LIG, EVEX_K; 2638 def rm : SI<0x10, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src), 2639 !strconcat(asm, "\t{$src, $dst|$dst, $src}"), 2640 [(set RC:$dst, (mem_pat addr:$src))], IIC_SSE_MOV_S_RM>, 2641 EVEX, VEX_LIG; 2642 let mayStore = 1 in { 2643 def mr: SI<0x11, MRMDestMem, (outs), (ins x86memop:$dst, RC:$src), 2644 !strconcat(asm, "\t{$src, $dst|$dst, $src}"), 2645 [(store RC:$src, addr:$dst)], IIC_SSE_MOV_S_MR>, 2646 EVEX, VEX_LIG; 2647 def mrk: SI<0x11, MRMDestMem, (outs), (ins x86memop:$dst, VK1WM:$mask, RC:$src), 2648 !strconcat(asm, "\t{$src, $dst {${mask}}|$dst {${mask}}, $src}"), 2649 [], IIC_SSE_MOV_S_MR>, 2650 EVEX, VEX_LIG, EVEX_K; 2651 } // mayStore 2652 } //hasSideEffects = 0 2653} 2654 2655let ExeDomain = SSEPackedSingle in 2656defm VMOVSSZ : avx512_move_scalar<"movss", FR32X, X86Movss, v4f32, f32mem, 2657 loadf32>, XS, EVEX_CD8<32, CD8VT1>; 2658 2659let ExeDomain = SSEPackedDouble in 2660defm VMOVSDZ : avx512_move_scalar<"movsd", FR64X, X86Movsd, v2f64, f64mem, 2661 loadf64>, XD, VEX_W, EVEX_CD8<64, CD8VT1>; 2662 2663def : Pat<(f32 (X86select VK1WM:$mask, (f32 FR32X:$src1), (f32 FR32X:$src2))), 2664 (COPY_TO_REGCLASS (VMOVSSZrrk (COPY_TO_REGCLASS FR32X:$src2, VR128X), 2665 VK1WM:$mask, (f32 (IMPLICIT_DEF)), FR32X:$src1), FR32X)>; 2666 2667def : Pat<(f64 (X86select VK1WM:$mask, (f64 FR64X:$src1), (f64 FR64X:$src2))), 2668 (COPY_TO_REGCLASS (VMOVSDZrrk (COPY_TO_REGCLASS FR64X:$src2, VR128X), 2669 VK1WM:$mask, (f64 (IMPLICIT_DEF)), FR64X:$src1), FR64X)>; 2670 2671def : Pat<(int_x86_avx512_mask_store_ss addr:$dst, VR128X:$src, GR8:$mask), 2672 (VMOVSSZmrk addr:$dst, (i1 (COPY_TO_REGCLASS GR8:$mask, VK1WM)), 2673 (COPY_TO_REGCLASS VR128X:$src, FR32X))>; 2674 2675// For the disassembler 2676let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0 in { 2677 def VMOVSSZrr_REV : SI<0x11, MRMDestReg, (outs VR128X:$dst), 2678 (ins VR128X:$src1, FR32X:$src2), 2679 "movss\t{$src2, $src1, $dst|$dst, $src1, $src2}", [], 2680 IIC_SSE_MOV_S_RR>, 2681 XS, EVEX_4V, VEX_LIG; 2682 def VMOVSDZrr_REV : SI<0x11, MRMDestReg, (outs VR128X:$dst), 2683 (ins VR128X:$src1, FR64X:$src2), 2684 "movsd\t{$src2, $src1, $dst|$dst, $src1, $src2}", [], 2685 IIC_SSE_MOV_S_RR>, 2686 XD, EVEX_4V, VEX_LIG, VEX_W; 2687} 2688 2689let Predicates = [HasAVX512] in { 2690 let AddedComplexity = 15 in { 2691 // Move scalar to XMM zero-extended, zeroing a VR128X then do a 2692 // MOVS{S,D} to the lower bits. 2693 def : Pat<(v4f32 (X86vzmovl (v4f32 (scalar_to_vector FR32X:$src)))), 2694 (VMOVSSZrr (v4f32 (V_SET0)), FR32X:$src)>; 2695 def : Pat<(v4f32 (X86vzmovl (v4f32 VR128X:$src))), 2696 (VMOVSSZrr (v4f32 (V_SET0)), (COPY_TO_REGCLASS VR128X:$src, FR32X))>; 2697 def : Pat<(v4i32 (X86vzmovl (v4i32 VR128X:$src))), 2698 (VMOVSSZrr (v4i32 (V_SET0)), (COPY_TO_REGCLASS VR128X:$src, FR32X))>; 2699 def : Pat<(v2f64 (X86vzmovl (v2f64 (scalar_to_vector FR64X:$src)))), 2700 (VMOVSDZrr (v2f64 (V_SET0)), FR64X:$src)>; 2701 2702 // Move low f32 and clear high bits. 2703 def : Pat<(v8f32 (X86vzmovl (v8f32 VR256X:$src))), 2704 (SUBREG_TO_REG (i32 0), 2705 (VMOVSSZrr (v4f32 (V_SET0)), 2706 (EXTRACT_SUBREG (v8f32 VR256X:$src), sub_xmm)), sub_xmm)>; 2707 def : Pat<(v8i32 (X86vzmovl (v8i32 VR256X:$src))), 2708 (SUBREG_TO_REG (i32 0), 2709 (VMOVSSZrr (v4i32 (V_SET0)), 2710 (EXTRACT_SUBREG (v8i32 VR256X:$src), sub_xmm)), sub_xmm)>; 2711 } 2712 2713 let AddedComplexity = 20 in { 2714 // MOVSSrm zeros the high parts of the register; represent this 2715 // with SUBREG_TO_REG. The AVX versions also write: DST[255:128] <- 0 2716 def : Pat<(v4f32 (X86vzmovl (v4f32 (scalar_to_vector (loadf32 addr:$src))))), 2717 (COPY_TO_REGCLASS (VMOVSSZrm addr:$src), VR128X)>; 2718 def : Pat<(v4f32 (scalar_to_vector (loadf32 addr:$src))), 2719 (COPY_TO_REGCLASS (VMOVSSZrm addr:$src), VR128X)>; 2720 def : Pat<(v4f32 (X86vzmovl (loadv4f32 addr:$src))), 2721 (COPY_TO_REGCLASS (VMOVSSZrm addr:$src), VR128X)>; 2722 2723 // MOVSDrm zeros the high parts of the register; represent this 2724 // with SUBREG_TO_REG. The AVX versions also write: DST[255:128] <- 0 2725 def : Pat<(v2f64 (X86vzmovl (v2f64 (scalar_to_vector (loadf64 addr:$src))))), 2726 (COPY_TO_REGCLASS (VMOVSDZrm addr:$src), VR128X)>; 2727 def : Pat<(v2f64 (scalar_to_vector (loadf64 addr:$src))), 2728 (COPY_TO_REGCLASS (VMOVSDZrm addr:$src), VR128X)>; 2729 def : Pat<(v2f64 (X86vzmovl (loadv2f64 addr:$src))), 2730 (COPY_TO_REGCLASS (VMOVSDZrm addr:$src), VR128X)>; 2731 def : Pat<(v2f64 (X86vzmovl (bc_v2f64 (loadv4f32 addr:$src)))), 2732 (COPY_TO_REGCLASS (VMOVSDZrm addr:$src), VR128X)>; 2733 def : Pat<(v2f64 (X86vzload addr:$src)), 2734 (COPY_TO_REGCLASS (VMOVSDZrm addr:$src), VR128X)>; 2735 2736 // Represent the same patterns above but in the form they appear for 2737 // 256-bit types 2738 def : Pat<(v8i32 (X86vzmovl (insert_subvector undef, 2739 (v4i32 (scalar_to_vector (loadi32 addr:$src))), (iPTR 0)))), 2740 (SUBREG_TO_REG (i32 0), (VMOVDI2PDIZrm addr:$src), sub_xmm)>; 2741 def : Pat<(v8f32 (X86vzmovl (insert_subvector undef, 2742 (v4f32 (scalar_to_vector (loadf32 addr:$src))), (iPTR 0)))), 2743 (SUBREG_TO_REG (i32 0), (VMOVSSZrm addr:$src), sub_xmm)>; 2744 def : Pat<(v4f64 (X86vzmovl (insert_subvector undef, 2745 (v2f64 (scalar_to_vector (loadf64 addr:$src))), (iPTR 0)))), 2746 (SUBREG_TO_REG (i32 0), (VMOVSDZrm addr:$src), sub_xmm)>; 2747 } 2748 def : Pat<(v8f32 (X86vzmovl (insert_subvector undef, 2749 (v4f32 (scalar_to_vector FR32X:$src)), (iPTR 0)))), 2750 (SUBREG_TO_REG (i32 0), (v4f32 (VMOVSSZrr (v4f32 (V_SET0)), 2751 FR32X:$src)), sub_xmm)>; 2752 def : Pat<(v4f64 (X86vzmovl (insert_subvector undef, 2753 (v2f64 (scalar_to_vector FR64X:$src)), (iPTR 0)))), 2754 (SUBREG_TO_REG (i64 0), (v2f64 (VMOVSDZrr (v2f64 (V_SET0)), 2755 FR64X:$src)), sub_xmm)>; 2756 def : Pat<(v4i64 (X86vzmovl (insert_subvector undef, 2757 (v2i64 (scalar_to_vector (loadi64 addr:$src))), (iPTR 0)))), 2758 (SUBREG_TO_REG (i64 0), (VMOVQI2PQIZrm addr:$src), sub_xmm)>; 2759 2760 // Move low f64 and clear high bits. 2761 def : Pat<(v4f64 (X86vzmovl (v4f64 VR256X:$src))), 2762 (SUBREG_TO_REG (i32 0), 2763 (VMOVSDZrr (v2f64 (V_SET0)), 2764 (EXTRACT_SUBREG (v4f64 VR256X:$src), sub_xmm)), sub_xmm)>; 2765 2766 def : Pat<(v4i64 (X86vzmovl (v4i64 VR256X:$src))), 2767 (SUBREG_TO_REG (i32 0), (VMOVSDZrr (v2i64 (V_SET0)), 2768 (EXTRACT_SUBREG (v4i64 VR256X:$src), sub_xmm)), sub_xmm)>; 2769 2770 // Extract and store. 2771 def : Pat<(store (f32 (vector_extract (v4f32 VR128X:$src), (iPTR 0))), 2772 addr:$dst), 2773 (VMOVSSZmr addr:$dst, (COPY_TO_REGCLASS (v4f32 VR128X:$src), FR32X))>; 2774 def : Pat<(store (f64 (vector_extract (v2f64 VR128X:$src), (iPTR 0))), 2775 addr:$dst), 2776 (VMOVSDZmr addr:$dst, (COPY_TO_REGCLASS (v2f64 VR128X:$src), FR64X))>; 2777 2778 // Shuffle with VMOVSS 2779 def : Pat<(v4i32 (X86Movss VR128X:$src1, VR128X:$src2)), 2780 (VMOVSSZrr (v4i32 VR128X:$src1), 2781 (COPY_TO_REGCLASS (v4i32 VR128X:$src2), FR32X))>; 2782 def : Pat<(v4f32 (X86Movss VR128X:$src1, VR128X:$src2)), 2783 (VMOVSSZrr (v4f32 VR128X:$src1), 2784 (COPY_TO_REGCLASS (v4f32 VR128X:$src2), FR32X))>; 2785 2786 // 256-bit variants 2787 def : Pat<(v8i32 (X86Movss VR256X:$src1, VR256X:$src2)), 2788 (SUBREG_TO_REG (i32 0), 2789 (VMOVSSZrr (EXTRACT_SUBREG (v8i32 VR256X:$src1), sub_xmm), 2790 (EXTRACT_SUBREG (v8i32 VR256X:$src2), sub_xmm)), 2791 sub_xmm)>; 2792 def : Pat<(v8f32 (X86Movss VR256X:$src1, VR256X:$src2)), 2793 (SUBREG_TO_REG (i32 0), 2794 (VMOVSSZrr (EXTRACT_SUBREG (v8f32 VR256X:$src1), sub_xmm), 2795 (EXTRACT_SUBREG (v8f32 VR256X:$src2), sub_xmm)), 2796 sub_xmm)>; 2797 2798 // Shuffle with VMOVSD 2799 def : Pat<(v2i64 (X86Movsd VR128X:$src1, VR128X:$src2)), 2800 (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>; 2801 def : Pat<(v2f64 (X86Movsd VR128X:$src1, VR128X:$src2)), 2802 (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>; 2803 def : Pat<(v4f32 (X86Movsd VR128X:$src1, VR128X:$src2)), 2804 (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>; 2805 def : Pat<(v4i32 (X86Movsd VR128X:$src1, VR128X:$src2)), 2806 (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>; 2807 2808 // 256-bit variants 2809 def : Pat<(v4i64 (X86Movsd VR256X:$src1, VR256X:$src2)), 2810 (SUBREG_TO_REG (i32 0), 2811 (VMOVSDZrr (EXTRACT_SUBREG (v4i64 VR256X:$src1), sub_xmm), 2812 (EXTRACT_SUBREG (v4i64 VR256X:$src2), sub_xmm)), 2813 sub_xmm)>; 2814 def : Pat<(v4f64 (X86Movsd VR256X:$src1, VR256X:$src2)), 2815 (SUBREG_TO_REG (i32 0), 2816 (VMOVSDZrr (EXTRACT_SUBREG (v4f64 VR256X:$src1), sub_xmm), 2817 (EXTRACT_SUBREG (v4f64 VR256X:$src2), sub_xmm)), 2818 sub_xmm)>; 2819 2820 def : Pat<(v2f64 (X86Movlpd VR128X:$src1, VR128X:$src2)), 2821 (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>; 2822 def : Pat<(v2i64 (X86Movlpd VR128X:$src1, VR128X:$src2)), 2823 (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>; 2824 def : Pat<(v4f32 (X86Movlps VR128X:$src1, VR128X:$src2)), 2825 (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>; 2826 def : Pat<(v4i32 (X86Movlps VR128X:$src1, VR128X:$src2)), 2827 (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS VR128X:$src2, FR64X))>; 2828} 2829 2830let AddedComplexity = 15 in 2831def VMOVZPQILo2PQIZrr : AVX512XSI<0x7E, MRMSrcReg, (outs VR128X:$dst), 2832 (ins VR128X:$src), 2833 "vmovq\t{$src, $dst|$dst, $src}", 2834 [(set VR128X:$dst, (v2i64 (X86vzmovl 2835 (v2i64 VR128X:$src))))], 2836 IIC_SSE_MOVQ_RR>, EVEX, VEX_W; 2837 2838let AddedComplexity = 20 in 2839def VMOVZPQILo2PQIZrm : AVX512XSI<0x7E, MRMSrcMem, (outs VR128X:$dst), 2840 (ins i128mem:$src), 2841 "vmovq\t{$src, $dst|$dst, $src}", 2842 [(set VR128X:$dst, (v2i64 (X86vzmovl 2843 (loadv2i64 addr:$src))))], 2844 IIC_SSE_MOVDQ>, EVEX, VEX_W, 2845 EVEX_CD8<8, CD8VT8>; 2846 2847let Predicates = [HasAVX512] in { 2848 // AVX 128-bit movd/movq instruction write zeros in the high 128-bit part. 2849 let AddedComplexity = 20 in { 2850 def : Pat<(v4i32 (X86vzmovl (v4i32 (scalar_to_vector (loadi32 addr:$src))))), 2851 (VMOVDI2PDIZrm addr:$src)>; 2852 def : Pat<(v2i64 (X86vzmovl (v2i64 (scalar_to_vector GR64:$src)))), 2853 (VMOV64toPQIZrr GR64:$src)>; 2854 def : Pat<(v4i32 (X86vzmovl (v4i32 (scalar_to_vector GR32:$src)))), 2855 (VMOVDI2PDIZrr GR32:$src)>; 2856 2857 def : Pat<(v4i32 (X86vzmovl (bc_v4i32 (loadv4f32 addr:$src)))), 2858 (VMOVDI2PDIZrm addr:$src)>; 2859 def : Pat<(v4i32 (X86vzmovl (bc_v4i32 (loadv2i64 addr:$src)))), 2860 (VMOVDI2PDIZrm addr:$src)>; 2861 def : Pat<(v2i64 (X86vzmovl (loadv2i64 addr:$src))), 2862 (VMOVZPQILo2PQIZrm addr:$src)>; 2863 def : Pat<(v2f64 (X86vzmovl (v2f64 VR128X:$src))), 2864 (VMOVZPQILo2PQIZrr VR128X:$src)>; 2865 def : Pat<(v2i64 (X86vzload addr:$src)), 2866 (VMOVZPQILo2PQIZrm addr:$src)>; 2867 } 2868 2869 // Use regular 128-bit instructions to match 256-bit scalar_to_vec+zext. 2870 def : Pat<(v8i32 (X86vzmovl (insert_subvector undef, 2871 (v4i32 (scalar_to_vector GR32:$src)),(iPTR 0)))), 2872 (SUBREG_TO_REG (i32 0), (VMOVDI2PDIZrr GR32:$src), sub_xmm)>; 2873 def : Pat<(v4i64 (X86vzmovl (insert_subvector undef, 2874 (v2i64 (scalar_to_vector GR64:$src)),(iPTR 0)))), 2875 (SUBREG_TO_REG (i64 0), (VMOV64toPQIZrr GR64:$src), sub_xmm)>; 2876} 2877 2878def : Pat<(v16i32 (X86Vinsert (v16i32 immAllZerosV), GR32:$src2, (iPTR 0))), 2879 (SUBREG_TO_REG (i32 0), (VMOVDI2PDIZrr GR32:$src2), sub_xmm)>; 2880 2881def : Pat<(v8i64 (X86Vinsert (bc_v8i64 (v16i32 immAllZerosV)), GR64:$src2, (iPTR 0))), 2882 (SUBREG_TO_REG (i32 0), (VMOV64toPQIZrr GR64:$src2), sub_xmm)>; 2883 2884def : Pat<(v16i32 (X86Vinsert undef, GR32:$src2, (iPTR 0))), 2885 (SUBREG_TO_REG (i32 0), (VMOVDI2PDIZrr GR32:$src2), sub_xmm)>; 2886 2887def : Pat<(v8i64 (X86Vinsert undef, GR64:$src2, (iPTR 0))), 2888 (SUBREG_TO_REG (i32 0), (VMOV64toPQIZrr GR64:$src2), sub_xmm)>; 2889 2890//===----------------------------------------------------------------------===// 2891// AVX-512 - Non-temporals 2892//===----------------------------------------------------------------------===// 2893let SchedRW = [WriteLoad] in { 2894 def VMOVNTDQAZrm : AVX512PI<0x2A, MRMSrcMem, (outs VR512:$dst), 2895 (ins i512mem:$src), "vmovntdqa\t{$src, $dst|$dst, $src}", 2896 [(set VR512:$dst, (int_x86_avx512_movntdqa addr:$src))], 2897 SSEPackedInt>, EVEX, T8PD, EVEX_V512, 2898 EVEX_CD8<64, CD8VF>; 2899 2900 let Predicates = [HasAVX512, HasVLX] in { 2901 def VMOVNTDQAZ256rm : AVX512PI<0x2A, MRMSrcMem, (outs VR256X:$dst), 2902 (ins i256mem:$src), 2903 "vmovntdqa\t{$src, $dst|$dst, $src}", [], 2904 SSEPackedInt>, EVEX, T8PD, EVEX_V256, 2905 EVEX_CD8<64, CD8VF>; 2906 2907 def VMOVNTDQAZ128rm : AVX512PI<0x2A, MRMSrcMem, (outs VR128X:$dst), 2908 (ins i128mem:$src), 2909 "vmovntdqa\t{$src, $dst|$dst, $src}", [], 2910 SSEPackedInt>, EVEX, T8PD, EVEX_V128, 2911 EVEX_CD8<64, CD8VF>; 2912 } 2913} 2914 2915multiclass avx512_movnt<bits<8> opc, string OpcodeStr, PatFrag st_frag, 2916 ValueType OpVT, RegisterClass RC, X86MemOperand memop, 2917 Domain d, InstrItinClass itin = IIC_SSE_MOVNT> { 2918 let SchedRW = [WriteStore], mayStore = 1, 2919 AddedComplexity = 400 in 2920 def mr : AVX512PI<opc, MRMDestMem, (outs), (ins memop:$dst, RC:$src), 2921 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 2922 [(st_frag (OpVT RC:$src), addr:$dst)], d, itin>, EVEX; 2923} 2924 2925multiclass avx512_movnt_vl<bits<8> opc, string OpcodeStr, PatFrag st_frag, 2926 string elty, string elsz, string vsz512, 2927 string vsz256, string vsz128, Domain d, 2928 Predicate prd, InstrItinClass itin = IIC_SSE_MOVNT> { 2929 let Predicates = [prd] in 2930 defm Z : avx512_movnt<opc, OpcodeStr, st_frag, 2931 !cast<ValueType>("v"##vsz512##elty##elsz), VR512, 2932 !cast<X86MemOperand>(elty##"512mem"), d, itin>, 2933 EVEX_V512; 2934 2935 let Predicates = [prd, HasVLX] in { 2936 defm Z256 : avx512_movnt<opc, OpcodeStr, st_frag, 2937 !cast<ValueType>("v"##vsz256##elty##elsz), VR256X, 2938 !cast<X86MemOperand>(elty##"256mem"), d, itin>, 2939 EVEX_V256; 2940 2941 defm Z128 : avx512_movnt<opc, OpcodeStr, st_frag, 2942 !cast<ValueType>("v"##vsz128##elty##elsz), VR128X, 2943 !cast<X86MemOperand>(elty##"128mem"), d, itin>, 2944 EVEX_V128; 2945 } 2946} 2947 2948defm VMOVNTDQ : avx512_movnt_vl<0xE7, "vmovntdq", alignednontemporalstore, 2949 "i", "64", "8", "4", "2", SSEPackedInt, 2950 HasAVX512>, PD, EVEX_CD8<64, CD8VF>; 2951 2952defm VMOVNTPD : avx512_movnt_vl<0x2B, "vmovntpd", alignednontemporalstore, 2953 "f", "64", "8", "4", "2", SSEPackedDouble, 2954 HasAVX512>, PD, VEX_W, EVEX_CD8<64, CD8VF>; 2955 2956defm VMOVNTPS : avx512_movnt_vl<0x2B, "vmovntps", alignednontemporalstore, 2957 "f", "32", "16", "8", "4", SSEPackedSingle, 2958 HasAVX512>, PS, EVEX_CD8<32, CD8VF>; 2959 2960//===----------------------------------------------------------------------===// 2961// AVX-512 - Integer arithmetic 2962// 2963multiclass avx512_binop_rm<bits<8> opc, string OpcodeStr, SDNode OpNode, 2964 X86VectorVTInfo _, OpndItins itins, 2965 bit IsCommutable = 0> { 2966 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), 2967 (ins _.RC:$src1, _.RC:$src2), OpcodeStr##_.Suffix, 2968 "$src2, $src1", "$src1, $src2", 2969 (_.VT (OpNode _.RC:$src1, _.RC:$src2)), 2970 itins.rr, IsCommutable>, 2971 AVX512BIBase, EVEX_4V; 2972 2973 let mayLoad = 1 in 2974 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), 2975 (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr##_.Suffix, 2976 "$src2, $src1", "$src1, $src2", 2977 (_.VT (OpNode _.RC:$src1, 2978 (bitconvert (_.LdFrag addr:$src2)))), 2979 itins.rm>, 2980 AVX512BIBase, EVEX_4V; 2981} 2982 2983multiclass avx512_binop_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode, 2984 X86VectorVTInfo _, OpndItins itins, 2985 bit IsCommutable = 0> : 2986 avx512_binop_rm<opc, OpcodeStr, OpNode, _, itins, IsCommutable> { 2987 let mayLoad = 1 in 2988 defm rmb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), 2989 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr##_.Suffix, 2990 "${src2}"##_.BroadcastStr##", $src1", 2991 "$src1, ${src2}"##_.BroadcastStr, 2992 (_.VT (OpNode _.RC:$src1, 2993 (X86VBroadcast 2994 (_.ScalarLdFrag addr:$src2)))), 2995 itins.rm>, 2996 AVX512BIBase, EVEX_4V, EVEX_B; 2997} 2998 2999multiclass avx512_binop_rm_vl<bits<8> opc, string OpcodeStr, SDNode OpNode, 3000 AVX512VLVectorVTInfo VTInfo, OpndItins itins, 3001 Predicate prd, bit IsCommutable = 0> { 3002 let Predicates = [prd] in 3003 defm Z : avx512_binop_rm<opc, OpcodeStr, OpNode, VTInfo.info512, itins, 3004 IsCommutable>, EVEX_V512; 3005 3006 let Predicates = [prd, HasVLX] in { 3007 defm Z256 : avx512_binop_rm<opc, OpcodeStr, OpNode, VTInfo.info256, itins, 3008 IsCommutable>, EVEX_V256; 3009 defm Z128 : avx512_binop_rm<opc, OpcodeStr, OpNode, VTInfo.info128, itins, 3010 IsCommutable>, EVEX_V128; 3011 } 3012} 3013 3014multiclass avx512_binop_rmb_vl<bits<8> opc, string OpcodeStr, SDNode OpNode, 3015 AVX512VLVectorVTInfo VTInfo, OpndItins itins, 3016 Predicate prd, bit IsCommutable = 0> { 3017 let Predicates = [prd] in 3018 defm Z : avx512_binop_rmb<opc, OpcodeStr, OpNode, VTInfo.info512, itins, 3019 IsCommutable>, EVEX_V512; 3020 3021 let Predicates = [prd, HasVLX] in { 3022 defm Z256 : avx512_binop_rmb<opc, OpcodeStr, OpNode, VTInfo.info256, itins, 3023 IsCommutable>, EVEX_V256; 3024 defm Z128 : avx512_binop_rmb<opc, OpcodeStr, OpNode, VTInfo.info128, itins, 3025 IsCommutable>, EVEX_V128; 3026 } 3027} 3028 3029multiclass avx512_binop_rm_vl_q<bits<8> opc, string OpcodeStr, SDNode OpNode, 3030 OpndItins itins, Predicate prd, 3031 bit IsCommutable = 0> { 3032 defm NAME : avx512_binop_rmb_vl<opc, OpcodeStr, OpNode, avx512vl_i64_info, 3033 itins, prd, IsCommutable>, 3034 VEX_W, EVEX_CD8<64, CD8VF>; 3035} 3036 3037multiclass avx512_binop_rm_vl_d<bits<8> opc, string OpcodeStr, SDNode OpNode, 3038 OpndItins itins, Predicate prd, 3039 bit IsCommutable = 0> { 3040 defm NAME : avx512_binop_rmb_vl<opc, OpcodeStr, OpNode, avx512vl_i32_info, 3041 itins, prd, IsCommutable>, EVEX_CD8<32, CD8VF>; 3042} 3043 3044multiclass avx512_binop_rm_vl_w<bits<8> opc, string OpcodeStr, SDNode OpNode, 3045 OpndItins itins, Predicate prd, 3046 bit IsCommutable = 0> { 3047 defm NAME : avx512_binop_rm_vl<opc, OpcodeStr, OpNode, avx512vl_i16_info, 3048 itins, prd, IsCommutable>, EVEX_CD8<16, CD8VF>; 3049} 3050 3051multiclass avx512_binop_rm_vl_b<bits<8> opc, string OpcodeStr, SDNode OpNode, 3052 OpndItins itins, Predicate prd, 3053 bit IsCommutable = 0> { 3054 defm NAME : avx512_binop_rm_vl<opc, OpcodeStr, OpNode, avx512vl_i8_info, 3055 itins, prd, IsCommutable>, EVEX_CD8<8, CD8VF>; 3056} 3057 3058multiclass avx512_binop_rm_vl_dq<bits<8> opc_d, bits<8> opc_q, string OpcodeStr, 3059 SDNode OpNode, OpndItins itins, Predicate prd, 3060 bit IsCommutable = 0> { 3061 defm Q : avx512_binop_rm_vl_q<opc_q, OpcodeStr, OpNode, itins, prd, 3062 IsCommutable>; 3063 3064 defm D : avx512_binop_rm_vl_d<opc_d, OpcodeStr, OpNode, itins, prd, 3065 IsCommutable>; 3066} 3067 3068multiclass avx512_binop_rm_vl_bw<bits<8> opc_b, bits<8> opc_w, string OpcodeStr, 3069 SDNode OpNode, OpndItins itins, Predicate prd, 3070 bit IsCommutable = 0> { 3071 defm W : avx512_binop_rm_vl_w<opc_w, OpcodeStr, OpNode, itins, prd, 3072 IsCommutable>; 3073 3074 defm B : avx512_binop_rm_vl_b<opc_b, OpcodeStr, OpNode, itins, prd, 3075 IsCommutable>; 3076} 3077 3078multiclass avx512_binop_rm_vl_all<bits<8> opc_b, bits<8> opc_w, 3079 bits<8> opc_d, bits<8> opc_q, 3080 string OpcodeStr, SDNode OpNode, 3081 OpndItins itins, bit IsCommutable = 0> { 3082 defm NAME : avx512_binop_rm_vl_dq<opc_d, opc_q, OpcodeStr, OpNode, 3083 itins, HasAVX512, IsCommutable>, 3084 avx512_binop_rm_vl_bw<opc_b, opc_w, OpcodeStr, OpNode, 3085 itins, HasBWI, IsCommutable>; 3086} 3087 3088multiclass avx512_binop_rm2<bits<8> opc, string OpcodeStr, OpndItins itins, 3089 SDNode OpNode,X86VectorVTInfo _Src, 3090 X86VectorVTInfo _Dst, bit IsCommutable = 0> { 3091 defm rr : AVX512_maskable<opc, MRMSrcReg, _Dst, (outs _Dst.RC:$dst), 3092 (ins _Src.RC:$src1, _Src.RC:$src2), OpcodeStr, 3093 "$src2, $src1","$src1, $src2", 3094 (_Dst.VT (OpNode 3095 (_Src.VT _Src.RC:$src1), 3096 (_Src.VT _Src.RC:$src2))), 3097 itins.rr, IsCommutable>, 3098 AVX512BIBase, EVEX_4V; 3099 let mayLoad = 1 in { 3100 defm rm : AVX512_maskable<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst), 3101 (ins _Src.RC:$src1, _Src.MemOp:$src2), OpcodeStr, 3102 "$src2, $src1", "$src1, $src2", 3103 (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1), 3104 (bitconvert (_Src.LdFrag addr:$src2)))), 3105 itins.rm>, 3106 AVX512BIBase, EVEX_4V; 3107 3108 defm rmb : AVX512_maskable<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst), 3109 (ins _Src.RC:$src1, _Dst.ScalarMemOp:$src2), 3110 OpcodeStr, 3111 "${src2}"##_Dst.BroadcastStr##", $src1", 3112 "$src1, ${src2}"##_Dst.BroadcastStr, 3113 (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1), (bitconvert 3114 (_Dst.VT (X86VBroadcast 3115 (_Dst.ScalarLdFrag addr:$src2)))))), 3116 itins.rm>, 3117 AVX512BIBase, EVEX_4V, EVEX_B; 3118 } 3119} 3120 3121defm VPADD : avx512_binop_rm_vl_all<0xFC, 0xFD, 0xFE, 0xD4, "vpadd", add, 3122 SSE_INTALU_ITINS_P, 1>; 3123defm VPSUB : avx512_binop_rm_vl_all<0xF8, 0xF9, 0xFA, 0xFB, "vpsub", sub, 3124 SSE_INTALU_ITINS_P, 0>; 3125defm VPADDS : avx512_binop_rm_vl_bw<0xEC, 0xED, "vpadds", X86adds, 3126 SSE_INTALU_ITINS_P, HasBWI, 1>; 3127defm VPSUBS : avx512_binop_rm_vl_bw<0xE8, 0xE9, "vpsubs", X86subs, 3128 SSE_INTALU_ITINS_P, HasBWI, 0>; 3129defm VPADDUS : avx512_binop_rm_vl_bw<0xDC, 0xDD, "vpaddus", X86addus, 3130 SSE_INTALU_ITINS_P, HasBWI, 1>; 3131defm VPSUBUS : avx512_binop_rm_vl_bw<0xD8, 0xD9, "vpsubus", X86subus, 3132 SSE_INTALU_ITINS_P, HasBWI, 0>; 3133defm VPMULLD : avx512_binop_rm_vl_d<0x40, "vpmull", mul, 3134 SSE_INTALU_ITINS_P, HasAVX512, 1>, T8PD; 3135defm VPMULLW : avx512_binop_rm_vl_w<0xD5, "vpmull", mul, 3136 SSE_INTALU_ITINS_P, HasBWI, 1>; 3137defm VPMULLQ : avx512_binop_rm_vl_q<0x40, "vpmull", mul, 3138 SSE_INTALU_ITINS_P, HasDQI, 1>, T8PD; 3139defm VPMULHW : avx512_binop_rm_vl_w<0xE5, "vpmulh", mulhs, SSE_INTALU_ITINS_P, 3140 HasBWI, 1>; 3141defm VPMULHUW : avx512_binop_rm_vl_w<0xE4, "vpmulhu", mulhu, SSE_INTMUL_ITINS_P, 3142 HasBWI, 1>; 3143defm VPMULHRSW : avx512_binop_rm_vl_w<0x0B, "vpmulhrs", X86mulhrs, SSE_INTMUL_ITINS_P, 3144 HasBWI, 1>, T8PD; 3145defm VPAVG : avx512_binop_rm_vl_bw<0xE0, 0xE3, "vpavg", X86avg, 3146 SSE_INTALU_ITINS_P, HasBWI, 1>; 3147 3148multiclass avx512_binop_all<bits<8> opc, string OpcodeStr, OpndItins itins, 3149 SDNode OpNode, bit IsCommutable = 0> { 3150 3151 defm NAME#Z : avx512_binop_rm2<opc, OpcodeStr, itins, OpNode, 3152 v16i32_info, v8i64_info, IsCommutable>, 3153 EVEX_V512, EVEX_CD8<64, CD8VF>, VEX_W; 3154 let Predicates = [HasVLX] in { 3155 defm NAME#Z256 : avx512_binop_rm2<opc, OpcodeStr, itins, OpNode, 3156 v8i32x_info, v4i64x_info, IsCommutable>, 3157 EVEX_V256, EVEX_CD8<64, CD8VF>, VEX_W; 3158 defm NAME#Z128 : avx512_binop_rm2<opc, OpcodeStr, itins, OpNode, 3159 v4i32x_info, v2i64x_info, IsCommutable>, 3160 EVEX_V128, EVEX_CD8<64, CD8VF>, VEX_W; 3161 } 3162} 3163 3164defm VPMULDQ : avx512_binop_all<0x28, "vpmuldq", SSE_INTALU_ITINS_P, 3165 X86pmuldq, 1>,T8PD; 3166defm VPMULUDQ : avx512_binop_all<0xF4, "vpmuludq", SSE_INTMUL_ITINS_P, 3167 X86pmuludq, 1>; 3168 3169multiclass avx512_packs_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode, 3170 X86VectorVTInfo _Src, X86VectorVTInfo _Dst> { 3171 let mayLoad = 1 in { 3172 defm rmb : AVX512_maskable<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst), 3173 (ins _Src.RC:$src1, _Src.ScalarMemOp:$src2), 3174 OpcodeStr, 3175 "${src2}"##_Src.BroadcastStr##", $src1", 3176 "$src1, ${src2}"##_Src.BroadcastStr, 3177 (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1), (bitconvert 3178 (_Src.VT (X86VBroadcast 3179 (_Src.ScalarLdFrag addr:$src2))))))>, 3180 EVEX_4V, EVEX_B, EVEX_CD8<_Src.EltSize, CD8VF>; 3181 } 3182} 3183 3184multiclass avx512_packs_rm<bits<8> opc, string OpcodeStr, 3185 SDNode OpNode,X86VectorVTInfo _Src, 3186 X86VectorVTInfo _Dst> { 3187 defm rr : AVX512_maskable<opc, MRMSrcReg, _Dst, (outs _Dst.RC:$dst), 3188 (ins _Src.RC:$src1, _Src.RC:$src2), OpcodeStr, 3189 "$src2, $src1","$src1, $src2", 3190 (_Dst.VT (OpNode 3191 (_Src.VT _Src.RC:$src1), 3192 (_Src.VT _Src.RC:$src2)))>, 3193 EVEX_CD8<_Src.EltSize, CD8VF>, EVEX_4V; 3194 let mayLoad = 1 in { 3195 defm rm : AVX512_maskable<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst), 3196 (ins _Src.RC:$src1, _Src.MemOp:$src2), OpcodeStr, 3197 "$src2, $src1", "$src1, $src2", 3198 (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1), 3199 (bitconvert (_Src.LdFrag addr:$src2))))>, 3200 EVEX_4V, EVEX_CD8<_Src.EltSize, CD8VF>; 3201 } 3202} 3203 3204multiclass avx512_packs_all_i32_i16<bits<8> opc, string OpcodeStr, 3205 SDNode OpNode> { 3206 defm NAME#Z : avx512_packs_rm<opc, OpcodeStr, OpNode, v16i32_info, 3207 v32i16_info>, 3208 avx512_packs_rmb<opc, OpcodeStr, OpNode, v16i32_info, 3209 v32i16_info>, EVEX_V512; 3210 let Predicates = [HasVLX] in { 3211 defm NAME#Z256 : avx512_packs_rm<opc, OpcodeStr, OpNode, v8i32x_info, 3212 v16i16x_info>, 3213 avx512_packs_rmb<opc, OpcodeStr, OpNode, v8i32x_info, 3214 v16i16x_info>, EVEX_V256; 3215 defm NAME#Z128 : avx512_packs_rm<opc, OpcodeStr, OpNode, v4i32x_info, 3216 v8i16x_info>, 3217 avx512_packs_rmb<opc, OpcodeStr, OpNode, v4i32x_info, 3218 v8i16x_info>, EVEX_V128; 3219 } 3220} 3221multiclass avx512_packs_all_i16_i8<bits<8> opc, string OpcodeStr, 3222 SDNode OpNode> { 3223 defm NAME#Z : avx512_packs_rm<opc, OpcodeStr, OpNode, v32i16_info, 3224 v64i8_info>, EVEX_V512; 3225 let Predicates = [HasVLX] in { 3226 defm NAME#Z256 : avx512_packs_rm<opc, OpcodeStr, OpNode, v16i16x_info, 3227 v32i8x_info>, EVEX_V256; 3228 defm NAME#Z128 : avx512_packs_rm<opc, OpcodeStr, OpNode, v8i16x_info, 3229 v16i8x_info>, EVEX_V128; 3230 } 3231} 3232let Predicates = [HasBWI] in { 3233 defm VPACKSSDW : avx512_packs_all_i32_i16<0x6B, "vpackssdw", X86Packss>, PD; 3234 defm VPACKUSDW : avx512_packs_all_i32_i16<0x2b, "vpackusdw", X86Packus>, T8PD; 3235 defm VPACKSSWB : avx512_packs_all_i16_i8 <0x63, "vpacksswb", X86Packss>, AVX512BIBase, VEX_W; 3236 defm VPACKUSWB : avx512_packs_all_i16_i8 <0x67, "vpackuswb", X86Packus>, AVX512BIBase, VEX_W; 3237} 3238 3239defm VPMAXSB : avx512_binop_rm_vl_b<0x3C, "vpmaxs", smax, 3240 SSE_INTALU_ITINS_P, HasBWI, 1>, T8PD; 3241defm VPMAXSW : avx512_binop_rm_vl_w<0xEE, "vpmaxs", smax, 3242 SSE_INTALU_ITINS_P, HasBWI, 1>; 3243defm VPMAXS : avx512_binop_rm_vl_dq<0x3D, 0x3D, "vpmaxs", smax, 3244 SSE_INTALU_ITINS_P, HasAVX512, 1>, T8PD; 3245 3246defm VPMAXUB : avx512_binop_rm_vl_b<0xDE, "vpmaxu", umax, 3247 SSE_INTALU_ITINS_P, HasBWI, 1>; 3248defm VPMAXUW : avx512_binop_rm_vl_w<0x3E, "vpmaxu", umax, 3249 SSE_INTALU_ITINS_P, HasBWI, 1>, T8PD; 3250defm VPMAXU : avx512_binop_rm_vl_dq<0x3F, 0x3F, "vpmaxu", umax, 3251 SSE_INTALU_ITINS_P, HasAVX512, 1>, T8PD; 3252 3253defm VPMINSB : avx512_binop_rm_vl_b<0x38, "vpmins", smin, 3254 SSE_INTALU_ITINS_P, HasBWI, 1>, T8PD; 3255defm VPMINSW : avx512_binop_rm_vl_w<0xEA, "vpmins", smin, 3256 SSE_INTALU_ITINS_P, HasBWI, 1>; 3257defm VPMINS : avx512_binop_rm_vl_dq<0x39, 0x39, "vpmins", smin, 3258 SSE_INTALU_ITINS_P, HasAVX512, 1>, T8PD; 3259 3260defm VPMINUB : avx512_binop_rm_vl_b<0xDA, "vpminu", umin, 3261 SSE_INTALU_ITINS_P, HasBWI, 1>; 3262defm VPMINUW : avx512_binop_rm_vl_w<0x3A, "vpminu", umin, 3263 SSE_INTALU_ITINS_P, HasBWI, 1>, T8PD; 3264defm VPMINU : avx512_binop_rm_vl_dq<0x3B, 0x3B, "vpminu", umin, 3265 SSE_INTALU_ITINS_P, HasAVX512, 1>, T8PD; 3266 3267//===----------------------------------------------------------------------===// 3268// AVX-512 - Unpack Instructions 3269//===----------------------------------------------------------------------===// 3270 3271multiclass avx512_unpack_fp<bits<8> opc, SDNode OpNode, ValueType vt, 3272 PatFrag mem_frag, RegisterClass RC, 3273 X86MemOperand x86memop, string asm, 3274 Domain d> { 3275 def rr : AVX512PI<opc, MRMSrcReg, 3276 (outs RC:$dst), (ins RC:$src1, RC:$src2), 3277 asm, [(set RC:$dst, 3278 (vt (OpNode RC:$src1, RC:$src2)))], 3279 d>, EVEX_4V; 3280 def rm : AVX512PI<opc, MRMSrcMem, 3281 (outs RC:$dst), (ins RC:$src1, x86memop:$src2), 3282 asm, [(set RC:$dst, 3283 (vt (OpNode RC:$src1, 3284 (bitconvert (mem_frag addr:$src2)))))], 3285 d>, EVEX_4V; 3286} 3287 3288defm VUNPCKHPSZ: avx512_unpack_fp<0x15, X86Unpckh, v16f32, loadv8f64, 3289 VR512, f512mem, "vunpckhps\t{$src2, $src1, $dst|$dst, $src1, $src2}", 3290 SSEPackedSingle>, PS, EVEX_V512, EVEX_CD8<32, CD8VF>; 3291defm VUNPCKHPDZ: avx512_unpack_fp<0x15, X86Unpckh, v8f64, loadv8f64, 3292 VR512, f512mem, "vunpckhpd\t{$src2, $src1, $dst|$dst, $src1, $src2}", 3293 SSEPackedDouble>, PD, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; 3294defm VUNPCKLPSZ: avx512_unpack_fp<0x14, X86Unpckl, v16f32, loadv8f64, 3295 VR512, f512mem, "vunpcklps\t{$src2, $src1, $dst|$dst, $src1, $src2}", 3296 SSEPackedSingle>, PS, EVEX_V512, EVEX_CD8<32, CD8VF>; 3297defm VUNPCKLPDZ: avx512_unpack_fp<0x14, X86Unpckl, v8f64, loadv8f64, 3298 VR512, f512mem, "vunpcklpd\t{$src2, $src1, $dst|$dst, $src1, $src2}", 3299 SSEPackedDouble>, PD, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; 3300 3301multiclass avx512_unpack_int<bits<8> opc, string OpcodeStr, SDNode OpNode, 3302 ValueType OpVT, RegisterClass RC, PatFrag memop_frag, 3303 X86MemOperand x86memop> { 3304 def rr : AVX512BI<opc, MRMSrcReg, (outs RC:$dst), 3305 (ins RC:$src1, RC:$src2), 3306 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 3307 [(set RC:$dst, (OpVT (OpNode (OpVT RC:$src1), (OpVT RC:$src2))))], 3308 IIC_SSE_UNPCK>, EVEX_4V; 3309 def rm : AVX512BI<opc, MRMSrcMem, (outs RC:$dst), 3310 (ins RC:$src1, x86memop:$src2), 3311 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 3312 [(set RC:$dst, (OpVT (OpNode (OpVT RC:$src1), 3313 (bitconvert (memop_frag addr:$src2)))))], 3314 IIC_SSE_UNPCK>, EVEX_4V; 3315} 3316defm VPUNPCKLDQZ : avx512_unpack_int<0x62, "vpunpckldq", X86Unpckl, v16i32, 3317 VR512, loadv16i32, i512mem>, EVEX_V512, 3318 EVEX_CD8<32, CD8VF>; 3319defm VPUNPCKLQDQZ : avx512_unpack_int<0x6C, "vpunpcklqdq", X86Unpckl, v8i64, 3320 VR512, loadv8i64, i512mem>, EVEX_V512, 3321 VEX_W, EVEX_CD8<64, CD8VF>; 3322defm VPUNPCKHDQZ : avx512_unpack_int<0x6A, "vpunpckhdq", X86Unpckh, v16i32, 3323 VR512, loadv16i32, i512mem>, EVEX_V512, 3324 EVEX_CD8<32, CD8VF>; 3325defm VPUNPCKHQDQZ : avx512_unpack_int<0x6D, "vpunpckhqdq", X86Unpckh, v8i64, 3326 VR512, loadv8i64, i512mem>, EVEX_V512, 3327 VEX_W, EVEX_CD8<64, CD8VF>; 3328//===----------------------------------------------------------------------===// 3329// AVX-512 Logical Instructions 3330//===----------------------------------------------------------------------===// 3331 3332defm VPAND : avx512_binop_rm_vl_dq<0xDB, 0xDB, "vpand", and, 3333 SSE_INTALU_ITINS_P, HasAVX512, 1>; 3334defm VPOR : avx512_binop_rm_vl_dq<0xEB, 0xEB, "vpor", or, 3335 SSE_INTALU_ITINS_P, HasAVX512, 1>; 3336defm VPXOR : avx512_binop_rm_vl_dq<0xEF, 0xEF, "vpxor", xor, 3337 SSE_INTALU_ITINS_P, HasAVX512, 1>; 3338defm VPANDN : avx512_binop_rm_vl_dq<0xDF, 0xDF, "vpandn", X86andnp, 3339 SSE_INTALU_ITINS_P, HasAVX512, 0>; 3340 3341//===----------------------------------------------------------------------===// 3342// AVX-512 FP arithmetic 3343//===----------------------------------------------------------------------===// 3344multiclass avx512_fp_scalar<bits<8> opc, string OpcodeStr,X86VectorVTInfo _, 3345 SDNode OpNode, SDNode VecNode, OpndItins itins, 3346 bit IsCommutable> { 3347 3348 defm rr_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), 3349 (ins _.RC:$src1, _.RC:$src2), OpcodeStr, 3350 "$src2, $src1", "$src1, $src2", 3351 (VecNode (_.VT _.RC:$src1), (_.VT _.RC:$src2), 3352 (i32 FROUND_CURRENT)), 3353 itins.rr, IsCommutable>; 3354 3355 defm rm_Int : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst), 3356 (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr, 3357 "$src2, $src1", "$src1, $src2", 3358 (VecNode (_.VT _.RC:$src1), 3359 (_.VT (scalar_to_vector (_.ScalarLdFrag addr:$src2))), 3360 (i32 FROUND_CURRENT)), 3361 itins.rm, IsCommutable>; 3362 let isCodeGenOnly = 1, isCommutable = IsCommutable, 3363 Predicates = [HasAVX512] in { 3364 def rr : I< opc, MRMSrcReg, (outs _.FRC:$dst), 3365 (ins _.FRC:$src1, _.FRC:$src2), 3366 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}", 3367 [(set _.FRC:$dst, (OpNode _.FRC:$src1, _.FRC:$src2))], 3368 itins.rr>; 3369 def rm : I< opc, MRMSrcMem, (outs _.FRC:$dst), 3370 (ins _.FRC:$src1, _.ScalarMemOp:$src2), 3371 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}", 3372 [(set _.FRC:$dst, (OpNode _.FRC:$src1, 3373 (_.ScalarLdFrag addr:$src2)))], itins.rr>; 3374 } 3375} 3376 3377multiclass avx512_fp_scalar_round<bits<8> opc, string OpcodeStr,X86VectorVTInfo _, 3378 SDNode VecNode, OpndItins itins, bit IsCommutable> { 3379 3380 defm rrb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), 3381 (ins _.RC:$src1, _.RC:$src2, AVX512RC:$rc), OpcodeStr, 3382 "$rc, $src2, $src1", "$src1, $src2, $rc", 3383 (VecNode (_.VT _.RC:$src1), (_.VT _.RC:$src2), 3384 (i32 imm:$rc)), itins.rr, IsCommutable>, 3385 EVEX_B, EVEX_RC; 3386} 3387multiclass avx512_fp_scalar_sae<bits<8> opc, string OpcodeStr,X86VectorVTInfo _, 3388 SDNode VecNode, OpndItins itins, bit IsCommutable> { 3389 3390 defm rrb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), 3391 (ins _.RC:$src1, _.RC:$src2), OpcodeStr, 3392 "{sae}, $src2, $src1", "$src1, $src2, {sae}", 3393 (VecNode (_.VT _.RC:$src1), (_.VT _.RC:$src2), 3394 (i32 FROUND_NO_EXC))>, EVEX_B; 3395} 3396 3397multiclass avx512_binop_s_round<bits<8> opc, string OpcodeStr, SDNode OpNode, 3398 SDNode VecNode, 3399 SizeItins itins, bit IsCommutable> { 3400 defm SSZ : avx512_fp_scalar<opc, OpcodeStr#"ss", f32x_info, OpNode, VecNode, 3401 itins.s, IsCommutable>, 3402 avx512_fp_scalar_round<opc, OpcodeStr#"ss", f32x_info, VecNode, 3403 itins.s, IsCommutable>, 3404 XS, EVEX_4V, VEX_LIG, EVEX_CD8<32, CD8VT1>; 3405 defm SDZ : avx512_fp_scalar<opc, OpcodeStr#"sd", f64x_info, OpNode, VecNode, 3406 itins.d, IsCommutable>, 3407 avx512_fp_scalar_round<opc, OpcodeStr#"sd", f64x_info, VecNode, 3408 itins.d, IsCommutable>, 3409 XD, VEX_W, EVEX_4V, VEX_LIG, EVEX_CD8<64, CD8VT1>; 3410} 3411 3412multiclass avx512_binop_s_sae<bits<8> opc, string OpcodeStr, SDNode OpNode, 3413 SDNode VecNode, 3414 SizeItins itins, bit IsCommutable> { 3415 defm SSZ : avx512_fp_scalar<opc, OpcodeStr#"ss", f32x_info, OpNode, VecNode, 3416 itins.s, IsCommutable>, 3417 avx512_fp_scalar_sae<opc, OpcodeStr#"ss", f32x_info, VecNode, 3418 itins.s, IsCommutable>, 3419 XS, EVEX_4V, VEX_LIG, EVEX_CD8<32, CD8VT1>; 3420 defm SDZ : avx512_fp_scalar<opc, OpcodeStr#"sd", f64x_info, OpNode, VecNode, 3421 itins.d, IsCommutable>, 3422 avx512_fp_scalar_sae<opc, OpcodeStr#"sd", f64x_info, VecNode, 3423 itins.d, IsCommutable>, 3424 XD, VEX_W, EVEX_4V, VEX_LIG, EVEX_CD8<64, CD8VT1>; 3425} 3426defm VADD : avx512_binop_s_round<0x58, "vadd", fadd, X86faddRnd, SSE_ALU_ITINS_S, 1>; 3427defm VMUL : avx512_binop_s_round<0x59, "vmul", fmul, X86fmulRnd, SSE_ALU_ITINS_S, 1>; 3428defm VSUB : avx512_binop_s_round<0x5C, "vsub", fsub, X86fsubRnd, SSE_ALU_ITINS_S, 0>; 3429defm VDIV : avx512_binop_s_round<0x5E, "vdiv", fdiv, X86fdivRnd, SSE_ALU_ITINS_S, 0>; 3430defm VMIN : avx512_binop_s_sae <0x5D, "vmin", X86fmin, X86fminRnd, SSE_ALU_ITINS_S, 1>; 3431defm VMAX : avx512_binop_s_sae <0x5F, "vmax", X86fmax, X86fmaxRnd, SSE_ALU_ITINS_S, 1>; 3432 3433multiclass avx512_fp_packed<bits<8> opc, string OpcodeStr, SDNode OpNode, 3434 X86VectorVTInfo _, bit IsCommutable> { 3435 defm rr: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), 3436 (ins _.RC:$src1, _.RC:$src2), OpcodeStr##_.Suffix, 3437 "$src2, $src1", "$src1, $src2", 3438 (_.VT (OpNode _.RC:$src1, _.RC:$src2))>, EVEX_4V; 3439 let mayLoad = 1 in { 3440 defm rm: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), 3441 (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr##_.Suffix, 3442 "$src2, $src1", "$src1, $src2", 3443 (OpNode _.RC:$src1, (_.LdFrag addr:$src2))>, EVEX_4V; 3444 defm rmb: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), 3445 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr##_.Suffix, 3446 "${src2}"##_.BroadcastStr##", $src1", 3447 "$src1, ${src2}"##_.BroadcastStr, 3448 (OpNode _.RC:$src1, (_.VT (X86VBroadcast 3449 (_.ScalarLdFrag addr:$src2))))>, 3450 EVEX_4V, EVEX_B; 3451 }//let mayLoad = 1 3452} 3453 3454multiclass avx512_fp_round_packed<bits<8> opc, string OpcodeStr, SDNode OpNodeRnd, 3455 X86VectorVTInfo _> { 3456 defm rb: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), 3457 (ins _.RC:$src1, _.RC:$src2, AVX512RC:$rc), OpcodeStr##_.Suffix, 3458 "$rc, $src2, $src1", "$src1, $src2, $rc", 3459 (_.VT (OpNodeRnd _.RC:$src1, _.RC:$src2, (i32 imm:$rc)))>, 3460 EVEX_4V, EVEX_B, EVEX_RC; 3461} 3462 3463 3464multiclass avx512_fp_sae_packed<bits<8> opc, string OpcodeStr, SDNode OpNodeRnd, 3465 X86VectorVTInfo _> { 3466 defm rb: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), 3467 (ins _.RC:$src1, _.RC:$src2), OpcodeStr##_.Suffix, 3468 "{sae}, $src2, $src1", "$src1, $src2, {sae}", 3469 (_.VT (OpNodeRnd _.RC:$src1, _.RC:$src2, (i32 FROUND_NO_EXC)))>, 3470 EVEX_4V, EVEX_B; 3471} 3472 3473multiclass avx512_fp_binop_p<bits<8> opc, string OpcodeStr, SDNode OpNode, 3474 bit IsCommutable = 0> { 3475 defm PSZ : avx512_fp_packed<opc, OpcodeStr, OpNode, v16f32_info, 3476 IsCommutable>, EVEX_V512, PS, 3477 EVEX_CD8<32, CD8VF>; 3478 defm PDZ : avx512_fp_packed<opc, OpcodeStr, OpNode, v8f64_info, 3479 IsCommutable>, EVEX_V512, PD, VEX_W, 3480 EVEX_CD8<64, CD8VF>; 3481 3482 // Define only if AVX512VL feature is present. 3483 let Predicates = [HasVLX] in { 3484 defm PSZ128 : avx512_fp_packed<opc, OpcodeStr, OpNode, v4f32x_info, 3485 IsCommutable>, EVEX_V128, PS, 3486 EVEX_CD8<32, CD8VF>; 3487 defm PSZ256 : avx512_fp_packed<opc, OpcodeStr, OpNode, v8f32x_info, 3488 IsCommutable>, EVEX_V256, PS, 3489 EVEX_CD8<32, CD8VF>; 3490 defm PDZ128 : avx512_fp_packed<opc, OpcodeStr, OpNode, v2f64x_info, 3491 IsCommutable>, EVEX_V128, PD, VEX_W, 3492 EVEX_CD8<64, CD8VF>; 3493 defm PDZ256 : avx512_fp_packed<opc, OpcodeStr, OpNode, v4f64x_info, 3494 IsCommutable>, EVEX_V256, PD, VEX_W, 3495 EVEX_CD8<64, CD8VF>; 3496 } 3497} 3498 3499multiclass avx512_fp_binop_p_round<bits<8> opc, string OpcodeStr, SDNode OpNodeRnd> { 3500 defm PSZ : avx512_fp_round_packed<opc, OpcodeStr, OpNodeRnd, v16f32_info>, 3501 EVEX_V512, PS, EVEX_CD8<32, CD8VF>; 3502 defm PDZ : avx512_fp_round_packed<opc, OpcodeStr, OpNodeRnd, v8f64_info>, 3503 EVEX_V512, PD, VEX_W,EVEX_CD8<64, CD8VF>; 3504} 3505 3506multiclass avx512_fp_binop_p_sae<bits<8> opc, string OpcodeStr, SDNode OpNodeRnd> { 3507 defm PSZ : avx512_fp_sae_packed<opc, OpcodeStr, OpNodeRnd, v16f32_info>, 3508 EVEX_V512, PS, EVEX_CD8<32, CD8VF>; 3509 defm PDZ : avx512_fp_sae_packed<opc, OpcodeStr, OpNodeRnd, v8f64_info>, 3510 EVEX_V512, PD, VEX_W,EVEX_CD8<64, CD8VF>; 3511} 3512 3513defm VADD : avx512_fp_binop_p<0x58, "vadd", fadd, 1>, 3514 avx512_fp_binop_p_round<0x58, "vadd", X86faddRnd>; 3515defm VMUL : avx512_fp_binop_p<0x59, "vmul", fmul, 1>, 3516 avx512_fp_binop_p_round<0x59, "vmul", X86fmulRnd>; 3517defm VSUB : avx512_fp_binop_p<0x5C, "vsub", fsub>, 3518 avx512_fp_binop_p_round<0x5C, "vsub", X86fsubRnd>; 3519defm VDIV : avx512_fp_binop_p<0x5E, "vdiv", fdiv>, 3520 avx512_fp_binop_p_round<0x5E, "vdiv", X86fdivRnd>; 3521defm VMIN : avx512_fp_binop_p<0x5D, "vmin", X86fmin, 1>, 3522 avx512_fp_binop_p_sae<0x5D, "vmin", X86fminRnd>; 3523defm VMAX : avx512_fp_binop_p<0x5F, "vmax", X86fmax, 1>, 3524 avx512_fp_binop_p_sae<0x5F, "vmax", X86fmaxRnd>; 3525let Predicates = [HasDQI] in { 3526 defm VAND : avx512_fp_binop_p<0x54, "vand", X86fand, 1>; 3527 defm VANDN : avx512_fp_binop_p<0x55, "vandn", X86fandn, 0>; 3528 defm VOR : avx512_fp_binop_p<0x56, "vor", X86for, 1>; 3529 defm VXOR : avx512_fp_binop_p<0x57, "vxor", X86fxor, 1>; 3530} 3531 3532multiclass avx512_fp_scalef_p<bits<8> opc, string OpcodeStr, SDNode OpNode, 3533 X86VectorVTInfo _> { 3534 defm rr: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), 3535 (ins _.RC:$src1, _.RC:$src2), OpcodeStr##_.Suffix, 3536 "$src2, $src1", "$src1, $src2", 3537 (_.VT (OpNode _.RC:$src1, _.RC:$src2, (i32 FROUND_CURRENT)))>, EVEX_4V; 3538 let mayLoad = 1 in { 3539 defm rm: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), 3540 (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr##_.Suffix, 3541 "$src2, $src1", "$src1, $src2", 3542 (OpNode _.RC:$src1, (_.LdFrag addr:$src2), (i32 FROUND_CURRENT))>, EVEX_4V; 3543 defm rmb: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), 3544 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr##_.Suffix, 3545 "${src2}"##_.BroadcastStr##", $src1", 3546 "$src1, ${src2}"##_.BroadcastStr, 3547 (OpNode _.RC:$src1, (_.VT (X86VBroadcast 3548 (_.ScalarLdFrag addr:$src2))), (i32 FROUND_CURRENT))>, 3549 EVEX_4V, EVEX_B; 3550 }//let mayLoad = 1 3551} 3552 3553multiclass avx512_fp_scalef_all<bits<8> opc, string OpcodeStr, SDNode OpNode> { 3554 defm PSZ : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v16f32_info>, 3555 avx512_fp_round_packed<opc, OpcodeStr, OpNode, v16f32_info>, 3556 EVEX_V512, EVEX_CD8<32, CD8VF>; 3557 defm PDZ : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v8f64_info>, 3558 avx512_fp_round_packed<opc, OpcodeStr, OpNode, v8f64_info>, 3559 EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; 3560 // Define only if AVX512VL feature is present. 3561 let Predicates = [HasVLX] in { 3562 defm PSZ128 : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v4f32x_info>, 3563 EVEX_V128, EVEX_CD8<32, CD8VF>; 3564 defm PSZ256 : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v8f32x_info>, 3565 EVEX_V256, EVEX_CD8<32, CD8VF>; 3566 defm PDZ128 : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v2f64x_info>, 3567 EVEX_V128, VEX_W, EVEX_CD8<64, CD8VF>; 3568 defm PDZ256 : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, v4f64x_info>, 3569 EVEX_V256, VEX_W, EVEX_CD8<64, CD8VF>; 3570 } 3571} 3572defm VSCALEF : avx512_fp_scalef_all<0x2C, "vscalef", X86scalef>, T8PD; 3573 3574//===----------------------------------------------------------------------===// 3575// AVX-512 VPTESTM instructions 3576//===----------------------------------------------------------------------===// 3577 3578multiclass avx512_vptest<bits<8> opc, string OpcodeStr, SDNode OpNode, 3579 X86VectorVTInfo _> { 3580 defm rr : AVX512_maskable_cmp<opc, MRMSrcReg, _, (outs _.KRC:$dst), 3581 (ins _.RC:$src1, _.RC:$src2), OpcodeStr, 3582 "$src2, $src1", "$src1, $src2", 3583 (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2))>, 3584 EVEX_4V; 3585 let mayLoad = 1 in 3586 defm rm : AVX512_maskable_cmp<opc, MRMSrcMem, _, (outs _.KRC:$dst), 3587 (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr, 3588 "$src2, $src1", "$src1, $src2", 3589 (OpNode (_.VT _.RC:$src1), 3590 (_.VT (bitconvert (_.LdFrag addr:$src2))))>, 3591 EVEX_4V, 3592 EVEX_CD8<_.EltSize, CD8VF>; 3593} 3594 3595multiclass avx512_vptest_mb<bits<8> opc, string OpcodeStr, SDNode OpNode, 3596 X86VectorVTInfo _> { 3597 let mayLoad = 1 in 3598 defm rmb : AVX512_maskable_cmp<opc, MRMSrcMem, _, (outs _.KRC:$dst), 3599 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr, 3600 "${src2}"##_.BroadcastStr##", $src1", 3601 "$src1, ${src2}"##_.BroadcastStr, 3602 (OpNode (_.VT _.RC:$src1), (_.VT (X86VBroadcast 3603 (_.ScalarLdFrag addr:$src2))))>, 3604 EVEX_B, EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>; 3605} 3606multiclass avx512_vptest_dq_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode, 3607 AVX512VLVectorVTInfo _> { 3608 let Predicates = [HasAVX512] in 3609 defm Z : avx512_vptest<opc, OpcodeStr, OpNode, _.info512>, 3610 avx512_vptest_mb<opc, OpcodeStr, OpNode, _.info512>, EVEX_V512; 3611 3612 let Predicates = [HasAVX512, HasVLX] in { 3613 defm Z256 : avx512_vptest<opc, OpcodeStr, OpNode, _.info256>, 3614 avx512_vptest_mb<opc, OpcodeStr, OpNode, _.info256>, EVEX_V256; 3615 defm Z128 : avx512_vptest<opc, OpcodeStr, OpNode, _.info128>, 3616 avx512_vptest_mb<opc, OpcodeStr, OpNode, _.info128>, EVEX_V128; 3617 } 3618} 3619 3620multiclass avx512_vptest_dq<bits<8> opc, string OpcodeStr, SDNode OpNode> { 3621 defm D : avx512_vptest_dq_sizes<opc, OpcodeStr#"d", OpNode, 3622 avx512vl_i32_info>; 3623 defm Q : avx512_vptest_dq_sizes<opc, OpcodeStr#"q", OpNode, 3624 avx512vl_i64_info>, VEX_W; 3625} 3626 3627multiclass avx512_vptest_wb<bits<8> opc, string OpcodeStr, 3628 SDNode OpNode> { 3629 let Predicates = [HasBWI] in { 3630 defm WZ: avx512_vptest<opc, OpcodeStr#"w", OpNode, v32i16_info>, 3631 EVEX_V512, VEX_W; 3632 defm BZ: avx512_vptest<opc, OpcodeStr#"b", OpNode, v64i8_info>, 3633 EVEX_V512; 3634 } 3635 let Predicates = [HasVLX, HasBWI] in { 3636 3637 defm WZ256: avx512_vptest<opc, OpcodeStr#"w", OpNode, v16i16x_info>, 3638 EVEX_V256, VEX_W; 3639 defm WZ128: avx512_vptest<opc, OpcodeStr#"w", OpNode, v8i16x_info>, 3640 EVEX_V128, VEX_W; 3641 defm BZ256: avx512_vptest<opc, OpcodeStr#"b", OpNode, v32i8x_info>, 3642 EVEX_V256; 3643 defm BZ128: avx512_vptest<opc, OpcodeStr#"b", OpNode, v16i8x_info>, 3644 EVEX_V128; 3645 } 3646} 3647 3648multiclass avx512_vptest_all_forms<bits<8> opc_wb, bits<8> opc_dq, string OpcodeStr, 3649 SDNode OpNode> : 3650 avx512_vptest_wb <opc_wb, OpcodeStr, OpNode>, 3651 avx512_vptest_dq<opc_dq, OpcodeStr, OpNode>; 3652 3653defm VPTESTM : avx512_vptest_all_forms<0x26, 0x27, "vptestm", X86testm>, T8PD; 3654defm VPTESTNM : avx512_vptest_all_forms<0x26, 0x27, "vptestnm", X86testnm>, T8XS; 3655 3656def : Pat <(i16 (int_x86_avx512_mask_ptestm_d_512 (v16i32 VR512:$src1), 3657 (v16i32 VR512:$src2), (i16 -1))), 3658 (COPY_TO_REGCLASS (VPTESTMDZrr VR512:$src1, VR512:$src2), GR16)>; 3659 3660def : Pat <(i8 (int_x86_avx512_mask_ptestm_q_512 (v8i64 VR512:$src1), 3661 (v8i64 VR512:$src2), (i8 -1))), 3662 (COPY_TO_REGCLASS (VPTESTMQZrr VR512:$src1, VR512:$src2), GR8)>; 3663 3664//===----------------------------------------------------------------------===// 3665// AVX-512 Shift instructions 3666//===----------------------------------------------------------------------===// 3667multiclass avx512_shift_rmi<bits<8> opc, Format ImmFormR, Format ImmFormM, 3668 string OpcodeStr, SDNode OpNode, X86VectorVTInfo _> { 3669 defm ri : AVX512_maskable<opc, ImmFormR, _, (outs _.RC:$dst), 3670 (ins _.RC:$src1, u8imm:$src2), OpcodeStr, 3671 "$src2, $src1", "$src1, $src2", 3672 (_.VT (OpNode _.RC:$src1, (i8 imm:$src2))), 3673 SSE_INTSHIFT_ITINS_P.rr>; 3674 let mayLoad = 1 in 3675 defm mi : AVX512_maskable<opc, ImmFormM, _, (outs _.RC:$dst), 3676 (ins _.MemOp:$src1, u8imm:$src2), OpcodeStr, 3677 "$src2, $src1", "$src1, $src2", 3678 (_.VT (OpNode (_.VT (bitconvert (_.LdFrag addr:$src1))), 3679 (i8 imm:$src2))), 3680 SSE_INTSHIFT_ITINS_P.rm>; 3681} 3682 3683multiclass avx512_shift_rmbi<bits<8> opc, Format ImmFormM, 3684 string OpcodeStr, SDNode OpNode, X86VectorVTInfo _> { 3685 let mayLoad = 1 in 3686 defm mbi : AVX512_maskable<opc, ImmFormM, _, (outs _.RC:$dst), 3687 (ins _.ScalarMemOp:$src1, u8imm:$src2), OpcodeStr, 3688 "$src2, ${src1}"##_.BroadcastStr, "${src1}"##_.BroadcastStr##", $src2", 3689 (_.VT (OpNode (X86VBroadcast (_.ScalarLdFrag addr:$src1)), (i8 imm:$src2))), 3690 SSE_INTSHIFT_ITINS_P.rm>, EVEX_B; 3691} 3692 3693multiclass avx512_shift_rrm<bits<8> opc, string OpcodeStr, SDNode OpNode, 3694 ValueType SrcVT, PatFrag bc_frag, X86VectorVTInfo _> { 3695 // src2 is always 128-bit 3696 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), 3697 (ins _.RC:$src1, VR128X:$src2), OpcodeStr, 3698 "$src2, $src1", "$src1, $src2", 3699 (_.VT (OpNode _.RC:$src1, (SrcVT VR128X:$src2))), 3700 SSE_INTSHIFT_ITINS_P.rr>, AVX512BIBase, EVEX_4V; 3701 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), 3702 (ins _.RC:$src1, i128mem:$src2), OpcodeStr, 3703 "$src2, $src1", "$src1, $src2", 3704 (_.VT (OpNode _.RC:$src1, (bc_frag (loadv2i64 addr:$src2)))), 3705 SSE_INTSHIFT_ITINS_P.rm>, AVX512BIBase, 3706 EVEX_4V; 3707} 3708 3709multiclass avx512_shift_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode, 3710 ValueType SrcVT, PatFrag bc_frag, 3711 AVX512VLVectorVTInfo VTInfo, Predicate prd> { 3712 let Predicates = [prd] in 3713 defm Z : avx512_shift_rrm<opc, OpcodeStr, OpNode, SrcVT, bc_frag, 3714 VTInfo.info512>, EVEX_V512, 3715 EVEX_CD8<VTInfo.info512.EltSize, CD8VQ> ; 3716 let Predicates = [prd, HasVLX] in { 3717 defm Z256 : avx512_shift_rrm<opc, OpcodeStr, OpNode, SrcVT, bc_frag, 3718 VTInfo.info256>, EVEX_V256, 3719 EVEX_CD8<VTInfo.info256.EltSize, CD8VH>; 3720 defm Z128 : avx512_shift_rrm<opc, OpcodeStr, OpNode, SrcVT, bc_frag, 3721 VTInfo.info128>, EVEX_V128, 3722 EVEX_CD8<VTInfo.info128.EltSize, CD8VF>; 3723 } 3724} 3725 3726multiclass avx512_shift_types<bits<8> opcd, bits<8> opcq, bits<8> opcw, 3727 string OpcodeStr, SDNode OpNode> { 3728 defm D : avx512_shift_sizes<opcd, OpcodeStr#"d", OpNode, v4i32, bc_v4i32, 3729 avx512vl_i32_info, HasAVX512>; 3730 defm Q : avx512_shift_sizes<opcq, OpcodeStr#"q", OpNode, v2i64, bc_v2i64, 3731 avx512vl_i64_info, HasAVX512>, VEX_W; 3732 defm W : avx512_shift_sizes<opcw, OpcodeStr#"w", OpNode, v8i16, bc_v8i16, 3733 avx512vl_i16_info, HasBWI>; 3734} 3735 3736multiclass avx512_shift_rmi_sizes<bits<8> opc, Format ImmFormR, Format ImmFormM, 3737 string OpcodeStr, SDNode OpNode, 3738 AVX512VLVectorVTInfo VTInfo> { 3739 let Predicates = [HasAVX512] in 3740 defm Z: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode, 3741 VTInfo.info512>, 3742 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode, 3743 VTInfo.info512>, EVEX_V512; 3744 let Predicates = [HasAVX512, HasVLX] in { 3745 defm Z256: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode, 3746 VTInfo.info256>, 3747 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode, 3748 VTInfo.info256>, EVEX_V256; 3749 defm Z128: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode, 3750 VTInfo.info128>, 3751 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode, 3752 VTInfo.info128>, EVEX_V128; 3753 } 3754} 3755 3756multiclass avx512_shift_rmi_w<bits<8> opcw, 3757 Format ImmFormR, Format ImmFormM, 3758 string OpcodeStr, SDNode OpNode> { 3759 let Predicates = [HasBWI] in 3760 defm WZ: avx512_shift_rmi<opcw, ImmFormR, ImmFormM, OpcodeStr, OpNode, 3761 v32i16_info>, EVEX_V512; 3762 let Predicates = [HasVLX, HasBWI] in { 3763 defm WZ256: avx512_shift_rmi<opcw, ImmFormR, ImmFormM, OpcodeStr, OpNode, 3764 v16i16x_info>, EVEX_V256; 3765 defm WZ128: avx512_shift_rmi<opcw, ImmFormR, ImmFormM, OpcodeStr, OpNode, 3766 v8i16x_info>, EVEX_V128; 3767 } 3768} 3769 3770multiclass avx512_shift_rmi_dq<bits<8> opcd, bits<8> opcq, 3771 Format ImmFormR, Format ImmFormM, 3772 string OpcodeStr, SDNode OpNode> { 3773 defm D: avx512_shift_rmi_sizes<opcd, ImmFormR, ImmFormM, OpcodeStr#"d", OpNode, 3774 avx512vl_i32_info>, EVEX_CD8<32, CD8VF>; 3775 defm Q: avx512_shift_rmi_sizes<opcq, ImmFormR, ImmFormM, OpcodeStr#"q", OpNode, 3776 avx512vl_i64_info>, EVEX_CD8<64, CD8VF>, VEX_W; 3777} 3778 3779defm VPSRL : avx512_shift_rmi_dq<0x72, 0x73, MRM2r, MRM2m, "vpsrl", X86vsrli>, 3780 avx512_shift_rmi_w<0x71, MRM2r, MRM2m, "vpsrlw", X86vsrli>, AVX512BIi8Base, EVEX_4V; 3781 3782defm VPSLL : avx512_shift_rmi_dq<0x72, 0x73, MRM6r, MRM6m, "vpsll", X86vshli>, 3783 avx512_shift_rmi_w<0x71, MRM6r, MRM6m, "vpsllw", X86vshli>, AVX512BIi8Base, EVEX_4V; 3784 3785defm VPSRA : avx512_shift_rmi_dq<0x72, 0x72, MRM4r, MRM4m, "vpsra", X86vsrai>, 3786 avx512_shift_rmi_w<0x71, MRM4r, MRM4m, "vpsraw", X86vsrai>, AVX512BIi8Base, EVEX_4V; 3787 3788defm VPROR : avx512_shift_rmi_dq<0x72, 0x72, MRM0r, MRM0m, "vpror", rotr>, AVX512BIi8Base, EVEX_4V; 3789defm VPROL : avx512_shift_rmi_dq<0x72, 0x72, MRM1r, MRM1m, "vprol", rotl>, AVX512BIi8Base, EVEX_4V; 3790 3791defm VPSLL : avx512_shift_types<0xF2, 0xF3, 0xF1, "vpsll", X86vshl>; 3792defm VPSRA : avx512_shift_types<0xE2, 0xE2, 0xE1, "vpsra", X86vsra>; 3793defm VPSRL : avx512_shift_types<0xD2, 0xD3, 0xD1, "vpsrl", X86vsrl>; 3794 3795//===-------------------------------------------------------------------===// 3796// Variable Bit Shifts 3797//===-------------------------------------------------------------------===// 3798multiclass avx512_var_shift<bits<8> opc, string OpcodeStr, SDNode OpNode, 3799 X86VectorVTInfo _> { 3800 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), 3801 (ins _.RC:$src1, _.RC:$src2), OpcodeStr, 3802 "$src2, $src1", "$src1, $src2", 3803 (_.VT (OpNode _.RC:$src1, (_.VT _.RC:$src2))), 3804 SSE_INTSHIFT_ITINS_P.rr>, AVX5128IBase, EVEX_4V; 3805 let mayLoad = 1 in 3806 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), 3807 (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr, 3808 "$src2, $src1", "$src1, $src2", 3809 (_.VT (OpNode _.RC:$src1, 3810 (_.VT (bitconvert (_.LdFrag addr:$src2))))), 3811 SSE_INTSHIFT_ITINS_P.rm>, AVX5128IBase, EVEX_4V, 3812 EVEX_CD8<_.EltSize, CD8VF>; 3813} 3814 3815multiclass avx512_var_shift_mb<bits<8> opc, string OpcodeStr, SDNode OpNode, 3816 X86VectorVTInfo _> { 3817 let mayLoad = 1 in 3818 defm rmb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), 3819 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr, 3820 "${src2}"##_.BroadcastStr##", $src1", 3821 "$src1, ${src2}"##_.BroadcastStr, 3822 (_.VT (OpNode _.RC:$src1, (_.VT (X86VBroadcast 3823 (_.ScalarLdFrag addr:$src2))))), 3824 SSE_INTSHIFT_ITINS_P.rm>, AVX5128IBase, EVEX_B, 3825 EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>; 3826} 3827multiclass avx512_var_shift_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode, 3828 AVX512VLVectorVTInfo _> { 3829 let Predicates = [HasAVX512] in 3830 defm Z : avx512_var_shift<opc, OpcodeStr, OpNode, _.info512>, 3831 avx512_var_shift_mb<opc, OpcodeStr, OpNode, _.info512>, EVEX_V512; 3832 3833 let Predicates = [HasAVX512, HasVLX] in { 3834 defm Z256 : avx512_var_shift<opc, OpcodeStr, OpNode, _.info256>, 3835 avx512_var_shift_mb<opc, OpcodeStr, OpNode, _.info256>, EVEX_V256; 3836 defm Z128 : avx512_var_shift<opc, OpcodeStr, OpNode, _.info128>, 3837 avx512_var_shift_mb<opc, OpcodeStr, OpNode, _.info128>, EVEX_V128; 3838 } 3839} 3840 3841multiclass avx512_var_shift_types<bits<8> opc, string OpcodeStr, 3842 SDNode OpNode> { 3843 defm D : avx512_var_shift_sizes<opc, OpcodeStr#"d", OpNode, 3844 avx512vl_i32_info>; 3845 defm Q : avx512_var_shift_sizes<opc, OpcodeStr#"q", OpNode, 3846 avx512vl_i64_info>, VEX_W; 3847} 3848 3849multiclass avx512_var_shift_w<bits<8> opc, string OpcodeStr, 3850 SDNode OpNode> { 3851 let Predicates = [HasBWI] in 3852 defm WZ: avx512_var_shift<opc, OpcodeStr, OpNode, v32i16_info>, 3853 EVEX_V512, VEX_W; 3854 let Predicates = [HasVLX, HasBWI] in { 3855 3856 defm WZ256: avx512_var_shift<opc, OpcodeStr, OpNode, v16i16x_info>, 3857 EVEX_V256, VEX_W; 3858 defm WZ128: avx512_var_shift<opc, OpcodeStr, OpNode, v8i16x_info>, 3859 EVEX_V128, VEX_W; 3860 } 3861} 3862 3863defm VPSLLV : avx512_var_shift_types<0x47, "vpsllv", shl>, 3864 avx512_var_shift_w<0x12, "vpsllvw", shl>; 3865defm VPSRAV : avx512_var_shift_types<0x46, "vpsrav", sra>, 3866 avx512_var_shift_w<0x11, "vpsravw", sra>; 3867defm VPSRLV : avx512_var_shift_types<0x45, "vpsrlv", srl>, 3868 avx512_var_shift_w<0x10, "vpsrlvw", srl>; 3869defm VPRORV : avx512_var_shift_types<0x14, "vprorv", rotr>; 3870defm VPROLV : avx512_var_shift_types<0x15, "vprolv", rotl>; 3871 3872//===-------------------------------------------------------------------===// 3873// 1-src variable permutation VPERMW/D/Q 3874//===-------------------------------------------------------------------===// 3875multiclass avx512_vperm_dq_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode, 3876 AVX512VLVectorVTInfo _> { 3877 let Predicates = [HasAVX512] in 3878 defm Z : avx512_var_shift<opc, OpcodeStr, OpNode, _.info512>, 3879 avx512_var_shift_mb<opc, OpcodeStr, OpNode, _.info512>, EVEX_V512; 3880 3881 let Predicates = [HasAVX512, HasVLX] in 3882 defm Z256 : avx512_var_shift<opc, OpcodeStr, OpNode, _.info256>, 3883 avx512_var_shift_mb<opc, OpcodeStr, OpNode, _.info256>, EVEX_V256; 3884} 3885 3886multiclass avx512_vpermi_dq_sizes<bits<8> opc, Format ImmFormR, Format ImmFormM, 3887 string OpcodeStr, SDNode OpNode, 3888 AVX512VLVectorVTInfo VTInfo> { 3889 let Predicates = [HasAVX512] in 3890 defm Z: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode, 3891 VTInfo.info512>, 3892 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode, 3893 VTInfo.info512>, EVEX_V512; 3894 let Predicates = [HasAVX512, HasVLX] in 3895 defm Z256: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode, 3896 VTInfo.info256>, 3897 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode, 3898 VTInfo.info256>, EVEX_V256; 3899} 3900 3901 3902defm VPERM : avx512_var_shift_w<0x8D, "vpermw", X86VPermv>; 3903 3904defm VPERMD : avx512_vperm_dq_sizes<0x36, "vpermd", X86VPermv, 3905 avx512vl_i32_info>; 3906defm VPERMQ : avx512_vperm_dq_sizes<0x36, "vpermq", X86VPermv, 3907 avx512vl_i64_info>, VEX_W; 3908defm VPERMPS : avx512_vperm_dq_sizes<0x16, "vpermps", X86VPermv, 3909 avx512vl_f32_info>; 3910defm VPERMPD : avx512_vperm_dq_sizes<0x16, "vpermpd", X86VPermv, 3911 avx512vl_f64_info>, VEX_W; 3912 3913defm VPERMQ : avx512_vpermi_dq_sizes<0x00, MRMSrcReg, MRMSrcMem, "vpermq", 3914 X86VPermi, avx512vl_i64_info>, 3915 EVEX, AVX512AIi8Base, EVEX_CD8<64, CD8VF>, VEX_W; 3916defm VPERMPD : avx512_vpermi_dq_sizes<0x01, MRMSrcReg, MRMSrcMem, "vpermpd", 3917 X86VPermi, avx512vl_f64_info>, 3918 EVEX, AVX512AIi8Base, EVEX_CD8<64, CD8VF>, VEX_W; 3919 3920//===----------------------------------------------------------------------===// 3921// AVX-512 - VPSHUFD, VPSHUFLW, VPSHUFHW 3922//===----------------------------------------------------------------------===// 3923 3924defm VPSHUFD : avx512_shift_rmi_sizes<0x70, MRMSrcReg, MRMSrcMem, "vpshufd", 3925 X86PShufd, avx512vl_i32_info>, 3926 EVEX, AVX512BIi8Base, EVEX_CD8<32, CD8VF>; 3927defm VPSHUFH : avx512_shift_rmi_w<0x70, MRMSrcReg, MRMSrcMem, "vpshufhw", 3928 X86PShufhw>, EVEX, AVX512XSIi8Base, VEX_W; 3929defm VPSHUFL : avx512_shift_rmi_w<0x70, MRMSrcReg, MRMSrcMem, "vpshuflw", 3930 X86PShuflw>, EVEX, AVX512XDIi8Base, VEX_W; 3931 3932multiclass avx512_pshufb_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode> { 3933 let Predicates = [HasBWI] in 3934 defm Z: avx512_var_shift<opc, OpcodeStr, OpNode, v64i8_info>, EVEX_V512; 3935 3936 let Predicates = [HasVLX, HasBWI] in { 3937 defm Z256: avx512_var_shift<opc, OpcodeStr, OpNode, v32i8x_info>, EVEX_V256; 3938 defm Z128: avx512_var_shift<opc, OpcodeStr, OpNode, v16i8x_info>, EVEX_V128; 3939 } 3940} 3941 3942defm VPSHUFB: avx512_pshufb_sizes<0x00, "vpshufb", X86pshufb>; 3943 3944//===----------------------------------------------------------------------===// 3945// AVX-512 - MOVDDUP 3946//===----------------------------------------------------------------------===// 3947 3948multiclass avx512_movddup<string OpcodeStr, RegisterClass RC, ValueType VT, 3949 X86MemOperand x86memop, PatFrag memop_frag> { 3950def rr : AVX512PDI<0x12, MRMSrcReg, (outs RC:$dst), (ins RC:$src), 3951 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 3952 [(set RC:$dst, (VT (X86Movddup RC:$src)))]>, EVEX; 3953def rm : AVX512PDI<0x12, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src), 3954 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 3955 [(set RC:$dst, 3956 (VT (X86Movddup (memop_frag addr:$src))))]>, EVEX; 3957} 3958 3959defm VMOVDDUPZ : avx512_movddup<"vmovddup", VR512, v8f64, f512mem, loadv8f64>, 3960 VEX_W, EVEX_V512, EVEX_CD8<64, CD8VF>; 3961def : Pat<(X86Movddup (v8f64 (scalar_to_vector (loadf64 addr:$src)))), 3962 (VMOVDDUPZrm addr:$src)>; 3963 3964//===---------------------------------------------------------------------===// 3965// Replicate Single FP - MOVSHDUP and MOVSLDUP 3966//===---------------------------------------------------------------------===// 3967multiclass avx512_replicate_sfp<bits<8> op, SDNode OpNode, string OpcodeStr, 3968 ValueType vt, RegisterClass RC, PatFrag mem_frag, 3969 X86MemOperand x86memop> { 3970 def rr : AVX512XSI<op, MRMSrcReg, (outs RC:$dst), (ins RC:$src), 3971 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 3972 [(set RC:$dst, (vt (OpNode RC:$src)))]>, EVEX; 3973 let mayLoad = 1 in 3974 def rm : AVX512XSI<op, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src), 3975 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 3976 [(set RC:$dst, (OpNode (mem_frag addr:$src)))]>, EVEX; 3977} 3978 3979defm VMOVSHDUPZ : avx512_replicate_sfp<0x16, X86Movshdup, "vmovshdup", 3980 v16f32, VR512, loadv16f32, f512mem>, EVEX_V512, 3981 EVEX_CD8<32, CD8VF>; 3982defm VMOVSLDUPZ : avx512_replicate_sfp<0x12, X86Movsldup, "vmovsldup", 3983 v16f32, VR512, loadv16f32, f512mem>, EVEX_V512, 3984 EVEX_CD8<32, CD8VF>; 3985 3986def : Pat<(v16i32 (X86Movshdup VR512:$src)), (VMOVSHDUPZrr VR512:$src)>; 3987def : Pat<(v16i32 (X86Movshdup (loadv16i32 addr:$src))), 3988 (VMOVSHDUPZrm addr:$src)>; 3989def : Pat<(v16i32 (X86Movsldup VR512:$src)), (VMOVSLDUPZrr VR512:$src)>; 3990def : Pat<(v16i32 (X86Movsldup (loadv16i32 addr:$src))), 3991 (VMOVSLDUPZrm addr:$src)>; 3992 3993//===----------------------------------------------------------------------===// 3994// Move Low to High and High to Low packed FP Instructions 3995//===----------------------------------------------------------------------===// 3996def VMOVLHPSZrr : AVX512PSI<0x16, MRMSrcReg, (outs VR128X:$dst), 3997 (ins VR128X:$src1, VR128X:$src2), 3998 "vmovlhps\t{$src2, $src1, $dst|$dst, $src1, $src2}", 3999 [(set VR128X:$dst, (v4f32 (X86Movlhps VR128X:$src1, VR128X:$src2)))], 4000 IIC_SSE_MOV_LH>, EVEX_4V; 4001def VMOVHLPSZrr : AVX512PSI<0x12, MRMSrcReg, (outs VR128X:$dst), 4002 (ins VR128X:$src1, VR128X:$src2), 4003 "vmovhlps\t{$src2, $src1, $dst|$dst, $src1, $src2}", 4004 [(set VR128X:$dst, (v4f32 (X86Movhlps VR128X:$src1, VR128X:$src2)))], 4005 IIC_SSE_MOV_LH>, EVEX_4V; 4006 4007let Predicates = [HasAVX512] in { 4008 // MOVLHPS patterns 4009 def : Pat<(v4i32 (X86Movlhps VR128X:$src1, VR128X:$src2)), 4010 (VMOVLHPSZrr VR128X:$src1, VR128X:$src2)>; 4011 def : Pat<(v2i64 (X86Movlhps VR128X:$src1, VR128X:$src2)), 4012 (VMOVLHPSZrr (v2i64 VR128X:$src1), VR128X:$src2)>; 4013 4014 // MOVHLPS patterns 4015 def : Pat<(v4i32 (X86Movhlps VR128X:$src1, VR128X:$src2)), 4016 (VMOVHLPSZrr VR128X:$src1, VR128X:$src2)>; 4017} 4018 4019//===----------------------------------------------------------------------===// 4020// FMA - Fused Multiply Operations 4021// 4022 4023let Constraints = "$src1 = $dst" in { 4024multiclass avx512_fma3p_213_rm<bits<8> opc, string OpcodeStr, SDNode OpNode, 4025 X86VectorVTInfo _> { 4026 defm r: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst), 4027 (ins _.RC:$src2, _.RC:$src3), 4028 OpcodeStr, "$src3, $src2", "$src2, $src3", 4029 (_.VT (OpNode _.RC:$src1, _.RC:$src2, _.RC:$src3))>, 4030 AVX512FMA3Base; 4031 4032 let mayLoad = 1 in { 4033 defm m: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst), 4034 (ins _.RC:$src2, _.MemOp:$src3), 4035 OpcodeStr, "$src3, $src2", "$src2, $src3", 4036 (_.VT (OpNode _.RC:$src1, _.RC:$src2, (_.LdFrag addr:$src3)))>, 4037 AVX512FMA3Base; 4038 4039 defm mb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst), 4040 (ins _.RC:$src2, _.ScalarMemOp:$src3), 4041 OpcodeStr, !strconcat("${src3}", _.BroadcastStr,", $src2"), 4042 !strconcat("$src2, ${src3}", _.BroadcastStr ), 4043 (OpNode _.RC:$src1, 4044 _.RC:$src2,(_.VT (X86VBroadcast (_.ScalarLdFrag addr:$src3))))>, 4045 AVX512FMA3Base, EVEX_B; 4046 } 4047} 4048 4049multiclass avx512_fma3_213_round<bits<8> opc, string OpcodeStr, SDNode OpNode, 4050 X86VectorVTInfo _> { 4051 defm rb: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst), 4052 (ins _.RC:$src2, _.RC:$src3, AVX512RC:$rc), 4053 OpcodeStr, "$rc, $src3, $src2", "$src2, $src3, $rc", 4054 (_.VT ( OpNode _.RC:$src1, _.RC:$src2, _.RC:$src3, (i32 imm:$rc)))>, 4055 AVX512FMA3Base, EVEX_B, EVEX_RC; 4056} 4057} // Constraints = "$src1 = $dst" 4058 4059multiclass avx512_fma3p_213_common<bits<8> opc, string OpcodeStr, SDNode OpNode, 4060 SDNode OpNodeRnd, AVX512VLVectorVTInfo _> { 4061 let Predicates = [HasAVX512] in { 4062 defm Z : avx512_fma3p_213_rm<opc, OpcodeStr, OpNode, _.info512>, 4063 avx512_fma3_213_round<opc, OpcodeStr, OpNodeRnd, _.info512>, 4064 EVEX_V512, EVEX_CD8<_.info512.EltSize, CD8VF>; 4065 } 4066 let Predicates = [HasVLX, HasAVX512] in { 4067 defm Z256 : avx512_fma3p_213_rm<opc, OpcodeStr, OpNode, _.info256>, 4068 EVEX_V256, EVEX_CD8<_.info256.EltSize, CD8VF>; 4069 defm Z128 : avx512_fma3p_213_rm<opc, OpcodeStr, OpNode, _.info128>, 4070 EVEX_V128, EVEX_CD8<_.info128.EltSize, CD8VF>; 4071 } 4072} 4073 4074multiclass avx512_fma3p_213_f<bits<8> opc, string OpcodeStr, SDNode OpNode, 4075 SDNode OpNodeRnd > { 4076 defm PS : avx512_fma3p_213_common<opc, OpcodeStr#"ps", OpNode, OpNodeRnd, 4077 avx512vl_f32_info>; 4078 defm PD : avx512_fma3p_213_common<opc, OpcodeStr#"pd", OpNode, OpNodeRnd, 4079 avx512vl_f64_info>, VEX_W; 4080} 4081 4082defm VFMADD213 : avx512_fma3p_213_f<0xA8, "vfmadd213", X86Fmadd, X86FmaddRnd>; 4083defm VFMSUB213 : avx512_fma3p_213_f<0xAA, "vfmsub213", X86Fmsub, X86FmsubRnd>; 4084defm VFMADDSUB213 : avx512_fma3p_213_f<0xA6, "vfmaddsub213", X86Fmaddsub, X86FmaddsubRnd>; 4085defm VFMSUBADD213 : avx512_fma3p_213_f<0xA7, "vfmsubadd213", X86Fmsubadd, X86FmsubaddRnd>; 4086defm VFNMADD213 : avx512_fma3p_213_f<0xAC, "vfnmadd213", X86Fnmadd, X86FnmaddRnd>; 4087defm VFNMSUB213 : avx512_fma3p_213_f<0xAE, "vfnmsub213", X86Fnmsub, X86FnmsubRnd>; 4088 4089 4090let Constraints = "$src1 = $dst" in { 4091multiclass avx512_fma3p_231_rm<bits<8> opc, string OpcodeStr, SDNode OpNode, 4092 X86VectorVTInfo _> { 4093 defm r: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst), 4094 (ins _.RC:$src2, _.RC:$src3), 4095 OpcodeStr, "$src3, $src2", "$src2, $src3", 4096 (_.VT (OpNode _.RC:$src2, _.RC:$src3, _.RC:$src1))>, 4097 AVX512FMA3Base; 4098 4099 let mayLoad = 1 in { 4100 defm m: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst), 4101 (ins _.RC:$src2, _.MemOp:$src3), 4102 OpcodeStr, "$src3, $src2", "$src2, $src3", 4103 (_.VT (OpNode _.RC:$src2, (_.LdFrag addr:$src3), _.RC:$src1))>, 4104 AVX512FMA3Base; 4105 4106 defm mb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst), 4107 (ins _.RC:$src2, _.ScalarMemOp:$src3), 4108 OpcodeStr, "${src3}"##_.BroadcastStr##", $src2", 4109 "$src2, ${src3}"##_.BroadcastStr, 4110 (_.VT (OpNode _.RC:$src2, 4111 (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src3))), 4112 _.RC:$src1))>, AVX512FMA3Base, EVEX_B; 4113 } 4114} 4115 4116multiclass avx512_fma3_231_round<bits<8> opc, string OpcodeStr, SDNode OpNode, 4117 X86VectorVTInfo _> { 4118 defm rb: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst), 4119 (ins _.RC:$src2, _.RC:$src3, AVX512RC:$rc), 4120 OpcodeStr, "$rc, $src3, $src2", "$src2, $src3, $rc", 4121 (_.VT ( OpNode _.RC:$src2, _.RC:$src3, _.RC:$src1, (i32 imm:$rc)))>, 4122 AVX512FMA3Base, EVEX_B, EVEX_RC; 4123} 4124} // Constraints = "$src1 = $dst" 4125 4126multiclass avx512_fma3p_231_common<bits<8> opc, string OpcodeStr, SDNode OpNode, 4127 SDNode OpNodeRnd, AVX512VLVectorVTInfo _> { 4128 let Predicates = [HasAVX512] in { 4129 defm Z : avx512_fma3p_231_rm<opc, OpcodeStr, OpNode, _.info512>, 4130 avx512_fma3_231_round<opc, OpcodeStr, OpNodeRnd, _.info512>, 4131 EVEX_V512, EVEX_CD8<_.info512.EltSize, CD8VF>; 4132 } 4133 let Predicates = [HasVLX, HasAVX512] in { 4134 defm Z256 : avx512_fma3p_231_rm<opc, OpcodeStr, OpNode, _.info256>, 4135 EVEX_V256, EVEX_CD8<_.info256.EltSize, CD8VF>; 4136 defm Z128 : avx512_fma3p_231_rm<opc, OpcodeStr, OpNode, _.info128>, 4137 EVEX_V128, EVEX_CD8<_.info128.EltSize, CD8VF>; 4138 } 4139} 4140 4141multiclass avx512_fma3p_231_f<bits<8> opc, string OpcodeStr, SDNode OpNode, 4142 SDNode OpNodeRnd > { 4143 defm PS : avx512_fma3p_231_common<opc, OpcodeStr#"ps", OpNode, OpNodeRnd, 4144 avx512vl_f32_info>; 4145 defm PD : avx512_fma3p_231_common<opc, OpcodeStr#"pd", OpNode, OpNodeRnd, 4146 avx512vl_f64_info>, VEX_W; 4147} 4148 4149defm VFMADD231 : avx512_fma3p_231_f<0xB8, "vfmadd231", X86Fmadd, X86FmaddRnd>; 4150defm VFMSUB231 : avx512_fma3p_231_f<0xBA, "vfmsub231", X86Fmsub, X86FmsubRnd>; 4151defm VFMADDSUB231 : avx512_fma3p_231_f<0xB6, "vfmaddsub231", X86Fmaddsub, X86FmaddsubRnd>; 4152defm VFMSUBADD231 : avx512_fma3p_231_f<0xB7, "vfmsubadd231", X86Fmsubadd, X86FmsubaddRnd>; 4153defm VFNMADD231 : avx512_fma3p_231_f<0xBC, "vfnmadd231", X86Fnmadd, X86FnmaddRnd>; 4154defm VFNMSUB231 : avx512_fma3p_231_f<0xBE, "vfnmsub231", X86Fnmsub, X86FnmsubRnd>; 4155 4156let Constraints = "$src1 = $dst" in { 4157multiclass avx512_fma3p_132_rm<bits<8> opc, string OpcodeStr, SDNode OpNode, 4158 X86VectorVTInfo _> { 4159 defm r: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst), 4160 (ins _.RC:$src3, _.RC:$src2), 4161 OpcodeStr, "$src2, $src3", "$src3, $src2", 4162 (_.VT (OpNode _.RC:$src1, _.RC:$src2, _.RC:$src3))>, 4163 AVX512FMA3Base; 4164 4165 let mayLoad = 1 in { 4166 defm m: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst), 4167 (ins _.RC:$src3, _.MemOp:$src2), 4168 OpcodeStr, "$src2, $src3", "$src3, $src2", 4169 (_.VT (OpNode _.RC:$src1, (_.LdFrag addr:$src2), _.RC:$src3))>, 4170 AVX512FMA3Base; 4171 4172 defm mb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst), 4173 (ins _.RC:$src3, _.ScalarMemOp:$src2), 4174 OpcodeStr, "${src2}"##_.BroadcastStr##", $src3", 4175 "$src3, ${src2}"##_.BroadcastStr, 4176 (_.VT (OpNode _.RC:$src1, 4177 (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src2))), 4178 _.RC:$src3))>, AVX512FMA3Base, EVEX_B; 4179 } 4180} 4181 4182multiclass avx512_fma3_132_round<bits<8> opc, string OpcodeStr, SDNode OpNode, 4183 X86VectorVTInfo _> { 4184 defm rb: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst), 4185 (ins _.RC:$src3, _.RC:$src2, AVX512RC:$rc), 4186 OpcodeStr, "$rc, $src2, $src3", "$src3, $src2, $rc", 4187 (_.VT ( OpNode _.RC:$src1, _.RC:$src2, _.RC:$src3, (i32 imm:$rc)))>, 4188 AVX512FMA3Base, EVEX_B, EVEX_RC; 4189} 4190} // Constraints = "$src1 = $dst" 4191 4192multiclass avx512_fma3p_132_common<bits<8> opc, string OpcodeStr, SDNode OpNode, 4193 SDNode OpNodeRnd, AVX512VLVectorVTInfo _> { 4194 let Predicates = [HasAVX512] in { 4195 defm Z : avx512_fma3p_132_rm<opc, OpcodeStr, OpNode, _.info512>, 4196 avx512_fma3_132_round<opc, OpcodeStr, OpNodeRnd, _.info512>, 4197 EVEX_V512, EVEX_CD8<_.info512.EltSize, CD8VF>; 4198 } 4199 let Predicates = [HasVLX, HasAVX512] in { 4200 defm Z256 : avx512_fma3p_132_rm<opc, OpcodeStr, OpNode, _.info256>, 4201 EVEX_V256, EVEX_CD8<_.info256.EltSize, CD8VF>; 4202 defm Z128 : avx512_fma3p_132_rm<opc, OpcodeStr, OpNode, _.info128>, 4203 EVEX_V128, EVEX_CD8<_.info128.EltSize, CD8VF>; 4204 } 4205} 4206 4207multiclass avx512_fma3p_132_f<bits<8> opc, string OpcodeStr, SDNode OpNode, 4208 SDNode OpNodeRnd > { 4209 defm PS : avx512_fma3p_132_common<opc, OpcodeStr#"ps", OpNode, OpNodeRnd, 4210 avx512vl_f32_info>; 4211 defm PD : avx512_fma3p_132_common<opc, OpcodeStr#"pd", OpNode, OpNodeRnd, 4212 avx512vl_f64_info>, VEX_W; 4213} 4214 4215defm VFMADD132 : avx512_fma3p_132_f<0x98, "vfmadd132", X86Fmadd, X86FmaddRnd>; 4216defm VFMSUB132 : avx512_fma3p_132_f<0x9A, "vfmsub132", X86Fmsub, X86FmsubRnd>; 4217defm VFMADDSUB132 : avx512_fma3p_132_f<0x96, "vfmaddsub132", X86Fmaddsub, X86FmaddsubRnd>; 4218defm VFMSUBADD132 : avx512_fma3p_132_f<0x97, "vfmsubadd132", X86Fmsubadd, X86FmsubaddRnd>; 4219defm VFNMADD132 : avx512_fma3p_132_f<0x9C, "vfnmadd132", X86Fnmadd, X86FnmaddRnd>; 4220defm VFNMSUB132 : avx512_fma3p_132_f<0x9E, "vfnmsub132", X86Fnmsub, X86FnmsubRnd>; 4221 4222// Scalar FMA 4223let Constraints = "$src1 = $dst" in { 4224multiclass avx512_fma3s_common<bits<8> opc, string OpcodeStr, X86VectorVTInfo _, 4225 dag RHS_VEC_r, dag RHS_VEC_m, dag RHS_VEC_rb, 4226 dag RHS_r, dag RHS_m > { 4227 defm r_Int: AVX512_maskable_3src_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), 4228 (ins _.RC:$src2, _.RC:$src3), OpcodeStr, 4229 "$src3, $src2", "$src2, $src3", RHS_VEC_r>, AVX512FMA3Base; 4230 4231 let mayLoad = 1 in 4232 defm m_Int: AVX512_maskable_3src_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst), 4233 (ins _.RC:$src2, _.MemOp:$src3), OpcodeStr, 4234 "$src3, $src2", "$src2, $src3", RHS_VEC_m>, AVX512FMA3Base; 4235 4236 defm rb_Int: AVX512_maskable_3src_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), 4237 (ins _.RC:$src2, _.RC:$src3, AVX512RC:$rc), 4238 OpcodeStr, "$rc, $src3, $src2", "$src2, $src3, $rc", RHS_VEC_rb>, 4239 AVX512FMA3Base, EVEX_B, EVEX_RC; 4240 4241 let isCodeGenOnly = 1 in { 4242 def r : AVX512FMA3<opc, MRMSrcReg, (outs _.FRC:$dst), 4243 (ins _.FRC:$src1, _.FRC:$src2, _.FRC:$src3), 4244 !strconcat(OpcodeStr, 4245 "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 4246 [RHS_r]>; 4247 let mayLoad = 1 in 4248 def m : AVX512FMA3<opc, MRMSrcMem, (outs _.FRC:$dst), 4249 (ins _.FRC:$src1, _.FRC:$src2, _.ScalarMemOp:$src3), 4250 !strconcat(OpcodeStr, 4251 "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 4252 [RHS_m]>; 4253 }// isCodeGenOnly = 1 4254} 4255}// Constraints = "$src1 = $dst" 4256 4257multiclass avx512_fma3s_all<bits<8> opc213, bits<8> opc231, bits<8> opc132, 4258 string OpcodeStr, SDNode OpNode, SDNode OpNodeRnd, X86VectorVTInfo _ , 4259 string SUFF> { 4260 4261 defm NAME#213#SUFF: avx512_fma3s_common<opc213, OpcodeStr#"213"#_.Suffix , _ , 4262 (_.VT (OpNode _.RC:$src2, _.RC:$src1, _.RC:$src3)), 4263 (_.VT (OpNode _.RC:$src2, _.RC:$src1, 4264 (_.VT (scalar_to_vector(_.ScalarLdFrag addr:$src3))))), 4265 (_.VT ( OpNodeRnd _.RC:$src2, _.RC:$src1, _.RC:$src3, 4266 (i32 imm:$rc))), 4267 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src2, _.FRC:$src1, 4268 _.FRC:$src3))), 4269 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src2, _.FRC:$src1, 4270 (_.ScalarLdFrag addr:$src3))))>; 4271 4272 defm NAME#231#SUFF: avx512_fma3s_common<opc231, OpcodeStr#"231"#_.Suffix , _ , 4273 (_.VT (OpNode _.RC:$src2, _.RC:$src3, _.RC:$src1)), 4274 (_.VT (OpNode _.RC:$src2, 4275 (_.VT (scalar_to_vector(_.ScalarLdFrag addr:$src3))), 4276 _.RC:$src1)), 4277 (_.VT ( OpNodeRnd _.RC:$src2, _.RC:$src3, _.RC:$src1, 4278 (i32 imm:$rc))), 4279 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src2, _.FRC:$src3, 4280 _.FRC:$src1))), 4281 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src2, 4282 (_.ScalarLdFrag addr:$src3), _.FRC:$src1)))>; 4283 4284 defm NAME#132#SUFF: avx512_fma3s_common<opc132, OpcodeStr#"132"#_.Suffix , _ , 4285 (_.VT (OpNode _.RC:$src1, _.RC:$src3, _.RC:$src2)), 4286 (_.VT (OpNode _.RC:$src1, 4287 (_.VT (scalar_to_vector(_.ScalarLdFrag addr:$src3))), 4288 _.RC:$src2)), 4289 (_.VT ( OpNodeRnd _.RC:$src1, _.RC:$src3, _.RC:$src2, 4290 (i32 imm:$rc))), 4291 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src1, _.FRC:$src3, 4292 _.FRC:$src2))), 4293 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src1, 4294 (_.ScalarLdFrag addr:$src3), _.FRC:$src2)))>; 4295} 4296 4297multiclass avx512_fma3s<bits<8> opc213, bits<8> opc231, bits<8> opc132, 4298 string OpcodeStr, SDNode OpNode, SDNode OpNodeRnd>{ 4299 let Predicates = [HasAVX512] in { 4300 defm NAME : avx512_fma3s_all<opc213, opc231, opc132, OpcodeStr, OpNode, 4301 OpNodeRnd, f32x_info, "SS">, 4302 EVEX_CD8<32, CD8VT1>, VEX_LIG; 4303 defm NAME : avx512_fma3s_all<opc213, opc231, opc132, OpcodeStr, OpNode, 4304 OpNodeRnd, f64x_info, "SD">, 4305 EVEX_CD8<64, CD8VT1>, VEX_LIG, VEX_W; 4306 } 4307} 4308 4309defm VFMADD : avx512_fma3s<0xA9, 0xB9, 0x99, "vfmadd", X86Fmadd, X86FmaddRnd>; 4310defm VFMSUB : avx512_fma3s<0xAB, 0xBB, 0x9B, "vfmsub", X86Fmsub, X86FmsubRnd>; 4311defm VFNMADD : avx512_fma3s<0xAD, 0xBD, 0x9D, "vfnmadd", X86Fnmadd, X86FnmaddRnd>; 4312defm VFNMSUB : avx512_fma3s<0xAF, 0xBF, 0x9F, "vfnmsub", X86Fnmsub, X86FnmsubRnd>; 4313 4314//===----------------------------------------------------------------------===// 4315// AVX-512 Scalar convert from sign integer to float/double 4316//===----------------------------------------------------------------------===// 4317 4318multiclass avx512_vcvtsi<bits<8> opc, SDNode OpNode, RegisterClass SrcRC, 4319 X86VectorVTInfo DstVT, X86MemOperand x86memop, 4320 PatFrag ld_frag, string asm> { 4321 let hasSideEffects = 0 in { 4322 def rr : SI<opc, MRMSrcReg, (outs DstVT.FRC:$dst), 4323 (ins DstVT.FRC:$src1, SrcRC:$src), 4324 !strconcat(asm,"\t{$src, $src1, $dst|$dst, $src1, $src}"), []>, 4325 EVEX_4V; 4326 let mayLoad = 1 in 4327 def rm : SI<opc, MRMSrcMem, (outs DstVT.FRC:$dst), 4328 (ins DstVT.FRC:$src1, x86memop:$src), 4329 !strconcat(asm,"\t{$src, $src1, $dst|$dst, $src1, $src}"), []>, 4330 EVEX_4V; 4331 } // hasSideEffects = 0 4332 let isCodeGenOnly = 1 in { 4333 def rr_Int : SI<opc, MRMSrcReg, (outs DstVT.RC:$dst), 4334 (ins DstVT.RC:$src1, SrcRC:$src2), 4335 !strconcat(asm,"\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 4336 [(set DstVT.RC:$dst, 4337 (OpNode (DstVT.VT DstVT.RC:$src1), 4338 SrcRC:$src2, 4339 (i32 FROUND_CURRENT)))]>, EVEX_4V; 4340 4341 def rm_Int : SI<opc, MRMSrcMem, (outs DstVT.RC:$dst), 4342 (ins DstVT.RC:$src1, x86memop:$src2), 4343 !strconcat(asm,"\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 4344 [(set DstVT.RC:$dst, 4345 (OpNode (DstVT.VT DstVT.RC:$src1), 4346 (ld_frag addr:$src2), 4347 (i32 FROUND_CURRENT)))]>, EVEX_4V; 4348 }//isCodeGenOnly = 1 4349} 4350 4351multiclass avx512_vcvtsi_round<bits<8> opc, SDNode OpNode, RegisterClass SrcRC, 4352 X86VectorVTInfo DstVT, string asm> { 4353 def rrb_Int : SI<opc, MRMSrcReg, (outs DstVT.RC:$dst), 4354 (ins DstVT.RC:$src1, SrcRC:$src2, AVX512RC:$rc), 4355 !strconcat(asm, 4356 "\t{$src2, $rc, $src1, $dst|$dst, $src1, $rc, $src2}"), 4357 [(set DstVT.RC:$dst, 4358 (OpNode (DstVT.VT DstVT.RC:$src1), 4359 SrcRC:$src2, 4360 (i32 imm:$rc)))]>, EVEX_4V, EVEX_B, EVEX_RC; 4361} 4362 4363multiclass avx512_vcvtsi_common<bits<8> opc, SDNode OpNode, RegisterClass SrcRC, 4364 X86VectorVTInfo DstVT, X86MemOperand x86memop, 4365 PatFrag ld_frag, string asm> { 4366 defm NAME : avx512_vcvtsi_round<opc, OpNode, SrcRC, DstVT, asm>, 4367 avx512_vcvtsi<opc, OpNode, SrcRC, DstVT, x86memop, ld_frag, asm>, 4368 VEX_LIG; 4369} 4370 4371let Predicates = [HasAVX512] in { 4372defm VCVTSI2SSZ : avx512_vcvtsi_common<0x2A, X86SintToFpRnd, GR32, 4373 v4f32x_info, i32mem, loadi32, "cvtsi2ss{l}">, 4374 XS, EVEX_CD8<32, CD8VT1>; 4375defm VCVTSI642SSZ: avx512_vcvtsi_common<0x2A, X86SintToFpRnd, GR64, 4376 v4f32x_info, i64mem, loadi64, "cvtsi2ss{q}">, 4377 XS, VEX_W, EVEX_CD8<64, CD8VT1>; 4378defm VCVTSI2SDZ : avx512_vcvtsi_common<0x2A, X86SintToFpRnd, GR32, 4379 v2f64x_info, i32mem, loadi32, "cvtsi2sd{l}">, 4380 XD, EVEX_CD8<32, CD8VT1>; 4381defm VCVTSI642SDZ: avx512_vcvtsi_common<0x2A, X86SintToFpRnd, GR64, 4382 v2f64x_info, i64mem, loadi64, "cvtsi2sd{q}">, 4383 XD, VEX_W, EVEX_CD8<64, CD8VT1>; 4384 4385def : Pat<(f32 (sint_to_fp (loadi32 addr:$src))), 4386 (VCVTSI2SSZrm (f32 (IMPLICIT_DEF)), addr:$src)>; 4387def : Pat<(f32 (sint_to_fp (loadi64 addr:$src))), 4388 (VCVTSI642SSZrm (f32 (IMPLICIT_DEF)), addr:$src)>; 4389def : Pat<(f64 (sint_to_fp (loadi32 addr:$src))), 4390 (VCVTSI2SDZrm (f64 (IMPLICIT_DEF)), addr:$src)>; 4391def : Pat<(f64 (sint_to_fp (loadi64 addr:$src))), 4392 (VCVTSI642SDZrm (f64 (IMPLICIT_DEF)), addr:$src)>; 4393 4394def : Pat<(f32 (sint_to_fp GR32:$src)), 4395 (VCVTSI2SSZrr (f32 (IMPLICIT_DEF)), GR32:$src)>; 4396def : Pat<(f32 (sint_to_fp GR64:$src)), 4397 (VCVTSI642SSZrr (f32 (IMPLICIT_DEF)), GR64:$src)>; 4398def : Pat<(f64 (sint_to_fp GR32:$src)), 4399 (VCVTSI2SDZrr (f64 (IMPLICIT_DEF)), GR32:$src)>; 4400def : Pat<(f64 (sint_to_fp GR64:$src)), 4401 (VCVTSI642SDZrr (f64 (IMPLICIT_DEF)), GR64:$src)>; 4402 4403defm VCVTUSI2SSZ : avx512_vcvtsi_common<0x7B, X86UintToFpRnd, GR32, 4404 v4f32x_info, i32mem, loadi32, 4405 "cvtusi2ss{l}">, XS, EVEX_CD8<32, CD8VT1>; 4406defm VCVTUSI642SSZ : avx512_vcvtsi_common<0x7B, X86UintToFpRnd, GR64, 4407 v4f32x_info, i64mem, loadi64, "cvtusi2ss{q}">, 4408 XS, VEX_W, EVEX_CD8<64, CD8VT1>; 4409defm VCVTUSI2SDZ : avx512_vcvtsi<0x7B, X86UintToFpRnd, GR32, v2f64x_info, 4410 i32mem, loadi32, "cvtusi2sd{l}">, 4411 XD, VEX_LIG, EVEX_CD8<32, CD8VT1>; 4412defm VCVTUSI642SDZ : avx512_vcvtsi_common<0x7B, X86UintToFpRnd, GR64, 4413 v2f64x_info, i64mem, loadi64, "cvtusi2sd{q}">, 4414 XD, VEX_W, EVEX_CD8<64, CD8VT1>; 4415 4416def : Pat<(f32 (uint_to_fp (loadi32 addr:$src))), 4417 (VCVTUSI2SSZrm (f32 (IMPLICIT_DEF)), addr:$src)>; 4418def : Pat<(f32 (uint_to_fp (loadi64 addr:$src))), 4419 (VCVTUSI642SSZrm (f32 (IMPLICIT_DEF)), addr:$src)>; 4420def : Pat<(f64 (uint_to_fp (loadi32 addr:$src))), 4421 (VCVTUSI2SDZrm (f64 (IMPLICIT_DEF)), addr:$src)>; 4422def : Pat<(f64 (uint_to_fp (loadi64 addr:$src))), 4423 (VCVTUSI642SDZrm (f64 (IMPLICIT_DEF)), addr:$src)>; 4424 4425def : Pat<(f32 (uint_to_fp GR32:$src)), 4426 (VCVTUSI2SSZrr (f32 (IMPLICIT_DEF)), GR32:$src)>; 4427def : Pat<(f32 (uint_to_fp GR64:$src)), 4428 (VCVTUSI642SSZrr (f32 (IMPLICIT_DEF)), GR64:$src)>; 4429def : Pat<(f64 (uint_to_fp GR32:$src)), 4430 (VCVTUSI2SDZrr (f64 (IMPLICIT_DEF)), GR32:$src)>; 4431def : Pat<(f64 (uint_to_fp GR64:$src)), 4432 (VCVTUSI642SDZrr (f64 (IMPLICIT_DEF)), GR64:$src)>; 4433} 4434 4435//===----------------------------------------------------------------------===// 4436// AVX-512 Scalar convert from float/double to integer 4437//===----------------------------------------------------------------------===// 4438multiclass avx512_cvt_s_int<bits<8> opc, RegisterClass SrcRC, RegisterClass DstRC, 4439 Intrinsic Int, Operand memop, ComplexPattern mem_cpat, 4440 string asm> { 4441let hasSideEffects = 0 in { 4442 def rr : SI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src), 4443 !strconcat(asm,"\t{$src, $dst|$dst, $src}"), 4444 [(set DstRC:$dst, (Int SrcRC:$src))]>, EVEX, VEX_LIG, 4445 Requires<[HasAVX512]>; 4446 let mayLoad = 1 in 4447 def rm : SI<opc, MRMSrcMem, (outs DstRC:$dst), (ins memop:$src), 4448 !strconcat(asm,"\t{$src, $dst|$dst, $src}"), []>, EVEX, VEX_LIG, 4449 Requires<[HasAVX512]>; 4450} // hasSideEffects = 0 4451} 4452let Predicates = [HasAVX512] in { 4453// Convert float/double to signed/unsigned int 32/64 4454defm VCVTSS2SIZ: avx512_cvt_s_int<0x2D, VR128X, GR32, int_x86_sse_cvtss2si, 4455 ssmem, sse_load_f32, "cvtss2si">, 4456 XS, EVEX_CD8<32, CD8VT1>; 4457defm VCVTSS2SI64Z: avx512_cvt_s_int<0x2D, VR128X, GR64, int_x86_sse_cvtss2si64, 4458 ssmem, sse_load_f32, "cvtss2si">, 4459 XS, VEX_W, EVEX_CD8<32, CD8VT1>; 4460defm VCVTSS2USIZ: avx512_cvt_s_int<0x79, VR128X, GR32, int_x86_avx512_cvtss2usi, 4461 ssmem, sse_load_f32, "cvtss2usi">, 4462 XS, EVEX_CD8<32, CD8VT1>; 4463defm VCVTSS2USI64Z: avx512_cvt_s_int<0x79, VR128X, GR64, 4464 int_x86_avx512_cvtss2usi64, ssmem, 4465 sse_load_f32, "cvtss2usi">, XS, VEX_W, 4466 EVEX_CD8<32, CD8VT1>; 4467defm VCVTSD2SIZ: avx512_cvt_s_int<0x2D, VR128X, GR32, int_x86_sse2_cvtsd2si, 4468 sdmem, sse_load_f64, "cvtsd2si">, 4469 XD, EVEX_CD8<64, CD8VT1>; 4470defm VCVTSD2SI64Z: avx512_cvt_s_int<0x2D, VR128X, GR64, int_x86_sse2_cvtsd2si64, 4471 sdmem, sse_load_f64, "cvtsd2si">, 4472 XD, VEX_W, EVEX_CD8<64, CD8VT1>; 4473defm VCVTSD2USIZ: avx512_cvt_s_int<0x79, VR128X, GR32, int_x86_avx512_cvtsd2usi, 4474 sdmem, sse_load_f64, "cvtsd2usi">, 4475 XD, EVEX_CD8<64, CD8VT1>; 4476defm VCVTSD2USI64Z: avx512_cvt_s_int<0x79, VR128X, GR64, 4477 int_x86_avx512_cvtsd2usi64, sdmem, 4478 sse_load_f64, "cvtsd2usi">, XD, VEX_W, 4479 EVEX_CD8<64, CD8VT1>; 4480 4481let isCodeGenOnly = 1 in { 4482 defm Int_VCVTSI2SSZ : sse12_cvt_sint_3addr<0x2A, GR32, VR128X, 4483 int_x86_sse_cvtsi2ss, i32mem, loadi32, "cvtsi2ss{l}", 4484 SSE_CVT_Scalar, 0>, XS, EVEX_4V; 4485 defm Int_VCVTSI2SS64Z : sse12_cvt_sint_3addr<0x2A, GR64, VR128X, 4486 int_x86_sse_cvtsi642ss, i64mem, loadi64, "cvtsi2ss{q}", 4487 SSE_CVT_Scalar, 0>, XS, EVEX_4V, VEX_W; 4488 defm Int_VCVTSI2SDZ : sse12_cvt_sint_3addr<0x2A, GR32, VR128X, 4489 int_x86_sse2_cvtsi2sd, i32mem, loadi32, "cvtsi2sd{l}", 4490 SSE_CVT_Scalar, 0>, XD, EVEX_4V; 4491 defm Int_VCVTSI2SD64Z : sse12_cvt_sint_3addr<0x2A, GR64, VR128X, 4492 int_x86_sse2_cvtsi642sd, i64mem, loadi64, "cvtsi2sd{q}", 4493 SSE_CVT_Scalar, 0>, XD, EVEX_4V, VEX_W; 4494 4495 defm Int_VCVTUSI2SDZ : sse12_cvt_sint_3addr<0x2A, GR32, VR128X, 4496 int_x86_avx512_cvtusi2sd, i32mem, loadi32, "cvtusi2sd{l}", 4497 SSE_CVT_Scalar, 0>, XD, EVEX_4V; 4498} // isCodeGenOnly = 1 4499 4500// Convert float/double to signed/unsigned int 32/64 with truncation 4501let isCodeGenOnly = 1 in { 4502 defm Int_VCVTTSS2SIZ : avx512_cvt_s_int<0x2C, VR128X, GR32, int_x86_sse_cvttss2si, 4503 ssmem, sse_load_f32, "cvttss2si">, 4504 XS, EVEX_CD8<32, CD8VT1>; 4505 defm Int_VCVTTSS2SI64Z : avx512_cvt_s_int<0x2C, VR128X, GR64, 4506 int_x86_sse_cvttss2si64, ssmem, sse_load_f32, 4507 "cvttss2si">, XS, VEX_W, 4508 EVEX_CD8<32, CD8VT1>; 4509 defm Int_VCVTTSD2SIZ : avx512_cvt_s_int<0x2C, VR128X, GR32, int_x86_sse2_cvttsd2si, 4510 sdmem, sse_load_f64, "cvttsd2si">, XD, 4511 EVEX_CD8<64, CD8VT1>; 4512 defm Int_VCVTTSD2SI64Z : avx512_cvt_s_int<0x2C, VR128X, GR64, 4513 int_x86_sse2_cvttsd2si64, sdmem, sse_load_f64, 4514 "cvttsd2si">, XD, VEX_W, 4515 EVEX_CD8<64, CD8VT1>; 4516 defm Int_VCVTTSS2USIZ : avx512_cvt_s_int<0x78, VR128X, GR32, 4517 int_x86_avx512_cvttss2usi, ssmem, sse_load_f32, 4518 "cvttss2usi">, XS, EVEX_CD8<32, CD8VT1>; 4519 defm Int_VCVTTSS2USI64Z : avx512_cvt_s_int<0x78, VR128X, GR64, 4520 int_x86_avx512_cvttss2usi64, ssmem, 4521 sse_load_f32, "cvttss2usi">, XS, VEX_W, 4522 EVEX_CD8<32, CD8VT1>; 4523 defm Int_VCVTTSD2USIZ : avx512_cvt_s_int<0x78, VR128X, GR32, 4524 int_x86_avx512_cvttsd2usi, 4525 sdmem, sse_load_f64, "cvttsd2usi">, XD, 4526 EVEX_CD8<64, CD8VT1>; 4527 defm Int_VCVTTSD2USI64Z : avx512_cvt_s_int<0x78, VR128X, GR64, 4528 int_x86_avx512_cvttsd2usi64, sdmem, 4529 sse_load_f64, "cvttsd2usi">, XD, VEX_W, 4530 EVEX_CD8<64, CD8VT1>; 4531} // isCodeGenOnly = 1 4532 4533multiclass avx512_cvt_s<bits<8> opc, RegisterClass SrcRC, RegisterClass DstRC, 4534 SDNode OpNode, X86MemOperand x86memop, PatFrag ld_frag, 4535 string asm> { 4536 def rr : SI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src), 4537 !strconcat(asm,"\t{$src, $dst|$dst, $src}"), 4538 [(set DstRC:$dst, (OpNode SrcRC:$src))]>, EVEX; 4539 def rm : SI<opc, MRMSrcMem, (outs DstRC:$dst), (ins x86memop:$src), 4540 !strconcat(asm,"\t{$src, $dst|$dst, $src}"), 4541 [(set DstRC:$dst, (OpNode (ld_frag addr:$src)))]>, EVEX; 4542} 4543 4544defm VCVTTSS2SIZ : avx512_cvt_s<0x2C, FR32X, GR32, fp_to_sint, f32mem, 4545 loadf32, "cvttss2si">, XS, 4546 EVEX_CD8<32, CD8VT1>; 4547defm VCVTTSS2USIZ : avx512_cvt_s<0x78, FR32X, GR32, fp_to_uint, f32mem, 4548 loadf32, "cvttss2usi">, XS, 4549 EVEX_CD8<32, CD8VT1>; 4550defm VCVTTSS2SI64Z : avx512_cvt_s<0x2C, FR32X, GR64, fp_to_sint, f32mem, 4551 loadf32, "cvttss2si">, XS, VEX_W, 4552 EVEX_CD8<32, CD8VT1>; 4553defm VCVTTSS2USI64Z : avx512_cvt_s<0x78, FR32X, GR64, fp_to_uint, f32mem, 4554 loadf32, "cvttss2usi">, XS, VEX_W, 4555 EVEX_CD8<32, CD8VT1>; 4556defm VCVTTSD2SIZ : avx512_cvt_s<0x2C, FR64X, GR32, fp_to_sint, f64mem, 4557 loadf64, "cvttsd2si">, XD, 4558 EVEX_CD8<64, CD8VT1>; 4559defm VCVTTSD2USIZ : avx512_cvt_s<0x78, FR64X, GR32, fp_to_uint, f64mem, 4560 loadf64, "cvttsd2usi">, XD, 4561 EVEX_CD8<64, CD8VT1>; 4562defm VCVTTSD2SI64Z : avx512_cvt_s<0x2C, FR64X, GR64, fp_to_sint, f64mem, 4563 loadf64, "cvttsd2si">, XD, VEX_W, 4564 EVEX_CD8<64, CD8VT1>; 4565defm VCVTTSD2USI64Z : avx512_cvt_s<0x78, FR64X, GR64, fp_to_uint, f64mem, 4566 loadf64, "cvttsd2usi">, XD, VEX_W, 4567 EVEX_CD8<64, CD8VT1>; 4568} // HasAVX512 4569//===----------------------------------------------------------------------===// 4570// AVX-512 Convert form float to double and back 4571//===----------------------------------------------------------------------===// 4572let hasSideEffects = 0 in { 4573def VCVTSS2SDZrr : AVX512XSI<0x5A, MRMSrcReg, (outs FR64X:$dst), 4574 (ins FR32X:$src1, FR32X:$src2), 4575 "vcvtss2sd\t{$src2, $src1, $dst|$dst, $src1, $src2}", 4576 []>, EVEX_4V, VEX_LIG, Sched<[WriteCvtF2F]>; 4577let mayLoad = 1 in 4578def VCVTSS2SDZrm : AVX512XSI<0x5A, MRMSrcMem, (outs FR64X:$dst), 4579 (ins FR32X:$src1, f32mem:$src2), 4580 "vcvtss2sd\t{$src2, $src1, $dst|$dst, $src1, $src2}", 4581 []>, EVEX_4V, VEX_LIG, Sched<[WriteCvtF2FLd, ReadAfterLd]>, 4582 EVEX_CD8<32, CD8VT1>; 4583 4584// Convert scalar double to scalar single 4585def VCVTSD2SSZrr : AVX512XDI<0x5A, MRMSrcReg, (outs FR32X:$dst), 4586 (ins FR64X:$src1, FR64X:$src2), 4587 "vcvtsd2ss\t{$src2, $src1, $dst|$dst, $src1, $src2}", 4588 []>, EVEX_4V, VEX_LIG, VEX_W, Sched<[WriteCvtF2F]>; 4589let mayLoad = 1 in 4590def VCVTSD2SSZrm : AVX512XDI<0x5A, MRMSrcMem, (outs FR32X:$dst), 4591 (ins FR64X:$src1, f64mem:$src2), 4592 "vcvtsd2ss\t{$src2, $src1, $dst|$dst, $src1, $src2}", 4593 []>, EVEX_4V, VEX_LIG, VEX_W, 4594 Sched<[WriteCvtF2FLd, ReadAfterLd]>, EVEX_CD8<64, CD8VT1>; 4595} 4596 4597def : Pat<(f64 (fextend FR32X:$src)), (VCVTSS2SDZrr FR32X:$src, FR32X:$src)>, 4598 Requires<[HasAVX512]>; 4599def : Pat<(fextend (loadf32 addr:$src)), 4600 (VCVTSS2SDZrm (f32 (IMPLICIT_DEF)), addr:$src)>, Requires<[HasAVX512]>; 4601 4602def : Pat<(extloadf32 addr:$src), 4603 (VCVTSS2SDZrm (f32 (IMPLICIT_DEF)), addr:$src)>, 4604 Requires<[HasAVX512, OptForSize]>; 4605 4606def : Pat<(extloadf32 addr:$src), 4607 (VCVTSS2SDZrr (f32 (IMPLICIT_DEF)), (VMOVSSZrm addr:$src))>, 4608 Requires<[HasAVX512, OptForSpeed]>; 4609 4610def : Pat<(f32 (fround FR64X:$src)), (VCVTSD2SSZrr FR64X:$src, FR64X:$src)>, 4611 Requires<[HasAVX512]>; 4612 4613//===----------------------------------------------------------------------===// 4614// AVX-512 Vector convert from signed/unsigned integer to float/double 4615// and from float/double to signed/unsigned integer 4616//===----------------------------------------------------------------------===// 4617 4618multiclass avx512_vcvt_fp<bits<8> opc, string OpcodeStr, X86VectorVTInfo _, 4619 X86VectorVTInfo _Src, SDNode OpNode, 4620 string Broadcast = _.BroadcastStr, 4621 string Alias = ""> { 4622 4623 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), 4624 (ins _Src.RC:$src), OpcodeStr, "$src", "$src", 4625 (_.VT (OpNode (_Src.VT _Src.RC:$src)))>, EVEX; 4626 4627 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), 4628 (ins _Src.MemOp:$src), OpcodeStr#Alias, "$src", "$src", 4629 (_.VT (OpNode (_Src.VT 4630 (bitconvert (_Src.LdFrag addr:$src)))))>, EVEX; 4631 4632 defm rmb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), 4633 (ins _Src.MemOp:$src), OpcodeStr, 4634 "${src}"##Broadcast, "${src}"##Broadcast, 4635 (_.VT (OpNode (_Src.VT 4636 (X86VBroadcast (_Src.ScalarLdFrag addr:$src))) 4637 ))>, EVEX, EVEX_B; 4638} 4639// Coversion with SAE - suppress all exceptions 4640multiclass avx512_vcvt_fp_sae<bits<8> opc, string OpcodeStr, X86VectorVTInfo _, 4641 X86VectorVTInfo _Src, SDNode OpNodeRnd> { 4642 defm rrb : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), 4643 (ins _Src.RC:$src), OpcodeStr, 4644 "{sae}, $src", "$src, {sae}", 4645 (_.VT (OpNodeRnd (_Src.VT _Src.RC:$src), 4646 (i32 FROUND_NO_EXC)))>, 4647 EVEX, EVEX_B; 4648} 4649 4650// Conversion with rounding control (RC) 4651multiclass avx512_vcvt_fp_rc<bits<8> opc, string OpcodeStr, X86VectorVTInfo _, 4652 X86VectorVTInfo _Src, SDNode OpNodeRnd> { 4653 defm rrb : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), 4654 (ins _Src.RC:$src, AVX512RC:$rc), OpcodeStr, 4655 "$rc, $src", "$src, $rc", 4656 (_.VT (OpNodeRnd (_Src.VT _Src.RC:$src), (i32 imm:$rc)))>, 4657 EVEX, EVEX_B, EVEX_RC; 4658} 4659 4660// Extend Float to Double 4661multiclass avx512_cvtps2pd<bits<8> opc, string OpcodeStr> { 4662 let Predicates = [HasAVX512] in { 4663 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f64_info, v8f32x_info, fextend>, 4664 avx512_vcvt_fp_sae<opc, OpcodeStr, v8f64_info, v8f32x_info, 4665 X86vfpextRnd>, EVEX_V512; 4666 } 4667 let Predicates = [HasVLX] in { 4668 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2f64x_info, v4f32x_info, 4669 X86vfpext, "{1to2}">, EVEX_V128; 4670 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f64x_info, v4f32x_info, fextend>, 4671 EVEX_V256; 4672 } 4673} 4674 4675// Truncate Double to Float 4676multiclass avx512_cvtpd2ps<bits<8> opc, string OpcodeStr> { 4677 let Predicates = [HasAVX512] in { 4678 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f32x_info, v8f64_info, fround>, 4679 avx512_vcvt_fp_rc<opc, OpcodeStr, v8f32x_info, v8f64_info, 4680 X86vfproundRnd>, EVEX_V512; 4681 } 4682 let Predicates = [HasVLX] in { 4683 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v2f64x_info, 4684 X86vfpround, "{1to2}", "{x}">, EVEX_V128; 4685 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v4f64x_info, fround, 4686 "{1to4}", "{y}">, EVEX_V256; 4687 } 4688} 4689 4690defm VCVTPD2PS : avx512_cvtpd2ps<0x5A, "vcvtpd2ps">, 4691 VEX_W, PD, EVEX_CD8<64, CD8VF>; 4692defm VCVTPS2PD : avx512_cvtps2pd<0x5A, "vcvtps2pd">, 4693 PS, EVEX_CD8<32, CD8VH>; 4694 4695def : Pat<(v8f64 (extloadv8f32 addr:$src)), 4696 (VCVTPS2PDZrm addr:$src)>; 4697 4698let Predicates = [HasVLX] in { 4699 def : Pat<(v4f64 (extloadv4f32 addr:$src)), 4700 (VCVTPS2PDZ256rm addr:$src)>; 4701} 4702 4703// Convert Signed/Unsigned Doubleword to Double 4704multiclass avx512_cvtdq2pd<bits<8> opc, string OpcodeStr, SDNode OpNode, 4705 SDNode OpNode128> { 4706 // No rounding in this op 4707 let Predicates = [HasAVX512] in 4708 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f64_info, v8i32x_info, OpNode>, 4709 EVEX_V512; 4710 4711 let Predicates = [HasVLX] in { 4712 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2f64x_info, v4i32x_info, 4713 OpNode128, "{1to2}">, EVEX_V128; 4714 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f64x_info, v4i32x_info, OpNode>, 4715 EVEX_V256; 4716 } 4717} 4718 4719// Convert Signed/Unsigned Doubleword to Float 4720multiclass avx512_cvtdq2ps<bits<8> opc, string OpcodeStr, SDNode OpNode, 4721 SDNode OpNodeRnd> { 4722 let Predicates = [HasAVX512] in 4723 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v16f32_info, v16i32_info, OpNode>, 4724 avx512_vcvt_fp_rc<opc, OpcodeStr, v16f32_info, v16i32_info, 4725 OpNodeRnd>, EVEX_V512; 4726 4727 let Predicates = [HasVLX] in { 4728 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v4i32x_info, OpNode>, 4729 EVEX_V128; 4730 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v8f32x_info, v8i32x_info, OpNode>, 4731 EVEX_V256; 4732 } 4733} 4734 4735// Convert Float to Signed/Unsigned Doubleword with truncation 4736multiclass avx512_cvttps2dq<bits<8> opc, string OpcodeStr, 4737 SDNode OpNode, SDNode OpNodeRnd> { 4738 let Predicates = [HasAVX512] in { 4739 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v16i32_info, v16f32_info, OpNode>, 4740 avx512_vcvt_fp_sae<opc, OpcodeStr, v16i32_info, v16f32_info, 4741 OpNodeRnd>, EVEX_V512; 4742 } 4743 let Predicates = [HasVLX] in { 4744 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f32x_info, OpNode>, 4745 EVEX_V128; 4746 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f32x_info, OpNode>, 4747 EVEX_V256; 4748 } 4749} 4750 4751// Convert Float to Signed/Unsigned Doubleword 4752multiclass avx512_cvtps2dq<bits<8> opc, string OpcodeStr, 4753 SDNode OpNode, SDNode OpNodeRnd> { 4754 let Predicates = [HasAVX512] in { 4755 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v16i32_info, v16f32_info, OpNode>, 4756 avx512_vcvt_fp_rc<opc, OpcodeStr, v16i32_info, v16f32_info, 4757 OpNodeRnd>, EVEX_V512; 4758 } 4759 let Predicates = [HasVLX] in { 4760 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f32x_info, OpNode>, 4761 EVEX_V128; 4762 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f32x_info, OpNode>, 4763 EVEX_V256; 4764 } 4765} 4766 4767// Convert Double to Signed/Unsigned Doubleword with truncation 4768multiclass avx512_cvttpd2dq<bits<8> opc, string OpcodeStr, 4769 SDNode OpNode, SDNode OpNodeRnd> { 4770 let Predicates = [HasAVX512] in { 4771 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f64_info, OpNode>, 4772 avx512_vcvt_fp_sae<opc, OpcodeStr, v8i32x_info, v8f64_info, 4773 OpNodeRnd>, EVEX_V512; 4774 } 4775 let Predicates = [HasVLX] in { 4776 // we need "x"/"y" suffixes in order to distinguish between 128 and 256 4777 // memory forms of these instructions in Asm Parcer. They have the same 4778 // dest type - 'v4i32x_info'. We also specify the broadcast string explicitly 4779 // due to the same reason. 4780 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v2f64x_info, OpNode, 4781 "{1to2}", "{x}">, EVEX_V128; 4782 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f64x_info, OpNode, 4783 "{1to4}", "{y}">, EVEX_V256; 4784 } 4785} 4786 4787// Convert Double to Signed/Unsigned Doubleword 4788multiclass avx512_cvtpd2dq<bits<8> opc, string OpcodeStr, 4789 SDNode OpNode, SDNode OpNodeRnd> { 4790 let Predicates = [HasAVX512] in { 4791 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f64_info, OpNode>, 4792 avx512_vcvt_fp_rc<opc, OpcodeStr, v8i32x_info, v8f64_info, 4793 OpNodeRnd>, EVEX_V512; 4794 } 4795 let Predicates = [HasVLX] in { 4796 // we need "x"/"y" suffixes in order to distinguish between 128 and 256 4797 // memory forms of these instructions in Asm Parcer. They have the same 4798 // dest type - 'v4i32x_info'. We also specify the broadcast string explicitly 4799 // due to the same reason. 4800 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v2f64x_info, OpNode, 4801 "{1to2}", "{x}">, EVEX_V128; 4802 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f64x_info, OpNode, 4803 "{1to4}", "{y}">, EVEX_V256; 4804 } 4805} 4806 4807// Convert Double to Signed/Unsigned Quardword 4808multiclass avx512_cvtpd2qq<bits<8> opc, string OpcodeStr, 4809 SDNode OpNode, SDNode OpNodeRnd> { 4810 let Predicates = [HasDQI] in { 4811 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f64_info, OpNode>, 4812 avx512_vcvt_fp_rc<opc, OpcodeStr, v8i64_info, v8f64_info, 4813 OpNodeRnd>, EVEX_V512; 4814 } 4815 let Predicates = [HasDQI, HasVLX] in { 4816 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v2f64x_info, OpNode>, 4817 EVEX_V128; 4818 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f64x_info, OpNode>, 4819 EVEX_V256; 4820 } 4821} 4822 4823// Convert Double to Signed/Unsigned Quardword with truncation 4824multiclass avx512_cvttpd2qq<bits<8> opc, string OpcodeStr, 4825 SDNode OpNode, SDNode OpNodeRnd> { 4826 let Predicates = [HasDQI] in { 4827 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f64_info, OpNode>, 4828 avx512_vcvt_fp_sae<opc, OpcodeStr, v8i64_info, v8f64_info, 4829 OpNodeRnd>, EVEX_V512; 4830 } 4831 let Predicates = [HasDQI, HasVLX] in { 4832 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v2f64x_info, OpNode>, 4833 EVEX_V128; 4834 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f64x_info, OpNode>, 4835 EVEX_V256; 4836 } 4837} 4838 4839// Convert Signed/Unsigned Quardword to Double 4840multiclass avx512_cvtqq2pd<bits<8> opc, string OpcodeStr, 4841 SDNode OpNode, SDNode OpNodeRnd> { 4842 let Predicates = [HasDQI] in { 4843 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f64_info, v8i64_info, OpNode>, 4844 avx512_vcvt_fp_rc<opc, OpcodeStr, v8f64_info, v8i64_info, 4845 OpNodeRnd>, EVEX_V512; 4846 } 4847 let Predicates = [HasDQI, HasVLX] in { 4848 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2f64x_info, v2i64x_info, OpNode>, 4849 EVEX_V128; 4850 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f64x_info, v4i64x_info, OpNode>, 4851 EVEX_V256; 4852 } 4853} 4854 4855// Convert Float to Signed/Unsigned Quardword 4856multiclass avx512_cvtps2qq<bits<8> opc, string OpcodeStr, 4857 SDNode OpNode, SDNode OpNodeRnd> { 4858 let Predicates = [HasDQI] in { 4859 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f32x_info, OpNode>, 4860 avx512_vcvt_fp_rc<opc, OpcodeStr, v8i64_info, v8f32x_info, 4861 OpNodeRnd>, EVEX_V512; 4862 } 4863 let Predicates = [HasDQI, HasVLX] in { 4864 // Explicitly specified broadcast string, since we take only 2 elements 4865 // from v4f32x_info source 4866 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v4f32x_info, OpNode, 4867 "{1to2}">, EVEX_V128; 4868 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f32x_info, OpNode>, 4869 EVEX_V256; 4870 } 4871} 4872 4873// Convert Float to Signed/Unsigned Quardword with truncation 4874multiclass avx512_cvttps2qq<bits<8> opc, string OpcodeStr, 4875 SDNode OpNode, SDNode OpNodeRnd> { 4876 let Predicates = [HasDQI] in { 4877 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f32x_info, OpNode>, 4878 avx512_vcvt_fp_sae<opc, OpcodeStr, v8i64_info, v8f32x_info, 4879 OpNodeRnd>, EVEX_V512; 4880 } 4881 let Predicates = [HasDQI, HasVLX] in { 4882 // Explicitly specified broadcast string, since we take only 2 elements 4883 // from v4f32x_info source 4884 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v4f32x_info, OpNode, 4885 "{1to2}">, EVEX_V128; 4886 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f32x_info, OpNode>, 4887 EVEX_V256; 4888 } 4889} 4890 4891// Convert Signed/Unsigned Quardword to Float 4892multiclass avx512_cvtqq2ps<bits<8> opc, string OpcodeStr, 4893 SDNode OpNode, SDNode OpNodeRnd> { 4894 let Predicates = [HasDQI] in { 4895 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f32x_info, v8i64_info, OpNode>, 4896 avx512_vcvt_fp_rc<opc, OpcodeStr, v8f32x_info, v8i64_info, 4897 OpNodeRnd>, EVEX_V512; 4898 } 4899 let Predicates = [HasDQI, HasVLX] in { 4900 // we need "x"/"y" suffixes in order to distinguish between 128 and 256 4901 // memory forms of these instructions in Asm Parcer. They have the same 4902 // dest type - 'v4i32x_info'. We also specify the broadcast string explicitly 4903 // due to the same reason. 4904 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v2i64x_info, OpNode, 4905 "{1to2}", "{x}">, EVEX_V128; 4906 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v4i64x_info, OpNode, 4907 "{1to4}", "{y}">, EVEX_V256; 4908 } 4909} 4910 4911defm VCVTDQ2PD : avx512_cvtdq2pd<0xE6, "vcvtdq2pd", sint_to_fp, X86cvtdq2pd>, XS, 4912 EVEX_CD8<32, CD8VH>; 4913 4914defm VCVTDQ2PS : avx512_cvtdq2ps<0x5B, "vcvtdq2ps", sint_to_fp, 4915 X86VSintToFpRnd>, 4916 PS, EVEX_CD8<32, CD8VF>; 4917 4918defm VCVTTPS2DQ : avx512_cvttps2dq<0x5B, "vcvttps2dq", fp_to_sint, 4919 X86VFpToSintRnd>, 4920 XS, EVEX_CD8<32, CD8VF>; 4921 4922defm VCVTTPD2DQ : avx512_cvttpd2dq<0xE6, "vcvttpd2dq", fp_to_sint, 4923 X86VFpToSintRnd>, 4924 PD, VEX_W, EVEX_CD8<64, CD8VF>; 4925 4926defm VCVTTPS2UDQ : avx512_cvttps2dq<0x78, "vcvttps2udq", fp_to_uint, 4927 X86VFpToUintRnd>, PS, 4928 EVEX_CD8<32, CD8VF>; 4929 4930defm VCVTTPD2UDQ : avx512_cvttpd2dq<0x78, "vcvttpd2udq", fp_to_uint, 4931 X86VFpToUintRnd>, PS, VEX_W, 4932 EVEX_CD8<64, CD8VF>; 4933 4934defm VCVTUDQ2PD : avx512_cvtdq2pd<0x7A, "vcvtudq2pd", uint_to_fp, X86cvtudq2pd>, 4935 XS, EVEX_CD8<32, CD8VH>; 4936 4937defm VCVTUDQ2PS : avx512_cvtdq2ps<0x7A, "vcvtudq2ps", uint_to_fp, 4938 X86VUintToFpRnd>, XD, 4939 EVEX_CD8<32, CD8VF>; 4940 4941defm VCVTPS2DQ : avx512_cvtps2dq<0x5B, "vcvtps2dq", X86cvtps2Int, 4942 X86cvtps2IntRnd>, PD, EVEX_CD8<32, CD8VF>; 4943 4944defm VCVTPD2DQ : avx512_cvtpd2dq<0xE6, "vcvtpd2dq", X86cvtpd2Int, 4945 X86cvtpd2IntRnd>, XD, VEX_W, 4946 EVEX_CD8<64, CD8VF>; 4947 4948defm VCVTPS2UDQ : avx512_cvtps2dq<0x79, "vcvtps2udq", X86cvtps2UInt, 4949 X86cvtps2UIntRnd>, 4950 PS, EVEX_CD8<32, CD8VF>; 4951defm VCVTPD2UDQ : avx512_cvtpd2dq<0x79, "vcvtpd2udq", X86cvtpd2UInt, 4952 X86cvtpd2UIntRnd>, VEX_W, 4953 PS, EVEX_CD8<64, CD8VF>; 4954 4955defm VCVTPD2QQ : avx512_cvtpd2qq<0x7B, "vcvtpd2qq", X86cvtpd2Int, 4956 X86cvtpd2IntRnd>, VEX_W, 4957 PD, EVEX_CD8<64, CD8VF>; 4958 4959defm VCVTPS2QQ : avx512_cvtps2qq<0x7B, "vcvtps2qq", X86cvtps2Int, 4960 X86cvtps2IntRnd>, PD, EVEX_CD8<32, CD8VH>; 4961 4962defm VCVTPD2UQQ : avx512_cvtpd2qq<0x79, "vcvtpd2uqq", X86cvtpd2UInt, 4963 X86cvtpd2UIntRnd>, VEX_W, 4964 PD, EVEX_CD8<64, CD8VF>; 4965 4966defm VCVTPS2UQQ : avx512_cvtps2qq<0x79, "vcvtps2uqq", X86cvtps2UInt, 4967 X86cvtps2UIntRnd>, PD, EVEX_CD8<32, CD8VH>; 4968 4969defm VCVTTPD2QQ : avx512_cvttpd2qq<0x7A, "vcvttpd2qq", fp_to_sint, 4970 X86VFpToSlongRnd>, VEX_W, 4971 PD, EVEX_CD8<64, CD8VF>; 4972 4973defm VCVTTPS2QQ : avx512_cvttps2qq<0x7A, "vcvttps2qq", fp_to_sint, 4974 X86VFpToSlongRnd>, PD, EVEX_CD8<32, CD8VH>; 4975 4976defm VCVTTPD2UQQ : avx512_cvttpd2qq<0x78, "vcvttpd2uqq", fp_to_uint, 4977 X86VFpToUlongRnd>, VEX_W, 4978 PD, EVEX_CD8<64, CD8VF>; 4979 4980defm VCVTTPS2UQQ : avx512_cvttps2qq<0x78, "vcvttps2uqq", fp_to_uint, 4981 X86VFpToUlongRnd>, PD, EVEX_CD8<32, CD8VH>; 4982 4983defm VCVTQQ2PD : avx512_cvtqq2pd<0xE6, "vcvtqq2pd", sint_to_fp, 4984 X86VSlongToFpRnd>, VEX_W, XS, EVEX_CD8<64, CD8VF>; 4985 4986defm VCVTUQQ2PD : avx512_cvtqq2pd<0x7A, "vcvtuqq2pd", uint_to_fp, 4987 X86VUlongToFpRnd>, VEX_W, XS, EVEX_CD8<64, CD8VF>; 4988 4989defm VCVTQQ2PS : avx512_cvtqq2ps<0x5B, "vcvtqq2ps", sint_to_fp, 4990 X86VSlongToFpRnd>, VEX_W, PS, EVEX_CD8<64, CD8VF>; 4991 4992defm VCVTUQQ2PS : avx512_cvtqq2ps<0x7A, "vcvtuqq2ps", uint_to_fp, 4993 X86VUlongToFpRnd>, VEX_W, XD, EVEX_CD8<64, CD8VF>; 4994 4995let Predicates = [NoVLX] in { 4996def : Pat<(v8i32 (fp_to_uint (v8f32 VR256X:$src1))), 4997 (EXTRACT_SUBREG (v16i32 (VCVTTPS2UDQZrr 4998 (v16f32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)))), sub_ymm)>; 4999 5000def : Pat<(v4i32 (fp_to_uint (v4f32 VR128X:$src1))), 5001 (EXTRACT_SUBREG (v16i32 (VCVTTPS2UDQZrr 5002 (v16f32 (SUBREG_TO_REG (i32 0), VR128X:$src1, sub_xmm)))), sub_xmm)>; 5003 5004def : Pat<(v8f32 (uint_to_fp (v8i32 VR256X:$src1))), 5005 (EXTRACT_SUBREG (v16f32 (VCVTUDQ2PSZrr 5006 (v16i32 (SUBREG_TO_REG (i32 0), VR256X:$src1, sub_ymm)))), sub_ymm)>; 5007 5008def : Pat<(v4f32 (uint_to_fp (v4i32 VR128X:$src1))), 5009 (EXTRACT_SUBREG (v16f32 (VCVTUDQ2PSZrr 5010 (v16i32 (SUBREG_TO_REG (i32 0), VR128X:$src1, sub_xmm)))), sub_xmm)>; 5011 5012def : Pat<(v4f64 (uint_to_fp (v4i32 VR128X:$src1))), 5013 (EXTRACT_SUBREG (v8f64 (VCVTUDQ2PDZrr 5014 (v8i32 (SUBREG_TO_REG (i32 0), VR128X:$src1, sub_xmm)))), sub_ymm)>; 5015} 5016 5017let Predicates = [HasAVX512] in { 5018 def : Pat<(v8f32 (fround (loadv8f64 addr:$src))), 5019 (VCVTPD2PSZrm addr:$src)>; 5020 def : Pat<(v8f64 (extloadv8f32 addr:$src)), 5021 (VCVTPS2PDZrm addr:$src)>; 5022} 5023 5024//===----------------------------------------------------------------------===// 5025// Half precision conversion instructions 5026//===----------------------------------------------------------------------===// 5027multiclass avx512_cvtph2ps<RegisterClass destRC, RegisterClass srcRC, 5028 X86MemOperand x86memop> { 5029 def rr : AVX5128I<0x13, MRMSrcReg, (outs destRC:$dst), (ins srcRC:$src), 5030 "vcvtph2ps\t{$src, $dst|$dst, $src}", 5031 []>, EVEX; 5032 let hasSideEffects = 0, mayLoad = 1 in 5033 def rm : AVX5128I<0x13, MRMSrcMem, (outs destRC:$dst), (ins x86memop:$src), 5034 "vcvtph2ps\t{$src, $dst|$dst, $src}", []>, EVEX; 5035} 5036 5037multiclass avx512_cvtps2ph<RegisterClass destRC, RegisterClass srcRC, 5038 X86MemOperand x86memop> { 5039 def rr : AVX512AIi8<0x1D, MRMDestReg, (outs destRC:$dst), 5040 (ins srcRC:$src1, i32u8imm:$src2), 5041 "vcvtps2ph\t{$src2, $src1, $dst|$dst, $src1, $src2}", 5042 []>, EVEX; 5043 let hasSideEffects = 0, mayStore = 1 in 5044 def mr : AVX512AIi8<0x1D, MRMDestMem, (outs), 5045 (ins x86memop:$dst, srcRC:$src1, i32u8imm:$src2), 5046 "vcvtps2ph\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>, EVEX; 5047} 5048 5049defm VCVTPH2PSZ : avx512_cvtph2ps<VR512, VR256X, f256mem>, EVEX_V512, 5050 EVEX_CD8<32, CD8VH>; 5051defm VCVTPS2PHZ : avx512_cvtps2ph<VR256X, VR512, f256mem>, EVEX_V512, 5052 EVEX_CD8<32, CD8VH>; 5053 5054def : Pat<(v16i16 (int_x86_avx512_mask_vcvtps2ph_512 (v16f32 VR512:$src), 5055 imm:$rc, (bc_v16i16(v8i32 immAllZerosV)), (i16 -1))), 5056 (VCVTPS2PHZrr VR512:$src, imm:$rc)>; 5057 5058def : Pat<(v16f32 (int_x86_avx512_mask_vcvtph2ps_512 (v16i16 VR256X:$src), 5059 (bc_v16f32(v16i32 immAllZerosV)), (i16 -1), (i32 FROUND_CURRENT))), 5060 (VCVTPH2PSZrr VR256X:$src)>; 5061 5062let Defs = [EFLAGS], Predicates = [HasAVX512] in { 5063 defm VUCOMISSZ : sse12_ord_cmp<0x2E, FR32X, X86cmp, f32, f32mem, loadf32, 5064 "ucomiss">, PS, EVEX, VEX_LIG, 5065 EVEX_CD8<32, CD8VT1>; 5066 defm VUCOMISDZ : sse12_ord_cmp<0x2E, FR64X, X86cmp, f64, f64mem, loadf64, 5067 "ucomisd">, PD, EVEX, 5068 VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>; 5069 let Pattern = []<dag> in { 5070 defm VCOMISSZ : sse12_ord_cmp<0x2F, VR128X, undef, v4f32, f128mem, load, 5071 "comiss">, PS, EVEX, VEX_LIG, 5072 EVEX_CD8<32, CD8VT1>; 5073 defm VCOMISDZ : sse12_ord_cmp<0x2F, VR128X, undef, v2f64, f128mem, load, 5074 "comisd">, PD, EVEX, 5075 VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>; 5076 } 5077 let isCodeGenOnly = 1 in { 5078 defm Int_VUCOMISSZ : sse12_ord_cmp<0x2E, VR128X, X86ucomi, v4f32, f128mem, 5079 load, "ucomiss">, PS, EVEX, VEX_LIG, 5080 EVEX_CD8<32, CD8VT1>; 5081 defm Int_VUCOMISDZ : sse12_ord_cmp<0x2E, VR128X, X86ucomi, v2f64, f128mem, 5082 load, "ucomisd">, PD, EVEX, 5083 VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>; 5084 5085 defm Int_VCOMISSZ : sse12_ord_cmp<0x2F, VR128X, X86comi, v4f32, f128mem, 5086 load, "comiss">, PS, EVEX, VEX_LIG, 5087 EVEX_CD8<32, CD8VT1>; 5088 defm Int_VCOMISDZ : sse12_ord_cmp<0x2F, VR128X, X86comi, v2f64, f128mem, 5089 load, "comisd">, PD, EVEX, 5090 VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>; 5091 } 5092} 5093 5094/// avx512_fp14_s rcp14ss, rcp14sd, rsqrt14ss, rsqrt14sd 5095multiclass avx512_fp14_s<bits<8> opc, string OpcodeStr, RegisterClass RC, 5096 X86MemOperand x86memop> { 5097 let hasSideEffects = 0 in { 5098 def rr : AVX5128I<opc, MRMSrcReg, (outs RC:$dst), 5099 (ins RC:$src1, RC:$src2), 5100 !strconcat(OpcodeStr, 5101 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>, EVEX_4V; 5102 let mayLoad = 1 in { 5103 def rm : AVX5128I<opc, MRMSrcMem, (outs RC:$dst), 5104 (ins RC:$src1, x86memop:$src2), 5105 !strconcat(OpcodeStr, 5106 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>, EVEX_4V; 5107 } 5108} 5109} 5110 5111defm VRCP14SS : avx512_fp14_s<0x4D, "vrcp14ss", FR32X, f32mem>, 5112 EVEX_CD8<32, CD8VT1>; 5113defm VRCP14SD : avx512_fp14_s<0x4D, "vrcp14sd", FR64X, f64mem>, 5114 VEX_W, EVEX_CD8<64, CD8VT1>; 5115defm VRSQRT14SS : avx512_fp14_s<0x4F, "vrsqrt14ss", FR32X, f32mem>, 5116 EVEX_CD8<32, CD8VT1>; 5117defm VRSQRT14SD : avx512_fp14_s<0x4F, "vrsqrt14sd", FR64X, f64mem>, 5118 VEX_W, EVEX_CD8<64, CD8VT1>; 5119 5120def : Pat <(v4f32 (int_x86_avx512_rcp14_ss (v4f32 VR128X:$src1), 5121 (v4f32 VR128X:$src2), (bc_v4f32 (v4i32 immAllZerosV)), (i8 -1))), 5122 (COPY_TO_REGCLASS (VRCP14SSrr (COPY_TO_REGCLASS VR128X:$src1, FR32X), 5123 (COPY_TO_REGCLASS VR128X:$src2, FR32X)), VR128X)>; 5124 5125def : Pat <(v2f64 (int_x86_avx512_rcp14_sd (v2f64 VR128X:$src1), 5126 (v2f64 VR128X:$src2), (bc_v2f64 (v4i32 immAllZerosV)), (i8 -1))), 5127 (COPY_TO_REGCLASS (VRCP14SDrr (COPY_TO_REGCLASS VR128X:$src1, FR64X), 5128 (COPY_TO_REGCLASS VR128X:$src2, FR64X)), VR128X)>; 5129 5130def : Pat <(v4f32 (int_x86_avx512_rsqrt14_ss (v4f32 VR128X:$src1), 5131 (v4f32 VR128X:$src2), (bc_v4f32 (v4i32 immAllZerosV)), (i8 -1))), 5132 (COPY_TO_REGCLASS (VRSQRT14SSrr (COPY_TO_REGCLASS VR128X:$src1, FR32X), 5133 (COPY_TO_REGCLASS VR128X:$src2, FR32X)), VR128X)>; 5134 5135def : Pat <(v2f64 (int_x86_avx512_rsqrt14_sd (v2f64 VR128X:$src1), 5136 (v2f64 VR128X:$src2), (bc_v2f64 (v4i32 immAllZerosV)), (i8 -1))), 5137 (COPY_TO_REGCLASS (VRSQRT14SDrr (COPY_TO_REGCLASS VR128X:$src1, FR64X), 5138 (COPY_TO_REGCLASS VR128X:$src2, FR64X)), VR128X)>; 5139 5140/// avx512_fp14_p rcp14ps, rcp14pd, rsqrt14ps, rsqrt14pd 5141multiclass avx512_fp14_p<bits<8> opc, string OpcodeStr, SDNode OpNode, 5142 X86VectorVTInfo _> { 5143 defm r: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), 5144 (ins _.RC:$src), OpcodeStr, "$src", "$src", 5145 (_.FloatVT (OpNode _.RC:$src))>, EVEX, T8PD; 5146 let mayLoad = 1 in { 5147 defm m: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), 5148 (ins _.MemOp:$src), OpcodeStr, "$src", "$src", 5149 (OpNode (_.FloatVT 5150 (bitconvert (_.LdFrag addr:$src))))>, EVEX, T8PD; 5151 defm mb: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), 5152 (ins _.ScalarMemOp:$src), OpcodeStr, 5153 "${src}"##_.BroadcastStr, "${src}"##_.BroadcastStr, 5154 (OpNode (_.FloatVT 5155 (X86VBroadcast (_.ScalarLdFrag addr:$src))))>, 5156 EVEX, T8PD, EVEX_B; 5157 } 5158} 5159 5160multiclass avx512_fp14_p_vl_all<bits<8> opc, string OpcodeStr, SDNode OpNode> { 5161 defm PSZ : avx512_fp14_p<opc, !strconcat(OpcodeStr, "ps"), OpNode, v16f32_info>, 5162 EVEX_V512, EVEX_CD8<32, CD8VF>; 5163 defm PDZ : avx512_fp14_p<opc, !strconcat(OpcodeStr, "pd"), OpNode, v8f64_info>, 5164 EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; 5165 5166 // Define only if AVX512VL feature is present. 5167 let Predicates = [HasVLX] in { 5168 defm PSZ128 : avx512_fp14_p<opc, !strconcat(OpcodeStr, "ps"), 5169 OpNode, v4f32x_info>, 5170 EVEX_V128, EVEX_CD8<32, CD8VF>; 5171 defm PSZ256 : avx512_fp14_p<opc, !strconcat(OpcodeStr, "ps"), 5172 OpNode, v8f32x_info>, 5173 EVEX_V256, EVEX_CD8<32, CD8VF>; 5174 defm PDZ128 : avx512_fp14_p<opc, !strconcat(OpcodeStr, "pd"), 5175 OpNode, v2f64x_info>, 5176 EVEX_V128, VEX_W, EVEX_CD8<64, CD8VF>; 5177 defm PDZ256 : avx512_fp14_p<opc, !strconcat(OpcodeStr, "pd"), 5178 OpNode, v4f64x_info>, 5179 EVEX_V256, VEX_W, EVEX_CD8<64, CD8VF>; 5180 } 5181} 5182 5183defm VRSQRT14 : avx512_fp14_p_vl_all<0x4E, "vrsqrt14", X86frsqrt>; 5184defm VRCP14 : avx512_fp14_p_vl_all<0x4C, "vrcp14", X86frcp>; 5185 5186def : Pat <(v16f32 (int_x86_avx512_rsqrt14_ps_512 (v16f32 VR512:$src), 5187 (bc_v16f32 (v16i32 immAllZerosV)), (i16 -1))), 5188 (VRSQRT14PSZr VR512:$src)>; 5189def : Pat <(v8f64 (int_x86_avx512_rsqrt14_pd_512 (v8f64 VR512:$src), 5190 (bc_v8f64 (v16i32 immAllZerosV)), (i8 -1))), 5191 (VRSQRT14PDZr VR512:$src)>; 5192 5193def : Pat <(v16f32 (int_x86_avx512_rcp14_ps_512 (v16f32 VR512:$src), 5194 (bc_v16f32 (v16i32 immAllZerosV)), (i16 -1))), 5195 (VRCP14PSZr VR512:$src)>; 5196def : Pat <(v8f64 (int_x86_avx512_rcp14_pd_512 (v8f64 VR512:$src), 5197 (bc_v8f64 (v16i32 immAllZerosV)), (i8 -1))), 5198 (VRCP14PDZr VR512:$src)>; 5199 5200/// avx512_fp28_s rcp28ss, rcp28sd, rsqrt28ss, rsqrt28sd 5201multiclass avx512_fp28_s<bits<8> opc, string OpcodeStr,X86VectorVTInfo _, 5202 SDNode OpNode> { 5203 5204 defm r : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), 5205 (ins _.RC:$src1, _.RC:$src2), OpcodeStr, 5206 "$src2, $src1", "$src1, $src2", 5207 (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2), 5208 (i32 FROUND_CURRENT))>; 5209 5210 defm rb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), 5211 (ins _.RC:$src1, _.RC:$src2), OpcodeStr, 5212 "{sae}, $src2, $src1", "$src1, $src2, {sae}", 5213 (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2), 5214 (i32 FROUND_NO_EXC))>, EVEX_B; 5215 5216 defm m : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst), 5217 (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr, 5218 "$src2, $src1", "$src1, $src2", 5219 (OpNode (_.VT _.RC:$src1), 5220 (_.VT (scalar_to_vector (_.ScalarLdFrag addr:$src2))), 5221 (i32 FROUND_CURRENT))>; 5222} 5223 5224multiclass avx512_eri_s<bits<8> opc, string OpcodeStr, SDNode OpNode> { 5225 defm SS : avx512_fp28_s<opc, OpcodeStr#"ss", f32x_info, OpNode>, 5226 EVEX_CD8<32, CD8VT1>; 5227 defm SD : avx512_fp28_s<opc, OpcodeStr#"sd", f64x_info, OpNode>, 5228 EVEX_CD8<64, CD8VT1>, VEX_W; 5229} 5230 5231let hasSideEffects = 0, Predicates = [HasERI] in { 5232 defm VRCP28 : avx512_eri_s<0xCB, "vrcp28", X86rcp28s>, T8PD, EVEX_4V; 5233 defm VRSQRT28 : avx512_eri_s<0xCD, "vrsqrt28", X86rsqrt28s>, T8PD, EVEX_4V; 5234} 5235/// avx512_fp28_p rcp28ps, rcp28pd, rsqrt28ps, rsqrt28pd 5236 5237multiclass avx512_fp28_p<bits<8> opc, string OpcodeStr, X86VectorVTInfo _, 5238 SDNode OpNode> { 5239 5240 defm r : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), 5241 (ins _.RC:$src), OpcodeStr, "$src", "$src", 5242 (OpNode (_.VT _.RC:$src), (i32 FROUND_CURRENT))>; 5243 5244 defm m : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), 5245 (ins _.MemOp:$src), OpcodeStr, "$src", "$src", 5246 (OpNode (_.FloatVT 5247 (bitconvert (_.LdFrag addr:$src))), 5248 (i32 FROUND_CURRENT))>; 5249 5250 defm mb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), 5251 (ins _.MemOp:$src), OpcodeStr, 5252 "${src}"##_.BroadcastStr, "${src}"##_.BroadcastStr, 5253 (OpNode (_.FloatVT 5254 (X86VBroadcast (_.ScalarLdFrag addr:$src))), 5255 (i32 FROUND_CURRENT))>, EVEX_B; 5256} 5257multiclass avx512_fp28_p_round<bits<8> opc, string OpcodeStr, X86VectorVTInfo _, 5258 SDNode OpNode> { 5259 defm rb : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), 5260 (ins _.RC:$src), OpcodeStr, 5261 "{sae}, $src", "$src, {sae}", 5262 (OpNode (_.VT _.RC:$src), (i32 FROUND_NO_EXC))>, EVEX_B; 5263} 5264 5265multiclass avx512_eri<bits<8> opc, string OpcodeStr, SDNode OpNode> { 5266 defm PS : avx512_fp28_p<opc, OpcodeStr#"ps", v16f32_info, OpNode>, 5267 avx512_fp28_p_round<opc, OpcodeStr#"ps", v16f32_info, OpNode>, 5268 T8PD, EVEX_V512, EVEX_CD8<32, CD8VF>; 5269 defm PD : avx512_fp28_p<opc, OpcodeStr#"pd", v8f64_info, OpNode>, 5270 avx512_fp28_p_round<opc, OpcodeStr#"pd", v8f64_info, OpNode>, 5271 T8PD, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; 5272} 5273 5274multiclass avx512_fp_unaryop_packed<bits<8> opc, string OpcodeStr, 5275 SDNode OpNode> { 5276 // Define only if AVX512VL feature is present. 5277 let Predicates = [HasVLX] in { 5278 defm PSZ128 : avx512_fp28_p<opc, OpcodeStr#"ps", v4f32x_info, OpNode>, 5279 EVEX_V128, T8PD, EVEX_CD8<32, CD8VF>; 5280 defm PSZ256 : avx512_fp28_p<opc, OpcodeStr#"ps", v8f32x_info, OpNode>, 5281 EVEX_V256, T8PD, EVEX_CD8<32, CD8VF>; 5282 defm PDZ128 : avx512_fp28_p<opc, OpcodeStr#"pd", v2f64x_info, OpNode>, 5283 EVEX_V128, VEX_W, T8PD, EVEX_CD8<64, CD8VF>; 5284 defm PDZ256 : avx512_fp28_p<opc, OpcodeStr#"pd", v4f64x_info, OpNode>, 5285 EVEX_V256, VEX_W, T8PD, EVEX_CD8<64, CD8VF>; 5286 } 5287} 5288let Predicates = [HasERI], hasSideEffects = 0 in { 5289 5290 defm VRSQRT28 : avx512_eri<0xCC, "vrsqrt28", X86rsqrt28>, EVEX; 5291 defm VRCP28 : avx512_eri<0xCA, "vrcp28", X86rcp28>, EVEX; 5292 defm VEXP2 : avx512_eri<0xC8, "vexp2", X86exp2>, EVEX; 5293} 5294defm VGETEXP : avx512_eri<0x42, "vgetexp", X86fgetexpRnd>, 5295 avx512_fp_unaryop_packed<0x42, "vgetexp", X86fgetexpRnd> , EVEX; 5296 5297multiclass avx512_sqrt_packed_round<bits<8> opc, string OpcodeStr, 5298 SDNode OpNodeRnd, X86VectorVTInfo _>{ 5299 defm rb: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), 5300 (ins _.RC:$src, AVX512RC:$rc), OpcodeStr, "$rc, $src", "$src, $rc", 5301 (_.VT (OpNodeRnd _.RC:$src, (i32 imm:$rc)))>, 5302 EVEX, EVEX_B, EVEX_RC; 5303} 5304 5305multiclass avx512_sqrt_packed<bits<8> opc, string OpcodeStr, 5306 SDNode OpNode, X86VectorVTInfo _>{ 5307 defm r: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), 5308 (ins _.RC:$src), OpcodeStr, "$src", "$src", 5309 (_.FloatVT (OpNode _.RC:$src))>, EVEX; 5310 let mayLoad = 1 in { 5311 defm m: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), 5312 (ins _.MemOp:$src), OpcodeStr, "$src", "$src", 5313 (OpNode (_.FloatVT 5314 (bitconvert (_.LdFrag addr:$src))))>, EVEX; 5315 5316 defm mb: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), 5317 (ins _.ScalarMemOp:$src), OpcodeStr, 5318 "${src}"##_.BroadcastStr, "${src}"##_.BroadcastStr, 5319 (OpNode (_.FloatVT 5320 (X86VBroadcast (_.ScalarLdFrag addr:$src))))>, 5321 EVEX, EVEX_B; 5322 } 5323} 5324 5325multiclass avx512_sqrt_scalar<bits<8> opc, string OpcodeStr, 5326 Intrinsic F32Int, Intrinsic F64Int, 5327 OpndItins itins_s, OpndItins itins_d> { 5328 def SSZr : SI<opc, MRMSrcReg, (outs FR32X:$dst), 5329 (ins FR32X:$src1, FR32X:$src2), 5330 !strconcat(OpcodeStr, 5331 "ss\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 5332 [], itins_s.rr>, XS, EVEX_4V; 5333 let isCodeGenOnly = 1 in 5334 def SSZr_Int : SIi8<opc, MRMSrcReg, (outs VR128X:$dst), 5335 (ins VR128X:$src1, VR128X:$src2), 5336 !strconcat(OpcodeStr, 5337 "ss\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 5338 [(set VR128X:$dst, 5339 (F32Int VR128X:$src1, VR128X:$src2))], 5340 itins_s.rr>, XS, EVEX_4V; 5341 let mayLoad = 1 in { 5342 def SSZm : SI<opc, MRMSrcMem, (outs FR32X:$dst), 5343 (ins FR32X:$src1, f32mem:$src2), 5344 !strconcat(OpcodeStr, 5345 "ss\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 5346 [], itins_s.rm>, XS, EVEX_4V, EVEX_CD8<32, CD8VT1>; 5347 let isCodeGenOnly = 1 in 5348 def SSZm_Int : SIi8<opc, MRMSrcMem, (outs VR128X:$dst), 5349 (ins VR128X:$src1, ssmem:$src2), 5350 !strconcat(OpcodeStr, 5351 "ss\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 5352 [(set VR128X:$dst, 5353 (F32Int VR128X:$src1, sse_load_f32:$src2))], 5354 itins_s.rm>, XS, EVEX_4V, EVEX_CD8<32, CD8VT1>; 5355 } 5356 def SDZr : SI<opc, MRMSrcReg, (outs FR64X:$dst), 5357 (ins FR64X:$src1, FR64X:$src2), 5358 !strconcat(OpcodeStr, 5359 "sd\t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>, 5360 XD, EVEX_4V, VEX_W; 5361 let isCodeGenOnly = 1 in 5362 def SDZr_Int : SIi8<opc, MRMSrcReg, (outs VR128X:$dst), 5363 (ins VR128X:$src1, VR128X:$src2), 5364 !strconcat(OpcodeStr, 5365 "sd\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 5366 [(set VR128X:$dst, 5367 (F64Int VR128X:$src1, VR128X:$src2))], 5368 itins_s.rr>, XD, EVEX_4V, VEX_W; 5369 let mayLoad = 1 in { 5370 def SDZm : SI<opc, MRMSrcMem, (outs FR64X:$dst), 5371 (ins FR64X:$src1, f64mem:$src2), 5372 !strconcat(OpcodeStr, 5373 "sd\t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>, 5374 XD, EVEX_4V, VEX_W, EVEX_CD8<64, CD8VT1>; 5375 let isCodeGenOnly = 1 in 5376 def SDZm_Int : SIi8<opc, MRMSrcMem, (outs VR128X:$dst), 5377 (ins VR128X:$src1, sdmem:$src2), 5378 !strconcat(OpcodeStr, 5379 "sd\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 5380 [(set VR128X:$dst, 5381 (F64Int VR128X:$src1, sse_load_f64:$src2))]>, 5382 XD, EVEX_4V, VEX_W, EVEX_CD8<64, CD8VT1>; 5383 } 5384} 5385 5386multiclass avx512_sqrt_packed_all<bits<8> opc, string OpcodeStr, 5387 SDNode OpNode> { 5388 defm PSZ : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "ps"), OpNode, 5389 v16f32_info>, 5390 EVEX_V512, PS, EVEX_CD8<32, CD8VF>; 5391 defm PDZ : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "pd"), OpNode, 5392 v8f64_info>, 5393 EVEX_V512, VEX_W, PD, EVEX_CD8<64, CD8VF>; 5394 // Define only if AVX512VL feature is present. 5395 let Predicates = [HasVLX] in { 5396 defm PSZ128 : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "ps"), 5397 OpNode, v4f32x_info>, 5398 EVEX_V128, PS, EVEX_CD8<32, CD8VF>; 5399 defm PSZ256 : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "ps"), 5400 OpNode, v8f32x_info>, 5401 EVEX_V256, PS, EVEX_CD8<32, CD8VF>; 5402 defm PDZ128 : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "pd"), 5403 OpNode, v2f64x_info>, 5404 EVEX_V128, VEX_W, PD, EVEX_CD8<64, CD8VF>; 5405 defm PDZ256 : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "pd"), 5406 OpNode, v4f64x_info>, 5407 EVEX_V256, VEX_W, PD, EVEX_CD8<64, CD8VF>; 5408 } 5409} 5410 5411multiclass avx512_sqrt_packed_all_round<bits<8> opc, string OpcodeStr, 5412 SDNode OpNodeRnd> { 5413 defm PSZ : avx512_sqrt_packed_round<opc, !strconcat(OpcodeStr, "ps"), OpNodeRnd, 5414 v16f32_info>, EVEX_V512, PS, EVEX_CD8<32, CD8VF>; 5415 defm PDZ : avx512_sqrt_packed_round<opc, !strconcat(OpcodeStr, "pd"), OpNodeRnd, 5416 v8f64_info>, EVEX_V512, VEX_W, PD, EVEX_CD8<64, CD8VF>; 5417} 5418 5419defm VSQRT : avx512_sqrt_packed_all<0x51, "vsqrt", fsqrt>, 5420 avx512_sqrt_packed_all_round<0x51, "vsqrt", X86fsqrtRnd>; 5421 5422defm VSQRT : avx512_sqrt_scalar<0x51, "sqrt", 5423 int_x86_avx512_sqrt_ss, int_x86_avx512_sqrt_sd, 5424 SSE_SQRTSS, SSE_SQRTSD>; 5425 5426let Predicates = [HasAVX512] in { 5427 def : Pat<(f32 (fsqrt FR32X:$src)), 5428 (VSQRTSSZr (f32 (IMPLICIT_DEF)), FR32X:$src)>; 5429 def : Pat<(f32 (fsqrt (load addr:$src))), 5430 (VSQRTSSZm (f32 (IMPLICIT_DEF)), addr:$src)>, 5431 Requires<[OptForSize]>; 5432 def : Pat<(f64 (fsqrt FR64X:$src)), 5433 (VSQRTSDZr (f64 (IMPLICIT_DEF)), FR64X:$src)>; 5434 def : Pat<(f64 (fsqrt (load addr:$src))), 5435 (VSQRTSDZm (f64 (IMPLICIT_DEF)), addr:$src)>, 5436 Requires<[OptForSize]>; 5437 5438 def : Pat<(f32 (X86frsqrt FR32X:$src)), 5439 (VRSQRT14SSrr (f32 (IMPLICIT_DEF)), FR32X:$src)>; 5440 def : Pat<(f32 (X86frsqrt (load addr:$src))), 5441 (VRSQRT14SSrm (f32 (IMPLICIT_DEF)), addr:$src)>, 5442 Requires<[OptForSize]>; 5443 5444 def : Pat<(f32 (X86frcp FR32X:$src)), 5445 (VRCP14SSrr (f32 (IMPLICIT_DEF)), FR32X:$src)>; 5446 def : Pat<(f32 (X86frcp (load addr:$src))), 5447 (VRCP14SSrm (f32 (IMPLICIT_DEF)), addr:$src)>, 5448 Requires<[OptForSize]>; 5449 5450 def : Pat<(int_x86_sse_sqrt_ss VR128X:$src), 5451 (COPY_TO_REGCLASS (VSQRTSSZr (f32 (IMPLICIT_DEF)), 5452 (COPY_TO_REGCLASS VR128X:$src, FR32)), 5453 VR128X)>; 5454 def : Pat<(int_x86_sse_sqrt_ss sse_load_f32:$src), 5455 (VSQRTSSZm_Int (v4f32 (IMPLICIT_DEF)), sse_load_f32:$src)>; 5456 5457 def : Pat<(int_x86_sse2_sqrt_sd VR128X:$src), 5458 (COPY_TO_REGCLASS (VSQRTSDZr (f64 (IMPLICIT_DEF)), 5459 (COPY_TO_REGCLASS VR128X:$src, FR64)), 5460 VR128X)>; 5461 def : Pat<(int_x86_sse2_sqrt_sd sse_load_f64:$src), 5462 (VSQRTSDZm_Int (v2f64 (IMPLICIT_DEF)), sse_load_f64:$src)>; 5463} 5464 5465 5466multiclass avx512_rndscale<bits<8> opc, string OpcodeStr, 5467 X86MemOperand x86memop, RegisterClass RC, 5468 PatFrag mem_frag, Domain d> { 5469let ExeDomain = d in { 5470 // Intrinsic operation, reg. 5471 // Vector intrinsic operation, reg 5472 def r : AVX512AIi8<opc, MRMSrcReg, 5473 (outs RC:$dst), (ins RC:$src1, i32u8imm:$src2), 5474 !strconcat(OpcodeStr, 5475 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 5476 []>, EVEX; 5477 5478 // Vector intrinsic operation, mem 5479 def m : AVX512AIi8<opc, MRMSrcMem, 5480 (outs RC:$dst), (ins x86memop:$src1, i32u8imm:$src2), 5481 !strconcat(OpcodeStr, 5482 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 5483 []>, EVEX; 5484} // ExeDomain 5485} 5486 5487defm VRNDSCALEPSZ : avx512_rndscale<0x08, "vrndscaleps", f512mem, VR512, 5488 loadv16f32, SSEPackedSingle>, EVEX_V512, 5489 EVEX_CD8<32, CD8VF>; 5490 5491def : Pat<(v16f32 (int_x86_avx512_mask_rndscale_ps_512 (v16f32 VR512:$src1), 5492 imm:$src2, (v16f32 VR512:$src1), (i16 -1), 5493 FROUND_CURRENT)), 5494 (VRNDSCALEPSZr VR512:$src1, imm:$src2)>; 5495 5496 5497defm VRNDSCALEPDZ : avx512_rndscale<0x09, "vrndscalepd", f512mem, VR512, 5498 loadv8f64, SSEPackedDouble>, EVEX_V512, 5499 VEX_W, EVEX_CD8<64, CD8VF>; 5500 5501def : Pat<(v8f64 (int_x86_avx512_mask_rndscale_pd_512 (v8f64 VR512:$src1), 5502 imm:$src2, (v8f64 VR512:$src1), (i8 -1), 5503 FROUND_CURRENT)), 5504 (VRNDSCALEPDZr VR512:$src1, imm:$src2)>; 5505 5506multiclass 5507avx512_rndscale_scalar<bits<8> opc, string OpcodeStr, X86VectorVTInfo _> { 5508 5509 let ExeDomain = _.ExeDomain in { 5510 defm r : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), 5511 (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3), OpcodeStr, 5512 "$src3, $src2, $src1", "$src1, $src2, $src3", 5513 (_.VT (X86RndScale (_.VT _.RC:$src1), (_.VT _.RC:$src2), 5514 (i32 imm:$src3), (i32 FROUND_CURRENT)))>; 5515 5516 defm rb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), 5517 (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3), OpcodeStr, 5518 "{sae}, $src3, $src2, $src1", "$src1, $src2, $src3, {sae}", 5519 (_.VT (X86RndScale (_.VT _.RC:$src1), (_.VT _.RC:$src2), 5520 (i32 imm:$src3), (i32 FROUND_NO_EXC)))>, EVEX_B; 5521 5522 let mayLoad = 1 in 5523 defm m : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst), 5524 (ins _.RC:$src1, _.MemOp:$src2, i32u8imm:$src3), OpcodeStr, 5525 "$src3, $src2, $src1", "$src1, $src2, $src3", 5526 (_.VT (X86RndScale (_.VT _.RC:$src1), 5527 (_.VT (scalar_to_vector (_.ScalarLdFrag addr:$src2))), 5528 (i32 imm:$src3), (i32 FROUND_CURRENT)))>; 5529 } 5530 let Predicates = [HasAVX512] in { 5531 def : Pat<(ffloor _.FRC:$src), (COPY_TO_REGCLASS 5532 (_.VT (!cast<Instruction>(NAME##r) (_.VT (IMPLICIT_DEF)), 5533 (_.VT (COPY_TO_REGCLASS _.FRC:$src, _.RC)), (i32 0x1))), _.FRC)>; 5534 def : Pat<(fceil _.FRC:$src), (COPY_TO_REGCLASS 5535 (_.VT (!cast<Instruction>(NAME##r) (_.VT (IMPLICIT_DEF)), 5536 (_.VT (COPY_TO_REGCLASS _.FRC:$src, _.RC)), (i32 0x2))), _.FRC)>; 5537 def : Pat<(ftrunc _.FRC:$src), (COPY_TO_REGCLASS 5538 (_.VT (!cast<Instruction>(NAME##r) (_.VT (IMPLICIT_DEF)), 5539 (_.VT (COPY_TO_REGCLASS _.FRC:$src, _.RC)), (i32 0x3))), _.FRC)>; 5540 def : Pat<(frint _.FRC:$src), (COPY_TO_REGCLASS 5541 (_.VT (!cast<Instruction>(NAME##r) (_.VT (IMPLICIT_DEF)), 5542 (_.VT (COPY_TO_REGCLASS _.FRC:$src, _.RC)), (i32 0x4))), _.FRC)>; 5543 def : Pat<(fnearbyint _.FRC:$src), (COPY_TO_REGCLASS 5544 (_.VT (!cast<Instruction>(NAME##r) (_.VT (IMPLICIT_DEF)), 5545 (_.VT (COPY_TO_REGCLASS _.FRC:$src, _.RC)), (i32 0xc))), _.FRC)>; 5546 5547 def : Pat<(ffloor (_.ScalarLdFrag addr:$src)), (COPY_TO_REGCLASS 5548 (_.VT (!cast<Instruction>(NAME##m) (_.VT (IMPLICIT_DEF)), 5549 addr:$src, (i32 0x1))), _.FRC)>; 5550 def : Pat<(fceil (_.ScalarLdFrag addr:$src)), (COPY_TO_REGCLASS 5551 (_.VT (!cast<Instruction>(NAME##m) (_.VT (IMPLICIT_DEF)), 5552 addr:$src, (i32 0x2))), _.FRC)>; 5553 def : Pat<(ftrunc (_.ScalarLdFrag addr:$src)), (COPY_TO_REGCLASS 5554 (_.VT (!cast<Instruction>(NAME##m) (_.VT (IMPLICIT_DEF)), 5555 addr:$src, (i32 0x3))), _.FRC)>; 5556 def : Pat<(frint (_.ScalarLdFrag addr:$src)), (COPY_TO_REGCLASS 5557 (_.VT (!cast<Instruction>(NAME##m) (_.VT (IMPLICIT_DEF)), 5558 addr:$src, (i32 0x4))), _.FRC)>; 5559 def : Pat<(fnearbyint (_.ScalarLdFrag addr:$src)), (COPY_TO_REGCLASS 5560 (_.VT (!cast<Instruction>(NAME##m) (_.VT (IMPLICIT_DEF)), 5561 addr:$src, (i32 0xc))), _.FRC)>; 5562 } 5563} 5564 5565defm VRNDSCALESS : avx512_rndscale_scalar<0x0A, "vrndscaless", f32x_info>, 5566 AVX512AIi8Base, EVEX_4V, EVEX_CD8<32, CD8VT1>; 5567 5568defm VRNDSCALESD : avx512_rndscale_scalar<0x0B, "vrndscalesd", f64x_info>, VEX_W, 5569 AVX512AIi8Base, EVEX_4V, EVEX_CD8<64, CD8VT1>; 5570 5571let Predicates = [HasAVX512] in { 5572def : Pat<(v16f32 (ffloor VR512:$src)), 5573 (VRNDSCALEPSZr VR512:$src, (i32 0x1))>; 5574def : Pat<(v16f32 (fnearbyint VR512:$src)), 5575 (VRNDSCALEPSZr VR512:$src, (i32 0xC))>; 5576def : Pat<(v16f32 (fceil VR512:$src)), 5577 (VRNDSCALEPSZr VR512:$src, (i32 0x2))>; 5578def : Pat<(v16f32 (frint VR512:$src)), 5579 (VRNDSCALEPSZr VR512:$src, (i32 0x4))>; 5580def : Pat<(v16f32 (ftrunc VR512:$src)), 5581 (VRNDSCALEPSZr VR512:$src, (i32 0x3))>; 5582 5583def : Pat<(v8f64 (ffloor VR512:$src)), 5584 (VRNDSCALEPDZr VR512:$src, (i32 0x1))>; 5585def : Pat<(v8f64 (fnearbyint VR512:$src)), 5586 (VRNDSCALEPDZr VR512:$src, (i32 0xC))>; 5587def : Pat<(v8f64 (fceil VR512:$src)), 5588 (VRNDSCALEPDZr VR512:$src, (i32 0x2))>; 5589def : Pat<(v8f64 (frint VR512:$src)), 5590 (VRNDSCALEPDZr VR512:$src, (i32 0x4))>; 5591def : Pat<(v8f64 (ftrunc VR512:$src)), 5592 (VRNDSCALEPDZr VR512:$src, (i32 0x3))>; 5593} 5594//------------------------------------------------- 5595// Integer truncate and extend operations 5596//------------------------------------------------- 5597 5598multiclass avx512_trunc_sat<bits<8> opc, string OpcodeStr, 5599 RegisterClass dstRC, RegisterClass srcRC, 5600 RegisterClass KRC, X86MemOperand x86memop> { 5601 def rr : AVX512XS8I<opc, MRMDestReg, (outs dstRC:$dst), 5602 (ins srcRC:$src), 5603 !strconcat(OpcodeStr,"\t{$src, $dst|$dst, $src}"), 5604 []>, EVEX; 5605 5606 def rrk : AVX512XS8I<opc, MRMDestReg, (outs dstRC:$dst), 5607 (ins KRC:$mask, srcRC:$src), 5608 !strconcat(OpcodeStr, 5609 "\t{$src, ${dst} {${mask}}|${dst} {${mask}}, $src}"), 5610 []>, EVEX, EVEX_K; 5611 5612 def rrkz : AVX512XS8I<opc, MRMDestReg, (outs dstRC:$dst), 5613 (ins KRC:$mask, srcRC:$src), 5614 !strconcat(OpcodeStr, 5615 "\t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}"), 5616 []>, EVEX, EVEX_KZ; 5617 5618 def mr : AVX512XS8I<opc, MRMDestMem, (outs), (ins x86memop:$dst, srcRC:$src), 5619 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 5620 []>, EVEX; 5621 5622 def mrk : AVX512XS8I<opc, MRMDestMem, (outs), 5623 (ins x86memop:$dst, KRC:$mask, srcRC:$src), 5624 !strconcat(OpcodeStr, "\t{$src, $dst {${mask}}|${dst} {${mask}}, $src}"), 5625 []>, EVEX, EVEX_K; 5626 5627} 5628defm VPMOVQB : avx512_trunc_sat<0x32, "vpmovqb", VR128X, VR512, VK8WM, 5629 i128mem>, EVEX_V512, EVEX_CD8<8, CD8VO>; 5630defm VPMOVSQB : avx512_trunc_sat<0x22, "vpmovsqb", VR128X, VR512, VK8WM, 5631 i128mem>, EVEX_V512, EVEX_CD8<8, CD8VO>; 5632defm VPMOVUSQB : avx512_trunc_sat<0x12, "vpmovusqb", VR128X, VR512, VK8WM, 5633 i128mem>, EVEX_V512, EVEX_CD8<8, CD8VO>; 5634defm VPMOVQW : avx512_trunc_sat<0x34, "vpmovqw", VR128X, VR512, VK8WM, 5635 i128mem>, EVEX_V512, EVEX_CD8<16, CD8VQ>; 5636defm VPMOVSQW : avx512_trunc_sat<0x24, "vpmovsqw", VR128X, VR512, VK8WM, 5637 i128mem>, EVEX_V512, EVEX_CD8<16, CD8VQ>; 5638defm VPMOVUSQW : avx512_trunc_sat<0x14, "vpmovusqw", VR128X, VR512, VK8WM, 5639 i128mem>, EVEX_V512, EVEX_CD8<16, CD8VQ>; 5640defm VPMOVQD : avx512_trunc_sat<0x35, "vpmovqd", VR256X, VR512, VK8WM, 5641 i256mem>, EVEX_V512, EVEX_CD8<32, CD8VH>; 5642defm VPMOVSQD : avx512_trunc_sat<0x25, "vpmovsqd", VR256X, VR512, VK8WM, 5643 i256mem>, EVEX_V512, EVEX_CD8<32, CD8VH>; 5644defm VPMOVUSQD : avx512_trunc_sat<0x15, "vpmovusqd", VR256X, VR512, VK8WM, 5645 i256mem>, EVEX_V512, EVEX_CD8<32, CD8VH>; 5646defm VPMOVDW : avx512_trunc_sat<0x33, "vpmovdw", VR256X, VR512, VK16WM, 5647 i256mem>, EVEX_V512, EVEX_CD8<16, CD8VH>; 5648defm VPMOVSDW : avx512_trunc_sat<0x23, "vpmovsdw", VR256X, VR512, VK16WM, 5649 i256mem>, EVEX_V512, EVEX_CD8<16, CD8VH>; 5650defm VPMOVUSDW : avx512_trunc_sat<0x13, "vpmovusdw", VR256X, VR512, VK16WM, 5651 i256mem>, EVEX_V512, EVEX_CD8<16, CD8VH>; 5652defm VPMOVDB : avx512_trunc_sat<0x31, "vpmovdb", VR128X, VR512, VK16WM, 5653 i128mem>, EVEX_V512, EVEX_CD8<8, CD8VQ>; 5654defm VPMOVSDB : avx512_trunc_sat<0x21, "vpmovsdb", VR128X, VR512, VK16WM, 5655 i128mem>, EVEX_V512, EVEX_CD8<8, CD8VQ>; 5656defm VPMOVUSDB : avx512_trunc_sat<0x11, "vpmovusdb", VR128X, VR512, VK16WM, 5657 i128mem>, EVEX_V512, EVEX_CD8<8, CD8VQ>; 5658 5659def : Pat<(v16i8 (X86vtrunc (v8i64 VR512:$src))), (VPMOVQBrr VR512:$src)>; 5660def : Pat<(v8i16 (X86vtrunc (v8i64 VR512:$src))), (VPMOVQWrr VR512:$src)>; 5661def : Pat<(v16i16 (X86vtrunc (v16i32 VR512:$src))), (VPMOVDWrr VR512:$src)>; 5662def : Pat<(v16i8 (X86vtrunc (v16i32 VR512:$src))), (VPMOVDBrr VR512:$src)>; 5663def : Pat<(v8i32 (X86vtrunc (v8i64 VR512:$src))), (VPMOVQDrr VR512:$src)>; 5664 5665def : Pat<(v16i8 (X86vtruncm VK16WM:$mask, (v16i32 VR512:$src))), 5666 (VPMOVDBrrkz VK16WM:$mask, VR512:$src)>; 5667def : Pat<(v16i16 (X86vtruncm VK16WM:$mask, (v16i32 VR512:$src))), 5668 (VPMOVDWrrkz VK16WM:$mask, VR512:$src)>; 5669def : Pat<(v8i16 (X86vtruncm VK8WM:$mask, (v8i64 VR512:$src))), 5670 (VPMOVQWrrkz VK8WM:$mask, VR512:$src)>; 5671def : Pat<(v8i32 (X86vtruncm VK8WM:$mask, (v8i64 VR512:$src))), 5672 (VPMOVQDrrkz VK8WM:$mask, VR512:$src)>; 5673 5674 5675multiclass avx512_extend_common<bits<8> opc, string OpcodeStr, 5676 X86VectorVTInfo DestInfo, X86VectorVTInfo SrcInfo, 5677 X86MemOperand x86memop, PatFrag LdFrag, SDNode OpNode>{ 5678 5679 defm rr : AVX512_maskable<opc, MRMSrcReg, DestInfo, (outs DestInfo.RC:$dst), 5680 (ins SrcInfo.RC:$src), OpcodeStr ,"$src", "$src", 5681 (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src)))>, 5682 EVEX; 5683 5684 let mayLoad = 1 in { 5685 defm rm : AVX512_maskable<opc, MRMSrcMem, DestInfo, (outs DestInfo.RC:$dst), 5686 (ins x86memop:$src), OpcodeStr ,"$src", "$src", 5687 (DestInfo.VT (LdFrag addr:$src))>, 5688 EVEX; 5689 } 5690} 5691 5692multiclass avx512_extend_BW<bits<8> opc, string OpcodeStr, SDNode OpNode, 5693 string ExtTy,PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi8")> { 5694 let Predicates = [HasVLX, HasBWI] in { 5695 defm Z128: avx512_extend_common<opc, OpcodeStr, v8i16x_info, 5696 v16i8x_info, i64mem, LdFrag, OpNode>, 5697 EVEX_CD8<8, CD8VH>, T8PD, EVEX_V128; 5698 5699 defm Z256: avx512_extend_common<opc, OpcodeStr, v16i16x_info, 5700 v16i8x_info, i128mem, LdFrag, OpNode>, 5701 EVEX_CD8<8, CD8VH>, T8PD, EVEX_V256; 5702 } 5703 let Predicates = [HasBWI] in { 5704 defm Z : avx512_extend_common<opc, OpcodeStr, v32i16_info, 5705 v32i8x_info, i256mem, LdFrag, OpNode>, 5706 EVEX_CD8<8, CD8VH>, T8PD, EVEX_V512; 5707 } 5708} 5709 5710multiclass avx512_extend_BD<bits<8> opc, string OpcodeStr, SDNode OpNode, 5711 string ExtTy,PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi8")> { 5712 let Predicates = [HasVLX, HasAVX512] in { 5713 defm Z128: avx512_extend_common<opc, OpcodeStr, v4i32x_info, 5714 v16i8x_info, i32mem, LdFrag, OpNode>, 5715 EVEX_CD8<8, CD8VQ>, T8PD, EVEX_V128; 5716 5717 defm Z256: avx512_extend_common<opc, OpcodeStr, v8i32x_info, 5718 v16i8x_info, i64mem, LdFrag, OpNode>, 5719 EVEX_CD8<8, CD8VQ>, T8PD, EVEX_V256; 5720 } 5721 let Predicates = [HasAVX512] in { 5722 defm Z : avx512_extend_common<opc, OpcodeStr, v16i32_info, 5723 v16i8x_info, i128mem, LdFrag, OpNode>, 5724 EVEX_CD8<8, CD8VQ>, T8PD, EVEX_V512; 5725 } 5726} 5727 5728multiclass avx512_extend_BQ<bits<8> opc, string OpcodeStr, SDNode OpNode, 5729 string ExtTy,PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi8")> { 5730 let Predicates = [HasVLX, HasAVX512] in { 5731 defm Z128: avx512_extend_common<opc, OpcodeStr, v2i64x_info, 5732 v16i8x_info, i16mem, LdFrag, OpNode>, 5733 EVEX_CD8<8, CD8VO>, T8PD, EVEX_V128; 5734 5735 defm Z256: avx512_extend_common<opc, OpcodeStr, v4i64x_info, 5736 v16i8x_info, i32mem, LdFrag, OpNode>, 5737 EVEX_CD8<8, CD8VO>, T8PD, EVEX_V256; 5738 } 5739 let Predicates = [HasAVX512] in { 5740 defm Z : avx512_extend_common<opc, OpcodeStr, v8i64_info, 5741 v16i8x_info, i64mem, LdFrag, OpNode>, 5742 EVEX_CD8<8, CD8VO>, T8PD, EVEX_V512; 5743 } 5744} 5745 5746multiclass avx512_extend_WD<bits<8> opc, string OpcodeStr, SDNode OpNode, 5747 string ExtTy,PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi16")> { 5748 let Predicates = [HasVLX, HasAVX512] in { 5749 defm Z128: avx512_extend_common<opc, OpcodeStr, v4i32x_info, 5750 v8i16x_info, i64mem, LdFrag, OpNode>, 5751 EVEX_CD8<16, CD8VH>, T8PD, EVEX_V128; 5752 5753 defm Z256: avx512_extend_common<opc, OpcodeStr, v8i32x_info, 5754 v8i16x_info, i128mem, LdFrag, OpNode>, 5755 EVEX_CD8<16, CD8VH>, T8PD, EVEX_V256; 5756 } 5757 let Predicates = [HasAVX512] in { 5758 defm Z : avx512_extend_common<opc, OpcodeStr, v16i32_info, 5759 v16i16x_info, i256mem, LdFrag, OpNode>, 5760 EVEX_CD8<16, CD8VH>, T8PD, EVEX_V512; 5761 } 5762} 5763 5764multiclass avx512_extend_WQ<bits<8> opc, string OpcodeStr, SDNode OpNode, 5765 string ExtTy,PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi16")> { 5766 let Predicates = [HasVLX, HasAVX512] in { 5767 defm Z128: avx512_extend_common<opc, OpcodeStr, v2i64x_info, 5768 v8i16x_info, i32mem, LdFrag, OpNode>, 5769 EVEX_CD8<16, CD8VQ>, T8PD, EVEX_V128; 5770 5771 defm Z256: avx512_extend_common<opc, OpcodeStr, v4i64x_info, 5772 v8i16x_info, i64mem, LdFrag, OpNode>, 5773 EVEX_CD8<16, CD8VQ>, T8PD, EVEX_V256; 5774 } 5775 let Predicates = [HasAVX512] in { 5776 defm Z : avx512_extend_common<opc, OpcodeStr, v8i64_info, 5777 v8i16x_info, i128mem, LdFrag, OpNode>, 5778 EVEX_CD8<16, CD8VQ>, T8PD, EVEX_V512; 5779 } 5780} 5781 5782multiclass avx512_extend_DQ<bits<8> opc, string OpcodeStr, SDNode OpNode, 5783 string ExtTy,PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi32")> { 5784 5785 let Predicates = [HasVLX, HasAVX512] in { 5786 defm Z128: avx512_extend_common<opc, OpcodeStr, v2i64x_info, 5787 v4i32x_info, i64mem, LdFrag, OpNode>, 5788 EVEX_CD8<32, CD8VH>, T8PD, EVEX_V128; 5789 5790 defm Z256: avx512_extend_common<opc, OpcodeStr, v4i64x_info, 5791 v4i32x_info, i128mem, LdFrag, OpNode>, 5792 EVEX_CD8<32, CD8VH>, T8PD, EVEX_V256; 5793 } 5794 let Predicates = [HasAVX512] in { 5795 defm Z : avx512_extend_common<opc, OpcodeStr, v8i64_info, 5796 v8i32x_info, i256mem, LdFrag, OpNode>, 5797 EVEX_CD8<32, CD8VH>, T8PD, EVEX_V512; 5798 } 5799} 5800 5801defm VPMOVZXBW : avx512_extend_BW<0x30, "vpmovzxbw", X86vzext, "z">; 5802defm VPMOVZXBD : avx512_extend_BD<0x31, "vpmovzxbd", X86vzext, "z">; 5803defm VPMOVZXBQ : avx512_extend_BQ<0x32, "vpmovzxbq", X86vzext, "z">; 5804defm VPMOVZXWD : avx512_extend_WD<0x33, "vpmovzxwd", X86vzext, "z">; 5805defm VPMOVZXWQ : avx512_extend_WQ<0x34, "vpmovzxwq", X86vzext, "z">; 5806defm VPMOVZXDQ : avx512_extend_DQ<0x35, "vpmovzxdq", X86vzext, "z">; 5807 5808 5809defm VPMOVSXBW: avx512_extend_BW<0x20, "vpmovsxbw", X86vsext, "s">; 5810defm VPMOVSXBD: avx512_extend_BD<0x21, "vpmovsxbd", X86vsext, "s">; 5811defm VPMOVSXBQ: avx512_extend_BQ<0x22, "vpmovsxbq", X86vsext, "s">; 5812defm VPMOVSXWD: avx512_extend_WD<0x23, "vpmovsxwd", X86vsext, "s">; 5813defm VPMOVSXWQ: avx512_extend_WQ<0x24, "vpmovsxwq", X86vsext, "s">; 5814defm VPMOVSXDQ: avx512_extend_DQ<0x25, "vpmovsxdq", X86vsext, "s">; 5815 5816//===----------------------------------------------------------------------===// 5817// GATHER - SCATTER Operations 5818 5819multiclass avx512_gather<bits<8> opc, string OpcodeStr, X86VectorVTInfo _, 5820 X86MemOperand memop, PatFrag GatherNode> { 5821 let Constraints = "@earlyclobber $dst, $src1 = $dst, $mask = $mask_wb", 5822 ExeDomain = _.ExeDomain in 5823 def rm : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst, _.KRCWM:$mask_wb), 5824 (ins _.RC:$src1, _.KRCWM:$mask, memop:$src2), 5825 !strconcat(OpcodeStr#_.Suffix, 5826 "\t{$src2, ${dst} {${mask}}|${dst} {${mask}}, $src2}"), 5827 [(set _.RC:$dst, _.KRCWM:$mask_wb, 5828 (GatherNode (_.VT _.RC:$src1), _.KRCWM:$mask, 5829 vectoraddr:$src2))]>, EVEX, EVEX_K, 5830 EVEX_CD8<_.EltSize, CD8VT1>; 5831} 5832 5833multiclass avx512_gather_q_pd<bits<8> dopc, bits<8> qopc, 5834 AVX512VLVectorVTInfo _, string OpcodeStr, string SUFF> { 5835 defm NAME##D##SUFF##Z: avx512_gather<dopc, OpcodeStr##"d", _.info512, 5836 vy32xmem, mgatherv8i32>, EVEX_V512, VEX_W; 5837 defm NAME##Q##SUFF##Z: avx512_gather<qopc, OpcodeStr##"q", _.info512, 5838 vz64mem, mgatherv8i64>, EVEX_V512, VEX_W; 5839let Predicates = [HasVLX] in { 5840 defm NAME##D##SUFF##Z256: avx512_gather<dopc, OpcodeStr##"d", _.info256, 5841 vx32xmem, mgatherv4i32>, EVEX_V256, VEX_W; 5842 defm NAME##Q##SUFF##Z256: avx512_gather<qopc, OpcodeStr##"q", _.info256, 5843 vy64xmem, mgatherv4i64>, EVEX_V256, VEX_W; 5844 defm NAME##D##SUFF##Z128: avx512_gather<dopc, OpcodeStr##"d", _.info128, 5845 vx32xmem, mgatherv4i32>, EVEX_V128, VEX_W; 5846 defm NAME##Q##SUFF##Z128: avx512_gather<qopc, OpcodeStr##"q", _.info128, 5847 vx64xmem, mgatherv2i64>, EVEX_V128, VEX_W; 5848} 5849} 5850 5851multiclass avx512_gather_d_ps<bits<8> dopc, bits<8> qopc, 5852 AVX512VLVectorVTInfo _, string OpcodeStr, string SUFF> { 5853 defm NAME##D##SUFF##Z: avx512_gather<dopc, OpcodeStr##"d", _.info512, vz32mem, 5854 mgatherv16i32>, EVEX_V512; 5855 defm NAME##Q##SUFF##Z: avx512_gather<qopc, OpcodeStr##"q", _.info256, vz64mem, 5856 mgatherv8i64>, EVEX_V512; 5857let Predicates = [HasVLX] in { 5858 defm NAME##D##SUFF##Z256: avx512_gather<dopc, OpcodeStr##"d", _.info256, 5859 vy32xmem, mgatherv8i32>, EVEX_V256; 5860 defm NAME##Q##SUFF##Z256: avx512_gather<qopc, OpcodeStr##"q", _.info128, 5861 vy64xmem, mgatherv4i64>, EVEX_V256; 5862 defm NAME##D##SUFF##Z128: avx512_gather<dopc, OpcodeStr##"d", _.info128, 5863 vx32xmem, mgatherv4i32>, EVEX_V128; 5864 defm NAME##Q##SUFF##Z128: avx512_gather<qopc, OpcodeStr##"q", _.info128, 5865 vx64xmem, mgatherv2i64>, EVEX_V128; 5866} 5867} 5868 5869 5870defm VGATHER : avx512_gather_q_pd<0x92, 0x93, avx512vl_f64_info, "vgather", "PD">, 5871 avx512_gather_d_ps<0x92, 0x93, avx512vl_f32_info, "vgather", "PS">; 5872 5873defm VPGATHER : avx512_gather_q_pd<0x90, 0x91, avx512vl_i64_info, "vpgather", "Q">, 5874 avx512_gather_d_ps<0x90, 0x91, avx512vl_i32_info, "vpgather", "D">; 5875 5876multiclass avx512_scatter<bits<8> opc, string OpcodeStr, X86VectorVTInfo _, 5877 X86MemOperand memop, PatFrag ScatterNode> { 5878 5879let mayStore = 1, Constraints = "$mask = $mask_wb", ExeDomain = _.ExeDomain in 5880 5881 def mr : AVX5128I<opc, MRMDestMem, (outs _.KRCWM:$mask_wb), 5882 (ins memop:$dst, _.KRCWM:$mask, _.RC:$src), 5883 !strconcat(OpcodeStr#_.Suffix, 5884 "\t{$src, ${dst} {${mask}}|${dst} {${mask}}, $src}"), 5885 [(set _.KRCWM:$mask_wb, (ScatterNode (_.VT _.RC:$src), 5886 _.KRCWM:$mask, vectoraddr:$dst))]>, 5887 EVEX, EVEX_K, EVEX_CD8<_.EltSize, CD8VT1>; 5888} 5889 5890multiclass avx512_scatter_q_pd<bits<8> dopc, bits<8> qopc, 5891 AVX512VLVectorVTInfo _, string OpcodeStr, string SUFF> { 5892 defm NAME##D##SUFF##Z: avx512_scatter<dopc, OpcodeStr##"d", _.info512, 5893 vy32xmem, mscatterv8i32>, EVEX_V512, VEX_W; 5894 defm NAME##Q##SUFF##Z: avx512_scatter<qopc, OpcodeStr##"q", _.info512, 5895 vz64mem, mscatterv8i64>, EVEX_V512, VEX_W; 5896let Predicates = [HasVLX] in { 5897 defm NAME##D##SUFF##Z256: avx512_scatter<dopc, OpcodeStr##"d", _.info256, 5898 vx32xmem, mscatterv4i32>, EVEX_V256, VEX_W; 5899 defm NAME##Q##SUFF##Z256: avx512_scatter<qopc, OpcodeStr##"q", _.info256, 5900 vy64xmem, mscatterv4i64>, EVEX_V256, VEX_W; 5901 defm NAME##D##SUFF##Z128: avx512_scatter<dopc, OpcodeStr##"d", _.info128, 5902 vx32xmem, mscatterv4i32>, EVEX_V128, VEX_W; 5903 defm NAME##Q##SUFF##Z128: avx512_scatter<qopc, OpcodeStr##"q", _.info128, 5904 vx64xmem, mscatterv2i64>, EVEX_V128, VEX_W; 5905} 5906} 5907 5908multiclass avx512_scatter_d_ps<bits<8> dopc, bits<8> qopc, 5909 AVX512VLVectorVTInfo _, string OpcodeStr, string SUFF> { 5910 defm NAME##D##SUFF##Z: avx512_scatter<dopc, OpcodeStr##"d", _.info512, vz32mem, 5911 mscatterv16i32>, EVEX_V512; 5912 defm NAME##Q##SUFF##Z: avx512_scatter<qopc, OpcodeStr##"q", _.info256, vz64mem, 5913 mscatterv8i64>, EVEX_V512; 5914let Predicates = [HasVLX] in { 5915 defm NAME##D##SUFF##Z256: avx512_scatter<dopc, OpcodeStr##"d", _.info256, 5916 vy32xmem, mscatterv8i32>, EVEX_V256; 5917 defm NAME##Q##SUFF##Z256: avx512_scatter<qopc, OpcodeStr##"q", _.info128, 5918 vy64xmem, mscatterv4i64>, EVEX_V256; 5919 defm NAME##D##SUFF##Z128: avx512_scatter<dopc, OpcodeStr##"d", _.info128, 5920 vx32xmem, mscatterv4i32>, EVEX_V128; 5921 defm NAME##Q##SUFF##Z128: avx512_scatter<qopc, OpcodeStr##"q", _.info128, 5922 vx64xmem, mscatterv2i64>, EVEX_V128; 5923} 5924} 5925 5926defm VSCATTER : avx512_scatter_q_pd<0xA2, 0xA3, avx512vl_f64_info, "vscatter", "PD">, 5927 avx512_scatter_d_ps<0xA2, 0xA3, avx512vl_f32_info, "vscatter", "PS">; 5928 5929defm VPSCATTER : avx512_scatter_q_pd<0xA0, 0xA1, avx512vl_i64_info, "vpscatter", "Q">, 5930 avx512_scatter_d_ps<0xA0, 0xA1, avx512vl_i32_info, "vpscatter", "D">; 5931 5932// prefetch 5933multiclass avx512_gather_scatter_prefetch<bits<8> opc, Format F, string OpcodeStr, 5934 RegisterClass KRC, X86MemOperand memop> { 5935 let Predicates = [HasPFI], hasSideEffects = 1 in 5936 def m : AVX5128I<opc, F, (outs), (ins KRC:$mask, memop:$src), 5937 !strconcat(OpcodeStr, "\t{$src {${mask}}|{${mask}}, $src}"), 5938 []>, EVEX, EVEX_K; 5939} 5940 5941defm VGATHERPF0DPS: avx512_gather_scatter_prefetch<0xC6, MRM1m, "vgatherpf0dps", 5942 VK16WM, vz32mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>; 5943 5944defm VGATHERPF0QPS: avx512_gather_scatter_prefetch<0xC7, MRM1m, "vgatherpf0qps", 5945 VK8WM, vz64mem>, EVEX_V512, EVEX_CD8<64, CD8VT1>; 5946 5947defm VGATHERPF0DPD: avx512_gather_scatter_prefetch<0xC6, MRM1m, "vgatherpf0dpd", 5948 VK8WM, vy32mem>, EVEX_V512, VEX_W, EVEX_CD8<32, CD8VT1>; 5949 5950defm VGATHERPF0QPD: avx512_gather_scatter_prefetch<0xC7, MRM1m, "vgatherpf0qpd", 5951 VK8WM, vz64mem>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>; 5952 5953defm VGATHERPF1DPS: avx512_gather_scatter_prefetch<0xC6, MRM2m, "vgatherpf1dps", 5954 VK16WM, vz32mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>; 5955 5956defm VGATHERPF1QPS: avx512_gather_scatter_prefetch<0xC7, MRM2m, "vgatherpf1qps", 5957 VK8WM, vz64mem>, EVEX_V512, EVEX_CD8<64, CD8VT1>; 5958 5959defm VGATHERPF1DPD: avx512_gather_scatter_prefetch<0xC6, MRM2m, "vgatherpf1dpd", 5960 VK8WM, vy32mem>, EVEX_V512, VEX_W, EVEX_CD8<32, CD8VT1>; 5961 5962defm VGATHERPF1QPD: avx512_gather_scatter_prefetch<0xC7, MRM2m, "vgatherpf1qpd", 5963 VK8WM, vz64mem>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>; 5964 5965defm VSCATTERPF0DPS: avx512_gather_scatter_prefetch<0xC6, MRM5m, "vscatterpf0dps", 5966 VK16WM, vz32mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>; 5967 5968defm VSCATTERPF0QPS: avx512_gather_scatter_prefetch<0xC7, MRM5m, "vscatterpf0qps", 5969 VK8WM, vz64mem>, EVEX_V512, EVEX_CD8<64, CD8VT1>; 5970 5971defm VSCATTERPF0DPD: avx512_gather_scatter_prefetch<0xC6, MRM5m, "vscatterpf0dpd", 5972 VK8WM, vy32mem>, EVEX_V512, VEX_W, EVEX_CD8<32, CD8VT1>; 5973 5974defm VSCATTERPF0QPD: avx512_gather_scatter_prefetch<0xC7, MRM5m, "vscatterpf0qpd", 5975 VK8WM, vz64mem>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>; 5976 5977defm VSCATTERPF1DPS: avx512_gather_scatter_prefetch<0xC6, MRM6m, "vscatterpf1dps", 5978 VK16WM, vz32mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>; 5979 5980defm VSCATTERPF1QPS: avx512_gather_scatter_prefetch<0xC7, MRM6m, "vscatterpf1qps", 5981 VK8WM, vz64mem>, EVEX_V512, EVEX_CD8<64, CD8VT1>; 5982 5983defm VSCATTERPF1DPD: avx512_gather_scatter_prefetch<0xC6, MRM6m, "vscatterpf1dpd", 5984 VK8WM, vy32mem>, EVEX_V512, VEX_W, EVEX_CD8<32, CD8VT1>; 5985 5986defm VSCATTERPF1QPD: avx512_gather_scatter_prefetch<0xC7, MRM6m, "vscatterpf1qpd", 5987 VK8WM, vz64mem>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>; 5988//===----------------------------------------------------------------------===// 5989// VSHUFPS - VSHUFPD Operations 5990 5991multiclass avx512_shufp<RegisterClass RC, X86MemOperand x86memop, 5992 ValueType vt, string OpcodeStr, PatFrag mem_frag, 5993 Domain d> { 5994 def rmi : AVX512PIi8<0xC6, MRMSrcMem, (outs RC:$dst), 5995 (ins RC:$src1, x86memop:$src2, u8imm:$src3), 5996 !strconcat(OpcodeStr, 5997 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), 5998 [(set RC:$dst, (vt (X86Shufp RC:$src1, (mem_frag addr:$src2), 5999 (i8 imm:$src3))))], d, IIC_SSE_SHUFP>, 6000 EVEX_4V, Sched<[WriteShuffleLd, ReadAfterLd]>; 6001 def rri : AVX512PIi8<0xC6, MRMSrcReg, (outs RC:$dst), 6002 (ins RC:$src1, RC:$src2, u8imm:$src3), 6003 !strconcat(OpcodeStr, 6004 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), 6005 [(set RC:$dst, (vt (X86Shufp RC:$src1, RC:$src2, 6006 (i8 imm:$src3))))], d, IIC_SSE_SHUFP>, 6007 EVEX_4V, Sched<[WriteShuffle]>; 6008} 6009 6010defm VSHUFPSZ : avx512_shufp<VR512, f512mem, v16f32, "vshufps", loadv16f32, 6011 SSEPackedSingle>, PS, EVEX_V512, EVEX_CD8<32, CD8VF>; 6012defm VSHUFPDZ : avx512_shufp<VR512, f512mem, v8f64, "vshufpd", loadv8f64, 6013 SSEPackedDouble>, PD, VEX_W, EVEX_V512, EVEX_CD8<64, CD8VF>; 6014 6015def : Pat<(v16i32 (X86Shufp VR512:$src1, VR512:$src2, (i8 imm:$imm))), 6016 (VSHUFPSZrri VR512:$src1, VR512:$src2, imm:$imm)>; 6017def : Pat<(v16i32 (X86Shufp VR512:$src1, 6018 (loadv16i32 addr:$src2), (i8 imm:$imm))), 6019 (VSHUFPSZrmi VR512:$src1, addr:$src2, imm:$imm)>; 6020 6021def : Pat<(v8i64 (X86Shufp VR512:$src1, VR512:$src2, (i8 imm:$imm))), 6022 (VSHUFPDZrri VR512:$src1, VR512:$src2, imm:$imm)>; 6023def : Pat<(v8i64 (X86Shufp VR512:$src1, 6024 (loadv8i64 addr:$src2), (i8 imm:$imm))), 6025 (VSHUFPDZrmi VR512:$src1, addr:$src2, imm:$imm)>; 6026 6027// Helper fragments to match sext vXi1 to vXiY. 6028def v16i1sextv16i32 : PatLeaf<(v16i32 (X86vsrai VR512:$src, (i8 31)))>; 6029def v8i1sextv8i64 : PatLeaf<(v8i64 (X86vsrai VR512:$src, (i8 63)))>; 6030 6031multiclass avx512_conflict<bits<8> opc, string OpcodeStr, 6032 RegisterClass RC, RegisterClass KRC, 6033 X86MemOperand x86memop, 6034 X86MemOperand x86scalar_mop, string BrdcstStr> { 6035 let hasSideEffects = 0 in { 6036 def rr : AVX5128I<opc, MRMSrcReg, (outs RC:$dst), 6037 (ins RC:$src), 6038 !strconcat(OpcodeStr, "\t{$src, ${dst} |${dst}, $src}"), 6039 []>, EVEX; 6040 let mayLoad = 1 in 6041 def rm : AVX5128I<opc, MRMSrcMem, (outs RC:$dst), 6042 (ins x86memop:$src), 6043 !strconcat(OpcodeStr, "\t{$src, ${dst}|${dst}, $src}"), 6044 []>, EVEX; 6045 let mayLoad = 1 in 6046 def rmb : AVX5128I<opc, MRMSrcMem, (outs RC:$dst), 6047 (ins x86scalar_mop:$src), 6048 !strconcat(OpcodeStr, "\t{${src}", BrdcstStr, 6049 ", ${dst}|${dst}, ${src}", BrdcstStr, "}"), 6050 []>, EVEX, EVEX_B; 6051 def rrkz : AVX5128I<opc, MRMSrcReg, (outs RC:$dst), 6052 (ins KRC:$mask, RC:$src), 6053 !strconcat(OpcodeStr, 6054 "\t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}"), 6055 []>, EVEX, EVEX_KZ; 6056 let mayLoad = 1 in 6057 def rmkz : AVX5128I<opc, MRMSrcMem, (outs RC:$dst), 6058 (ins KRC:$mask, x86memop:$src), 6059 !strconcat(OpcodeStr, 6060 "\t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}"), 6061 []>, EVEX, EVEX_KZ; 6062 let mayLoad = 1 in 6063 def rmbkz : AVX5128I<opc, MRMSrcMem, (outs RC:$dst), 6064 (ins KRC:$mask, x86scalar_mop:$src), 6065 !strconcat(OpcodeStr, "\t{${src}", BrdcstStr, 6066 ", ${dst} {${mask}} {z}|${dst} {${mask}} {z}, ${src}", 6067 BrdcstStr, "}"), 6068 []>, EVEX, EVEX_KZ, EVEX_B; 6069 6070 let Constraints = "$src1 = $dst" in { 6071 def rrk : AVX5128I<opc, MRMSrcReg, (outs RC:$dst), 6072 (ins RC:$src1, KRC:$mask, RC:$src2), 6073 !strconcat(OpcodeStr, 6074 "\t{$src2, ${dst} {${mask}}|${dst} {${mask}}, $src2}"), 6075 []>, EVEX, EVEX_K; 6076 let mayLoad = 1 in 6077 def rmk : AVX5128I<opc, MRMSrcMem, (outs RC:$dst), 6078 (ins RC:$src1, KRC:$mask, x86memop:$src2), 6079 !strconcat(OpcodeStr, 6080 "\t{$src2, ${dst} {${mask}}|${dst} {${mask}}, $src2}"), 6081 []>, EVEX, EVEX_K; 6082 let mayLoad = 1 in 6083 def rmbk : AVX5128I<opc, MRMSrcMem, (outs RC:$dst), 6084 (ins RC:$src1, KRC:$mask, x86scalar_mop:$src2), 6085 !strconcat(OpcodeStr, "\t{${src2}", BrdcstStr, 6086 ", ${dst} {${mask}}|${dst} {${mask}}, ${src2}", BrdcstStr, "}"), 6087 []>, EVEX, EVEX_K, EVEX_B; 6088 } 6089 } 6090} 6091 6092let Predicates = [HasCDI] in { 6093defm VPCONFLICTD : avx512_conflict<0xC4, "vpconflictd", VR512, VK16WM, 6094 i512mem, i32mem, "{1to16}">, 6095 EVEX_V512, EVEX_CD8<32, CD8VF>; 6096 6097 6098defm VPCONFLICTQ : avx512_conflict<0xC4, "vpconflictq", VR512, VK8WM, 6099 i512mem, i64mem, "{1to8}">, 6100 EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; 6101 6102} 6103 6104def : Pat<(int_x86_avx512_mask_conflict_d_512 VR512:$src2, VR512:$src1, 6105 GR16:$mask), 6106 (VPCONFLICTDrrk VR512:$src1, 6107 (v16i1 (COPY_TO_REGCLASS GR16:$mask, VK16WM)), VR512:$src2)>; 6108 6109def : Pat<(int_x86_avx512_mask_conflict_q_512 VR512:$src2, VR512:$src1, 6110 GR8:$mask), 6111 (VPCONFLICTQrrk VR512:$src1, 6112 (v8i1 (COPY_TO_REGCLASS GR8:$mask, VK8WM)), VR512:$src2)>; 6113 6114let Predicates = [HasCDI] in { 6115defm VPLZCNTD : avx512_conflict<0x44, "vplzcntd", VR512, VK16WM, 6116 i512mem, i32mem, "{1to16}">, 6117 EVEX_V512, EVEX_CD8<32, CD8VF>; 6118 6119 6120defm VPLZCNTQ : avx512_conflict<0x44, "vplzcntq", VR512, VK8WM, 6121 i512mem, i64mem, "{1to8}">, 6122 EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>; 6123 6124} 6125 6126def : Pat<(int_x86_avx512_mask_lzcnt_d_512 VR512:$src2, VR512:$src1, 6127 GR16:$mask), 6128 (VPLZCNTDrrk VR512:$src1, 6129 (v16i1 (COPY_TO_REGCLASS GR16:$mask, VK16WM)), VR512:$src2)>; 6130 6131def : Pat<(int_x86_avx512_mask_lzcnt_q_512 VR512:$src2, VR512:$src1, 6132 GR8:$mask), 6133 (VPLZCNTQrrk VR512:$src1, 6134 (v8i1 (COPY_TO_REGCLASS GR8:$mask, VK8WM)), VR512:$src2)>; 6135 6136def : Pat<(v16i32 (ctlz (loadv16i32 addr:$src))), 6137 (VPLZCNTDrm addr:$src)>; 6138def : Pat<(v16i32 (ctlz (v16i32 VR512:$src))), 6139 (VPLZCNTDrr VR512:$src)>; 6140def : Pat<(v8i64 (ctlz (loadv8i64 addr:$src))), 6141 (VPLZCNTQrm addr:$src)>; 6142def : Pat<(v8i64 (ctlz (v8i64 VR512:$src))), 6143 (VPLZCNTQrr VR512:$src)>; 6144 6145def : Pat<(store (i1 -1), addr:$dst), (MOV8mi addr:$dst, (i8 1))>; 6146def : Pat<(store (i1 1), addr:$dst), (MOV8mi addr:$dst, (i8 1))>; 6147def : Pat<(store (i1 0), addr:$dst), (MOV8mi addr:$dst, (i8 0))>; 6148 6149def : Pat<(store VK1:$src, addr:$dst), 6150 (MOV8mr addr:$dst, 6151 (EXTRACT_SUBREG (KMOVWrk (COPY_TO_REGCLASS VK1:$src, VK16)), 6152 sub_8bit))>, Requires<[HasAVX512, NoDQI]>; 6153 6154def : Pat<(store VK8:$src, addr:$dst), 6155 (MOV8mr addr:$dst, 6156 (EXTRACT_SUBREG (KMOVWrk (COPY_TO_REGCLASS VK8:$src, VK16)), 6157 sub_8bit))>, Requires<[HasAVX512, NoDQI]>; 6158 6159def truncstorei1 : PatFrag<(ops node:$val, node:$ptr), 6160 (truncstore node:$val, node:$ptr), [{ 6161 return cast<StoreSDNode>(N)->getMemoryVT() == MVT::i1; 6162}]>; 6163 6164def : Pat<(truncstorei1 GR8:$src, addr:$dst), 6165 (MOV8mr addr:$dst, GR8:$src)>; 6166 6167multiclass cvt_by_vec_width<bits<8> opc, X86VectorVTInfo Vec, string OpcodeStr > { 6168def rr : AVX512XS8I<opc, MRMSrcReg, (outs Vec.RC:$dst), (ins Vec.KRC:$src), 6169 !strconcat(OpcodeStr##Vec.Suffix, "\t{$src, $dst|$dst, $src}"), 6170 [(set Vec.RC:$dst, (Vec.VT (X86vsext Vec.KRC:$src)))]>, EVEX; 6171} 6172 6173multiclass cvt_mask_by_elt_width<bits<8> opc, AVX512VLVectorVTInfo VTInfo, 6174 string OpcodeStr, Predicate prd> { 6175let Predicates = [prd] in 6176 defm Z : cvt_by_vec_width<opc, VTInfo.info512, OpcodeStr>, EVEX_V512; 6177 6178 let Predicates = [prd, HasVLX] in { 6179 defm Z256 : cvt_by_vec_width<opc, VTInfo.info256, OpcodeStr>, EVEX_V256; 6180 defm Z128 : cvt_by_vec_width<opc, VTInfo.info128, OpcodeStr>, EVEX_V128; 6181 } 6182} 6183 6184multiclass avx512_convert_mask_to_vector<string OpcodeStr> { 6185 defm NAME##B : cvt_mask_by_elt_width<0x28, avx512vl_i8_info, OpcodeStr, 6186 HasBWI>; 6187 defm NAME##W : cvt_mask_by_elt_width<0x28, avx512vl_i16_info, OpcodeStr, 6188 HasBWI>, VEX_W; 6189 defm NAME##D : cvt_mask_by_elt_width<0x38, avx512vl_i32_info, OpcodeStr, 6190 HasDQI>; 6191 defm NAME##Q : cvt_mask_by_elt_width<0x38, avx512vl_i64_info, OpcodeStr, 6192 HasDQI>, VEX_W; 6193} 6194 6195defm VPMOVM2 : avx512_convert_mask_to_vector<"vpmovm2">; 6196 6197multiclass convert_vector_to_mask_common<bits<8> opc, X86VectorVTInfo _, string OpcodeStr > { 6198def rr : AVX512XS8I<opc, MRMSrcReg, (outs _.KRC:$dst), (ins _.RC:$src), 6199 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 6200 [(set _.KRC:$dst, (trunc (_.VT _.RC:$src)))]>, EVEX; 6201} 6202 6203multiclass avx512_convert_vector_to_mask<bits<8> opc, string OpcodeStr, 6204 AVX512VLVectorVTInfo VTInfo, Predicate prd> { 6205let Predicates = [prd] in 6206 defm Z : convert_vector_to_mask_common <opc, VTInfo.info512, OpcodeStr>, 6207 EVEX_V512; 6208 6209 let Predicates = [prd, HasVLX] in { 6210 defm Z256 : convert_vector_to_mask_common<opc, VTInfo.info256, OpcodeStr>, 6211 EVEX_V256; 6212 defm Z128 : convert_vector_to_mask_common<opc, VTInfo.info128, OpcodeStr>, 6213 EVEX_V128; 6214 } 6215} 6216 6217defm VPMOVB2M : avx512_convert_vector_to_mask<0x29, "vpmovb2m", 6218 avx512vl_i8_info, HasBWI>; 6219defm VPMOVW2M : avx512_convert_vector_to_mask<0x29, "vpmovw2m", 6220 avx512vl_i16_info, HasBWI>, VEX_W; 6221defm VPMOVD2M : avx512_convert_vector_to_mask<0x39, "vpmovd2m", 6222 avx512vl_i32_info, HasDQI>; 6223defm VPMOVQ2M : avx512_convert_vector_to_mask<0x39, "vpmovq2m", 6224 avx512vl_i64_info, HasDQI>, VEX_W; 6225 6226//===----------------------------------------------------------------------===// 6227// AVX-512 - COMPRESS and EXPAND 6228// 6229 6230multiclass compress_by_vec_width<bits<8> opc, X86VectorVTInfo _, 6231 string OpcodeStr> { 6232 defm rr : AVX512_maskable<opc, MRMDestReg, _, (outs _.RC:$dst), 6233 (ins _.RC:$src1), OpcodeStr, "$src1", "$src1", 6234 (_.VT (X86compress _.RC:$src1))>, AVX5128IBase; 6235 6236 let mayStore = 1 in { 6237 def mr : AVX5128I<opc, MRMDestMem, (outs), 6238 (ins _.MemOp:$dst, _.RC:$src), 6239 OpcodeStr # "\t{$src, $dst |$dst, $src}", 6240 []>, EVEX_CD8<_.EltSize, CD8VT1>; 6241 6242 def mrk : AVX5128I<opc, MRMDestMem, (outs), 6243 (ins _.MemOp:$dst, _.KRCWM:$mask, _.RC:$src), 6244 OpcodeStr # "\t{$src, $dst {${mask}} |$dst {${mask}}, $src}", 6245 [(store (_.VT (vselect _.KRCWM:$mask, 6246 (_.VT (X86compress _.RC:$src)), _.ImmAllZerosV)), 6247 addr:$dst)]>, 6248 EVEX_K, EVEX_CD8<_.EltSize, CD8VT1>; 6249 } 6250} 6251 6252multiclass compress_by_elt_width<bits<8> opc, string OpcodeStr, 6253 AVX512VLVectorVTInfo VTInfo> { 6254 defm Z : compress_by_vec_width<opc, VTInfo.info512, OpcodeStr>, EVEX_V512; 6255 6256 let Predicates = [HasVLX] in { 6257 defm Z256 : compress_by_vec_width<opc, VTInfo.info256, OpcodeStr>, EVEX_V256; 6258 defm Z128 : compress_by_vec_width<opc, VTInfo.info128, OpcodeStr>, EVEX_V128; 6259 } 6260} 6261 6262defm VPCOMPRESSD : compress_by_elt_width <0x8B, "vpcompressd", avx512vl_i32_info>, 6263 EVEX; 6264defm VPCOMPRESSQ : compress_by_elt_width <0x8B, "vpcompressq", avx512vl_i64_info>, 6265 EVEX, VEX_W; 6266defm VCOMPRESSPS : compress_by_elt_width <0x8A, "vcompressps", avx512vl_f32_info>, 6267 EVEX; 6268defm VCOMPRESSPD : compress_by_elt_width <0x8A, "vcompresspd", avx512vl_f64_info>, 6269 EVEX, VEX_W; 6270 6271// expand 6272multiclass expand_by_vec_width<bits<8> opc, X86VectorVTInfo _, 6273 string OpcodeStr> { 6274 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), 6275 (ins _.RC:$src1), OpcodeStr, "$src1", "$src1", 6276 (_.VT (X86expand _.RC:$src1))>, AVX5128IBase; 6277 6278 let mayLoad = 1 in 6279 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), 6280 (ins _.MemOp:$src1), OpcodeStr, "$src1", "$src1", 6281 (_.VT (X86expand (_.VT (bitconvert 6282 (_.LdFrag addr:$src1)))))>, 6283 AVX5128IBase, EVEX_CD8<_.EltSize, CD8VT1>; 6284} 6285 6286multiclass expand_by_elt_width<bits<8> opc, string OpcodeStr, 6287 AVX512VLVectorVTInfo VTInfo> { 6288 defm Z : expand_by_vec_width<opc, VTInfo.info512, OpcodeStr>, EVEX_V512; 6289 6290 let Predicates = [HasVLX] in { 6291 defm Z256 : expand_by_vec_width<opc, VTInfo.info256, OpcodeStr>, EVEX_V256; 6292 defm Z128 : expand_by_vec_width<opc, VTInfo.info128, OpcodeStr>, EVEX_V128; 6293 } 6294} 6295 6296defm VPEXPANDD : expand_by_elt_width <0x89, "vpexpandd", avx512vl_i32_info>, 6297 EVEX; 6298defm VPEXPANDQ : expand_by_elt_width <0x89, "vpexpandq", avx512vl_i64_info>, 6299 EVEX, VEX_W; 6300defm VEXPANDPS : expand_by_elt_width <0x88, "vexpandps", avx512vl_f32_info>, 6301 EVEX; 6302defm VEXPANDPD : expand_by_elt_width <0x88, "vexpandpd", avx512vl_f64_info>, 6303 EVEX, VEX_W; 6304 6305//handle instruction reg_vec1 = op(reg_vec2,reg_vec3,imm) 6306// op(reg_vec2,mem_vec,imm) 6307// op(reg_vec2,broadcast(eltVt),imm) 6308//all instruction created with FROUND_CURRENT 6309multiclass avx512_fp_packed_imm<bits<8> opc, string OpcodeStr, SDNode OpNode, 6310 X86VectorVTInfo _>{ 6311 defm rri : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), 6312 (ins _.RC:$src1, _.RC:$src2, u8imm:$src3), 6313 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3", 6314 (OpNode (_.VT _.RC:$src1), 6315 (_.VT _.RC:$src2), 6316 (i8 imm:$src3), 6317 (i32 FROUND_CURRENT))>; 6318 let mayLoad = 1 in { 6319 defm rmi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), 6320 (ins _.RC:$src1, _.MemOp:$src2, u8imm:$src3), 6321 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3", 6322 (OpNode (_.VT _.RC:$src1), 6323 (_.VT (bitconvert (_.LdFrag addr:$src2))), 6324 (i8 imm:$src3), 6325 (i32 FROUND_CURRENT))>; 6326 defm rmbi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), 6327 (ins _.RC:$src1, _.ScalarMemOp:$src2, u8imm:$src3), 6328 OpcodeStr, "$src3, ${src2}"##_.BroadcastStr##", $src1", 6329 "$src1, ${src2}"##_.BroadcastStr##", $src3", 6330 (OpNode (_.VT _.RC:$src1), 6331 (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src2))), 6332 (i8 imm:$src3), 6333 (i32 FROUND_CURRENT))>, EVEX_B; 6334 } 6335} 6336 6337//handle instruction reg_vec1 = op(reg_vec2,reg_vec3,imm) 6338// op(reg_vec2,mem_vec,imm) 6339// op(reg_vec2,broadcast(eltVt),imm) 6340multiclass avx512_3Op_imm8<bits<8> opc, string OpcodeStr, SDNode OpNode, 6341 X86VectorVTInfo _>{ 6342 defm rri : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), 6343 (ins _.RC:$src1, _.RC:$src2, u8imm:$src3), 6344 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3", 6345 (OpNode (_.VT _.RC:$src1), 6346 (_.VT _.RC:$src2), 6347 (i8 imm:$src3))>; 6348 let mayLoad = 1 in { 6349 defm rmi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), 6350 (ins _.RC:$src1, _.MemOp:$src2, u8imm:$src3), 6351 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3", 6352 (OpNode (_.VT _.RC:$src1), 6353 (_.VT (bitconvert (_.LdFrag addr:$src2))), 6354 (i8 imm:$src3))>; 6355 defm rmbi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), 6356 (ins _.RC:$src1, _.ScalarMemOp:$src2, u8imm:$src3), 6357 OpcodeStr, "$src3, ${src2}"##_.BroadcastStr##", $src1", 6358 "$src1, ${src2}"##_.BroadcastStr##", $src3", 6359 (OpNode (_.VT _.RC:$src1), 6360 (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src2))), 6361 (i8 imm:$src3))>, EVEX_B; 6362 } 6363} 6364 6365//handle scalar instruction reg_vec1 = op(reg_vec2,reg_vec3,imm) 6366// op(reg_vec2,mem_scalar,imm) 6367//all instruction created with FROUND_CURRENT 6368multiclass avx512_fp_scalar_imm<bits<8> opc, string OpcodeStr, SDNode OpNode, 6369 X86VectorVTInfo _> { 6370 6371 defm rri : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst), 6372 (ins _.RC:$src1, _.RC:$src2, u8imm:$src3), 6373 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3", 6374 (OpNode (_.VT _.RC:$src1), 6375 (_.VT _.RC:$src2), 6376 (i8 imm:$src3), 6377 (i32 FROUND_CURRENT))>; 6378 let mayLoad = 1 in { 6379 defm rmi : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst), 6380 (ins _.RC:$src1, _.MemOp:$src2, u8imm:$src3), 6381 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3", 6382 (OpNode (_.VT _.RC:$src1), 6383 (_.VT (scalar_to_vector 6384 (_.ScalarLdFrag addr:$src2))), 6385 (i8 imm:$src3), 6386 (i32 FROUND_CURRENT))>; 6387 6388 let isAsmParserOnly = 1 in { 6389 defm rmi_alt :AVX512_maskable_in_asm<opc, MRMSrcMem, _, (outs _.FRC:$dst), 6390 (ins _.FRC:$src1, _.ScalarMemOp:$src2, u8imm:$src3), 6391 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3", 6392 []>; 6393 } 6394 } 6395} 6396 6397//handle instruction reg_vec1 = op(reg_vec2,reg_vec3,imm),{sae} 6398multiclass avx512_fp_sae_packed_imm<bits<8> opc, string OpcodeStr, 6399 SDNode OpNode, X86VectorVTInfo _>{ 6400 defm rrib : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), 6401 (ins _.RC:$src1, _.RC:$src2, u8imm:$src3), 6402 OpcodeStr, "$src3,{sae}, $src2, $src1", 6403 "$src1, $src2,{sae}, $src3", 6404 (OpNode (_.VT _.RC:$src1), 6405 (_.VT _.RC:$src2), 6406 (i8 imm:$src3), 6407 (i32 FROUND_NO_EXC))>, EVEX_B; 6408} 6409//handle scalar instruction reg_vec1 = op(reg_vec2,reg_vec3,imm),{sae} 6410multiclass avx512_fp_sae_scalar_imm<bits<8> opc, string OpcodeStr, 6411 SDNode OpNode, X86VectorVTInfo _> { 6412 defm NAME: avx512_fp_sae_packed_imm<opc, OpcodeStr, OpNode, _>; 6413} 6414 6415multiclass avx512_common_fp_sae_packed_imm<string OpcodeStr, 6416 AVX512VLVectorVTInfo _, bits<8> opc, SDNode OpNode, Predicate prd>{ 6417 let Predicates = [prd] in { 6418 defm Z : avx512_fp_packed_imm<opc, OpcodeStr, OpNode, _.info512>, 6419 avx512_fp_sae_packed_imm<opc, OpcodeStr, OpNode, _.info512>, 6420 EVEX_V512; 6421 6422 } 6423 let Predicates = [prd, HasVLX] in { 6424 defm Z128 : avx512_fp_packed_imm<opc, OpcodeStr, OpNode, _.info128>, 6425 EVEX_V128; 6426 defm Z256 : avx512_fp_packed_imm<opc, OpcodeStr, OpNode, _.info256>, 6427 EVEX_V256; 6428 } 6429} 6430 6431multiclass avx512_common_3Op_imm8<string OpcodeStr, AVX512VLVectorVTInfo _, 6432 bits<8> opc, SDNode OpNode>{ 6433 let Predicates = [HasAVX512] in { 6434 defm Z : avx512_3Op_imm8<opc, OpcodeStr, OpNode, _.info512>, EVEX_V512; 6435 } 6436 let Predicates = [HasAVX512, HasVLX] in { 6437 defm Z128 : avx512_3Op_imm8<opc, OpcodeStr, OpNode, _.info128>, EVEX_V128; 6438 defm Z256 : avx512_3Op_imm8<opc, OpcodeStr, OpNode, _.info256>, EVEX_V256; 6439 } 6440} 6441 6442multiclass avx512_common_fp_sae_scalar_imm<string OpcodeStr, 6443 X86VectorVTInfo _, bits<8> opc, SDNode OpNode, Predicate prd>{ 6444 let Predicates = [prd] in { 6445 defm Z128 : avx512_fp_scalar_imm<opc, OpcodeStr, OpNode, _>, 6446 avx512_fp_sae_scalar_imm<opc, OpcodeStr, OpNode, _>; 6447 } 6448} 6449 6450defm VFIXUPIMMPD : avx512_common_fp_sae_packed_imm<"vfixupimmpd", 6451 avx512vl_f64_info, 0x54, X86VFixupimm, HasAVX512>, 6452 AVX512AIi8Base, EVEX_4V, EVEX_CD8<64, CD8VF>, VEX_W; 6453defm VFIXUPIMMPS : avx512_common_fp_sae_packed_imm<"vfixupimmps", 6454 avx512vl_f32_info, 0x54, X86VFixupimm, HasAVX512>, 6455 AVX512AIi8Base, EVEX_4V, EVEX_CD8<32, CD8VF>; 6456 6457defm VFIXUPIMMSD: avx512_common_fp_sae_scalar_imm<"vfixupimmsd", f64x_info, 6458 0x55, X86VFixupimm, HasAVX512>, 6459 AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<64, CD8VT1>, VEX_W; 6460defm VFIXUPIMMSS: avx512_common_fp_sae_scalar_imm<"vfixupimmss", f32x_info, 6461 0x55, X86VFixupimm, HasAVX512>, 6462 AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<32, CD8VT1>; 6463 6464defm VRANGEPD : avx512_common_fp_sae_packed_imm<"vrangepd", avx512vl_f64_info, 6465 0x50, X86VRange, HasDQI>, 6466 AVX512AIi8Base, EVEX_4V, EVEX_CD8<64, CD8VF>, VEX_W; 6467defm VRANGEPS : avx512_common_fp_sae_packed_imm<"vrangeps", avx512vl_f32_info, 6468 0x50, X86VRange, HasDQI>, 6469 AVX512AIi8Base, EVEX_4V, EVEX_CD8<32, CD8VF>; 6470 6471defm VRANGESD: avx512_common_fp_sae_scalar_imm<"vrangesd", f64x_info, 6472 0x51, X86VRange, HasDQI>, 6473 AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<64, CD8VT1>, VEX_W; 6474defm VRANGESS: avx512_common_fp_sae_scalar_imm<"vrangess", f32x_info, 6475 0x51, X86VRange, HasDQI>, 6476 AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<32, CD8VT1>; 6477 6478 6479multiclass avx512_shuff_packed_128<string OpcodeStr, AVX512VLVectorVTInfo _, 6480 bits<8> opc, SDNode OpNode = X86Shuf128>{ 6481 let Predicates = [HasAVX512] in { 6482 defm Z : avx512_3Op_imm8<opc, OpcodeStr, OpNode, _.info512>, EVEX_V512; 6483 6484 } 6485 let Predicates = [HasAVX512, HasVLX] in { 6486 defm Z256 : avx512_3Op_imm8<opc, OpcodeStr, OpNode, _.info256>, EVEX_V256; 6487 } 6488} 6489 6490defm VSHUFF32X4 : avx512_shuff_packed_128<"vshuff32x4",avx512vl_f32_info, 0x23>, 6491 AVX512AIi8Base, EVEX_4V, EVEX_CD8<32, CD8VF>; 6492defm VSHUFF64X2 : avx512_shuff_packed_128<"vshuff64x2",avx512vl_f64_info, 0x23>, 6493 AVX512AIi8Base, EVEX_4V, EVEX_CD8<64, CD8VF>, VEX_W; 6494defm VSHUFI32X4 : avx512_shuff_packed_128<"vshufi32x4",avx512vl_i32_info, 0x43>, 6495 AVX512AIi8Base, EVEX_4V, EVEX_CD8<32, CD8VF>; 6496defm VSHUFI64X2 : avx512_shuff_packed_128<"vshufi64x2",avx512vl_i64_info, 0x43>, 6497 AVX512AIi8Base, EVEX_4V, EVEX_CD8<64, CD8VF>, VEX_W; 6498 6499multiclass avx512_valign<string OpcodeStr, AVX512VLVectorVTInfo VTInfo_I, 6500 AVX512VLVectorVTInfo VTInfo_FP>{ 6501 defm NAME: avx512_common_3Op_imm8<OpcodeStr, VTInfo_I, 0x03, X86VAlign>, 6502 AVX512AIi8Base, EVEX_4V; 6503 let isCodeGenOnly = 1 in { 6504 defm NAME#_FP: avx512_common_3Op_imm8<OpcodeStr, VTInfo_FP, 0x03, X86VAlign>, 6505 AVX512AIi8Base, EVEX_4V; 6506 } 6507} 6508 6509defm VALIGND: avx512_valign<"valignd", avx512vl_i32_info, avx512vl_f32_info>, 6510 EVEX_CD8<32, CD8VF>; 6511defm VALIGNQ: avx512_valign<"valignq", avx512vl_i64_info, avx512vl_f64_info>, 6512 EVEX_CD8<64, CD8VF>, VEX_W; 6513 6514multiclass avx512_unary_rm<bits<8> opc, string OpcodeStr, SDNode OpNode, 6515 X86VectorVTInfo _> { 6516 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst), 6517 (ins _.RC:$src1), OpcodeStr##_.Suffix, 6518 "$src1", "$src1", 6519 (_.VT (OpNode _.RC:$src1))>, EVEX, AVX5128IBase; 6520 6521 let mayLoad = 1 in 6522 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), 6523 (ins _.MemOp:$src1), OpcodeStr##_.Suffix, 6524 "$src1", "$src1", 6525 (_.VT (OpNode (bitconvert (_.LdFrag addr:$src1))))>, 6526 EVEX, AVX5128IBase, EVEX_CD8<_.EltSize, CD8VF>; 6527} 6528 6529multiclass avx512_unary_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode, 6530 X86VectorVTInfo _> : 6531 avx512_unary_rm<opc, OpcodeStr, OpNode, _> { 6532 let mayLoad = 1 in 6533 defm rmb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst), 6534 (ins _.ScalarMemOp:$src1), OpcodeStr##_.Suffix, 6535 "${src1}"##_.BroadcastStr, 6536 "${src1}"##_.BroadcastStr, 6537 (_.VT (OpNode (X86VBroadcast 6538 (_.ScalarLdFrag addr:$src1))))>, 6539 EVEX, AVX5128IBase, EVEX_B, EVEX_CD8<_.EltSize, CD8VF>; 6540} 6541 6542multiclass avx512_unary_rm_vl<bits<8> opc, string OpcodeStr, SDNode OpNode, 6543 AVX512VLVectorVTInfo VTInfo, Predicate prd> { 6544 let Predicates = [prd] in 6545 defm Z : avx512_unary_rm<opc, OpcodeStr, OpNode, VTInfo.info512>, EVEX_V512; 6546 6547 let Predicates = [prd, HasVLX] in { 6548 defm Z256 : avx512_unary_rm<opc, OpcodeStr, OpNode, VTInfo.info256>, 6549 EVEX_V256; 6550 defm Z128 : avx512_unary_rm<opc, OpcodeStr, OpNode, VTInfo.info128>, 6551 EVEX_V128; 6552 } 6553} 6554 6555multiclass avx512_unary_rmb_vl<bits<8> opc, string OpcodeStr, SDNode OpNode, 6556 AVX512VLVectorVTInfo VTInfo, Predicate prd> { 6557 let Predicates = [prd] in 6558 defm Z : avx512_unary_rmb<opc, OpcodeStr, OpNode, VTInfo.info512>, 6559 EVEX_V512; 6560 6561 let Predicates = [prd, HasVLX] in { 6562 defm Z256 : avx512_unary_rmb<opc, OpcodeStr, OpNode, VTInfo.info256>, 6563 EVEX_V256; 6564 defm Z128 : avx512_unary_rmb<opc, OpcodeStr, OpNode, VTInfo.info128>, 6565 EVEX_V128; 6566 } 6567} 6568 6569multiclass avx512_unary_rm_vl_dq<bits<8> opc_d, bits<8> opc_q, string OpcodeStr, 6570 SDNode OpNode, Predicate prd> { 6571 defm Q : avx512_unary_rmb_vl<opc_q, OpcodeStr, OpNode, avx512vl_i64_info, 6572 prd>, VEX_W; 6573 defm D : avx512_unary_rmb_vl<opc_d, OpcodeStr, OpNode, avx512vl_i32_info, prd>; 6574} 6575 6576multiclass avx512_unary_rm_vl_bw<bits<8> opc_b, bits<8> opc_w, string OpcodeStr, 6577 SDNode OpNode, Predicate prd> { 6578 defm W : avx512_unary_rm_vl<opc_w, OpcodeStr, OpNode, avx512vl_i16_info, prd>; 6579 defm B : avx512_unary_rm_vl<opc_b, OpcodeStr, OpNode, avx512vl_i8_info, prd>; 6580} 6581 6582multiclass avx512_unary_rm_vl_all<bits<8> opc_b, bits<8> opc_w, 6583 bits<8> opc_d, bits<8> opc_q, 6584 string OpcodeStr, SDNode OpNode> { 6585 defm NAME : avx512_unary_rm_vl_dq<opc_d, opc_q, OpcodeStr, OpNode, 6586 HasAVX512>, 6587 avx512_unary_rm_vl_bw<opc_b, opc_w, OpcodeStr, OpNode, 6588 HasBWI>; 6589} 6590 6591defm VPABS : avx512_unary_rm_vl_all<0x1C, 0x1D, 0x1E, 0x1F, "vpabs", X86Abs>; 6592 6593def : Pat<(xor 6594 (bc_v16i32 (v16i1sextv16i32)), 6595 (bc_v16i32 (add (v16i32 VR512:$src), (v16i1sextv16i32)))), 6596 (VPABSDZrr VR512:$src)>; 6597def : Pat<(xor 6598 (bc_v8i64 (v8i1sextv8i64)), 6599 (bc_v8i64 (add (v8i64 VR512:$src), (v8i1sextv8i64)))), 6600 (VPABSQZrr VR512:$src)>; 6601