1//===-- X86InstrSSE.td - SSE Instruction Set ---------------*- tablegen -*-===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file describes the X86 SSE instruction set, defining the instructions, 11// and properties of the instructions which are needed for code generation, 12// machine code emission, and analysis. 13// 14//===----------------------------------------------------------------------===// 15 16class OpndItins<InstrItinClass arg_rr, InstrItinClass arg_rm> { 17 InstrItinClass rr = arg_rr; 18 InstrItinClass rm = arg_rm; 19 // InstrSchedModel info. 20 X86FoldableSchedWrite Sched = WriteFAdd; 21} 22 23class SizeItins<OpndItins arg_s, OpndItins arg_d> { 24 OpndItins s = arg_s; 25 OpndItins d = arg_d; 26} 27 28 29class ShiftOpndItins<InstrItinClass arg_rr, InstrItinClass arg_rm, 30 InstrItinClass arg_ri> { 31 InstrItinClass rr = arg_rr; 32 InstrItinClass rm = arg_rm; 33 InstrItinClass ri = arg_ri; 34} 35 36 37// scalar 38let Sched = WriteFAdd in { 39def SSE_ALU_F32S : OpndItins< 40 IIC_SSE_ALU_F32S_RR, IIC_SSE_ALU_F32S_RM 41>; 42 43def SSE_ALU_F64S : OpndItins< 44 IIC_SSE_ALU_F64S_RR, IIC_SSE_ALU_F64S_RM 45>; 46} 47 48def SSE_ALU_ITINS_S : SizeItins< 49 SSE_ALU_F32S, SSE_ALU_F64S 50>; 51 52let Sched = WriteFMul in { 53def SSE_MUL_F32S : OpndItins< 54 IIC_SSE_MUL_F32S_RR, IIC_SSE_MUL_F64S_RM 55>; 56 57def SSE_MUL_F64S : OpndItins< 58 IIC_SSE_MUL_F64S_RR, IIC_SSE_MUL_F64S_RM 59>; 60} 61 62def SSE_MUL_ITINS_S : SizeItins< 63 SSE_MUL_F32S, SSE_MUL_F64S 64>; 65 66let Sched = WriteFDiv in { 67def SSE_DIV_F32S : OpndItins< 68 IIC_SSE_DIV_F32S_RR, IIC_SSE_DIV_F64S_RM 69>; 70 71def SSE_DIV_F64S : OpndItins< 72 IIC_SSE_DIV_F64S_RR, IIC_SSE_DIV_F64S_RM 73>; 74} 75 76def SSE_DIV_ITINS_S : SizeItins< 77 SSE_DIV_F32S, SSE_DIV_F64S 78>; 79 80// parallel 81let Sched = WriteFAdd in { 82def SSE_ALU_F32P : OpndItins< 83 IIC_SSE_ALU_F32P_RR, IIC_SSE_ALU_F32P_RM 84>; 85 86def SSE_ALU_F64P : OpndItins< 87 IIC_SSE_ALU_F64P_RR, IIC_SSE_ALU_F64P_RM 88>; 89} 90 91def SSE_ALU_ITINS_P : SizeItins< 92 SSE_ALU_F32P, SSE_ALU_F64P 93>; 94 95let Sched = WriteFMul in { 96def SSE_MUL_F32P : OpndItins< 97 IIC_SSE_MUL_F32P_RR, IIC_SSE_MUL_F64P_RM 98>; 99 100def SSE_MUL_F64P : OpndItins< 101 IIC_SSE_MUL_F64P_RR, IIC_SSE_MUL_F64P_RM 102>; 103} 104 105def SSE_MUL_ITINS_P : SizeItins< 106 SSE_MUL_F32P, SSE_MUL_F64P 107>; 108 109let Sched = WriteFDiv in { 110def SSE_DIV_F32P : OpndItins< 111 IIC_SSE_DIV_F32P_RR, IIC_SSE_DIV_F64P_RM 112>; 113 114def SSE_DIV_F64P : OpndItins< 115 IIC_SSE_DIV_F64P_RR, IIC_SSE_DIV_F64P_RM 116>; 117} 118 119def SSE_DIV_ITINS_P : SizeItins< 120 SSE_DIV_F32P, SSE_DIV_F64P 121>; 122 123let Sched = WriteVecLogic in 124def SSE_VEC_BIT_ITINS_P : OpndItins< 125 IIC_SSE_BIT_P_RR, IIC_SSE_BIT_P_RM 126>; 127 128def SSE_BIT_ITINS_P : OpndItins< 129 IIC_SSE_BIT_P_RR, IIC_SSE_BIT_P_RM 130>; 131 132let Sched = WriteVecALU in { 133def SSE_INTALU_ITINS_P : OpndItins< 134 IIC_SSE_INTALU_P_RR, IIC_SSE_INTALU_P_RM 135>; 136 137def SSE_INTALUQ_ITINS_P : OpndItins< 138 IIC_SSE_INTALUQ_P_RR, IIC_SSE_INTALUQ_P_RM 139>; 140} 141 142let Sched = WriteVecIMul in 143def SSE_INTMUL_ITINS_P : OpndItins< 144 IIC_SSE_INTMUL_P_RR, IIC_SSE_INTMUL_P_RM 145>; 146 147def SSE_INTSHIFT_ITINS_P : ShiftOpndItins< 148 IIC_SSE_INTSH_P_RR, IIC_SSE_INTSH_P_RM, IIC_SSE_INTSH_P_RI 149>; 150 151def SSE_MOVA_ITINS : OpndItins< 152 IIC_SSE_MOVA_P_RR, IIC_SSE_MOVA_P_RM 153>; 154 155def SSE_MOVU_ITINS : OpndItins< 156 IIC_SSE_MOVU_P_RR, IIC_SSE_MOVU_P_RM 157>; 158 159def SSE_DPPD_ITINS : OpndItins< 160 IIC_SSE_DPPD_RR, IIC_SSE_DPPD_RM 161>; 162 163def SSE_DPPS_ITINS : OpndItins< 164 IIC_SSE_DPPS_RR, IIC_SSE_DPPD_RM 165>; 166 167def DEFAULT_ITINS : OpndItins< 168 IIC_ALU_NONMEM, IIC_ALU_MEM 169>; 170 171def SSE_EXTRACT_ITINS : OpndItins< 172 IIC_SSE_EXTRACTPS_RR, IIC_SSE_EXTRACTPS_RM 173>; 174 175def SSE_INSERT_ITINS : OpndItins< 176 IIC_SSE_INSERTPS_RR, IIC_SSE_INSERTPS_RM 177>; 178 179let Sched = WriteMPSAD in 180def SSE_MPSADBW_ITINS : OpndItins< 181 IIC_SSE_MPSADBW_RR, IIC_SSE_MPSADBW_RM 182>; 183 184let Sched = WriteVecIMul in 185def SSE_PMULLD_ITINS : OpndItins< 186 IIC_SSE_PMULLD_RR, IIC_SSE_PMULLD_RM 187>; 188 189// Definitions for backward compatibility. 190// The instructions mapped on these definitions uses a different itinerary 191// than the actual scheduling model. 192let Sched = WriteShuffle in 193def DEFAULT_ITINS_SHUFFLESCHED : OpndItins< 194 IIC_ALU_NONMEM, IIC_ALU_MEM 195>; 196 197let Sched = WriteVecIMul in 198def DEFAULT_ITINS_VECIMULSCHED : OpndItins< 199 IIC_ALU_NONMEM, IIC_ALU_MEM 200>; 201 202let Sched = WriteShuffle in 203def SSE_INTALU_ITINS_SHUFF_P : OpndItins< 204 IIC_SSE_INTALU_P_RR, IIC_SSE_INTALU_P_RM 205>; 206 207let Sched = WriteMPSAD in 208def DEFAULT_ITINS_MPSADSCHED : OpndItins< 209 IIC_ALU_NONMEM, IIC_ALU_MEM 210>; 211 212let Sched = WriteFBlend in 213def DEFAULT_ITINS_FBLENDSCHED : OpndItins< 214 IIC_ALU_NONMEM, IIC_ALU_MEM 215>; 216 217let Sched = WriteBlend in 218def DEFAULT_ITINS_BLENDSCHED : OpndItins< 219 IIC_ALU_NONMEM, IIC_ALU_MEM 220>; 221 222let Sched = WriteVarBlend in 223def DEFAULT_ITINS_VARBLENDSCHED : OpndItins< 224 IIC_ALU_NONMEM, IIC_ALU_MEM 225>; 226 227let Sched = WriteFBlend in 228def SSE_INTALU_ITINS_FBLEND_P : OpndItins< 229 IIC_SSE_INTALU_P_RR, IIC_SSE_INTALU_P_RM 230>; 231 232let Sched = WriteBlend in 233def SSE_INTALU_ITINS_BLEND_P : OpndItins< 234 IIC_SSE_INTALU_P_RR, IIC_SSE_INTALU_P_RM 235>; 236 237//===----------------------------------------------------------------------===// 238// SSE 1 & 2 Instructions Classes 239//===----------------------------------------------------------------------===// 240 241/// sse12_fp_scalar - SSE 1 & 2 scalar instructions class 242multiclass sse12_fp_scalar<bits<8> opc, string OpcodeStr, SDNode OpNode, 243 RegisterClass RC, X86MemOperand x86memop, 244 Domain d, OpndItins itins, bit Is2Addr = 1> { 245 let isCommutable = 1 in { 246 def rr : SI<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2), 247 !if(Is2Addr, 248 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 249 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 250 [(set RC:$dst, (OpNode RC:$src1, RC:$src2))], itins.rr, d>, 251 Sched<[itins.Sched]>; 252 } 253 def rm : SI<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2), 254 !if(Is2Addr, 255 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 256 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 257 [(set RC:$dst, (OpNode RC:$src1, (load addr:$src2)))], itins.rm, d>, 258 Sched<[itins.Sched.Folded, ReadAfterLd]>; 259} 260 261/// sse12_fp_scalar_int - SSE 1 & 2 scalar instructions intrinsics class 262multiclass sse12_fp_scalar_int<bits<8> opc, string OpcodeStr, RegisterClass RC, 263 string asm, string SSEVer, string FPSizeStr, 264 Operand memopr, ComplexPattern mem_cpat, 265 Domain d, OpndItins itins, bit Is2Addr = 1> { 266let isCodeGenOnly = 1 in { 267 def rr_Int : SI_Int<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2), 268 !if(Is2Addr, 269 !strconcat(asm, "\t{$src2, $dst|$dst, $src2}"), 270 !strconcat(asm, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 271 [(set RC:$dst, (!cast<Intrinsic>( 272 !strconcat("int_x86_sse", SSEVer, "_", OpcodeStr, FPSizeStr)) 273 RC:$src1, RC:$src2))], itins.rr, d>, 274 Sched<[itins.Sched]>; 275 def rm_Int : SI_Int<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, memopr:$src2), 276 !if(Is2Addr, 277 !strconcat(asm, "\t{$src2, $dst|$dst, $src2}"), 278 !strconcat(asm, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 279 [(set RC:$dst, (!cast<Intrinsic>(!strconcat("int_x86_sse", 280 SSEVer, "_", OpcodeStr, FPSizeStr)) 281 RC:$src1, mem_cpat:$src2))], itins.rm, d>, 282 Sched<[itins.Sched.Folded, ReadAfterLd]>; 283} 284} 285 286/// sse12_fp_packed - SSE 1 & 2 packed instructions class 287multiclass sse12_fp_packed<bits<8> opc, string OpcodeStr, SDNode OpNode, 288 RegisterClass RC, ValueType vt, 289 X86MemOperand x86memop, PatFrag mem_frag, 290 Domain d, OpndItins itins, bit Is2Addr = 1> { 291 let isCommutable = 1 in 292 def rr : PI<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2), 293 !if(Is2Addr, 294 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 295 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 296 [(set RC:$dst, (vt (OpNode RC:$src1, RC:$src2)))], itins.rr, d>, 297 Sched<[itins.Sched]>; 298 let mayLoad = 1 in 299 def rm : PI<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2), 300 !if(Is2Addr, 301 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 302 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 303 [(set RC:$dst, (OpNode RC:$src1, (mem_frag addr:$src2)))], 304 itins.rm, d>, 305 Sched<[itins.Sched.Folded, ReadAfterLd]>; 306} 307 308/// sse12_fp_packed_logical_rm - SSE 1 & 2 packed instructions class 309multiclass sse12_fp_packed_logical_rm<bits<8> opc, RegisterClass RC, Domain d, 310 string OpcodeStr, X86MemOperand x86memop, 311 list<dag> pat_rr, list<dag> pat_rm, 312 bit Is2Addr = 1> { 313 let isCommutable = 1, hasSideEffects = 0 in 314 def rr : PI<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2), 315 !if(Is2Addr, 316 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 317 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 318 pat_rr, NoItinerary, d>, 319 Sched<[WriteVecLogic]>; 320 def rm : PI<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2), 321 !if(Is2Addr, 322 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 323 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 324 pat_rm, NoItinerary, d>, 325 Sched<[WriteVecLogicLd, ReadAfterLd]>; 326} 327 328//===----------------------------------------------------------------------===// 329// Non-instruction patterns 330//===----------------------------------------------------------------------===// 331 332// A vector extract of the first f32/f64 position is a subregister copy 333def : Pat<(f32 (vector_extract (v4f32 VR128:$src), (iPTR 0))), 334 (COPY_TO_REGCLASS (v4f32 VR128:$src), FR32)>; 335def : Pat<(f64 (vector_extract (v2f64 VR128:$src), (iPTR 0))), 336 (COPY_TO_REGCLASS (v2f64 VR128:$src), FR64)>; 337 338// A 128-bit subvector extract from the first 256-bit vector position 339// is a subregister copy that needs no instruction. 340def : Pat<(v4i32 (extract_subvector (v8i32 VR256:$src), (iPTR 0))), 341 (v4i32 (EXTRACT_SUBREG (v8i32 VR256:$src), sub_xmm))>; 342def : Pat<(v4f32 (extract_subvector (v8f32 VR256:$src), (iPTR 0))), 343 (v4f32 (EXTRACT_SUBREG (v8f32 VR256:$src), sub_xmm))>; 344 345def : Pat<(v2i64 (extract_subvector (v4i64 VR256:$src), (iPTR 0))), 346 (v2i64 (EXTRACT_SUBREG (v4i64 VR256:$src), sub_xmm))>; 347def : Pat<(v2f64 (extract_subvector (v4f64 VR256:$src), (iPTR 0))), 348 (v2f64 (EXTRACT_SUBREG (v4f64 VR256:$src), sub_xmm))>; 349 350def : Pat<(v8i16 (extract_subvector (v16i16 VR256:$src), (iPTR 0))), 351 (v8i16 (EXTRACT_SUBREG (v16i16 VR256:$src), sub_xmm))>; 352def : Pat<(v16i8 (extract_subvector (v32i8 VR256:$src), (iPTR 0))), 353 (v16i8 (EXTRACT_SUBREG (v32i8 VR256:$src), sub_xmm))>; 354 355// A 128-bit subvector insert to the first 256-bit vector position 356// is a subregister copy that needs no instruction. 357let AddedComplexity = 25 in { // to give priority over vinsertf128rm 358def : Pat<(insert_subvector undef, (v2i64 VR128:$src), (iPTR 0)), 359 (INSERT_SUBREG (v4i64 (IMPLICIT_DEF)), VR128:$src, sub_xmm)>; 360def : Pat<(insert_subvector undef, (v2f64 VR128:$src), (iPTR 0)), 361 (INSERT_SUBREG (v4f64 (IMPLICIT_DEF)), VR128:$src, sub_xmm)>; 362def : Pat<(insert_subvector undef, (v4i32 VR128:$src), (iPTR 0)), 363 (INSERT_SUBREG (v8i32 (IMPLICIT_DEF)), VR128:$src, sub_xmm)>; 364def : Pat<(insert_subvector undef, (v4f32 VR128:$src), (iPTR 0)), 365 (INSERT_SUBREG (v8f32 (IMPLICIT_DEF)), VR128:$src, sub_xmm)>; 366def : Pat<(insert_subvector undef, (v8i16 VR128:$src), (iPTR 0)), 367 (INSERT_SUBREG (v16i16 (IMPLICIT_DEF)), VR128:$src, sub_xmm)>; 368def : Pat<(insert_subvector undef, (v16i8 VR128:$src), (iPTR 0)), 369 (INSERT_SUBREG (v32i8 (IMPLICIT_DEF)), VR128:$src, sub_xmm)>; 370} 371 372// Implicitly promote a 32-bit scalar to a vector. 373def : Pat<(v4f32 (scalar_to_vector FR32:$src)), 374 (COPY_TO_REGCLASS FR32:$src, VR128)>; 375def : Pat<(v8f32 (scalar_to_vector FR32:$src)), 376 (COPY_TO_REGCLASS FR32:$src, VR128)>; 377// Implicitly promote a 64-bit scalar to a vector. 378def : Pat<(v2f64 (scalar_to_vector FR64:$src)), 379 (COPY_TO_REGCLASS FR64:$src, VR128)>; 380def : Pat<(v4f64 (scalar_to_vector FR64:$src)), 381 (COPY_TO_REGCLASS FR64:$src, VR128)>; 382 383// Bitcasts between 128-bit vector types. Return the original type since 384// no instruction is needed for the conversion 385let Predicates = [HasSSE2] in { 386 def : Pat<(v2i64 (bitconvert (v4i32 VR128:$src))), (v2i64 VR128:$src)>; 387 def : Pat<(v2i64 (bitconvert (v8i16 VR128:$src))), (v2i64 VR128:$src)>; 388 def : Pat<(v2i64 (bitconvert (v16i8 VR128:$src))), (v2i64 VR128:$src)>; 389 def : Pat<(v2i64 (bitconvert (v2f64 VR128:$src))), (v2i64 VR128:$src)>; 390 def : Pat<(v2i64 (bitconvert (v4f32 VR128:$src))), (v2i64 VR128:$src)>; 391 def : Pat<(v4i32 (bitconvert (v2i64 VR128:$src))), (v4i32 VR128:$src)>; 392 def : Pat<(v4i32 (bitconvert (v8i16 VR128:$src))), (v4i32 VR128:$src)>; 393 def : Pat<(v4i32 (bitconvert (v16i8 VR128:$src))), (v4i32 VR128:$src)>; 394 def : Pat<(v4i32 (bitconvert (v2f64 VR128:$src))), (v4i32 VR128:$src)>; 395 def : Pat<(v4i32 (bitconvert (v4f32 VR128:$src))), (v4i32 VR128:$src)>; 396 def : Pat<(v8i16 (bitconvert (v2i64 VR128:$src))), (v8i16 VR128:$src)>; 397 def : Pat<(v8i16 (bitconvert (v4i32 VR128:$src))), (v8i16 VR128:$src)>; 398 def : Pat<(v8i16 (bitconvert (v16i8 VR128:$src))), (v8i16 VR128:$src)>; 399 def : Pat<(v8i16 (bitconvert (v2f64 VR128:$src))), (v8i16 VR128:$src)>; 400 def : Pat<(v8i16 (bitconvert (v4f32 VR128:$src))), (v8i16 VR128:$src)>; 401 def : Pat<(v16i8 (bitconvert (v2i64 VR128:$src))), (v16i8 VR128:$src)>; 402 def : Pat<(v16i8 (bitconvert (v4i32 VR128:$src))), (v16i8 VR128:$src)>; 403 def : Pat<(v16i8 (bitconvert (v8i16 VR128:$src))), (v16i8 VR128:$src)>; 404 def : Pat<(v16i8 (bitconvert (v2f64 VR128:$src))), (v16i8 VR128:$src)>; 405 def : Pat<(v16i8 (bitconvert (v4f32 VR128:$src))), (v16i8 VR128:$src)>; 406 def : Pat<(v4f32 (bitconvert (v2i64 VR128:$src))), (v4f32 VR128:$src)>; 407 def : Pat<(v4f32 (bitconvert (v4i32 VR128:$src))), (v4f32 VR128:$src)>; 408 def : Pat<(v4f32 (bitconvert (v8i16 VR128:$src))), (v4f32 VR128:$src)>; 409 def : Pat<(v4f32 (bitconvert (v16i8 VR128:$src))), (v4f32 VR128:$src)>; 410 def : Pat<(v4f32 (bitconvert (v2f64 VR128:$src))), (v4f32 VR128:$src)>; 411 def : Pat<(v2f64 (bitconvert (v2i64 VR128:$src))), (v2f64 VR128:$src)>; 412 def : Pat<(v2f64 (bitconvert (v4i32 VR128:$src))), (v2f64 VR128:$src)>; 413 def : Pat<(v2f64 (bitconvert (v8i16 VR128:$src))), (v2f64 VR128:$src)>; 414 def : Pat<(v2f64 (bitconvert (v16i8 VR128:$src))), (v2f64 VR128:$src)>; 415 def : Pat<(v2f64 (bitconvert (v4f32 VR128:$src))), (v2f64 VR128:$src)>; 416} 417 418// Bitcasts between 256-bit vector types. Return the original type since 419// no instruction is needed for the conversion 420let Predicates = [HasAVX] in { 421 def : Pat<(v4f64 (bitconvert (v8f32 VR256:$src))), (v4f64 VR256:$src)>; 422 def : Pat<(v4f64 (bitconvert (v8i32 VR256:$src))), (v4f64 VR256:$src)>; 423 def : Pat<(v4f64 (bitconvert (v4i64 VR256:$src))), (v4f64 VR256:$src)>; 424 def : Pat<(v4f64 (bitconvert (v16i16 VR256:$src))), (v4f64 VR256:$src)>; 425 def : Pat<(v4f64 (bitconvert (v32i8 VR256:$src))), (v4f64 VR256:$src)>; 426 def : Pat<(v8f32 (bitconvert (v8i32 VR256:$src))), (v8f32 VR256:$src)>; 427 def : Pat<(v8f32 (bitconvert (v4i64 VR256:$src))), (v8f32 VR256:$src)>; 428 def : Pat<(v8f32 (bitconvert (v4f64 VR256:$src))), (v8f32 VR256:$src)>; 429 def : Pat<(v8f32 (bitconvert (v32i8 VR256:$src))), (v8f32 VR256:$src)>; 430 def : Pat<(v8f32 (bitconvert (v16i16 VR256:$src))), (v8f32 VR256:$src)>; 431 def : Pat<(v4i64 (bitconvert (v8f32 VR256:$src))), (v4i64 VR256:$src)>; 432 def : Pat<(v4i64 (bitconvert (v8i32 VR256:$src))), (v4i64 VR256:$src)>; 433 def : Pat<(v4i64 (bitconvert (v4f64 VR256:$src))), (v4i64 VR256:$src)>; 434 def : Pat<(v4i64 (bitconvert (v32i8 VR256:$src))), (v4i64 VR256:$src)>; 435 def : Pat<(v4i64 (bitconvert (v16i16 VR256:$src))), (v4i64 VR256:$src)>; 436 def : Pat<(v32i8 (bitconvert (v4f64 VR256:$src))), (v32i8 VR256:$src)>; 437 def : Pat<(v32i8 (bitconvert (v4i64 VR256:$src))), (v32i8 VR256:$src)>; 438 def : Pat<(v32i8 (bitconvert (v8f32 VR256:$src))), (v32i8 VR256:$src)>; 439 def : Pat<(v32i8 (bitconvert (v8i32 VR256:$src))), (v32i8 VR256:$src)>; 440 def : Pat<(v32i8 (bitconvert (v16i16 VR256:$src))), (v32i8 VR256:$src)>; 441 def : Pat<(v8i32 (bitconvert (v32i8 VR256:$src))), (v8i32 VR256:$src)>; 442 def : Pat<(v8i32 (bitconvert (v16i16 VR256:$src))), (v8i32 VR256:$src)>; 443 def : Pat<(v8i32 (bitconvert (v8f32 VR256:$src))), (v8i32 VR256:$src)>; 444 def : Pat<(v8i32 (bitconvert (v4i64 VR256:$src))), (v8i32 VR256:$src)>; 445 def : Pat<(v8i32 (bitconvert (v4f64 VR256:$src))), (v8i32 VR256:$src)>; 446 def : Pat<(v16i16 (bitconvert (v8f32 VR256:$src))), (v16i16 VR256:$src)>; 447 def : Pat<(v16i16 (bitconvert (v8i32 VR256:$src))), (v16i16 VR256:$src)>; 448 def : Pat<(v16i16 (bitconvert (v4i64 VR256:$src))), (v16i16 VR256:$src)>; 449 def : Pat<(v16i16 (bitconvert (v4f64 VR256:$src))), (v16i16 VR256:$src)>; 450 def : Pat<(v16i16 (bitconvert (v32i8 VR256:$src))), (v16i16 VR256:$src)>; 451} 452 453// Alias instructions that map fld0 to xorps for sse or vxorps for avx. 454// This is expanded by ExpandPostRAPseudos. 455let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1, 456 isPseudo = 1, SchedRW = [WriteZero] in { 457 def FsFLD0SS : I<0, Pseudo, (outs FR32:$dst), (ins), "", 458 [(set FR32:$dst, fp32imm0)]>, Requires<[HasSSE1]>; 459 def FsFLD0SD : I<0, Pseudo, (outs FR64:$dst), (ins), "", 460 [(set FR64:$dst, fpimm0)]>, Requires<[HasSSE2]>; 461} 462 463//===----------------------------------------------------------------------===// 464// AVX & SSE - Zero/One Vectors 465//===----------------------------------------------------------------------===// 466 467// Alias instruction that maps zero vector to pxor / xorp* for sse. 468// This is expanded by ExpandPostRAPseudos to an xorps / vxorps, and then 469// swizzled by ExecutionDepsFix to pxor. 470// We set canFoldAsLoad because this can be converted to a constant-pool 471// load of an all-zeros value if folding it would be beneficial. 472let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1, 473 isPseudo = 1, SchedRW = [WriteZero] in { 474def V_SET0 : I<0, Pseudo, (outs VR128:$dst), (ins), "", 475 [(set VR128:$dst, (v4f32 immAllZerosV))]>; 476} 477 478def : Pat<(v2f64 immAllZerosV), (V_SET0)>; 479def : Pat<(v4i32 immAllZerosV), (V_SET0)>; 480def : Pat<(v2i64 immAllZerosV), (V_SET0)>; 481def : Pat<(v8i16 immAllZerosV), (V_SET0)>; 482def : Pat<(v16i8 immAllZerosV), (V_SET0)>; 483 484 485// The same as done above but for AVX. The 256-bit AVX1 ISA doesn't support PI, 486// and doesn't need it because on sandy bridge the register is set to zero 487// at the rename stage without using any execution unit, so SET0PSY 488// and SET0PDY can be used for vector int instructions without penalty 489let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1, 490 isPseudo = 1, Predicates = [HasAVX], SchedRW = [WriteZero] in { 491def AVX_SET0 : I<0, Pseudo, (outs VR256:$dst), (ins), "", 492 [(set VR256:$dst, (v8f32 immAllZerosV))]>; 493} 494 495let Predicates = [HasAVX] in 496 def : Pat<(v4f64 immAllZerosV), (AVX_SET0)>; 497 498let Predicates = [HasAVX2] in { 499 def : Pat<(v4i64 immAllZerosV), (AVX_SET0)>; 500 def : Pat<(v8i32 immAllZerosV), (AVX_SET0)>; 501 def : Pat<(v16i16 immAllZerosV), (AVX_SET0)>; 502 def : Pat<(v32i8 immAllZerosV), (AVX_SET0)>; 503} 504 505// AVX1 has no support for 256-bit integer instructions, but since the 128-bit 506// VPXOR instruction writes zero to its upper part, it's safe build zeros. 507let Predicates = [HasAVX1Only] in { 508def : Pat<(v32i8 immAllZerosV), (SUBREG_TO_REG (i8 0), (V_SET0), sub_xmm)>; 509def : Pat<(bc_v32i8 (v8f32 immAllZerosV)), 510 (SUBREG_TO_REG (i8 0), (V_SET0), sub_xmm)>; 511 512def : Pat<(v16i16 immAllZerosV), (SUBREG_TO_REG (i16 0), (V_SET0), sub_xmm)>; 513def : Pat<(bc_v16i16 (v8f32 immAllZerosV)), 514 (SUBREG_TO_REG (i16 0), (V_SET0), sub_xmm)>; 515 516def : Pat<(v8i32 immAllZerosV), (SUBREG_TO_REG (i32 0), (V_SET0), sub_xmm)>; 517def : Pat<(bc_v8i32 (v8f32 immAllZerosV)), 518 (SUBREG_TO_REG (i32 0), (V_SET0), sub_xmm)>; 519 520def : Pat<(v4i64 immAllZerosV), (SUBREG_TO_REG (i64 0), (V_SET0), sub_xmm)>; 521def : Pat<(bc_v4i64 (v8f32 immAllZerosV)), 522 (SUBREG_TO_REG (i64 0), (V_SET0), sub_xmm)>; 523} 524 525// We set canFoldAsLoad because this can be converted to a constant-pool 526// load of an all-ones value if folding it would be beneficial. 527let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1, 528 isPseudo = 1, SchedRW = [WriteZero] in { 529 def V_SETALLONES : I<0, Pseudo, (outs VR128:$dst), (ins), "", 530 [(set VR128:$dst, (v4i32 immAllOnesV))]>; 531 let Predicates = [HasAVX2] in 532 def AVX2_SETALLONES : I<0, Pseudo, (outs VR256:$dst), (ins), "", 533 [(set VR256:$dst, (v8i32 immAllOnesV))]>; 534} 535 536 537//===----------------------------------------------------------------------===// 538// SSE 1 & 2 - Move FP Scalar Instructions 539// 540// Move Instructions. Register-to-register movss/movsd is not used for FR32/64 541// register copies because it's a partial register update; Register-to-register 542// movss/movsd is not modeled as an INSERT_SUBREG because INSERT_SUBREG requires 543// that the insert be implementable in terms of a copy, and just mentioned, we 544// don't use movss/movsd for copies. 545//===----------------------------------------------------------------------===// 546 547multiclass sse12_move_rr<RegisterClass RC, SDNode OpNode, ValueType vt, 548 X86MemOperand x86memop, string base_opc, 549 string asm_opr, Domain d = GenericDomain> { 550 def rr : SI<0x10, MRMSrcReg, (outs VR128:$dst), 551 (ins VR128:$src1, RC:$src2), 552 !strconcat(base_opc, asm_opr), 553 [(set VR128:$dst, (vt (OpNode VR128:$src1, 554 (scalar_to_vector RC:$src2))))], 555 IIC_SSE_MOV_S_RR, d>, Sched<[WriteFShuffle]>; 556 557 // For the disassembler 558 let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0 in 559 def rr_REV : SI<0x11, MRMDestReg, (outs VR128:$dst), 560 (ins VR128:$src1, RC:$src2), 561 !strconcat(base_opc, asm_opr), 562 [], IIC_SSE_MOV_S_RR>, Sched<[WriteFShuffle]>; 563} 564 565multiclass sse12_move<RegisterClass RC, SDNode OpNode, ValueType vt, 566 X86MemOperand x86memop, string OpcodeStr, 567 Domain d = GenericDomain> { 568 // AVX 569 defm V#NAME : sse12_move_rr<RC, OpNode, vt, x86memop, OpcodeStr, 570 "\t{$src2, $src1, $dst|$dst, $src1, $src2}", d>, 571 VEX_4V, VEX_LIG; 572 573 def V#NAME#mr : SI<0x11, MRMDestMem, (outs), (ins x86memop:$dst, RC:$src), 574 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 575 [(store RC:$src, addr:$dst)], IIC_SSE_MOV_S_MR, d>, 576 VEX, VEX_LIG, Sched<[WriteStore]>; 577 // SSE1 & 2 578 let Constraints = "$src1 = $dst" in { 579 defm NAME : sse12_move_rr<RC, OpNode, vt, x86memop, OpcodeStr, 580 "\t{$src2, $dst|$dst, $src2}", d>; 581 } 582 583 def NAME#mr : SI<0x11, MRMDestMem, (outs), (ins x86memop:$dst, RC:$src), 584 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 585 [(store RC:$src, addr:$dst)], IIC_SSE_MOV_S_MR, d>, 586 Sched<[WriteStore]>; 587} 588 589// Loading from memory automatically zeroing upper bits. 590multiclass sse12_move_rm<RegisterClass RC, X86MemOperand x86memop, 591 PatFrag mem_pat, string OpcodeStr, 592 Domain d = GenericDomain> { 593 def V#NAME#rm : SI<0x10, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src), 594 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 595 [(set RC:$dst, (mem_pat addr:$src))], 596 IIC_SSE_MOV_S_RM, d>, VEX, VEX_LIG, Sched<[WriteLoad]>; 597 def NAME#rm : SI<0x10, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src), 598 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 599 [(set RC:$dst, (mem_pat addr:$src))], 600 IIC_SSE_MOV_S_RM, d>, Sched<[WriteLoad]>; 601} 602 603defm MOVSS : sse12_move<FR32, X86Movss, v4f32, f32mem, "movss", 604 SSEPackedSingle>, XS; 605defm MOVSD : sse12_move<FR64, X86Movsd, v2f64, f64mem, "movsd", 606 SSEPackedDouble>, XD; 607 608let canFoldAsLoad = 1, isReMaterializable = 1 in { 609 defm MOVSS : sse12_move_rm<FR32, f32mem, loadf32, "movss", 610 SSEPackedSingle>, XS; 611 612 let AddedComplexity = 20 in 613 defm MOVSD : sse12_move_rm<FR64, f64mem, loadf64, "movsd", 614 SSEPackedDouble>, XD; 615} 616 617// Patterns 618let Predicates = [UseAVX] in { 619 let AddedComplexity = 20 in { 620 // MOVSSrm zeros the high parts of the register; represent this 621 // with SUBREG_TO_REG. The AVX versions also write: DST[255:128] <- 0 622 def : Pat<(v4f32 (X86vzmovl (v4f32 (scalar_to_vector (loadf32 addr:$src))))), 623 (COPY_TO_REGCLASS (VMOVSSrm addr:$src), VR128)>; 624 def : Pat<(v4f32 (scalar_to_vector (loadf32 addr:$src))), 625 (COPY_TO_REGCLASS (VMOVSSrm addr:$src), VR128)>; 626 def : Pat<(v4f32 (X86vzmovl (loadv4f32 addr:$src))), 627 (COPY_TO_REGCLASS (VMOVSSrm addr:$src), VR128)>; 628 629 // MOVSDrm zeros the high parts of the register; represent this 630 // with SUBREG_TO_REG. The AVX versions also write: DST[255:128] <- 0 631 def : Pat<(v2f64 (X86vzmovl (v2f64 (scalar_to_vector (loadf64 addr:$src))))), 632 (COPY_TO_REGCLASS (VMOVSDrm addr:$src), VR128)>; 633 def : Pat<(v2f64 (scalar_to_vector (loadf64 addr:$src))), 634 (COPY_TO_REGCLASS (VMOVSDrm addr:$src), VR128)>; 635 def : Pat<(v2f64 (X86vzmovl (loadv2f64 addr:$src))), 636 (COPY_TO_REGCLASS (VMOVSDrm addr:$src), VR128)>; 637 def : Pat<(v2f64 (X86vzmovl (bc_v2f64 (loadv4f32 addr:$src)))), 638 (COPY_TO_REGCLASS (VMOVSDrm addr:$src), VR128)>; 639 def : Pat<(v2f64 (X86vzload addr:$src)), 640 (COPY_TO_REGCLASS (VMOVSDrm addr:$src), VR128)>; 641 642 // Represent the same patterns above but in the form they appear for 643 // 256-bit types 644 def : Pat<(v8f32 (X86vzmovl (insert_subvector undef, 645 (v4f32 (scalar_to_vector (loadf32 addr:$src))), (iPTR 0)))), 646 (SUBREG_TO_REG (i32 0), (VMOVSSrm addr:$src), sub_xmm)>; 647 def : Pat<(v4f64 (X86vzmovl (insert_subvector undef, 648 (v2f64 (scalar_to_vector (loadf64 addr:$src))), (iPTR 0)))), 649 (SUBREG_TO_REG (i32 0), (VMOVSDrm addr:$src), sub_xmm)>; 650 } 651 652 // Extract and store. 653 def : Pat<(store (f32 (vector_extract (v4f32 VR128:$src), (iPTR 0))), 654 addr:$dst), 655 (VMOVSSmr addr:$dst, (COPY_TO_REGCLASS (v4f32 VR128:$src), FR32))>; 656 def : Pat<(store (f64 (vector_extract (v2f64 VR128:$src), (iPTR 0))), 657 addr:$dst), 658 (VMOVSDmr addr:$dst, (COPY_TO_REGCLASS (v2f64 VR128:$src), FR64))>; 659 660 // Shuffle with VMOVSS 661 def : Pat<(v4i32 (X86Movss VR128:$src1, VR128:$src2)), 662 (VMOVSSrr (v4i32 VR128:$src1), 663 (COPY_TO_REGCLASS (v4i32 VR128:$src2), FR32))>; 664 def : Pat<(v4f32 (X86Movss VR128:$src1, VR128:$src2)), 665 (VMOVSSrr (v4f32 VR128:$src1), 666 (COPY_TO_REGCLASS (v4f32 VR128:$src2), FR32))>; 667 668 // 256-bit variants 669 def : Pat<(v8i32 (X86Movss VR256:$src1, VR256:$src2)), 670 (SUBREG_TO_REG (i32 0), 671 (VMOVSSrr (EXTRACT_SUBREG (v8i32 VR256:$src1), sub_xmm), 672 (EXTRACT_SUBREG (v8i32 VR256:$src2), sub_xmm)), 673 sub_xmm)>; 674 def : Pat<(v8f32 (X86Movss VR256:$src1, VR256:$src2)), 675 (SUBREG_TO_REG (i32 0), 676 (VMOVSSrr (EXTRACT_SUBREG (v8f32 VR256:$src1), sub_xmm), 677 (EXTRACT_SUBREG (v8f32 VR256:$src2), sub_xmm)), 678 sub_xmm)>; 679 680 // Shuffle with VMOVSD 681 def : Pat<(v2i64 (X86Movsd VR128:$src1, VR128:$src2)), 682 (VMOVSDrr VR128:$src1, (COPY_TO_REGCLASS VR128:$src2, FR64))>; 683 def : Pat<(v2f64 (X86Movsd VR128:$src1, VR128:$src2)), 684 (VMOVSDrr VR128:$src1, (COPY_TO_REGCLASS VR128:$src2, FR64))>; 685 def : Pat<(v4f32 (X86Movsd VR128:$src1, VR128:$src2)), 686 (VMOVSDrr VR128:$src1, (COPY_TO_REGCLASS VR128:$src2, FR64))>; 687 def : Pat<(v4i32 (X86Movsd VR128:$src1, VR128:$src2)), 688 (VMOVSDrr VR128:$src1, (COPY_TO_REGCLASS VR128:$src2, FR64))>; 689 690 // 256-bit variants 691 def : Pat<(v4i64 (X86Movsd VR256:$src1, VR256:$src2)), 692 (SUBREG_TO_REG (i32 0), 693 (VMOVSDrr (EXTRACT_SUBREG (v4i64 VR256:$src1), sub_xmm), 694 (EXTRACT_SUBREG (v4i64 VR256:$src2), sub_xmm)), 695 sub_xmm)>; 696 def : Pat<(v4f64 (X86Movsd VR256:$src1, VR256:$src2)), 697 (SUBREG_TO_REG (i32 0), 698 (VMOVSDrr (EXTRACT_SUBREG (v4f64 VR256:$src1), sub_xmm), 699 (EXTRACT_SUBREG (v4f64 VR256:$src2), sub_xmm)), 700 sub_xmm)>; 701 702 // FIXME: Instead of a X86Movlps there should be a X86Movsd here, the problem 703 // is during lowering, where it's not possible to recognize the fold cause 704 // it has two uses through a bitcast. One use disappears at isel time and the 705 // fold opportunity reappears. 706 def : Pat<(v2f64 (X86Movlpd VR128:$src1, VR128:$src2)), 707 (VMOVSDrr VR128:$src1, (COPY_TO_REGCLASS VR128:$src2, FR64))>; 708 def : Pat<(v2i64 (X86Movlpd VR128:$src1, VR128:$src2)), 709 (VMOVSDrr VR128:$src1, (COPY_TO_REGCLASS VR128:$src2, FR64))>; 710 def : Pat<(v4f32 (X86Movlps VR128:$src1, VR128:$src2)), 711 (VMOVSDrr VR128:$src1, (COPY_TO_REGCLASS VR128:$src2, FR64))>; 712 def : Pat<(v4i32 (X86Movlps VR128:$src1, VR128:$src2)), 713 (VMOVSDrr VR128:$src1, (COPY_TO_REGCLASS VR128:$src2, FR64))>; 714} 715 716let Predicates = [UseSSE1] in { 717 let Predicates = [NoSSE41], AddedComplexity = 15 in { 718 // Move scalar to XMM zero-extended, zeroing a VR128 then do a 719 // MOVSS to the lower bits. 720 def : Pat<(v4f32 (X86vzmovl (v4f32 (scalar_to_vector FR32:$src)))), 721 (MOVSSrr (v4f32 (V_SET0)), FR32:$src)>; 722 def : Pat<(v4f32 (X86vzmovl (v4f32 VR128:$src))), 723 (MOVSSrr (v4f32 (V_SET0)), (COPY_TO_REGCLASS VR128:$src, FR32))>; 724 def : Pat<(v4i32 (X86vzmovl (v4i32 VR128:$src))), 725 (MOVSSrr (v4i32 (V_SET0)), (COPY_TO_REGCLASS VR128:$src, FR32))>; 726 } 727 728 let AddedComplexity = 20 in { 729 // MOVSSrm already zeros the high parts of the register. 730 def : Pat<(v4f32 (X86vzmovl (v4f32 (scalar_to_vector (loadf32 addr:$src))))), 731 (COPY_TO_REGCLASS (MOVSSrm addr:$src), VR128)>; 732 def : Pat<(v4f32 (scalar_to_vector (loadf32 addr:$src))), 733 (COPY_TO_REGCLASS (MOVSSrm addr:$src), VR128)>; 734 def : Pat<(v4f32 (X86vzmovl (loadv4f32 addr:$src))), 735 (COPY_TO_REGCLASS (MOVSSrm addr:$src), VR128)>; 736 } 737 738 // Extract and store. 739 def : Pat<(store (f32 (vector_extract (v4f32 VR128:$src), (iPTR 0))), 740 addr:$dst), 741 (MOVSSmr addr:$dst, (COPY_TO_REGCLASS VR128:$src, FR32))>; 742 743 // Shuffle with MOVSS 744 def : Pat<(v4i32 (X86Movss VR128:$src1, VR128:$src2)), 745 (MOVSSrr VR128:$src1, (COPY_TO_REGCLASS VR128:$src2, FR32))>; 746 def : Pat<(v4f32 (X86Movss VR128:$src1, VR128:$src2)), 747 (MOVSSrr VR128:$src1, (COPY_TO_REGCLASS VR128:$src2, FR32))>; 748} 749 750let Predicates = [UseSSE2] in { 751 let Predicates = [NoSSE41], AddedComplexity = 15 in { 752 // Move scalar to XMM zero-extended, zeroing a VR128 then do a 753 // MOVSD to the lower bits. 754 def : Pat<(v2f64 (X86vzmovl (v2f64 (scalar_to_vector FR64:$src)))), 755 (MOVSDrr (v2f64 (V_SET0)), FR64:$src)>; 756 } 757 758 let AddedComplexity = 20 in { 759 // MOVSDrm already zeros the high parts of the register. 760 def : Pat<(v2f64 (X86vzmovl (v2f64 (scalar_to_vector (loadf64 addr:$src))))), 761 (COPY_TO_REGCLASS (MOVSDrm addr:$src), VR128)>; 762 def : Pat<(v2f64 (scalar_to_vector (loadf64 addr:$src))), 763 (COPY_TO_REGCLASS (MOVSDrm addr:$src), VR128)>; 764 def : Pat<(v2f64 (X86vzmovl (loadv2f64 addr:$src))), 765 (COPY_TO_REGCLASS (MOVSDrm addr:$src), VR128)>; 766 def : Pat<(v2f64 (X86vzmovl (bc_v2f64 (loadv4f32 addr:$src)))), 767 (COPY_TO_REGCLASS (MOVSDrm addr:$src), VR128)>; 768 def : Pat<(v2f64 (X86vzload addr:$src)), 769 (COPY_TO_REGCLASS (MOVSDrm addr:$src), VR128)>; 770 } 771 772 // Extract and store. 773 def : Pat<(store (f64 (vector_extract (v2f64 VR128:$src), (iPTR 0))), 774 addr:$dst), 775 (MOVSDmr addr:$dst, (COPY_TO_REGCLASS VR128:$src, FR64))>; 776 777 // Shuffle with MOVSD 778 def : Pat<(v2i64 (X86Movsd VR128:$src1, VR128:$src2)), 779 (MOVSDrr VR128:$src1, (COPY_TO_REGCLASS VR128:$src2, FR64))>; 780 def : Pat<(v2f64 (X86Movsd VR128:$src1, VR128:$src2)), 781 (MOVSDrr VR128:$src1, (COPY_TO_REGCLASS VR128:$src2, FR64))>; 782 def : Pat<(v4f32 (X86Movsd VR128:$src1, VR128:$src2)), 783 (MOVSDrr VR128:$src1, (COPY_TO_REGCLASS VR128:$src2, FR64))>; 784 def : Pat<(v4i32 (X86Movsd VR128:$src1, VR128:$src2)), 785 (MOVSDrr VR128:$src1, (COPY_TO_REGCLASS VR128:$src2, FR64))>; 786 787 // FIXME: Instead of a X86Movlps there should be a X86Movsd here, the problem 788 // is during lowering, where it's not possible to recognize the fold because 789 // it has two uses through a bitcast. One use disappears at isel time and the 790 // fold opportunity reappears. 791 def : Pat<(v2f64 (X86Movlpd VR128:$src1, VR128:$src2)), 792 (MOVSDrr VR128:$src1, (COPY_TO_REGCLASS VR128:$src2, FR64))>; 793 def : Pat<(v2i64 (X86Movlpd VR128:$src1, VR128:$src2)), 794 (MOVSDrr VR128:$src1, (COPY_TO_REGCLASS VR128:$src2, FR64))>; 795 def : Pat<(v4f32 (X86Movlps VR128:$src1, VR128:$src2)), 796 (MOVSDrr VR128:$src1, (COPY_TO_REGCLASS VR128:$src2, FR64))>; 797 def : Pat<(v4i32 (X86Movlps VR128:$src1, VR128:$src2)), 798 (MOVSDrr VR128:$src1, (COPY_TO_REGCLASS VR128:$src2, FR64))>; 799} 800 801//===----------------------------------------------------------------------===// 802// SSE 1 & 2 - Move Aligned/Unaligned FP Instructions 803//===----------------------------------------------------------------------===// 804 805multiclass sse12_mov_packed<bits<8> opc, RegisterClass RC, 806 X86MemOperand x86memop, PatFrag ld_frag, 807 string asm, Domain d, 808 OpndItins itins, 809 bit IsReMaterializable = 1> { 810let hasSideEffects = 0 in 811 def rr : PI<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src), 812 !strconcat(asm, "\t{$src, $dst|$dst, $src}"), [], itins.rr, d>, 813 Sched<[WriteFShuffle]>; 814let canFoldAsLoad = 1, isReMaterializable = IsReMaterializable in 815 def rm : PI<opc, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src), 816 !strconcat(asm, "\t{$src, $dst|$dst, $src}"), 817 [(set RC:$dst, (ld_frag addr:$src))], itins.rm, d>, 818 Sched<[WriteLoad]>; 819} 820 821let Predicates = [HasAVX, NoVLX] in { 822defm VMOVAPS : sse12_mov_packed<0x28, VR128, f128mem, alignedloadv4f32, 823 "movaps", SSEPackedSingle, SSE_MOVA_ITINS>, 824 PS, VEX; 825defm VMOVAPD : sse12_mov_packed<0x28, VR128, f128mem, alignedloadv2f64, 826 "movapd", SSEPackedDouble, SSE_MOVA_ITINS>, 827 PD, VEX; 828defm VMOVUPS : sse12_mov_packed<0x10, VR128, f128mem, loadv4f32, 829 "movups", SSEPackedSingle, SSE_MOVU_ITINS>, 830 PS, VEX; 831defm VMOVUPD : sse12_mov_packed<0x10, VR128, f128mem, loadv2f64, 832 "movupd", SSEPackedDouble, SSE_MOVU_ITINS, 0>, 833 PD, VEX; 834 835defm VMOVAPSY : sse12_mov_packed<0x28, VR256, f256mem, alignedloadv8f32, 836 "movaps", SSEPackedSingle, SSE_MOVA_ITINS>, 837 PS, VEX, VEX_L; 838defm VMOVAPDY : sse12_mov_packed<0x28, VR256, f256mem, alignedloadv4f64, 839 "movapd", SSEPackedDouble, SSE_MOVA_ITINS>, 840 PD, VEX, VEX_L; 841defm VMOVUPSY : sse12_mov_packed<0x10, VR256, f256mem, loadv8f32, 842 "movups", SSEPackedSingle, SSE_MOVU_ITINS>, 843 PS, VEX, VEX_L; 844defm VMOVUPDY : sse12_mov_packed<0x10, VR256, f256mem, loadv4f64, 845 "movupd", SSEPackedDouble, SSE_MOVU_ITINS, 0>, 846 PD, VEX, VEX_L; 847} 848 849let Predicates = [UseSSE1] in { 850defm MOVAPS : sse12_mov_packed<0x28, VR128, f128mem, alignedloadv4f32, 851 "movaps", SSEPackedSingle, SSE_MOVA_ITINS>, 852 PS; 853defm MOVUPS : sse12_mov_packed<0x10, VR128, f128mem, loadv4f32, 854 "movups", SSEPackedSingle, SSE_MOVU_ITINS>, 855 PS; 856} 857let Predicates = [UseSSE2] in { 858defm MOVAPD : sse12_mov_packed<0x28, VR128, f128mem, alignedloadv2f64, 859 "movapd", SSEPackedDouble, SSE_MOVA_ITINS>, 860 PD; 861defm MOVUPD : sse12_mov_packed<0x10, VR128, f128mem, loadv2f64, 862 "movupd", SSEPackedDouble, SSE_MOVU_ITINS, 0>, 863 PD; 864} 865 866let SchedRW = [WriteStore], Predicates = [HasAVX, NoVLX] in { 867def VMOVAPSmr : VPSI<0x29, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src), 868 "movaps\t{$src, $dst|$dst, $src}", 869 [(alignedstore (v4f32 VR128:$src), addr:$dst)], 870 IIC_SSE_MOVA_P_MR>, VEX; 871def VMOVAPDmr : VPDI<0x29, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src), 872 "movapd\t{$src, $dst|$dst, $src}", 873 [(alignedstore (v2f64 VR128:$src), addr:$dst)], 874 IIC_SSE_MOVA_P_MR>, VEX; 875def VMOVUPSmr : VPSI<0x11, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src), 876 "movups\t{$src, $dst|$dst, $src}", 877 [(store (v4f32 VR128:$src), addr:$dst)], 878 IIC_SSE_MOVU_P_MR>, VEX; 879def VMOVUPDmr : VPDI<0x11, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src), 880 "movupd\t{$src, $dst|$dst, $src}", 881 [(store (v2f64 VR128:$src), addr:$dst)], 882 IIC_SSE_MOVU_P_MR>, VEX; 883def VMOVAPSYmr : VPSI<0x29, MRMDestMem, (outs), (ins f256mem:$dst, VR256:$src), 884 "movaps\t{$src, $dst|$dst, $src}", 885 [(alignedstore256 (v8f32 VR256:$src), addr:$dst)], 886 IIC_SSE_MOVA_P_MR>, VEX, VEX_L; 887def VMOVAPDYmr : VPDI<0x29, MRMDestMem, (outs), (ins f256mem:$dst, VR256:$src), 888 "movapd\t{$src, $dst|$dst, $src}", 889 [(alignedstore256 (v4f64 VR256:$src), addr:$dst)], 890 IIC_SSE_MOVA_P_MR>, VEX, VEX_L; 891def VMOVUPSYmr : VPSI<0x11, MRMDestMem, (outs), (ins f256mem:$dst, VR256:$src), 892 "movups\t{$src, $dst|$dst, $src}", 893 [(store (v8f32 VR256:$src), addr:$dst)], 894 IIC_SSE_MOVU_P_MR>, VEX, VEX_L; 895def VMOVUPDYmr : VPDI<0x11, MRMDestMem, (outs), (ins f256mem:$dst, VR256:$src), 896 "movupd\t{$src, $dst|$dst, $src}", 897 [(store (v4f64 VR256:$src), addr:$dst)], 898 IIC_SSE_MOVU_P_MR>, VEX, VEX_L; 899} // SchedRW 900 901// For disassembler 902let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0, 903 SchedRW = [WriteFShuffle] in { 904 def VMOVAPSrr_REV : VPSI<0x29, MRMDestReg, (outs VR128:$dst), 905 (ins VR128:$src), 906 "movaps\t{$src, $dst|$dst, $src}", [], 907 IIC_SSE_MOVA_P_RR>, VEX; 908 def VMOVAPDrr_REV : VPDI<0x29, MRMDestReg, (outs VR128:$dst), 909 (ins VR128:$src), 910 "movapd\t{$src, $dst|$dst, $src}", [], 911 IIC_SSE_MOVA_P_RR>, VEX; 912 def VMOVUPSrr_REV : VPSI<0x11, MRMDestReg, (outs VR128:$dst), 913 (ins VR128:$src), 914 "movups\t{$src, $dst|$dst, $src}", [], 915 IIC_SSE_MOVU_P_RR>, VEX; 916 def VMOVUPDrr_REV : VPDI<0x11, MRMDestReg, (outs VR128:$dst), 917 (ins VR128:$src), 918 "movupd\t{$src, $dst|$dst, $src}", [], 919 IIC_SSE_MOVU_P_RR>, VEX; 920 def VMOVAPSYrr_REV : VPSI<0x29, MRMDestReg, (outs VR256:$dst), 921 (ins VR256:$src), 922 "movaps\t{$src, $dst|$dst, $src}", [], 923 IIC_SSE_MOVA_P_RR>, VEX, VEX_L; 924 def VMOVAPDYrr_REV : VPDI<0x29, MRMDestReg, (outs VR256:$dst), 925 (ins VR256:$src), 926 "movapd\t{$src, $dst|$dst, $src}", [], 927 IIC_SSE_MOVA_P_RR>, VEX, VEX_L; 928 def VMOVUPSYrr_REV : VPSI<0x11, MRMDestReg, (outs VR256:$dst), 929 (ins VR256:$src), 930 "movups\t{$src, $dst|$dst, $src}", [], 931 IIC_SSE_MOVU_P_RR>, VEX, VEX_L; 932 def VMOVUPDYrr_REV : VPDI<0x11, MRMDestReg, (outs VR256:$dst), 933 (ins VR256:$src), 934 "movupd\t{$src, $dst|$dst, $src}", [], 935 IIC_SSE_MOVU_P_RR>, VEX, VEX_L; 936} 937 938let Predicates = [HasAVX] in { 939def : Pat<(v8i32 (X86vzmovl 940 (insert_subvector undef, (v4i32 VR128:$src), (iPTR 0)))), 941 (SUBREG_TO_REG (i32 0), (VMOVAPSrr VR128:$src), sub_xmm)>; 942def : Pat<(v4i64 (X86vzmovl 943 (insert_subvector undef, (v2i64 VR128:$src), (iPTR 0)))), 944 (SUBREG_TO_REG (i32 0), (VMOVAPSrr VR128:$src), sub_xmm)>; 945def : Pat<(v8f32 (X86vzmovl 946 (insert_subvector undef, (v4f32 VR128:$src), (iPTR 0)))), 947 (SUBREG_TO_REG (i32 0), (VMOVAPSrr VR128:$src), sub_xmm)>; 948def : Pat<(v4f64 (X86vzmovl 949 (insert_subvector undef, (v2f64 VR128:$src), (iPTR 0)))), 950 (SUBREG_TO_REG (i32 0), (VMOVAPSrr VR128:$src), sub_xmm)>; 951} 952 953 954def : Pat<(int_x86_avx_storeu_ps_256 addr:$dst, VR256:$src), 955 (VMOVUPSYmr addr:$dst, VR256:$src)>; 956def : Pat<(int_x86_avx_storeu_pd_256 addr:$dst, VR256:$src), 957 (VMOVUPDYmr addr:$dst, VR256:$src)>; 958 959let SchedRW = [WriteStore] in { 960def MOVAPSmr : PSI<0x29, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src), 961 "movaps\t{$src, $dst|$dst, $src}", 962 [(alignedstore (v4f32 VR128:$src), addr:$dst)], 963 IIC_SSE_MOVA_P_MR>; 964def MOVAPDmr : PDI<0x29, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src), 965 "movapd\t{$src, $dst|$dst, $src}", 966 [(alignedstore (v2f64 VR128:$src), addr:$dst)], 967 IIC_SSE_MOVA_P_MR>; 968def MOVUPSmr : PSI<0x11, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src), 969 "movups\t{$src, $dst|$dst, $src}", 970 [(store (v4f32 VR128:$src), addr:$dst)], 971 IIC_SSE_MOVU_P_MR>; 972def MOVUPDmr : PDI<0x11, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src), 973 "movupd\t{$src, $dst|$dst, $src}", 974 [(store (v2f64 VR128:$src), addr:$dst)], 975 IIC_SSE_MOVU_P_MR>; 976} // SchedRW 977 978// For disassembler 979let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0, 980 SchedRW = [WriteFShuffle] in { 981 def MOVAPSrr_REV : PSI<0x29, MRMDestReg, (outs VR128:$dst), (ins VR128:$src), 982 "movaps\t{$src, $dst|$dst, $src}", [], 983 IIC_SSE_MOVA_P_RR>; 984 def MOVAPDrr_REV : PDI<0x29, MRMDestReg, (outs VR128:$dst), (ins VR128:$src), 985 "movapd\t{$src, $dst|$dst, $src}", [], 986 IIC_SSE_MOVA_P_RR>; 987 def MOVUPSrr_REV : PSI<0x11, MRMDestReg, (outs VR128:$dst), (ins VR128:$src), 988 "movups\t{$src, $dst|$dst, $src}", [], 989 IIC_SSE_MOVU_P_RR>; 990 def MOVUPDrr_REV : PDI<0x11, MRMDestReg, (outs VR128:$dst), (ins VR128:$src), 991 "movupd\t{$src, $dst|$dst, $src}", [], 992 IIC_SSE_MOVU_P_RR>; 993} 994 995let Predicates = [HasAVX] in { 996 def : Pat<(int_x86_sse_storeu_ps addr:$dst, VR128:$src), 997 (VMOVUPSmr addr:$dst, VR128:$src)>; 998 def : Pat<(int_x86_sse2_storeu_pd addr:$dst, VR128:$src), 999 (VMOVUPDmr addr:$dst, VR128:$src)>; 1000} 1001 1002let Predicates = [UseSSE1] in 1003 def : Pat<(int_x86_sse_storeu_ps addr:$dst, VR128:$src), 1004 (MOVUPSmr addr:$dst, VR128:$src)>; 1005let Predicates = [UseSSE2] in 1006 def : Pat<(int_x86_sse2_storeu_pd addr:$dst, VR128:$src), 1007 (MOVUPDmr addr:$dst, VR128:$src)>; 1008 1009// Use vmovaps/vmovups for AVX integer load/store. 1010let Predicates = [HasAVX, NoVLX] in { 1011 // 128-bit load/store 1012 def : Pat<(alignedloadv2i64 addr:$src), 1013 (VMOVAPSrm addr:$src)>; 1014 def : Pat<(loadv2i64 addr:$src), 1015 (VMOVUPSrm addr:$src)>; 1016 1017 def : Pat<(alignedstore (v2i64 VR128:$src), addr:$dst), 1018 (VMOVAPSmr addr:$dst, VR128:$src)>; 1019 def : Pat<(alignedstore (v4i32 VR128:$src), addr:$dst), 1020 (VMOVAPSmr addr:$dst, VR128:$src)>; 1021 def : Pat<(alignedstore (v8i16 VR128:$src), addr:$dst), 1022 (VMOVAPSmr addr:$dst, VR128:$src)>; 1023 def : Pat<(alignedstore (v16i8 VR128:$src), addr:$dst), 1024 (VMOVAPSmr addr:$dst, VR128:$src)>; 1025 def : Pat<(store (v2i64 VR128:$src), addr:$dst), 1026 (VMOVUPSmr addr:$dst, VR128:$src)>; 1027 def : Pat<(store (v4i32 VR128:$src), addr:$dst), 1028 (VMOVUPSmr addr:$dst, VR128:$src)>; 1029 def : Pat<(store (v8i16 VR128:$src), addr:$dst), 1030 (VMOVUPSmr addr:$dst, VR128:$src)>; 1031 def : Pat<(store (v16i8 VR128:$src), addr:$dst), 1032 (VMOVUPSmr addr:$dst, VR128:$src)>; 1033 1034 // 256-bit load/store 1035 def : Pat<(alignedloadv4i64 addr:$src), 1036 (VMOVAPSYrm addr:$src)>; 1037 def : Pat<(loadv4i64 addr:$src), 1038 (VMOVUPSYrm addr:$src)>; 1039 def : Pat<(alignedstore256 (v4i64 VR256:$src), addr:$dst), 1040 (VMOVAPSYmr addr:$dst, VR256:$src)>; 1041 def : Pat<(alignedstore256 (v8i32 VR256:$src), addr:$dst), 1042 (VMOVAPSYmr addr:$dst, VR256:$src)>; 1043 def : Pat<(alignedstore256 (v16i16 VR256:$src), addr:$dst), 1044 (VMOVAPSYmr addr:$dst, VR256:$src)>; 1045 def : Pat<(alignedstore256 (v32i8 VR256:$src), addr:$dst), 1046 (VMOVAPSYmr addr:$dst, VR256:$src)>; 1047 def : Pat<(store (v4i64 VR256:$src), addr:$dst), 1048 (VMOVUPSYmr addr:$dst, VR256:$src)>; 1049 def : Pat<(store (v8i32 VR256:$src), addr:$dst), 1050 (VMOVUPSYmr addr:$dst, VR256:$src)>; 1051 def : Pat<(store (v16i16 VR256:$src), addr:$dst), 1052 (VMOVUPSYmr addr:$dst, VR256:$src)>; 1053 def : Pat<(store (v32i8 VR256:$src), addr:$dst), 1054 (VMOVUPSYmr addr:$dst, VR256:$src)>; 1055 1056 // Special patterns for storing subvector extracts of lower 128-bits 1057 // Its cheaper to just use VMOVAPS/VMOVUPS instead of VEXTRACTF128mr 1058 def : Pat<(alignedstore (v2f64 (extract_subvector 1059 (v4f64 VR256:$src), (iPTR 0))), addr:$dst), 1060 (VMOVAPDmr addr:$dst, (v2f64 (EXTRACT_SUBREG VR256:$src,sub_xmm)))>; 1061 def : Pat<(alignedstore (v4f32 (extract_subvector 1062 (v8f32 VR256:$src), (iPTR 0))), addr:$dst), 1063 (VMOVAPSmr addr:$dst, (v4f32 (EXTRACT_SUBREG VR256:$src,sub_xmm)))>; 1064 def : Pat<(alignedstore (v2i64 (extract_subvector 1065 (v4i64 VR256:$src), (iPTR 0))), addr:$dst), 1066 (VMOVAPDmr addr:$dst, (v2i64 (EXTRACT_SUBREG VR256:$src,sub_xmm)))>; 1067 def : Pat<(alignedstore (v4i32 (extract_subvector 1068 (v8i32 VR256:$src), (iPTR 0))), addr:$dst), 1069 (VMOVAPSmr addr:$dst, (v4i32 (EXTRACT_SUBREG VR256:$src,sub_xmm)))>; 1070 def : Pat<(alignedstore (v8i16 (extract_subvector 1071 (v16i16 VR256:$src), (iPTR 0))), addr:$dst), 1072 (VMOVAPSmr addr:$dst, (v8i16 (EXTRACT_SUBREG VR256:$src,sub_xmm)))>; 1073 def : Pat<(alignedstore (v16i8 (extract_subvector 1074 (v32i8 VR256:$src), (iPTR 0))), addr:$dst), 1075 (VMOVAPSmr addr:$dst, (v16i8 (EXTRACT_SUBREG VR256:$src,sub_xmm)))>; 1076 1077 def : Pat<(store (v2f64 (extract_subvector 1078 (v4f64 VR256:$src), (iPTR 0))), addr:$dst), 1079 (VMOVUPDmr addr:$dst, (v2f64 (EXTRACT_SUBREG VR256:$src,sub_xmm)))>; 1080 def : Pat<(store (v4f32 (extract_subvector 1081 (v8f32 VR256:$src), (iPTR 0))), addr:$dst), 1082 (VMOVUPSmr addr:$dst, (v4f32 (EXTRACT_SUBREG VR256:$src,sub_xmm)))>; 1083 def : Pat<(store (v2i64 (extract_subvector 1084 (v4i64 VR256:$src), (iPTR 0))), addr:$dst), 1085 (VMOVUPDmr addr:$dst, (v2i64 (EXTRACT_SUBREG VR256:$src,sub_xmm)))>; 1086 def : Pat<(store (v4i32 (extract_subvector 1087 (v8i32 VR256:$src), (iPTR 0))), addr:$dst), 1088 (VMOVUPSmr addr:$dst, (v4i32 (EXTRACT_SUBREG VR256:$src,sub_xmm)))>; 1089 def : Pat<(store (v8i16 (extract_subvector 1090 (v16i16 VR256:$src), (iPTR 0))), addr:$dst), 1091 (VMOVUPSmr addr:$dst, (v8i16 (EXTRACT_SUBREG VR256:$src,sub_xmm)))>; 1092 def : Pat<(store (v16i8 (extract_subvector 1093 (v32i8 VR256:$src), (iPTR 0))), addr:$dst), 1094 (VMOVUPSmr addr:$dst, (v16i8 (EXTRACT_SUBREG VR256:$src,sub_xmm)))>; 1095} 1096 1097// Use movaps / movups for SSE integer load / store (one byte shorter). 1098// The instructions selected below are then converted to MOVDQA/MOVDQU 1099// during the SSE domain pass. 1100let Predicates = [UseSSE1] in { 1101 def : Pat<(alignedloadv2i64 addr:$src), 1102 (MOVAPSrm addr:$src)>; 1103 def : Pat<(loadv2i64 addr:$src), 1104 (MOVUPSrm addr:$src)>; 1105 1106 def : Pat<(alignedstore (v2i64 VR128:$src), addr:$dst), 1107 (MOVAPSmr addr:$dst, VR128:$src)>; 1108 def : Pat<(alignedstore (v4i32 VR128:$src), addr:$dst), 1109 (MOVAPSmr addr:$dst, VR128:$src)>; 1110 def : Pat<(alignedstore (v8i16 VR128:$src), addr:$dst), 1111 (MOVAPSmr addr:$dst, VR128:$src)>; 1112 def : Pat<(alignedstore (v16i8 VR128:$src), addr:$dst), 1113 (MOVAPSmr addr:$dst, VR128:$src)>; 1114 def : Pat<(store (v2i64 VR128:$src), addr:$dst), 1115 (MOVUPSmr addr:$dst, VR128:$src)>; 1116 def : Pat<(store (v4i32 VR128:$src), addr:$dst), 1117 (MOVUPSmr addr:$dst, VR128:$src)>; 1118 def : Pat<(store (v8i16 VR128:$src), addr:$dst), 1119 (MOVUPSmr addr:$dst, VR128:$src)>; 1120 def : Pat<(store (v16i8 VR128:$src), addr:$dst), 1121 (MOVUPSmr addr:$dst, VR128:$src)>; 1122} 1123 1124// Alias instruction to load FR32 or FR64 from f128mem using movaps. Upper 1125// bits are disregarded. FIXME: Set encoding to pseudo! 1126let canFoldAsLoad = 1, isReMaterializable = 1, SchedRW = [WriteLoad] in { 1127let isCodeGenOnly = 1 in { 1128 def FsVMOVAPSrm : VPSI<0x28, MRMSrcMem, (outs FR32:$dst), (ins f128mem:$src), 1129 "movaps\t{$src, $dst|$dst, $src}", 1130 [(set FR32:$dst, (alignedloadfsf32 addr:$src))], 1131 IIC_SSE_MOVA_P_RM>, VEX; 1132 def FsVMOVAPDrm : VPDI<0x28, MRMSrcMem, (outs FR64:$dst), (ins f128mem:$src), 1133 "movapd\t{$src, $dst|$dst, $src}", 1134 [(set FR64:$dst, (alignedloadfsf64 addr:$src))], 1135 IIC_SSE_MOVA_P_RM>, VEX; 1136 def FsMOVAPSrm : PSI<0x28, MRMSrcMem, (outs FR32:$dst), (ins f128mem:$src), 1137 "movaps\t{$src, $dst|$dst, $src}", 1138 [(set FR32:$dst, (alignedloadfsf32 addr:$src))], 1139 IIC_SSE_MOVA_P_RM>; 1140 def FsMOVAPDrm : PDI<0x28, MRMSrcMem, (outs FR64:$dst), (ins f128mem:$src), 1141 "movapd\t{$src, $dst|$dst, $src}", 1142 [(set FR64:$dst, (alignedloadfsf64 addr:$src))], 1143 IIC_SSE_MOVA_P_RM>; 1144} 1145} 1146 1147//===----------------------------------------------------------------------===// 1148// SSE 1 & 2 - Move Low packed FP Instructions 1149//===----------------------------------------------------------------------===// 1150 1151multiclass sse12_mov_hilo_packed_base<bits<8>opc, SDNode psnode, SDNode pdnode, 1152 string base_opc, string asm_opr, 1153 InstrItinClass itin> { 1154 def PSrm : PI<opc, MRMSrcMem, 1155 (outs VR128:$dst), (ins VR128:$src1, f64mem:$src2), 1156 !strconcat(base_opc, "s", asm_opr), 1157 [(set VR128:$dst, 1158 (psnode VR128:$src1, 1159 (bc_v4f32 (v2f64 (scalar_to_vector (loadf64 addr:$src2))))))], 1160 itin, SSEPackedSingle>, PS, 1161 Sched<[WriteFShuffleLd, ReadAfterLd]>; 1162 1163 def PDrm : PI<opc, MRMSrcMem, 1164 (outs VR128:$dst), (ins VR128:$src1, f64mem:$src2), 1165 !strconcat(base_opc, "d", asm_opr), 1166 [(set VR128:$dst, (v2f64 (pdnode VR128:$src1, 1167 (scalar_to_vector (loadf64 addr:$src2)))))], 1168 itin, SSEPackedDouble>, PD, 1169 Sched<[WriteFShuffleLd, ReadAfterLd]>; 1170 1171} 1172 1173multiclass sse12_mov_hilo_packed<bits<8>opc, SDNode psnode, SDNode pdnode, 1174 string base_opc, InstrItinClass itin> { 1175 defm V#NAME : sse12_mov_hilo_packed_base<opc, psnode, pdnode, base_opc, 1176 "\t{$src2, $src1, $dst|$dst, $src1, $src2}", 1177 itin>, VEX_4V; 1178 1179let Constraints = "$src1 = $dst" in 1180 defm NAME : sse12_mov_hilo_packed_base<opc, psnode, pdnode, base_opc, 1181 "\t{$src2, $dst|$dst, $src2}", 1182 itin>; 1183} 1184 1185let AddedComplexity = 20 in { 1186 defm MOVL : sse12_mov_hilo_packed<0x12, X86Movlps, X86Movlpd, "movlp", 1187 IIC_SSE_MOV_LH>; 1188} 1189 1190let SchedRW = [WriteStore] in { 1191def VMOVLPSmr : VPSI<0x13, MRMDestMem, (outs), (ins f64mem:$dst, VR128:$src), 1192 "movlps\t{$src, $dst|$dst, $src}", 1193 [(store (f64 (vector_extract (bc_v2f64 (v4f32 VR128:$src)), 1194 (iPTR 0))), addr:$dst)], 1195 IIC_SSE_MOV_LH>, VEX; 1196def VMOVLPDmr : VPDI<0x13, MRMDestMem, (outs), (ins f64mem:$dst, VR128:$src), 1197 "movlpd\t{$src, $dst|$dst, $src}", 1198 [(store (f64 (vector_extract (v2f64 VR128:$src), 1199 (iPTR 0))), addr:$dst)], 1200 IIC_SSE_MOV_LH>, VEX; 1201def MOVLPSmr : PSI<0x13, MRMDestMem, (outs), (ins f64mem:$dst, VR128:$src), 1202 "movlps\t{$src, $dst|$dst, $src}", 1203 [(store (f64 (vector_extract (bc_v2f64 (v4f32 VR128:$src)), 1204 (iPTR 0))), addr:$dst)], 1205 IIC_SSE_MOV_LH>; 1206def MOVLPDmr : PDI<0x13, MRMDestMem, (outs), (ins f64mem:$dst, VR128:$src), 1207 "movlpd\t{$src, $dst|$dst, $src}", 1208 [(store (f64 (vector_extract (v2f64 VR128:$src), 1209 (iPTR 0))), addr:$dst)], 1210 IIC_SSE_MOV_LH>; 1211} // SchedRW 1212 1213let Predicates = [HasAVX] in { 1214 // Shuffle with VMOVLPS 1215 def : Pat<(v4f32 (X86Movlps VR128:$src1, (load addr:$src2))), 1216 (VMOVLPSrm VR128:$src1, addr:$src2)>; 1217 def : Pat<(v4i32 (X86Movlps VR128:$src1, (load addr:$src2))), 1218 (VMOVLPSrm VR128:$src1, addr:$src2)>; 1219 1220 // Shuffle with VMOVLPD 1221 def : Pat<(v2f64 (X86Movlpd VR128:$src1, (load addr:$src2))), 1222 (VMOVLPDrm VR128:$src1, addr:$src2)>; 1223 def : Pat<(v2i64 (X86Movlpd VR128:$src1, (load addr:$src2))), 1224 (VMOVLPDrm VR128:$src1, addr:$src2)>; 1225 def : Pat<(v2f64 (X86Movsd VR128:$src1, 1226 (v2f64 (scalar_to_vector (loadf64 addr:$src2))))), 1227 (VMOVLPDrm VR128:$src1, addr:$src2)>; 1228 1229 // Store patterns 1230 def : Pat<(store (v4f32 (X86Movlps (load addr:$src1), VR128:$src2)), 1231 addr:$src1), 1232 (VMOVLPSmr addr:$src1, VR128:$src2)>; 1233 def : Pat<(store (v4i32 (X86Movlps 1234 (bc_v4i32 (loadv2i64 addr:$src1)), VR128:$src2)), addr:$src1), 1235 (VMOVLPSmr addr:$src1, VR128:$src2)>; 1236 def : Pat<(store (v2f64 (X86Movlpd (load addr:$src1), VR128:$src2)), 1237 addr:$src1), 1238 (VMOVLPDmr addr:$src1, VR128:$src2)>; 1239 def : Pat<(store (v2i64 (X86Movlpd (load addr:$src1), VR128:$src2)), 1240 addr:$src1), 1241 (VMOVLPDmr addr:$src1, VR128:$src2)>; 1242} 1243 1244let Predicates = [UseSSE1] in { 1245 // (store (vector_shuffle (load addr), v2, <4, 5, 2, 3>), addr) using MOVLPS 1246 def : Pat<(store (i64 (vector_extract (bc_v2i64 (v4f32 VR128:$src2)), 1247 (iPTR 0))), addr:$src1), 1248 (MOVLPSmr addr:$src1, VR128:$src2)>; 1249 1250 // Shuffle with MOVLPS 1251 def : Pat<(v4f32 (X86Movlps VR128:$src1, (load addr:$src2))), 1252 (MOVLPSrm VR128:$src1, addr:$src2)>; 1253 def : Pat<(v4i32 (X86Movlps VR128:$src1, (load addr:$src2))), 1254 (MOVLPSrm VR128:$src1, addr:$src2)>; 1255 def : Pat<(X86Movlps VR128:$src1, 1256 (bc_v4f32 (v2i64 (scalar_to_vector (loadi64 addr:$src2))))), 1257 (MOVLPSrm VR128:$src1, addr:$src2)>; 1258 1259 // Store patterns 1260 def : Pat<(store (v4f32 (X86Movlps (load addr:$src1), VR128:$src2)), 1261 addr:$src1), 1262 (MOVLPSmr addr:$src1, VR128:$src2)>; 1263 def : Pat<(store (v4i32 (X86Movlps 1264 (bc_v4i32 (loadv2i64 addr:$src1)), VR128:$src2)), 1265 addr:$src1), 1266 (MOVLPSmr addr:$src1, VR128:$src2)>; 1267} 1268 1269let Predicates = [UseSSE2] in { 1270 // Shuffle with MOVLPD 1271 def : Pat<(v2f64 (X86Movlpd VR128:$src1, (load addr:$src2))), 1272 (MOVLPDrm VR128:$src1, addr:$src2)>; 1273 def : Pat<(v2i64 (X86Movlpd VR128:$src1, (load addr:$src2))), 1274 (MOVLPDrm VR128:$src1, addr:$src2)>; 1275 def : Pat<(v2f64 (X86Movsd VR128:$src1, 1276 (v2f64 (scalar_to_vector (loadf64 addr:$src2))))), 1277 (MOVLPDrm VR128:$src1, addr:$src2)>; 1278 1279 // Store patterns 1280 def : Pat<(store (v2f64 (X86Movlpd (load addr:$src1), VR128:$src2)), 1281 addr:$src1), 1282 (MOVLPDmr addr:$src1, VR128:$src2)>; 1283 def : Pat<(store (v2i64 (X86Movlpd (load addr:$src1), VR128:$src2)), 1284 addr:$src1), 1285 (MOVLPDmr addr:$src1, VR128:$src2)>; 1286} 1287 1288//===----------------------------------------------------------------------===// 1289// SSE 1 & 2 - Move Hi packed FP Instructions 1290//===----------------------------------------------------------------------===// 1291 1292let AddedComplexity = 20 in { 1293 defm MOVH : sse12_mov_hilo_packed<0x16, X86Movlhps, X86Movlhpd, "movhp", 1294 IIC_SSE_MOV_LH>; 1295} 1296 1297let SchedRW = [WriteStore] in { 1298// v2f64 extract element 1 is always custom lowered to unpack high to low 1299// and extract element 0 so the non-store version isn't too horrible. 1300def VMOVHPSmr : VPSI<0x17, MRMDestMem, (outs), (ins f64mem:$dst, VR128:$src), 1301 "movhps\t{$src, $dst|$dst, $src}", 1302 [(store (f64 (vector_extract 1303 (X86Unpckh (bc_v2f64 (v4f32 VR128:$src)), 1304 (bc_v2f64 (v4f32 VR128:$src))), 1305 (iPTR 0))), addr:$dst)], IIC_SSE_MOV_LH>, VEX; 1306def VMOVHPDmr : VPDI<0x17, MRMDestMem, (outs), (ins f64mem:$dst, VR128:$src), 1307 "movhpd\t{$src, $dst|$dst, $src}", 1308 [(store (f64 (vector_extract 1309 (v2f64 (X86Unpckh VR128:$src, VR128:$src)), 1310 (iPTR 0))), addr:$dst)], IIC_SSE_MOV_LH>, VEX; 1311def MOVHPSmr : PSI<0x17, MRMDestMem, (outs), (ins f64mem:$dst, VR128:$src), 1312 "movhps\t{$src, $dst|$dst, $src}", 1313 [(store (f64 (vector_extract 1314 (X86Unpckh (bc_v2f64 (v4f32 VR128:$src)), 1315 (bc_v2f64 (v4f32 VR128:$src))), 1316 (iPTR 0))), addr:$dst)], IIC_SSE_MOV_LH>; 1317def MOVHPDmr : PDI<0x17, MRMDestMem, (outs), (ins f64mem:$dst, VR128:$src), 1318 "movhpd\t{$src, $dst|$dst, $src}", 1319 [(store (f64 (vector_extract 1320 (v2f64 (X86Unpckh VR128:$src, VR128:$src)), 1321 (iPTR 0))), addr:$dst)], IIC_SSE_MOV_LH>; 1322} // SchedRW 1323 1324let Predicates = [HasAVX] in { 1325 // VMOVHPS patterns 1326 def : Pat<(X86Movlhps VR128:$src1, 1327 (bc_v4f32 (v2i64 (scalar_to_vector (loadi64 addr:$src2))))), 1328 (VMOVHPSrm VR128:$src1, addr:$src2)>; 1329 def : Pat<(X86Movlhps VR128:$src1, 1330 (bc_v4i32 (v2i64 (X86vzload addr:$src2)))), 1331 (VMOVHPSrm VR128:$src1, addr:$src2)>; 1332 1333 // VMOVHPD patterns 1334 1335 // FIXME: Instead of X86Unpckl, there should be a X86Movlhpd here, the problem 1336 // is during lowering, where it's not possible to recognize the load fold 1337 // cause it has two uses through a bitcast. One use disappears at isel time 1338 // and the fold opportunity reappears. 1339 def : Pat<(v2f64 (X86Unpckl VR128:$src1, 1340 (scalar_to_vector (loadf64 addr:$src2)))), 1341 (VMOVHPDrm VR128:$src1, addr:$src2)>; 1342 // Also handle an i64 load because that may get selected as a faster way to 1343 // load the data. 1344 def : Pat<(v2f64 (X86Unpckl VR128:$src1, 1345 (bc_v2f64 (v2i64 (scalar_to_vector (loadi64 addr:$src2)))))), 1346 (VMOVHPDrm VR128:$src1, addr:$src2)>; 1347 1348 def : Pat<(store (f64 (vector_extract 1349 (v2f64 (X86VPermilpi VR128:$src, (i8 1))), 1350 (iPTR 0))), addr:$dst), 1351 (VMOVHPDmr addr:$dst, VR128:$src)>; 1352} 1353 1354let Predicates = [UseSSE1] in { 1355 // MOVHPS patterns 1356 def : Pat<(X86Movlhps VR128:$src1, 1357 (bc_v4f32 (v2i64 (scalar_to_vector (loadi64 addr:$src2))))), 1358 (MOVHPSrm VR128:$src1, addr:$src2)>; 1359 def : Pat<(X86Movlhps VR128:$src1, 1360 (bc_v4f32 (v2i64 (X86vzload addr:$src2)))), 1361 (MOVHPSrm VR128:$src1, addr:$src2)>; 1362} 1363 1364let Predicates = [UseSSE2] in { 1365 // MOVHPD patterns 1366 1367 // FIXME: Instead of X86Unpckl, there should be a X86Movlhpd here, the problem 1368 // is during lowering, where it's not possible to recognize the load fold 1369 // cause it has two uses through a bitcast. One use disappears at isel time 1370 // and the fold opportunity reappears. 1371 def : Pat<(v2f64 (X86Unpckl VR128:$src1, 1372 (scalar_to_vector (loadf64 addr:$src2)))), 1373 (MOVHPDrm VR128:$src1, addr:$src2)>; 1374 // Also handle an i64 load because that may get selected as a faster way to 1375 // load the data. 1376 def : Pat<(v2f64 (X86Unpckl VR128:$src1, 1377 (bc_v2f64 (v2i64 (scalar_to_vector (loadi64 addr:$src2)))))), 1378 (MOVHPDrm VR128:$src1, addr:$src2)>; 1379 1380 def : Pat<(store (f64 (vector_extract 1381 (v2f64 (X86Shufp VR128:$src, VR128:$src, (i8 1))), 1382 (iPTR 0))), addr:$dst), 1383 (MOVHPDmr addr:$dst, VR128:$src)>; 1384} 1385 1386//===----------------------------------------------------------------------===// 1387// SSE 1 & 2 - Move Low to High and High to Low packed FP Instructions 1388//===----------------------------------------------------------------------===// 1389 1390let AddedComplexity = 20, Predicates = [UseAVX] in { 1391 def VMOVLHPSrr : VPSI<0x16, MRMSrcReg, (outs VR128:$dst), 1392 (ins VR128:$src1, VR128:$src2), 1393 "movlhps\t{$src2, $src1, $dst|$dst, $src1, $src2}", 1394 [(set VR128:$dst, 1395 (v4f32 (X86Movlhps VR128:$src1, VR128:$src2)))], 1396 IIC_SSE_MOV_LH>, 1397 VEX_4V, Sched<[WriteFShuffle]>; 1398 def VMOVHLPSrr : VPSI<0x12, MRMSrcReg, (outs VR128:$dst), 1399 (ins VR128:$src1, VR128:$src2), 1400 "movhlps\t{$src2, $src1, $dst|$dst, $src1, $src2}", 1401 [(set VR128:$dst, 1402 (v4f32 (X86Movhlps VR128:$src1, VR128:$src2)))], 1403 IIC_SSE_MOV_LH>, 1404 VEX_4V, Sched<[WriteFShuffle]>; 1405} 1406let Constraints = "$src1 = $dst", AddedComplexity = 20 in { 1407 def MOVLHPSrr : PSI<0x16, MRMSrcReg, (outs VR128:$dst), 1408 (ins VR128:$src1, VR128:$src2), 1409 "movlhps\t{$src2, $dst|$dst, $src2}", 1410 [(set VR128:$dst, 1411 (v4f32 (X86Movlhps VR128:$src1, VR128:$src2)))], 1412 IIC_SSE_MOV_LH>, Sched<[WriteFShuffle]>; 1413 def MOVHLPSrr : PSI<0x12, MRMSrcReg, (outs VR128:$dst), 1414 (ins VR128:$src1, VR128:$src2), 1415 "movhlps\t{$src2, $dst|$dst, $src2}", 1416 [(set VR128:$dst, 1417 (v4f32 (X86Movhlps VR128:$src1, VR128:$src2)))], 1418 IIC_SSE_MOV_LH>, Sched<[WriteFShuffle]>; 1419} 1420 1421let Predicates = [UseAVX] in { 1422 // MOVLHPS patterns 1423 def : Pat<(v4i32 (X86Movlhps VR128:$src1, VR128:$src2)), 1424 (VMOVLHPSrr VR128:$src1, VR128:$src2)>; 1425 def : Pat<(v2i64 (X86Movlhps VR128:$src1, VR128:$src2)), 1426 (VMOVLHPSrr (v2i64 VR128:$src1), VR128:$src2)>; 1427 1428 // MOVHLPS patterns 1429 def : Pat<(v4i32 (X86Movhlps VR128:$src1, VR128:$src2)), 1430 (VMOVHLPSrr VR128:$src1, VR128:$src2)>; 1431} 1432 1433let Predicates = [UseSSE1] in { 1434 // MOVLHPS patterns 1435 def : Pat<(v4i32 (X86Movlhps VR128:$src1, VR128:$src2)), 1436 (MOVLHPSrr VR128:$src1, VR128:$src2)>; 1437 def : Pat<(v2i64 (X86Movlhps VR128:$src1, VR128:$src2)), 1438 (MOVLHPSrr (v2i64 VR128:$src1), VR128:$src2)>; 1439 1440 // MOVHLPS patterns 1441 def : Pat<(v4i32 (X86Movhlps VR128:$src1, VR128:$src2)), 1442 (MOVHLPSrr VR128:$src1, VR128:$src2)>; 1443} 1444 1445//===----------------------------------------------------------------------===// 1446// SSE 1 & 2 - Conversion Instructions 1447//===----------------------------------------------------------------------===// 1448 1449def SSE_CVT_PD : OpndItins< 1450 IIC_SSE_CVT_PD_RR, IIC_SSE_CVT_PD_RM 1451>; 1452 1453let Sched = WriteCvtI2F in 1454def SSE_CVT_PS : OpndItins< 1455 IIC_SSE_CVT_PS_RR, IIC_SSE_CVT_PS_RM 1456>; 1457 1458let Sched = WriteCvtI2F in 1459def SSE_CVT_Scalar : OpndItins< 1460 IIC_SSE_CVT_Scalar_RR, IIC_SSE_CVT_Scalar_RM 1461>; 1462 1463let Sched = WriteCvtF2I in 1464def SSE_CVT_SS2SI_32 : OpndItins< 1465 IIC_SSE_CVT_SS2SI32_RR, IIC_SSE_CVT_SS2SI32_RM 1466>; 1467 1468let Sched = WriteCvtF2I in 1469def SSE_CVT_SS2SI_64 : OpndItins< 1470 IIC_SSE_CVT_SS2SI64_RR, IIC_SSE_CVT_SS2SI64_RM 1471>; 1472 1473let Sched = WriteCvtF2I in 1474def SSE_CVT_SD2SI : OpndItins< 1475 IIC_SSE_CVT_SD2SI_RR, IIC_SSE_CVT_SD2SI_RM 1476>; 1477 1478multiclass sse12_cvt_s<bits<8> opc, RegisterClass SrcRC, RegisterClass DstRC, 1479 SDNode OpNode, X86MemOperand x86memop, PatFrag ld_frag, 1480 string asm, OpndItins itins> { 1481 def rr : SI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src), asm, 1482 [(set DstRC:$dst, (OpNode SrcRC:$src))], 1483 itins.rr>, Sched<[itins.Sched]>; 1484 def rm : SI<opc, MRMSrcMem, (outs DstRC:$dst), (ins x86memop:$src), asm, 1485 [(set DstRC:$dst, (OpNode (ld_frag addr:$src)))], 1486 itins.rm>, Sched<[itins.Sched.Folded]>; 1487} 1488 1489multiclass sse12_cvt_p<bits<8> opc, RegisterClass SrcRC, RegisterClass DstRC, 1490 X86MemOperand x86memop, string asm, Domain d, 1491 OpndItins itins> { 1492let hasSideEffects = 0 in { 1493 def rr : I<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src), asm, 1494 [], itins.rr, d>, Sched<[itins.Sched]>; 1495 let mayLoad = 1 in 1496 def rm : I<opc, MRMSrcMem, (outs DstRC:$dst), (ins x86memop:$src), asm, 1497 [], itins.rm, d>, Sched<[itins.Sched.Folded]>; 1498} 1499} 1500 1501multiclass sse12_vcvt_avx<bits<8> opc, RegisterClass SrcRC, RegisterClass DstRC, 1502 X86MemOperand x86memop, string asm> { 1503let hasSideEffects = 0, Predicates = [UseAVX] in { 1504 def rr : SI<opc, MRMSrcReg, (outs DstRC:$dst), (ins DstRC:$src1, SrcRC:$src), 1505 !strconcat(asm,"\t{$src, $src1, $dst|$dst, $src1, $src}"), []>, 1506 Sched<[WriteCvtI2F]>; 1507 let mayLoad = 1 in 1508 def rm : SI<opc, MRMSrcMem, (outs DstRC:$dst), 1509 (ins DstRC:$src1, x86memop:$src), 1510 !strconcat(asm,"\t{$src, $src1, $dst|$dst, $src1, $src}"), []>, 1511 Sched<[WriteCvtI2FLd, ReadAfterLd]>; 1512} // hasSideEffects = 0 1513} 1514 1515let Predicates = [UseAVX] in { 1516defm VCVTTSS2SI : sse12_cvt_s<0x2C, FR32, GR32, fp_to_sint, f32mem, loadf32, 1517 "cvttss2si\t{$src, $dst|$dst, $src}", 1518 SSE_CVT_SS2SI_32>, 1519 XS, VEX, VEX_LIG; 1520defm VCVTTSS2SI64 : sse12_cvt_s<0x2C, FR32, GR64, fp_to_sint, f32mem, loadf32, 1521 "cvttss2si\t{$src, $dst|$dst, $src}", 1522 SSE_CVT_SS2SI_64>, 1523 XS, VEX, VEX_W, VEX_LIG; 1524defm VCVTTSD2SI : sse12_cvt_s<0x2C, FR64, GR32, fp_to_sint, f64mem, loadf64, 1525 "cvttsd2si\t{$src, $dst|$dst, $src}", 1526 SSE_CVT_SD2SI>, 1527 XD, VEX, VEX_LIG; 1528defm VCVTTSD2SI64 : sse12_cvt_s<0x2C, FR64, GR64, fp_to_sint, f64mem, loadf64, 1529 "cvttsd2si\t{$src, $dst|$dst, $src}", 1530 SSE_CVT_SD2SI>, 1531 XD, VEX, VEX_W, VEX_LIG; 1532 1533def : InstAlias<"vcvttss2si{l}\t{$src, $dst|$dst, $src}", 1534 (VCVTTSS2SIrr GR32:$dst, FR32:$src), 0>; 1535def : InstAlias<"vcvttss2si{l}\t{$src, $dst|$dst, $src}", 1536 (VCVTTSS2SIrm GR32:$dst, f32mem:$src), 0>; 1537def : InstAlias<"vcvttsd2si{l}\t{$src, $dst|$dst, $src}", 1538 (VCVTTSD2SIrr GR32:$dst, FR64:$src), 0>; 1539def : InstAlias<"vcvttsd2si{l}\t{$src, $dst|$dst, $src}", 1540 (VCVTTSD2SIrm GR32:$dst, f64mem:$src), 0>; 1541def : InstAlias<"vcvttss2si{q}\t{$src, $dst|$dst, $src}", 1542 (VCVTTSS2SI64rr GR64:$dst, FR32:$src), 0>; 1543def : InstAlias<"vcvttss2si{q}\t{$src, $dst|$dst, $src}", 1544 (VCVTTSS2SI64rm GR64:$dst, f32mem:$src), 0>; 1545def : InstAlias<"vcvttsd2si{q}\t{$src, $dst|$dst, $src}", 1546 (VCVTTSD2SI64rr GR64:$dst, FR64:$src), 0>; 1547def : InstAlias<"vcvttsd2si{q}\t{$src, $dst|$dst, $src}", 1548 (VCVTTSD2SI64rm GR64:$dst, f64mem:$src), 0>; 1549} 1550// The assembler can recognize rr 64-bit instructions by seeing a rxx 1551// register, but the same isn't true when only using memory operands, 1552// provide other assembly "l" and "q" forms to address this explicitly 1553// where appropriate to do so. 1554defm VCVTSI2SS : sse12_vcvt_avx<0x2A, GR32, FR32, i32mem, "cvtsi2ss{l}">, 1555 XS, VEX_4V, VEX_LIG; 1556defm VCVTSI2SS64 : sse12_vcvt_avx<0x2A, GR64, FR32, i64mem, "cvtsi2ss{q}">, 1557 XS, VEX_4V, VEX_W, VEX_LIG; 1558defm VCVTSI2SD : sse12_vcvt_avx<0x2A, GR32, FR64, i32mem, "cvtsi2sd{l}">, 1559 XD, VEX_4V, VEX_LIG; 1560defm VCVTSI2SD64 : sse12_vcvt_avx<0x2A, GR64, FR64, i64mem, "cvtsi2sd{q}">, 1561 XD, VEX_4V, VEX_W, VEX_LIG; 1562 1563let Predicates = [UseAVX] in { 1564 def : InstAlias<"vcvtsi2ss\t{$src, $src1, $dst|$dst, $src1, $src}", 1565 (VCVTSI2SSrm FR64:$dst, FR64:$src1, i32mem:$src), 0>; 1566 def : InstAlias<"vcvtsi2sd\t{$src, $src1, $dst|$dst, $src1, $src}", 1567 (VCVTSI2SDrm FR64:$dst, FR64:$src1, i32mem:$src), 0>; 1568 1569 def : Pat<(f32 (sint_to_fp (loadi32 addr:$src))), 1570 (VCVTSI2SSrm (f32 (IMPLICIT_DEF)), addr:$src)>; 1571 def : Pat<(f32 (sint_to_fp (loadi64 addr:$src))), 1572 (VCVTSI2SS64rm (f32 (IMPLICIT_DEF)), addr:$src)>; 1573 def : Pat<(f64 (sint_to_fp (loadi32 addr:$src))), 1574 (VCVTSI2SDrm (f64 (IMPLICIT_DEF)), addr:$src)>; 1575 def : Pat<(f64 (sint_to_fp (loadi64 addr:$src))), 1576 (VCVTSI2SD64rm (f64 (IMPLICIT_DEF)), addr:$src)>; 1577 1578 def : Pat<(f32 (sint_to_fp GR32:$src)), 1579 (VCVTSI2SSrr (f32 (IMPLICIT_DEF)), GR32:$src)>; 1580 def : Pat<(f32 (sint_to_fp GR64:$src)), 1581 (VCVTSI2SS64rr (f32 (IMPLICIT_DEF)), GR64:$src)>; 1582 def : Pat<(f64 (sint_to_fp GR32:$src)), 1583 (VCVTSI2SDrr (f64 (IMPLICIT_DEF)), GR32:$src)>; 1584 def : Pat<(f64 (sint_to_fp GR64:$src)), 1585 (VCVTSI2SD64rr (f64 (IMPLICIT_DEF)), GR64:$src)>; 1586} 1587 1588defm CVTTSS2SI : sse12_cvt_s<0x2C, FR32, GR32, fp_to_sint, f32mem, loadf32, 1589 "cvttss2si\t{$src, $dst|$dst, $src}", 1590 SSE_CVT_SS2SI_32>, XS; 1591defm CVTTSS2SI64 : sse12_cvt_s<0x2C, FR32, GR64, fp_to_sint, f32mem, loadf32, 1592 "cvttss2si\t{$src, $dst|$dst, $src}", 1593 SSE_CVT_SS2SI_64>, XS, REX_W; 1594defm CVTTSD2SI : sse12_cvt_s<0x2C, FR64, GR32, fp_to_sint, f64mem, loadf64, 1595 "cvttsd2si\t{$src, $dst|$dst, $src}", 1596 SSE_CVT_SD2SI>, XD; 1597defm CVTTSD2SI64 : sse12_cvt_s<0x2C, FR64, GR64, fp_to_sint, f64mem, loadf64, 1598 "cvttsd2si\t{$src, $dst|$dst, $src}", 1599 SSE_CVT_SD2SI>, XD, REX_W; 1600defm CVTSI2SS : sse12_cvt_s<0x2A, GR32, FR32, sint_to_fp, i32mem, loadi32, 1601 "cvtsi2ss{l}\t{$src, $dst|$dst, $src}", 1602 SSE_CVT_Scalar>, XS; 1603defm CVTSI2SS64 : sse12_cvt_s<0x2A, GR64, FR32, sint_to_fp, i64mem, loadi64, 1604 "cvtsi2ss{q}\t{$src, $dst|$dst, $src}", 1605 SSE_CVT_Scalar>, XS, REX_W; 1606defm CVTSI2SD : sse12_cvt_s<0x2A, GR32, FR64, sint_to_fp, i32mem, loadi32, 1607 "cvtsi2sd{l}\t{$src, $dst|$dst, $src}", 1608 SSE_CVT_Scalar>, XD; 1609defm CVTSI2SD64 : sse12_cvt_s<0x2A, GR64, FR64, sint_to_fp, i64mem, loadi64, 1610 "cvtsi2sd{q}\t{$src, $dst|$dst, $src}", 1611 SSE_CVT_Scalar>, XD, REX_W; 1612 1613def : InstAlias<"cvttss2si{l}\t{$src, $dst|$dst, $src}", 1614 (CVTTSS2SIrr GR32:$dst, FR32:$src), 0>; 1615def : InstAlias<"cvttss2si{l}\t{$src, $dst|$dst, $src}", 1616 (CVTTSS2SIrm GR32:$dst, f32mem:$src), 0>; 1617def : InstAlias<"cvttsd2si{l}\t{$src, $dst|$dst, $src}", 1618 (CVTTSD2SIrr GR32:$dst, FR64:$src), 0>; 1619def : InstAlias<"cvttsd2si{l}\t{$src, $dst|$dst, $src}", 1620 (CVTTSD2SIrm GR32:$dst, f64mem:$src), 0>; 1621def : InstAlias<"cvttss2si{q}\t{$src, $dst|$dst, $src}", 1622 (CVTTSS2SI64rr GR64:$dst, FR32:$src), 0>; 1623def : InstAlias<"cvttss2si{q}\t{$src, $dst|$dst, $src}", 1624 (CVTTSS2SI64rm GR64:$dst, f32mem:$src), 0>; 1625def : InstAlias<"cvttsd2si{q}\t{$src, $dst|$dst, $src}", 1626 (CVTTSD2SI64rr GR64:$dst, FR64:$src), 0>; 1627def : InstAlias<"cvttsd2si{q}\t{$src, $dst|$dst, $src}", 1628 (CVTTSD2SI64rm GR64:$dst, f64mem:$src), 0>; 1629 1630def : InstAlias<"cvtsi2ss\t{$src, $dst|$dst, $src}", 1631 (CVTSI2SSrm FR64:$dst, i32mem:$src), 0>; 1632def : InstAlias<"cvtsi2sd\t{$src, $dst|$dst, $src}", 1633 (CVTSI2SDrm FR64:$dst, i32mem:$src), 0>; 1634 1635// Conversion Instructions Intrinsics - Match intrinsics which expect MM 1636// and/or XMM operand(s). 1637 1638multiclass sse12_cvt_sint<bits<8> opc, RegisterClass SrcRC, RegisterClass DstRC, 1639 Intrinsic Int, Operand memop, ComplexPattern mem_cpat, 1640 string asm, OpndItins itins> { 1641 def rr : SI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src), 1642 !strconcat(asm, "\t{$src, $dst|$dst, $src}"), 1643 [(set DstRC:$dst, (Int SrcRC:$src))], itins.rr>, 1644 Sched<[itins.Sched]>; 1645 def rm : SI<opc, MRMSrcMem, (outs DstRC:$dst), (ins memop:$src), 1646 !strconcat(asm, "\t{$src, $dst|$dst, $src}"), 1647 [(set DstRC:$dst, (Int mem_cpat:$src))], itins.rm>, 1648 Sched<[itins.Sched.Folded]>; 1649} 1650 1651multiclass sse12_cvt_sint_3addr<bits<8> opc, RegisterClass SrcRC, 1652 RegisterClass DstRC, Intrinsic Int, X86MemOperand x86memop, 1653 PatFrag ld_frag, string asm, OpndItins itins, 1654 bit Is2Addr = 1> { 1655 def rr : SI<opc, MRMSrcReg, (outs DstRC:$dst), (ins DstRC:$src1, SrcRC:$src2), 1656 !if(Is2Addr, 1657 !strconcat(asm, "\t{$src2, $dst|$dst, $src2}"), 1658 !strconcat(asm, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 1659 [(set DstRC:$dst, (Int DstRC:$src1, SrcRC:$src2))], 1660 itins.rr>, Sched<[itins.Sched]>; 1661 def rm : SI<opc, MRMSrcMem, (outs DstRC:$dst), 1662 (ins DstRC:$src1, x86memop:$src2), 1663 !if(Is2Addr, 1664 !strconcat(asm, "\t{$src2, $dst|$dst, $src2}"), 1665 !strconcat(asm, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 1666 [(set DstRC:$dst, (Int DstRC:$src1, (ld_frag addr:$src2)))], 1667 itins.rm>, Sched<[itins.Sched.Folded, ReadAfterLd]>; 1668} 1669 1670let Predicates = [UseAVX] in { 1671defm VCVTSD2SI : sse12_cvt_sint<0x2D, VR128, GR32, 1672 int_x86_sse2_cvtsd2si, sdmem, sse_load_f64, "cvtsd2si", 1673 SSE_CVT_SD2SI>, XD, VEX, VEX_LIG; 1674defm VCVTSD2SI64 : sse12_cvt_sint<0x2D, VR128, GR64, 1675 int_x86_sse2_cvtsd2si64, sdmem, sse_load_f64, "cvtsd2si", 1676 SSE_CVT_SD2SI>, XD, VEX, VEX_W, VEX_LIG; 1677} 1678defm CVTSD2SI : sse12_cvt_sint<0x2D, VR128, GR32, int_x86_sse2_cvtsd2si, 1679 sdmem, sse_load_f64, "cvtsd2si", SSE_CVT_SD2SI>, XD; 1680defm CVTSD2SI64 : sse12_cvt_sint<0x2D, VR128, GR64, int_x86_sse2_cvtsd2si64, 1681 sdmem, sse_load_f64, "cvtsd2si", SSE_CVT_SD2SI>, XD, REX_W; 1682 1683 1684let isCodeGenOnly = 1 in { 1685 let Predicates = [UseAVX] in { 1686 defm Int_VCVTSI2SS : sse12_cvt_sint_3addr<0x2A, GR32, VR128, 1687 int_x86_sse_cvtsi2ss, i32mem, loadi32, "cvtsi2ss{l}", 1688 SSE_CVT_Scalar, 0>, XS, VEX_4V; 1689 defm Int_VCVTSI2SS64 : sse12_cvt_sint_3addr<0x2A, GR64, VR128, 1690 int_x86_sse_cvtsi642ss, i64mem, loadi64, "cvtsi2ss{q}", 1691 SSE_CVT_Scalar, 0>, XS, VEX_4V, 1692 VEX_W; 1693 defm Int_VCVTSI2SD : sse12_cvt_sint_3addr<0x2A, GR32, VR128, 1694 int_x86_sse2_cvtsi2sd, i32mem, loadi32, "cvtsi2sd{l}", 1695 SSE_CVT_Scalar, 0>, XD, VEX_4V; 1696 defm Int_VCVTSI2SD64 : sse12_cvt_sint_3addr<0x2A, GR64, VR128, 1697 int_x86_sse2_cvtsi642sd, i64mem, loadi64, "cvtsi2sd{q}", 1698 SSE_CVT_Scalar, 0>, XD, 1699 VEX_4V, VEX_W; 1700 } 1701 let Constraints = "$src1 = $dst" in { 1702 defm Int_CVTSI2SS : sse12_cvt_sint_3addr<0x2A, GR32, VR128, 1703 int_x86_sse_cvtsi2ss, i32mem, loadi32, 1704 "cvtsi2ss{l}", SSE_CVT_Scalar>, XS; 1705 defm Int_CVTSI2SS64 : sse12_cvt_sint_3addr<0x2A, GR64, VR128, 1706 int_x86_sse_cvtsi642ss, i64mem, loadi64, 1707 "cvtsi2ss{q}", SSE_CVT_Scalar>, XS, REX_W; 1708 defm Int_CVTSI2SD : sse12_cvt_sint_3addr<0x2A, GR32, VR128, 1709 int_x86_sse2_cvtsi2sd, i32mem, loadi32, 1710 "cvtsi2sd{l}", SSE_CVT_Scalar>, XD; 1711 defm Int_CVTSI2SD64 : sse12_cvt_sint_3addr<0x2A, GR64, VR128, 1712 int_x86_sse2_cvtsi642sd, i64mem, loadi64, 1713 "cvtsi2sd{q}", SSE_CVT_Scalar>, XD, REX_W; 1714 } 1715} // isCodeGenOnly = 1 1716 1717/// SSE 1 Only 1718 1719// Aliases for intrinsics 1720let isCodeGenOnly = 1 in { 1721let Predicates = [UseAVX] in { 1722defm Int_VCVTTSS2SI : sse12_cvt_sint<0x2C, VR128, GR32, int_x86_sse_cvttss2si, 1723 ssmem, sse_load_f32, "cvttss2si", 1724 SSE_CVT_SS2SI_32>, XS, VEX; 1725defm Int_VCVTTSS2SI64 : sse12_cvt_sint<0x2C, VR128, GR64, 1726 int_x86_sse_cvttss2si64, ssmem, sse_load_f32, 1727 "cvttss2si", SSE_CVT_SS2SI_64>, 1728 XS, VEX, VEX_W; 1729defm Int_VCVTTSD2SI : sse12_cvt_sint<0x2C, VR128, GR32, int_x86_sse2_cvttsd2si, 1730 sdmem, sse_load_f64, "cvttsd2si", 1731 SSE_CVT_SD2SI>, XD, VEX; 1732defm Int_VCVTTSD2SI64 : sse12_cvt_sint<0x2C, VR128, GR64, 1733 int_x86_sse2_cvttsd2si64, sdmem, sse_load_f64, 1734 "cvttsd2si", SSE_CVT_SD2SI>, 1735 XD, VEX, VEX_W; 1736} 1737defm Int_CVTTSS2SI : sse12_cvt_sint<0x2C, VR128, GR32, int_x86_sse_cvttss2si, 1738 ssmem, sse_load_f32, "cvttss2si", 1739 SSE_CVT_SS2SI_32>, XS; 1740defm Int_CVTTSS2SI64 : sse12_cvt_sint<0x2C, VR128, GR64, 1741 int_x86_sse_cvttss2si64, ssmem, sse_load_f32, 1742 "cvttss2si", SSE_CVT_SS2SI_64>, XS, REX_W; 1743defm Int_CVTTSD2SI : sse12_cvt_sint<0x2C, VR128, GR32, int_x86_sse2_cvttsd2si, 1744 sdmem, sse_load_f64, "cvttsd2si", 1745 SSE_CVT_SD2SI>, XD; 1746defm Int_CVTTSD2SI64 : sse12_cvt_sint<0x2C, VR128, GR64, 1747 int_x86_sse2_cvttsd2si64, sdmem, sse_load_f64, 1748 "cvttsd2si", SSE_CVT_SD2SI>, XD, REX_W; 1749} // isCodeGenOnly = 1 1750 1751let Predicates = [UseAVX] in { 1752defm VCVTSS2SI : sse12_cvt_sint<0x2D, VR128, GR32, int_x86_sse_cvtss2si, 1753 ssmem, sse_load_f32, "cvtss2si", 1754 SSE_CVT_SS2SI_32>, XS, VEX, VEX_LIG; 1755defm VCVTSS2SI64 : sse12_cvt_sint<0x2D, VR128, GR64, int_x86_sse_cvtss2si64, 1756 ssmem, sse_load_f32, "cvtss2si", 1757 SSE_CVT_SS2SI_64>, XS, VEX, VEX_W, VEX_LIG; 1758} 1759defm CVTSS2SI : sse12_cvt_sint<0x2D, VR128, GR32, int_x86_sse_cvtss2si, 1760 ssmem, sse_load_f32, "cvtss2si", 1761 SSE_CVT_SS2SI_32>, XS; 1762defm CVTSS2SI64 : sse12_cvt_sint<0x2D, VR128, GR64, int_x86_sse_cvtss2si64, 1763 ssmem, sse_load_f32, "cvtss2si", 1764 SSE_CVT_SS2SI_64>, XS, REX_W; 1765 1766defm VCVTDQ2PS : sse12_cvt_p<0x5B, VR128, VR128, i128mem, 1767 "vcvtdq2ps\t{$src, $dst|$dst, $src}", 1768 SSEPackedSingle, SSE_CVT_PS>, 1769 PS, VEX, Requires<[HasAVX]>; 1770defm VCVTDQ2PSY : sse12_cvt_p<0x5B, VR256, VR256, i256mem, 1771 "vcvtdq2ps\t{$src, $dst|$dst, $src}", 1772 SSEPackedSingle, SSE_CVT_PS>, 1773 PS, VEX, VEX_L, Requires<[HasAVX]>; 1774 1775defm CVTDQ2PS : sse12_cvt_p<0x5B, VR128, VR128, i128mem, 1776 "cvtdq2ps\t{$src, $dst|$dst, $src}", 1777 SSEPackedSingle, SSE_CVT_PS>, 1778 PS, Requires<[UseSSE2]>; 1779 1780let Predicates = [UseAVX] in { 1781def : InstAlias<"vcvtss2si{l}\t{$src, $dst|$dst, $src}", 1782 (VCVTSS2SIrr GR32:$dst, VR128:$src), 0>; 1783def : InstAlias<"vcvtss2si{l}\t{$src, $dst|$dst, $src}", 1784 (VCVTSS2SIrm GR32:$dst, ssmem:$src), 0>; 1785def : InstAlias<"vcvtsd2si{l}\t{$src, $dst|$dst, $src}", 1786 (VCVTSD2SIrr GR32:$dst, VR128:$src), 0>; 1787def : InstAlias<"vcvtsd2si{l}\t{$src, $dst|$dst, $src}", 1788 (VCVTSD2SIrm GR32:$dst, sdmem:$src), 0>; 1789def : InstAlias<"vcvtss2si{q}\t{$src, $dst|$dst, $src}", 1790 (VCVTSS2SI64rr GR64:$dst, VR128:$src), 0>; 1791def : InstAlias<"vcvtss2si{q}\t{$src, $dst|$dst, $src}", 1792 (VCVTSS2SI64rm GR64:$dst, ssmem:$src), 0>; 1793def : InstAlias<"vcvtsd2si{q}\t{$src, $dst|$dst, $src}", 1794 (VCVTSD2SI64rr GR64:$dst, VR128:$src), 0>; 1795def : InstAlias<"vcvtsd2si{q}\t{$src, $dst|$dst, $src}", 1796 (VCVTSD2SI64rm GR64:$dst, sdmem:$src), 0>; 1797} 1798 1799def : InstAlias<"cvtss2si{l}\t{$src, $dst|$dst, $src}", 1800 (CVTSS2SIrr GR32:$dst, VR128:$src), 0>; 1801def : InstAlias<"cvtss2si{l}\t{$src, $dst|$dst, $src}", 1802 (CVTSS2SIrm GR32:$dst, ssmem:$src), 0>; 1803def : InstAlias<"cvtsd2si{l}\t{$src, $dst|$dst, $src}", 1804 (CVTSD2SIrr GR32:$dst, VR128:$src), 0>; 1805def : InstAlias<"cvtsd2si{l}\t{$src, $dst|$dst, $src}", 1806 (CVTSD2SIrm GR32:$dst, sdmem:$src), 0>; 1807def : InstAlias<"cvtss2si{q}\t{$src, $dst|$dst, $src}", 1808 (CVTSS2SI64rr GR64:$dst, VR128:$src), 0>; 1809def : InstAlias<"cvtss2si{q}\t{$src, $dst|$dst, $src}", 1810 (CVTSS2SI64rm GR64:$dst, ssmem:$src), 0>; 1811def : InstAlias<"cvtsd2si{q}\t{$src, $dst|$dst, $src}", 1812 (CVTSD2SI64rr GR64:$dst, VR128:$src), 0>; 1813def : InstAlias<"cvtsd2si{q}\t{$src, $dst|$dst, $src}", 1814 (CVTSD2SI64rm GR64:$dst, sdmem:$src)>; 1815 1816/// SSE 2 Only 1817 1818// Convert scalar double to scalar single 1819let hasSideEffects = 0, Predicates = [UseAVX] in { 1820def VCVTSD2SSrr : VSDI<0x5A, MRMSrcReg, (outs FR32:$dst), 1821 (ins FR64:$src1, FR64:$src2), 1822 "cvtsd2ss\t{$src2, $src1, $dst|$dst, $src1, $src2}", [], 1823 IIC_SSE_CVT_Scalar_RR>, VEX_4V, VEX_LIG, 1824 Sched<[WriteCvtF2F]>; 1825let mayLoad = 1 in 1826def VCVTSD2SSrm : I<0x5A, MRMSrcMem, (outs FR32:$dst), 1827 (ins FR64:$src1, f64mem:$src2), 1828 "vcvtsd2ss\t{$src2, $src1, $dst|$dst, $src1, $src2}", 1829 [], IIC_SSE_CVT_Scalar_RM>, 1830 XD, Requires<[HasAVX, OptForSize]>, VEX_4V, VEX_LIG, 1831 Sched<[WriteCvtF2FLd, ReadAfterLd]>; 1832} 1833 1834def : Pat<(f32 (fround FR64:$src)), (VCVTSD2SSrr FR64:$src, FR64:$src)>, 1835 Requires<[UseAVX]>; 1836 1837def CVTSD2SSrr : SDI<0x5A, MRMSrcReg, (outs FR32:$dst), (ins FR64:$src), 1838 "cvtsd2ss\t{$src, $dst|$dst, $src}", 1839 [(set FR32:$dst, (fround FR64:$src))], 1840 IIC_SSE_CVT_Scalar_RR>, Sched<[WriteCvtF2F]>; 1841def CVTSD2SSrm : I<0x5A, MRMSrcMem, (outs FR32:$dst), (ins f64mem:$src), 1842 "cvtsd2ss\t{$src, $dst|$dst, $src}", 1843 [(set FR32:$dst, (fround (loadf64 addr:$src)))], 1844 IIC_SSE_CVT_Scalar_RM>, 1845 XD, 1846 Requires<[UseSSE2, OptForSize]>, Sched<[WriteCvtF2FLd]>; 1847 1848let isCodeGenOnly = 1 in { 1849def Int_VCVTSD2SSrr: I<0x5A, MRMSrcReg, 1850 (outs VR128:$dst), (ins VR128:$src1, VR128:$src2), 1851 "vcvtsd2ss\t{$src2, $src1, $dst|$dst, $src1, $src2}", 1852 [(set VR128:$dst, 1853 (int_x86_sse2_cvtsd2ss VR128:$src1, VR128:$src2))], 1854 IIC_SSE_CVT_Scalar_RR>, XD, VEX_4V, Requires<[HasAVX]>, 1855 Sched<[WriteCvtF2F]>; 1856def Int_VCVTSD2SSrm: I<0x5A, MRMSrcReg, 1857 (outs VR128:$dst), (ins VR128:$src1, sdmem:$src2), 1858 "vcvtsd2ss\t{$src2, $src1, $dst|$dst, $src1, $src2}", 1859 [(set VR128:$dst, (int_x86_sse2_cvtsd2ss 1860 VR128:$src1, sse_load_f64:$src2))], 1861 IIC_SSE_CVT_Scalar_RM>, XD, VEX_4V, Requires<[HasAVX]>, 1862 Sched<[WriteCvtF2FLd, ReadAfterLd]>; 1863 1864let Constraints = "$src1 = $dst" in { 1865def Int_CVTSD2SSrr: I<0x5A, MRMSrcReg, 1866 (outs VR128:$dst), (ins VR128:$src1, VR128:$src2), 1867 "cvtsd2ss\t{$src2, $dst|$dst, $src2}", 1868 [(set VR128:$dst, 1869 (int_x86_sse2_cvtsd2ss VR128:$src1, VR128:$src2))], 1870 IIC_SSE_CVT_Scalar_RR>, XD, Requires<[UseSSE2]>, 1871 Sched<[WriteCvtF2F]>; 1872def Int_CVTSD2SSrm: I<0x5A, MRMSrcReg, 1873 (outs VR128:$dst), (ins VR128:$src1, sdmem:$src2), 1874 "cvtsd2ss\t{$src2, $dst|$dst, $src2}", 1875 [(set VR128:$dst, (int_x86_sse2_cvtsd2ss 1876 VR128:$src1, sse_load_f64:$src2))], 1877 IIC_SSE_CVT_Scalar_RM>, XD, Requires<[UseSSE2]>, 1878 Sched<[WriteCvtF2FLd, ReadAfterLd]>; 1879} 1880} // isCodeGenOnly = 1 1881 1882// Convert scalar single to scalar double 1883// SSE2 instructions with XS prefix 1884let hasSideEffects = 0, Predicates = [UseAVX] in { 1885def VCVTSS2SDrr : I<0x5A, MRMSrcReg, (outs FR64:$dst), 1886 (ins FR32:$src1, FR32:$src2), 1887 "vcvtss2sd\t{$src2, $src1, $dst|$dst, $src1, $src2}", 1888 [], IIC_SSE_CVT_Scalar_RR>, 1889 XS, Requires<[HasAVX]>, VEX_4V, VEX_LIG, 1890 Sched<[WriteCvtF2F]>; 1891let mayLoad = 1 in 1892def VCVTSS2SDrm : I<0x5A, MRMSrcMem, (outs FR64:$dst), 1893 (ins FR32:$src1, f32mem:$src2), 1894 "vcvtss2sd\t{$src2, $src1, $dst|$dst, $src1, $src2}", 1895 [], IIC_SSE_CVT_Scalar_RM>, 1896 XS, VEX_4V, VEX_LIG, Requires<[HasAVX, OptForSize]>, 1897 Sched<[WriteCvtF2FLd, ReadAfterLd]>; 1898} 1899 1900def : Pat<(f64 (fextend FR32:$src)), 1901 (VCVTSS2SDrr FR32:$src, FR32:$src)>, Requires<[UseAVX]>; 1902def : Pat<(fextend (loadf32 addr:$src)), 1903 (VCVTSS2SDrm (f32 (IMPLICIT_DEF)), addr:$src)>, Requires<[UseAVX]>; 1904 1905def : Pat<(extloadf32 addr:$src), 1906 (VCVTSS2SDrm (f32 (IMPLICIT_DEF)), addr:$src)>, 1907 Requires<[UseAVX, OptForSize]>; 1908def : Pat<(extloadf32 addr:$src), 1909 (VCVTSS2SDrr (f32 (IMPLICIT_DEF)), (VMOVSSrm addr:$src))>, 1910 Requires<[UseAVX, OptForSpeed]>; 1911 1912def CVTSS2SDrr : I<0x5A, MRMSrcReg, (outs FR64:$dst), (ins FR32:$src), 1913 "cvtss2sd\t{$src, $dst|$dst, $src}", 1914 [(set FR64:$dst, (fextend FR32:$src))], 1915 IIC_SSE_CVT_Scalar_RR>, XS, 1916 Requires<[UseSSE2]>, Sched<[WriteCvtF2F]>; 1917def CVTSS2SDrm : I<0x5A, MRMSrcMem, (outs FR64:$dst), (ins f32mem:$src), 1918 "cvtss2sd\t{$src, $dst|$dst, $src}", 1919 [(set FR64:$dst, (extloadf32 addr:$src))], 1920 IIC_SSE_CVT_Scalar_RM>, XS, 1921 Requires<[UseSSE2, OptForSize]>, Sched<[WriteCvtF2FLd]>; 1922 1923// extload f32 -> f64. This matches load+fextend because we have a hack in 1924// the isel (PreprocessForFPConvert) that can introduce loads after dag 1925// combine. 1926// Since these loads aren't folded into the fextend, we have to match it 1927// explicitly here. 1928def : Pat<(fextend (loadf32 addr:$src)), 1929 (CVTSS2SDrm addr:$src)>, Requires<[UseSSE2]>; 1930def : Pat<(extloadf32 addr:$src), 1931 (CVTSS2SDrr (MOVSSrm addr:$src))>, Requires<[UseSSE2, OptForSpeed]>; 1932 1933let isCodeGenOnly = 1 in { 1934def Int_VCVTSS2SDrr: I<0x5A, MRMSrcReg, 1935 (outs VR128:$dst), (ins VR128:$src1, VR128:$src2), 1936 "vcvtss2sd\t{$src2, $src1, $dst|$dst, $src1, $src2}", 1937 [(set VR128:$dst, 1938 (int_x86_sse2_cvtss2sd VR128:$src1, VR128:$src2))], 1939 IIC_SSE_CVT_Scalar_RR>, XS, VEX_4V, Requires<[HasAVX]>, 1940 Sched<[WriteCvtF2F]>; 1941def Int_VCVTSS2SDrm: I<0x5A, MRMSrcMem, 1942 (outs VR128:$dst), (ins VR128:$src1, ssmem:$src2), 1943 "vcvtss2sd\t{$src2, $src1, $dst|$dst, $src1, $src2}", 1944 [(set VR128:$dst, 1945 (int_x86_sse2_cvtss2sd VR128:$src1, sse_load_f32:$src2))], 1946 IIC_SSE_CVT_Scalar_RM>, XS, VEX_4V, Requires<[HasAVX]>, 1947 Sched<[WriteCvtF2FLd, ReadAfterLd]>; 1948let Constraints = "$src1 = $dst" in { // SSE2 instructions with XS prefix 1949def Int_CVTSS2SDrr: I<0x5A, MRMSrcReg, 1950 (outs VR128:$dst), (ins VR128:$src1, VR128:$src2), 1951 "cvtss2sd\t{$src2, $dst|$dst, $src2}", 1952 [(set VR128:$dst, 1953 (int_x86_sse2_cvtss2sd VR128:$src1, VR128:$src2))], 1954 IIC_SSE_CVT_Scalar_RR>, XS, Requires<[UseSSE2]>, 1955 Sched<[WriteCvtF2F]>; 1956def Int_CVTSS2SDrm: I<0x5A, MRMSrcMem, 1957 (outs VR128:$dst), (ins VR128:$src1, ssmem:$src2), 1958 "cvtss2sd\t{$src2, $dst|$dst, $src2}", 1959 [(set VR128:$dst, 1960 (int_x86_sse2_cvtss2sd VR128:$src1, sse_load_f32:$src2))], 1961 IIC_SSE_CVT_Scalar_RM>, XS, Requires<[UseSSE2]>, 1962 Sched<[WriteCvtF2FLd, ReadAfterLd]>; 1963} 1964} // isCodeGenOnly = 1 1965 1966// Convert packed single/double fp to doubleword 1967def VCVTPS2DQrr : VPDI<0x5B, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 1968 "cvtps2dq\t{$src, $dst|$dst, $src}", 1969 [(set VR128:$dst, (int_x86_sse2_cvtps2dq VR128:$src))], 1970 IIC_SSE_CVT_PS_RR>, VEX, Sched<[WriteCvtF2I]>; 1971def VCVTPS2DQrm : VPDI<0x5B, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src), 1972 "cvtps2dq\t{$src, $dst|$dst, $src}", 1973 [(set VR128:$dst, 1974 (int_x86_sse2_cvtps2dq (loadv4f32 addr:$src)))], 1975 IIC_SSE_CVT_PS_RM>, VEX, Sched<[WriteCvtF2ILd]>; 1976def VCVTPS2DQYrr : VPDI<0x5B, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src), 1977 "cvtps2dq\t{$src, $dst|$dst, $src}", 1978 [(set VR256:$dst, 1979 (int_x86_avx_cvt_ps2dq_256 VR256:$src))], 1980 IIC_SSE_CVT_PS_RR>, VEX, VEX_L, Sched<[WriteCvtF2I]>; 1981def VCVTPS2DQYrm : VPDI<0x5B, MRMSrcMem, (outs VR256:$dst), (ins f256mem:$src), 1982 "cvtps2dq\t{$src, $dst|$dst, $src}", 1983 [(set VR256:$dst, 1984 (int_x86_avx_cvt_ps2dq_256 (loadv8f32 addr:$src)))], 1985 IIC_SSE_CVT_PS_RM>, VEX, VEX_L, Sched<[WriteCvtF2ILd]>; 1986def CVTPS2DQrr : PDI<0x5B, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 1987 "cvtps2dq\t{$src, $dst|$dst, $src}", 1988 [(set VR128:$dst, (int_x86_sse2_cvtps2dq VR128:$src))], 1989 IIC_SSE_CVT_PS_RR>, Sched<[WriteCvtF2I]>; 1990def CVTPS2DQrm : PDI<0x5B, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src), 1991 "cvtps2dq\t{$src, $dst|$dst, $src}", 1992 [(set VR128:$dst, 1993 (int_x86_sse2_cvtps2dq (memopv4f32 addr:$src)))], 1994 IIC_SSE_CVT_PS_RM>, Sched<[WriteCvtF2ILd]>; 1995 1996 1997// Convert Packed Double FP to Packed DW Integers 1998let Predicates = [HasAVX] in { 1999// The assembler can recognize rr 256-bit instructions by seeing a ymm 2000// register, but the same isn't true when using memory operands instead. 2001// Provide other assembly rr and rm forms to address this explicitly. 2002def VCVTPD2DQrr : SDI<0xE6, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 2003 "vcvtpd2dq\t{$src, $dst|$dst, $src}", 2004 [(set VR128:$dst, (int_x86_sse2_cvtpd2dq VR128:$src))]>, 2005 VEX, Sched<[WriteCvtF2I]>; 2006 2007// XMM only 2008def : InstAlias<"vcvtpd2dqx\t{$src, $dst|$dst, $src}", 2009 (VCVTPD2DQrr VR128:$dst, VR128:$src), 0>; 2010def VCVTPD2DQXrm : SDI<0xE6, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src), 2011 "vcvtpd2dqx\t{$src, $dst|$dst, $src}", 2012 [(set VR128:$dst, 2013 (int_x86_sse2_cvtpd2dq (loadv2f64 addr:$src)))]>, VEX, 2014 Sched<[WriteCvtF2ILd]>; 2015 2016// YMM only 2017def VCVTPD2DQYrr : SDI<0xE6, MRMSrcReg, (outs VR128:$dst), (ins VR256:$src), 2018 "vcvtpd2dq{y}\t{$src, $dst|$dst, $src}", 2019 [(set VR128:$dst, 2020 (int_x86_avx_cvt_pd2dq_256 VR256:$src))]>, VEX, VEX_L, 2021 Sched<[WriteCvtF2I]>; 2022def VCVTPD2DQYrm : SDI<0xE6, MRMSrcMem, (outs VR128:$dst), (ins f256mem:$src), 2023 "vcvtpd2dq{y}\t{$src, $dst|$dst, $src}", 2024 [(set VR128:$dst, 2025 (int_x86_avx_cvt_pd2dq_256 (loadv4f64 addr:$src)))]>, 2026 VEX, VEX_L, Sched<[WriteCvtF2ILd]>; 2027def : InstAlias<"vcvtpd2dq\t{$src, $dst|$dst, $src}", 2028 (VCVTPD2DQYrr VR128:$dst, VR256:$src), 0>; 2029} 2030 2031def CVTPD2DQrm : SDI<0xE6, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src), 2032 "cvtpd2dq\t{$src, $dst|$dst, $src}", 2033 [(set VR128:$dst, 2034 (int_x86_sse2_cvtpd2dq (memopv2f64 addr:$src)))], 2035 IIC_SSE_CVT_PD_RM>, Sched<[WriteCvtF2ILd]>; 2036def CVTPD2DQrr : SDI<0xE6, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 2037 "cvtpd2dq\t{$src, $dst|$dst, $src}", 2038 [(set VR128:$dst, (int_x86_sse2_cvtpd2dq VR128:$src))], 2039 IIC_SSE_CVT_PD_RR>, Sched<[WriteCvtF2I]>; 2040 2041// Convert with truncation packed single/double fp to doubleword 2042// SSE2 packed instructions with XS prefix 2043def VCVTTPS2DQrr : VS2SI<0x5B, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 2044 "cvttps2dq\t{$src, $dst|$dst, $src}", 2045 [(set VR128:$dst, 2046 (int_x86_sse2_cvttps2dq VR128:$src))], 2047 IIC_SSE_CVT_PS_RR>, VEX, Sched<[WriteCvtF2I]>; 2048def VCVTTPS2DQrm : VS2SI<0x5B, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src), 2049 "cvttps2dq\t{$src, $dst|$dst, $src}", 2050 [(set VR128:$dst, (int_x86_sse2_cvttps2dq 2051 (loadv4f32 addr:$src)))], 2052 IIC_SSE_CVT_PS_RM>, VEX, Sched<[WriteCvtF2ILd]>; 2053def VCVTTPS2DQYrr : VS2SI<0x5B, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src), 2054 "cvttps2dq\t{$src, $dst|$dst, $src}", 2055 [(set VR256:$dst, 2056 (int_x86_avx_cvtt_ps2dq_256 VR256:$src))], 2057 IIC_SSE_CVT_PS_RR>, VEX, VEX_L, Sched<[WriteCvtF2I]>; 2058def VCVTTPS2DQYrm : VS2SI<0x5B, MRMSrcMem, (outs VR256:$dst), (ins f256mem:$src), 2059 "cvttps2dq\t{$src, $dst|$dst, $src}", 2060 [(set VR256:$dst, (int_x86_avx_cvtt_ps2dq_256 2061 (loadv8f32 addr:$src)))], 2062 IIC_SSE_CVT_PS_RM>, VEX, VEX_L, 2063 Sched<[WriteCvtF2ILd]>; 2064 2065def CVTTPS2DQrr : S2SI<0x5B, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 2066 "cvttps2dq\t{$src, $dst|$dst, $src}", 2067 [(set VR128:$dst, (int_x86_sse2_cvttps2dq VR128:$src))], 2068 IIC_SSE_CVT_PS_RR>, Sched<[WriteCvtF2I]>; 2069def CVTTPS2DQrm : S2SI<0x5B, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src), 2070 "cvttps2dq\t{$src, $dst|$dst, $src}", 2071 [(set VR128:$dst, 2072 (int_x86_sse2_cvttps2dq (memopv4f32 addr:$src)))], 2073 IIC_SSE_CVT_PS_RM>, Sched<[WriteCvtF2ILd]>; 2074 2075let Predicates = [HasAVX] in { 2076 def : Pat<(v4f32 (sint_to_fp (v4i32 VR128:$src))), 2077 (VCVTDQ2PSrr VR128:$src)>; 2078 def : Pat<(v4f32 (sint_to_fp (bc_v4i32 (loadv2i64 addr:$src)))), 2079 (VCVTDQ2PSrm addr:$src)>; 2080 2081 def : Pat<(int_x86_sse2_cvtdq2ps VR128:$src), 2082 (VCVTDQ2PSrr VR128:$src)>; 2083 def : Pat<(int_x86_sse2_cvtdq2ps (bc_v4i32 (loadv2i64 addr:$src))), 2084 (VCVTDQ2PSrm addr:$src)>; 2085 2086 def : Pat<(v4i32 (fp_to_sint (v4f32 VR128:$src))), 2087 (VCVTTPS2DQrr VR128:$src)>; 2088 def : Pat<(v4i32 (fp_to_sint (loadv4f32 addr:$src))), 2089 (VCVTTPS2DQrm addr:$src)>; 2090 2091 def : Pat<(v8f32 (sint_to_fp (v8i32 VR256:$src))), 2092 (VCVTDQ2PSYrr VR256:$src)>; 2093 def : Pat<(v8f32 (sint_to_fp (bc_v8i32 (loadv4i64 addr:$src)))), 2094 (VCVTDQ2PSYrm addr:$src)>; 2095 2096 def : Pat<(v8i32 (fp_to_sint (v8f32 VR256:$src))), 2097 (VCVTTPS2DQYrr VR256:$src)>; 2098 def : Pat<(v8i32 (fp_to_sint (loadv8f32 addr:$src))), 2099 (VCVTTPS2DQYrm addr:$src)>; 2100} 2101 2102let Predicates = [UseSSE2] in { 2103 def : Pat<(v4f32 (sint_to_fp (v4i32 VR128:$src))), 2104 (CVTDQ2PSrr VR128:$src)>; 2105 def : Pat<(v4f32 (sint_to_fp (bc_v4i32 (memopv2i64 addr:$src)))), 2106 (CVTDQ2PSrm addr:$src)>; 2107 2108 def : Pat<(int_x86_sse2_cvtdq2ps VR128:$src), 2109 (CVTDQ2PSrr VR128:$src)>; 2110 def : Pat<(int_x86_sse2_cvtdq2ps (bc_v4i32 (memopv2i64 addr:$src))), 2111 (CVTDQ2PSrm addr:$src)>; 2112 2113 def : Pat<(v4i32 (fp_to_sint (v4f32 VR128:$src))), 2114 (CVTTPS2DQrr VR128:$src)>; 2115 def : Pat<(v4i32 (fp_to_sint (memopv4f32 addr:$src))), 2116 (CVTTPS2DQrm addr:$src)>; 2117} 2118 2119def VCVTTPD2DQrr : VPDI<0xE6, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 2120 "cvttpd2dq\t{$src, $dst|$dst, $src}", 2121 [(set VR128:$dst, 2122 (int_x86_sse2_cvttpd2dq VR128:$src))], 2123 IIC_SSE_CVT_PD_RR>, VEX, Sched<[WriteCvtF2I]>; 2124 2125// The assembler can recognize rr 256-bit instructions by seeing a ymm 2126// register, but the same isn't true when using memory operands instead. 2127// Provide other assembly rr and rm forms to address this explicitly. 2128 2129// XMM only 2130def : InstAlias<"vcvttpd2dqx\t{$src, $dst|$dst, $src}", 2131 (VCVTTPD2DQrr VR128:$dst, VR128:$src), 0>; 2132def VCVTTPD2DQXrm : VPDI<0xE6, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src), 2133 "cvttpd2dqx\t{$src, $dst|$dst, $src}", 2134 [(set VR128:$dst, (int_x86_sse2_cvttpd2dq 2135 (loadv2f64 addr:$src)))], 2136 IIC_SSE_CVT_PD_RM>, VEX, Sched<[WriteCvtF2ILd]>; 2137 2138// YMM only 2139def VCVTTPD2DQYrr : VPDI<0xE6, MRMSrcReg, (outs VR128:$dst), (ins VR256:$src), 2140 "cvttpd2dq{y}\t{$src, $dst|$dst, $src}", 2141 [(set VR128:$dst, 2142 (int_x86_avx_cvtt_pd2dq_256 VR256:$src))], 2143 IIC_SSE_CVT_PD_RR>, VEX, VEX_L, Sched<[WriteCvtF2I]>; 2144def VCVTTPD2DQYrm : VPDI<0xE6, MRMSrcMem, (outs VR128:$dst), (ins f256mem:$src), 2145 "cvttpd2dq{y}\t{$src, $dst|$dst, $src}", 2146 [(set VR128:$dst, 2147 (int_x86_avx_cvtt_pd2dq_256 (loadv4f64 addr:$src)))], 2148 IIC_SSE_CVT_PD_RM>, VEX, VEX_L, Sched<[WriteCvtF2ILd]>; 2149def : InstAlias<"vcvttpd2dq\t{$src, $dst|$dst, $src}", 2150 (VCVTTPD2DQYrr VR128:$dst, VR256:$src), 0>; 2151 2152let Predicates = [HasAVX] in { 2153 def : Pat<(v4i32 (fp_to_sint (v4f64 VR256:$src))), 2154 (VCVTTPD2DQYrr VR256:$src)>; 2155 def : Pat<(v4i32 (fp_to_sint (loadv4f64 addr:$src))), 2156 (VCVTTPD2DQYrm addr:$src)>; 2157} // Predicates = [HasAVX] 2158 2159def CVTTPD2DQrr : PDI<0xE6, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 2160 "cvttpd2dq\t{$src, $dst|$dst, $src}", 2161 [(set VR128:$dst, (int_x86_sse2_cvttpd2dq VR128:$src))], 2162 IIC_SSE_CVT_PD_RR>, Sched<[WriteCvtF2I]>; 2163def CVTTPD2DQrm : PDI<0xE6, MRMSrcMem, (outs VR128:$dst),(ins f128mem:$src), 2164 "cvttpd2dq\t{$src, $dst|$dst, $src}", 2165 [(set VR128:$dst, (int_x86_sse2_cvttpd2dq 2166 (memopv2f64 addr:$src)))], 2167 IIC_SSE_CVT_PD_RM>, 2168 Sched<[WriteCvtF2ILd]>; 2169 2170// Convert packed single to packed double 2171let Predicates = [HasAVX] in { 2172 // SSE2 instructions without OpSize prefix 2173def VCVTPS2PDrr : I<0x5A, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 2174 "vcvtps2pd\t{$src, $dst|$dst, $src}", 2175 [(set VR128:$dst, (int_x86_sse2_cvtps2pd VR128:$src))], 2176 IIC_SSE_CVT_PD_RR>, PS, VEX, Sched<[WriteCvtF2F]>; 2177def VCVTPS2PDrm : I<0x5A, MRMSrcMem, (outs VR128:$dst), (ins f64mem:$src), 2178 "vcvtps2pd\t{$src, $dst|$dst, $src}", 2179 [(set VR128:$dst, (v2f64 (extloadv2f32 addr:$src)))], 2180 IIC_SSE_CVT_PD_RM>, PS, VEX, Sched<[WriteCvtF2FLd]>; 2181def VCVTPS2PDYrr : I<0x5A, MRMSrcReg, (outs VR256:$dst), (ins VR128:$src), 2182 "vcvtps2pd\t{$src, $dst|$dst, $src}", 2183 [(set VR256:$dst, 2184 (int_x86_avx_cvt_ps2_pd_256 VR128:$src))], 2185 IIC_SSE_CVT_PD_RR>, PS, VEX, VEX_L, Sched<[WriteCvtF2F]>; 2186def VCVTPS2PDYrm : I<0x5A, MRMSrcMem, (outs VR256:$dst), (ins f128mem:$src), 2187 "vcvtps2pd\t{$src, $dst|$dst, $src}", 2188 [(set VR256:$dst, 2189 (int_x86_avx_cvt_ps2_pd_256 (loadv4f32 addr:$src)))], 2190 IIC_SSE_CVT_PD_RM>, PS, VEX, VEX_L, Sched<[WriteCvtF2FLd]>; 2191} 2192 2193let Predicates = [UseSSE2] in { 2194def CVTPS2PDrr : I<0x5A, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 2195 "cvtps2pd\t{$src, $dst|$dst, $src}", 2196 [(set VR128:$dst, (int_x86_sse2_cvtps2pd VR128:$src))], 2197 IIC_SSE_CVT_PD_RR>, PS, Sched<[WriteCvtF2F]>; 2198def CVTPS2PDrm : I<0x5A, MRMSrcMem, (outs VR128:$dst), (ins f64mem:$src), 2199 "cvtps2pd\t{$src, $dst|$dst, $src}", 2200 [(set VR128:$dst, (v2f64 (extloadv2f32 addr:$src)))], 2201 IIC_SSE_CVT_PD_RM>, PS, Sched<[WriteCvtF2FLd]>; 2202} 2203 2204// Convert Packed DW Integers to Packed Double FP 2205let Predicates = [HasAVX] in { 2206let hasSideEffects = 0, mayLoad = 1 in 2207def VCVTDQ2PDrm : S2SI<0xE6, MRMSrcMem, (outs VR128:$dst), (ins i64mem:$src), 2208 "vcvtdq2pd\t{$src, $dst|$dst, $src}", 2209 []>, VEX, Sched<[WriteCvtI2FLd]>; 2210def VCVTDQ2PDrr : S2SI<0xE6, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 2211 "vcvtdq2pd\t{$src, $dst|$dst, $src}", 2212 [(set VR128:$dst, 2213 (int_x86_sse2_cvtdq2pd VR128:$src))]>, VEX, 2214 Sched<[WriteCvtI2F]>; 2215def VCVTDQ2PDYrm : S2SI<0xE6, MRMSrcMem, (outs VR256:$dst), (ins i128mem:$src), 2216 "vcvtdq2pd\t{$src, $dst|$dst, $src}", 2217 [(set VR256:$dst, 2218 (int_x86_avx_cvtdq2_pd_256 2219 (bitconvert (loadv2i64 addr:$src))))]>, VEX, VEX_L, 2220 Sched<[WriteCvtI2FLd]>; 2221def VCVTDQ2PDYrr : S2SI<0xE6, MRMSrcReg, (outs VR256:$dst), (ins VR128:$src), 2222 "vcvtdq2pd\t{$src, $dst|$dst, $src}", 2223 [(set VR256:$dst, 2224 (int_x86_avx_cvtdq2_pd_256 VR128:$src))]>, VEX, VEX_L, 2225 Sched<[WriteCvtI2F]>; 2226} 2227 2228let hasSideEffects = 0, mayLoad = 1 in 2229def CVTDQ2PDrm : S2SI<0xE6, MRMSrcMem, (outs VR128:$dst), (ins i64mem:$src), 2230 "cvtdq2pd\t{$src, $dst|$dst, $src}", [], 2231 IIC_SSE_CVT_PD_RR>, Sched<[WriteCvtI2FLd]>; 2232def CVTDQ2PDrr : S2SI<0xE6, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 2233 "cvtdq2pd\t{$src, $dst|$dst, $src}", 2234 [(set VR128:$dst, (int_x86_sse2_cvtdq2pd VR128:$src))], 2235 IIC_SSE_CVT_PD_RM>, Sched<[WriteCvtI2F]>; 2236 2237// AVX register conversion intrinsics 2238let Predicates = [HasAVX] in { 2239 def : Pat<(v2f64 (X86cvtdq2pd (v4i32 VR128:$src))), 2240 (VCVTDQ2PDrr VR128:$src)>; 2241 def : Pat<(v2f64 (X86cvtdq2pd (bc_v4i32 (loadv2i64 addr:$src)))), 2242 (VCVTDQ2PDrm addr:$src)>; 2243 2244 def : Pat<(v4f64 (sint_to_fp (v4i32 VR128:$src))), 2245 (VCVTDQ2PDYrr VR128:$src)>; 2246 def : Pat<(v4f64 (sint_to_fp (bc_v4i32 (loadv2i64 addr:$src)))), 2247 (VCVTDQ2PDYrm addr:$src)>; 2248} // Predicates = [HasAVX] 2249 2250// SSE2 register conversion intrinsics 2251let Predicates = [HasSSE2] in { 2252 def : Pat<(v2f64 (X86cvtdq2pd (v4i32 VR128:$src))), 2253 (CVTDQ2PDrr VR128:$src)>; 2254 def : Pat<(v2f64 (X86cvtdq2pd (bc_v4i32 (loadv2i64 addr:$src)))), 2255 (CVTDQ2PDrm addr:$src)>; 2256} // Predicates = [HasSSE2] 2257 2258// Convert packed double to packed single 2259// The assembler can recognize rr 256-bit instructions by seeing a ymm 2260// register, but the same isn't true when using memory operands instead. 2261// Provide other assembly rr and rm forms to address this explicitly. 2262def VCVTPD2PSrr : VPDI<0x5A, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 2263 "cvtpd2ps\t{$src, $dst|$dst, $src}", 2264 [(set VR128:$dst, (int_x86_sse2_cvtpd2ps VR128:$src))], 2265 IIC_SSE_CVT_PD_RR>, VEX, Sched<[WriteCvtF2F]>; 2266 2267// XMM only 2268def : InstAlias<"vcvtpd2psx\t{$src, $dst|$dst, $src}", 2269 (VCVTPD2PSrr VR128:$dst, VR128:$src), 0>; 2270def VCVTPD2PSXrm : VPDI<0x5A, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src), 2271 "cvtpd2psx\t{$src, $dst|$dst, $src}", 2272 [(set VR128:$dst, 2273 (int_x86_sse2_cvtpd2ps (loadv2f64 addr:$src)))], 2274 IIC_SSE_CVT_PD_RM>, VEX, Sched<[WriteCvtF2FLd]>; 2275 2276// YMM only 2277def VCVTPD2PSYrr : VPDI<0x5A, MRMSrcReg, (outs VR128:$dst), (ins VR256:$src), 2278 "cvtpd2ps{y}\t{$src, $dst|$dst, $src}", 2279 [(set VR128:$dst, 2280 (int_x86_avx_cvt_pd2_ps_256 VR256:$src))], 2281 IIC_SSE_CVT_PD_RR>, VEX, VEX_L, Sched<[WriteCvtF2F]>; 2282def VCVTPD2PSYrm : VPDI<0x5A, MRMSrcMem, (outs VR128:$dst), (ins f256mem:$src), 2283 "cvtpd2ps{y}\t{$src, $dst|$dst, $src}", 2284 [(set VR128:$dst, 2285 (int_x86_avx_cvt_pd2_ps_256 (loadv4f64 addr:$src)))], 2286 IIC_SSE_CVT_PD_RM>, VEX, VEX_L, Sched<[WriteCvtF2FLd]>; 2287def : InstAlias<"vcvtpd2ps\t{$src, $dst|$dst, $src}", 2288 (VCVTPD2PSYrr VR128:$dst, VR256:$src), 0>; 2289 2290def CVTPD2PSrr : PDI<0x5A, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 2291 "cvtpd2ps\t{$src, $dst|$dst, $src}", 2292 [(set VR128:$dst, (int_x86_sse2_cvtpd2ps VR128:$src))], 2293 IIC_SSE_CVT_PD_RR>, Sched<[WriteCvtF2F]>; 2294def CVTPD2PSrm : PDI<0x5A, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src), 2295 "cvtpd2ps\t{$src, $dst|$dst, $src}", 2296 [(set VR128:$dst, 2297 (int_x86_sse2_cvtpd2ps (memopv2f64 addr:$src)))], 2298 IIC_SSE_CVT_PD_RM>, Sched<[WriteCvtF2FLd]>; 2299 2300 2301// AVX 256-bit register conversion intrinsics 2302// FIXME: Migrate SSE conversion intrinsics matching to use patterns as below 2303// whenever possible to avoid declaring two versions of each one. 2304let Predicates = [HasAVX] in { 2305 def : Pat<(int_x86_avx_cvtdq2_ps_256 VR256:$src), 2306 (VCVTDQ2PSYrr VR256:$src)>; 2307 def : Pat<(int_x86_avx_cvtdq2_ps_256 (bitconvert (loadv4i64 addr:$src))), 2308 (VCVTDQ2PSYrm addr:$src)>; 2309 2310 // Match fround and fextend for 128/256-bit conversions 2311 def : Pat<(v4f32 (X86vfpround (v2f64 VR128:$src))), 2312 (VCVTPD2PSrr VR128:$src)>; 2313 def : Pat<(v4f32 (X86vfpround (loadv2f64 addr:$src))), 2314 (VCVTPD2PSXrm addr:$src)>; 2315 def : Pat<(v4f32 (fround (v4f64 VR256:$src))), 2316 (VCVTPD2PSYrr VR256:$src)>; 2317 def : Pat<(v4f32 (fround (loadv4f64 addr:$src))), 2318 (VCVTPD2PSYrm addr:$src)>; 2319 2320 def : Pat<(v2f64 (X86vfpext (v4f32 VR128:$src))), 2321 (VCVTPS2PDrr VR128:$src)>; 2322 def : Pat<(v4f64 (fextend (v4f32 VR128:$src))), 2323 (VCVTPS2PDYrr VR128:$src)>; 2324 def : Pat<(v4f64 (extloadv4f32 addr:$src)), 2325 (VCVTPS2PDYrm addr:$src)>; 2326} 2327 2328let Predicates = [UseSSE2] in { 2329 // Match fround and fextend for 128 conversions 2330 def : Pat<(v4f32 (X86vfpround (v2f64 VR128:$src))), 2331 (CVTPD2PSrr VR128:$src)>; 2332 def : Pat<(v4f32 (X86vfpround (memopv2f64 addr:$src))), 2333 (CVTPD2PSrm addr:$src)>; 2334 2335 def : Pat<(v2f64 (X86vfpext (v4f32 VR128:$src))), 2336 (CVTPS2PDrr VR128:$src)>; 2337} 2338 2339//===----------------------------------------------------------------------===// 2340// SSE 1 & 2 - Compare Instructions 2341//===----------------------------------------------------------------------===// 2342 2343// sse12_cmp_scalar - sse 1 & 2 compare scalar instructions 2344multiclass sse12_cmp_scalar<RegisterClass RC, X86MemOperand x86memop, 2345 Operand CC, SDNode OpNode, ValueType VT, 2346 PatFrag ld_frag, string asm, string asm_alt, 2347 OpndItins itins, ImmLeaf immLeaf> { 2348 def rr : SIi8<0xC2, MRMSrcReg, 2349 (outs RC:$dst), (ins RC:$src1, RC:$src2, CC:$cc), asm, 2350 [(set RC:$dst, (OpNode (VT RC:$src1), RC:$src2, immLeaf:$cc))], 2351 itins.rr>, Sched<[itins.Sched]>; 2352 def rm : SIi8<0xC2, MRMSrcMem, 2353 (outs RC:$dst), (ins RC:$src1, x86memop:$src2, CC:$cc), asm, 2354 [(set RC:$dst, (OpNode (VT RC:$src1), 2355 (ld_frag addr:$src2), immLeaf:$cc))], 2356 itins.rm>, 2357 Sched<[itins.Sched.Folded, ReadAfterLd]>; 2358 2359 // Accept explicit immediate argument form instead of comparison code. 2360 let isAsmParserOnly = 1, hasSideEffects = 0 in { 2361 def rr_alt : SIi8<0xC2, MRMSrcReg, (outs RC:$dst), 2362 (ins RC:$src1, RC:$src2, u8imm:$cc), asm_alt, [], 2363 IIC_SSE_ALU_F32S_RR>, Sched<[itins.Sched]>; 2364 let mayLoad = 1 in 2365 def rm_alt : SIi8<0xC2, MRMSrcMem, (outs RC:$dst), 2366 (ins RC:$src1, x86memop:$src2, u8imm:$cc), asm_alt, [], 2367 IIC_SSE_ALU_F32S_RM>, 2368 Sched<[itins.Sched.Folded, ReadAfterLd]>; 2369 } 2370} 2371 2372defm VCMPSS : sse12_cmp_scalar<FR32, f32mem, AVXCC, X86cmps, f32, loadf32, 2373 "cmp${cc}ss\t{$src2, $src1, $dst|$dst, $src1, $src2}", 2374 "cmpss\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}", 2375 SSE_ALU_F32S, i8immZExt5>, XS, VEX_4V, VEX_LIG; 2376defm VCMPSD : sse12_cmp_scalar<FR64, f64mem, AVXCC, X86cmps, f64, loadf64, 2377 "cmp${cc}sd\t{$src2, $src1, $dst|$dst, $src1, $src2}", 2378 "cmpsd\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}", 2379 SSE_ALU_F32S, i8immZExt5>, // same latency as 32 bit compare 2380 XD, VEX_4V, VEX_LIG; 2381 2382let Constraints = "$src1 = $dst" in { 2383 defm CMPSS : sse12_cmp_scalar<FR32, f32mem, SSECC, X86cmps, f32, loadf32, 2384 "cmp${cc}ss\t{$src2, $dst|$dst, $src2}", 2385 "cmpss\t{$cc, $src2, $dst|$dst, $src2, $cc}", SSE_ALU_F32S, 2386 i8immZExt3>, XS; 2387 defm CMPSD : sse12_cmp_scalar<FR64, f64mem, SSECC, X86cmps, f64, loadf64, 2388 "cmp${cc}sd\t{$src2, $dst|$dst, $src2}", 2389 "cmpsd\t{$cc, $src2, $dst|$dst, $src2, $cc}", 2390 SSE_ALU_F64S, i8immZExt3>, XD; 2391} 2392 2393multiclass sse12_cmp_scalar_int<X86MemOperand x86memop, Operand CC, 2394 Intrinsic Int, string asm, OpndItins itins, 2395 ImmLeaf immLeaf> { 2396 def rr : SIi8<0xC2, MRMSrcReg, (outs VR128:$dst), 2397 (ins VR128:$src1, VR128:$src, CC:$cc), asm, 2398 [(set VR128:$dst, (Int VR128:$src1, 2399 VR128:$src, immLeaf:$cc))], 2400 itins.rr>, 2401 Sched<[itins.Sched]>; 2402 def rm : SIi8<0xC2, MRMSrcMem, (outs VR128:$dst), 2403 (ins VR128:$src1, x86memop:$src, CC:$cc), asm, 2404 [(set VR128:$dst, (Int VR128:$src1, 2405 (load addr:$src), immLeaf:$cc))], 2406 itins.rm>, 2407 Sched<[itins.Sched.Folded, ReadAfterLd]>; 2408} 2409 2410let isCodeGenOnly = 1 in { 2411 // Aliases to match intrinsics which expect XMM operand(s). 2412 defm Int_VCMPSS : sse12_cmp_scalar_int<f32mem, AVXCC, int_x86_sse_cmp_ss, 2413 "cmp${cc}ss\t{$src, $src1, $dst|$dst, $src1, $src}", 2414 SSE_ALU_F32S, i8immZExt5>, 2415 XS, VEX_4V; 2416 defm Int_VCMPSD : sse12_cmp_scalar_int<f64mem, AVXCC, int_x86_sse2_cmp_sd, 2417 "cmp${cc}sd\t{$src, $src1, $dst|$dst, $src1, $src}", 2418 SSE_ALU_F32S, i8immZExt5>, // same latency as f32 2419 XD, VEX_4V; 2420 let Constraints = "$src1 = $dst" in { 2421 defm Int_CMPSS : sse12_cmp_scalar_int<f32mem, SSECC, int_x86_sse_cmp_ss, 2422 "cmp${cc}ss\t{$src, $dst|$dst, $src}", 2423 SSE_ALU_F32S, i8immZExt3>, XS; 2424 defm Int_CMPSD : sse12_cmp_scalar_int<f64mem, SSECC, int_x86_sse2_cmp_sd, 2425 "cmp${cc}sd\t{$src, $dst|$dst, $src}", 2426 SSE_ALU_F64S, i8immZExt3>, 2427 XD; 2428} 2429} 2430 2431 2432// sse12_ord_cmp - Unordered/Ordered scalar fp compare and set EFLAGS 2433multiclass sse12_ord_cmp<bits<8> opc, RegisterClass RC, SDNode OpNode, 2434 ValueType vt, X86MemOperand x86memop, 2435 PatFrag ld_frag, string OpcodeStr> { 2436 def rr: SI<opc, MRMSrcReg, (outs), (ins RC:$src1, RC:$src2), 2437 !strconcat(OpcodeStr, "\t{$src2, $src1|$src1, $src2}"), 2438 [(set EFLAGS, (OpNode (vt RC:$src1), RC:$src2))], 2439 IIC_SSE_COMIS_RR>, 2440 Sched<[WriteFAdd]>; 2441 def rm: SI<opc, MRMSrcMem, (outs), (ins RC:$src1, x86memop:$src2), 2442 !strconcat(OpcodeStr, "\t{$src2, $src1|$src1, $src2}"), 2443 [(set EFLAGS, (OpNode (vt RC:$src1), 2444 (ld_frag addr:$src2)))], 2445 IIC_SSE_COMIS_RM>, 2446 Sched<[WriteFAddLd, ReadAfterLd]>; 2447} 2448 2449let Defs = [EFLAGS] in { 2450 defm VUCOMISS : sse12_ord_cmp<0x2E, FR32, X86cmp, f32, f32mem, loadf32, 2451 "ucomiss">, PS, VEX, VEX_LIG; 2452 defm VUCOMISD : sse12_ord_cmp<0x2E, FR64, X86cmp, f64, f64mem, loadf64, 2453 "ucomisd">, PD, VEX, VEX_LIG; 2454 let Pattern = []<dag> in { 2455 defm VCOMISS : sse12_ord_cmp<0x2F, VR128, undef, v4f32, f128mem, load, 2456 "comiss">, PS, VEX, VEX_LIG; 2457 defm VCOMISD : sse12_ord_cmp<0x2F, VR128, undef, v2f64, f128mem, load, 2458 "comisd">, PD, VEX, VEX_LIG; 2459 } 2460 2461 let isCodeGenOnly = 1 in { 2462 defm Int_VUCOMISS : sse12_ord_cmp<0x2E, VR128, X86ucomi, v4f32, f128mem, 2463 load, "ucomiss">, PS, VEX; 2464 defm Int_VUCOMISD : sse12_ord_cmp<0x2E, VR128, X86ucomi, v2f64, f128mem, 2465 load, "ucomisd">, PD, VEX; 2466 2467 defm Int_VCOMISS : sse12_ord_cmp<0x2F, VR128, X86comi, v4f32, f128mem, 2468 load, "comiss">, PS, VEX; 2469 defm Int_VCOMISD : sse12_ord_cmp<0x2F, VR128, X86comi, v2f64, f128mem, 2470 load, "comisd">, PD, VEX; 2471 } 2472 defm UCOMISS : sse12_ord_cmp<0x2E, FR32, X86cmp, f32, f32mem, loadf32, 2473 "ucomiss">, PS; 2474 defm UCOMISD : sse12_ord_cmp<0x2E, FR64, X86cmp, f64, f64mem, loadf64, 2475 "ucomisd">, PD; 2476 2477 let Pattern = []<dag> in { 2478 defm COMISS : sse12_ord_cmp<0x2F, VR128, undef, v4f32, f128mem, load, 2479 "comiss">, PS; 2480 defm COMISD : sse12_ord_cmp<0x2F, VR128, undef, v2f64, f128mem, load, 2481 "comisd">, PD; 2482 } 2483 2484 let isCodeGenOnly = 1 in { 2485 defm Int_UCOMISS : sse12_ord_cmp<0x2E, VR128, X86ucomi, v4f32, f128mem, 2486 load, "ucomiss">, PS; 2487 defm Int_UCOMISD : sse12_ord_cmp<0x2E, VR128, X86ucomi, v2f64, f128mem, 2488 load, "ucomisd">, PD; 2489 2490 defm Int_COMISS : sse12_ord_cmp<0x2F, VR128, X86comi, v4f32, f128mem, load, 2491 "comiss">, PS; 2492 defm Int_COMISD : sse12_ord_cmp<0x2F, VR128, X86comi, v2f64, f128mem, load, 2493 "comisd">, PD; 2494 } 2495} // Defs = [EFLAGS] 2496 2497// sse12_cmp_packed - sse 1 & 2 compare packed instructions 2498multiclass sse12_cmp_packed<RegisterClass RC, X86MemOperand x86memop, 2499 Operand CC, Intrinsic Int, string asm, 2500 string asm_alt, Domain d, ImmLeaf immLeaf, 2501 PatFrag ld_frag, OpndItins itins = SSE_ALU_F32P> { 2502 let isCommutable = 1 in 2503 def rri : PIi8<0xC2, MRMSrcReg, 2504 (outs RC:$dst), (ins RC:$src1, RC:$src2, CC:$cc), asm, 2505 [(set RC:$dst, (Int RC:$src1, RC:$src2, immLeaf:$cc))], 2506 itins.rr, d>, 2507 Sched<[WriteFAdd]>; 2508 def rmi : PIi8<0xC2, MRMSrcMem, 2509 (outs RC:$dst), (ins RC:$src1, x86memop:$src2, CC:$cc), asm, 2510 [(set RC:$dst, (Int RC:$src1, (ld_frag addr:$src2), immLeaf:$cc))], 2511 itins.rm, d>, 2512 Sched<[WriteFAddLd, ReadAfterLd]>; 2513 2514 // Accept explicit immediate argument form instead of comparison code. 2515 let isAsmParserOnly = 1, hasSideEffects = 0 in { 2516 def rri_alt : PIi8<0xC2, MRMSrcReg, 2517 (outs RC:$dst), (ins RC:$src1, RC:$src2, u8imm:$cc), 2518 asm_alt, [], itins.rr, d>, Sched<[WriteFAdd]>; 2519 let mayLoad = 1 in 2520 def rmi_alt : PIi8<0xC2, MRMSrcMem, 2521 (outs RC:$dst), (ins RC:$src1, x86memop:$src2, u8imm:$cc), 2522 asm_alt, [], itins.rm, d>, 2523 Sched<[WriteFAddLd, ReadAfterLd]>; 2524 } 2525} 2526 2527defm VCMPPS : sse12_cmp_packed<VR128, f128mem, AVXCC, int_x86_sse_cmp_ps, 2528 "cmp${cc}ps\t{$src2, $src1, $dst|$dst, $src1, $src2}", 2529 "cmpps\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}", 2530 SSEPackedSingle, i8immZExt5, loadv4f32>, PS, VEX_4V; 2531defm VCMPPD : sse12_cmp_packed<VR128, f128mem, AVXCC, int_x86_sse2_cmp_pd, 2532 "cmp${cc}pd\t{$src2, $src1, $dst|$dst, $src1, $src2}", 2533 "cmppd\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}", 2534 SSEPackedDouble, i8immZExt5, loadv2f64>, PD, VEX_4V; 2535defm VCMPPSY : sse12_cmp_packed<VR256, f256mem, AVXCC, int_x86_avx_cmp_ps_256, 2536 "cmp${cc}ps\t{$src2, $src1, $dst|$dst, $src1, $src2}", 2537 "cmpps\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}", 2538 SSEPackedSingle, i8immZExt5, loadv8f32>, PS, VEX_4V, VEX_L; 2539defm VCMPPDY : sse12_cmp_packed<VR256, f256mem, AVXCC, int_x86_avx_cmp_pd_256, 2540 "cmp${cc}pd\t{$src2, $src1, $dst|$dst, $src1, $src2}", 2541 "cmppd\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}", 2542 SSEPackedDouble, i8immZExt5, loadv4f64>, PD, VEX_4V, VEX_L; 2543let Constraints = "$src1 = $dst" in { 2544 defm CMPPS : sse12_cmp_packed<VR128, f128mem, SSECC, int_x86_sse_cmp_ps, 2545 "cmp${cc}ps\t{$src2, $dst|$dst, $src2}", 2546 "cmpps\t{$cc, $src2, $dst|$dst, $src2, $cc}", 2547 SSEPackedSingle, i8immZExt5, memopv4f32, SSE_ALU_F32P>, PS; 2548 defm CMPPD : sse12_cmp_packed<VR128, f128mem, SSECC, int_x86_sse2_cmp_pd, 2549 "cmp${cc}pd\t{$src2, $dst|$dst, $src2}", 2550 "cmppd\t{$cc, $src2, $dst|$dst, $src2, $cc}", 2551 SSEPackedDouble, i8immZExt5, memopv2f64, SSE_ALU_F64P>, PD; 2552} 2553 2554let Predicates = [HasAVX] in { 2555def : Pat<(v4i32 (X86cmpp (v4f32 VR128:$src1), VR128:$src2, imm:$cc)), 2556 (VCMPPSrri (v4f32 VR128:$src1), (v4f32 VR128:$src2), imm:$cc)>; 2557def : Pat<(v4i32 (X86cmpp (v4f32 VR128:$src1), (loadv4f32 addr:$src2), imm:$cc)), 2558 (VCMPPSrmi (v4f32 VR128:$src1), addr:$src2, imm:$cc)>; 2559def : Pat<(v2i64 (X86cmpp (v2f64 VR128:$src1), VR128:$src2, imm:$cc)), 2560 (VCMPPDrri VR128:$src1, VR128:$src2, imm:$cc)>; 2561def : Pat<(v2i64 (X86cmpp (v2f64 VR128:$src1), (loadv2f64 addr:$src2), imm:$cc)), 2562 (VCMPPDrmi VR128:$src1, addr:$src2, imm:$cc)>; 2563 2564def : Pat<(v8i32 (X86cmpp (v8f32 VR256:$src1), VR256:$src2, imm:$cc)), 2565 (VCMPPSYrri (v8f32 VR256:$src1), (v8f32 VR256:$src2), imm:$cc)>; 2566def : Pat<(v8i32 (X86cmpp (v8f32 VR256:$src1), (loadv8f32 addr:$src2), imm:$cc)), 2567 (VCMPPSYrmi (v8f32 VR256:$src1), addr:$src2, imm:$cc)>; 2568def : Pat<(v4i64 (X86cmpp (v4f64 VR256:$src1), VR256:$src2, imm:$cc)), 2569 (VCMPPDYrri VR256:$src1, VR256:$src2, imm:$cc)>; 2570def : Pat<(v4i64 (X86cmpp (v4f64 VR256:$src1), (loadv4f64 addr:$src2), imm:$cc)), 2571 (VCMPPDYrmi VR256:$src1, addr:$src2, imm:$cc)>; 2572} 2573 2574let Predicates = [UseSSE1] in { 2575def : Pat<(v4i32 (X86cmpp (v4f32 VR128:$src1), VR128:$src2, imm:$cc)), 2576 (CMPPSrri (v4f32 VR128:$src1), (v4f32 VR128:$src2), imm:$cc)>; 2577def : Pat<(v4i32 (X86cmpp (v4f32 VR128:$src1), (memopv4f32 addr:$src2), imm:$cc)), 2578 (CMPPSrmi (v4f32 VR128:$src1), addr:$src2, imm:$cc)>; 2579} 2580 2581let Predicates = [UseSSE2] in { 2582def : Pat<(v2i64 (X86cmpp (v2f64 VR128:$src1), VR128:$src2, imm:$cc)), 2583 (CMPPDrri VR128:$src1, VR128:$src2, imm:$cc)>; 2584def : Pat<(v2i64 (X86cmpp (v2f64 VR128:$src1), (memopv2f64 addr:$src2), imm:$cc)), 2585 (CMPPDrmi VR128:$src1, addr:$src2, imm:$cc)>; 2586} 2587 2588//===----------------------------------------------------------------------===// 2589// SSE 1 & 2 - Shuffle Instructions 2590//===----------------------------------------------------------------------===// 2591 2592/// sse12_shuffle - sse 1 & 2 fp shuffle instructions 2593multiclass sse12_shuffle<RegisterClass RC, X86MemOperand x86memop, 2594 ValueType vt, string asm, PatFrag mem_frag, 2595 Domain d> { 2596 def rmi : PIi8<0xC6, MRMSrcMem, (outs RC:$dst), 2597 (ins RC:$src1, x86memop:$src2, u8imm:$src3), asm, 2598 [(set RC:$dst, (vt (X86Shufp RC:$src1, (mem_frag addr:$src2), 2599 (i8 imm:$src3))))], IIC_SSE_SHUFP, d>, 2600 Sched<[WriteFShuffleLd, ReadAfterLd]>; 2601 def rri : PIi8<0xC6, MRMSrcReg, (outs RC:$dst), 2602 (ins RC:$src1, RC:$src2, u8imm:$src3), asm, 2603 [(set RC:$dst, (vt (X86Shufp RC:$src1, RC:$src2, 2604 (i8 imm:$src3))))], IIC_SSE_SHUFP, d>, 2605 Sched<[WriteFShuffle]>; 2606} 2607 2608defm VSHUFPS : sse12_shuffle<VR128, f128mem, v4f32, 2609 "shufps\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", 2610 loadv4f32, SSEPackedSingle>, PS, VEX_4V; 2611defm VSHUFPSY : sse12_shuffle<VR256, f256mem, v8f32, 2612 "shufps\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", 2613 loadv8f32, SSEPackedSingle>, PS, VEX_4V, VEX_L; 2614defm VSHUFPD : sse12_shuffle<VR128, f128mem, v2f64, 2615 "shufpd\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", 2616 loadv2f64, SSEPackedDouble>, PD, VEX_4V; 2617defm VSHUFPDY : sse12_shuffle<VR256, f256mem, v4f64, 2618 "shufpd\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", 2619 loadv4f64, SSEPackedDouble>, PD, VEX_4V, VEX_L; 2620 2621let Constraints = "$src1 = $dst" in { 2622 defm SHUFPS : sse12_shuffle<VR128, f128mem, v4f32, 2623 "shufps\t{$src3, $src2, $dst|$dst, $src2, $src3}", 2624 memopv4f32, SSEPackedSingle>, PS; 2625 defm SHUFPD : sse12_shuffle<VR128, f128mem, v2f64, 2626 "shufpd\t{$src3, $src2, $dst|$dst, $src2, $src3}", 2627 memopv2f64, SSEPackedDouble>, PD; 2628} 2629 2630let Predicates = [HasAVX] in { 2631 def : Pat<(v4i32 (X86Shufp VR128:$src1, 2632 (bc_v4i32 (loadv2i64 addr:$src2)), (i8 imm:$imm))), 2633 (VSHUFPSrmi VR128:$src1, addr:$src2, imm:$imm)>; 2634 def : Pat<(v4i32 (X86Shufp VR128:$src1, VR128:$src2, (i8 imm:$imm))), 2635 (VSHUFPSrri VR128:$src1, VR128:$src2, imm:$imm)>; 2636 2637 def : Pat<(v2i64 (X86Shufp VR128:$src1, 2638 (loadv2i64 addr:$src2), (i8 imm:$imm))), 2639 (VSHUFPDrmi VR128:$src1, addr:$src2, imm:$imm)>; 2640 def : Pat<(v2i64 (X86Shufp VR128:$src1, VR128:$src2, (i8 imm:$imm))), 2641 (VSHUFPDrri VR128:$src1, VR128:$src2, imm:$imm)>; 2642 2643 // 256-bit patterns 2644 def : Pat<(v8i32 (X86Shufp VR256:$src1, VR256:$src2, (i8 imm:$imm))), 2645 (VSHUFPSYrri VR256:$src1, VR256:$src2, imm:$imm)>; 2646 def : Pat<(v8i32 (X86Shufp VR256:$src1, 2647 (bc_v8i32 (loadv4i64 addr:$src2)), (i8 imm:$imm))), 2648 (VSHUFPSYrmi VR256:$src1, addr:$src2, imm:$imm)>; 2649 2650 def : Pat<(v4i64 (X86Shufp VR256:$src1, VR256:$src2, (i8 imm:$imm))), 2651 (VSHUFPDYrri VR256:$src1, VR256:$src2, imm:$imm)>; 2652 def : Pat<(v4i64 (X86Shufp VR256:$src1, 2653 (loadv4i64 addr:$src2), (i8 imm:$imm))), 2654 (VSHUFPDYrmi VR256:$src1, addr:$src2, imm:$imm)>; 2655} 2656 2657let Predicates = [UseSSE1] in { 2658 def : Pat<(v4i32 (X86Shufp VR128:$src1, 2659 (bc_v4i32 (memopv2i64 addr:$src2)), (i8 imm:$imm))), 2660 (SHUFPSrmi VR128:$src1, addr:$src2, imm:$imm)>; 2661 def : Pat<(v4i32 (X86Shufp VR128:$src1, VR128:$src2, (i8 imm:$imm))), 2662 (SHUFPSrri VR128:$src1, VR128:$src2, imm:$imm)>; 2663} 2664 2665let Predicates = [UseSSE2] in { 2666 // Generic SHUFPD patterns 2667 def : Pat<(v2i64 (X86Shufp VR128:$src1, 2668 (memopv2i64 addr:$src2), (i8 imm:$imm))), 2669 (SHUFPDrmi VR128:$src1, addr:$src2, imm:$imm)>; 2670 def : Pat<(v2i64 (X86Shufp VR128:$src1, VR128:$src2, (i8 imm:$imm))), 2671 (SHUFPDrri VR128:$src1, VR128:$src2, imm:$imm)>; 2672} 2673 2674//===----------------------------------------------------------------------===// 2675// SSE 1 & 2 - Unpack FP Instructions 2676//===----------------------------------------------------------------------===// 2677 2678/// sse12_unpack_interleave - sse 1 & 2 fp unpack and interleave 2679multiclass sse12_unpack_interleave<bits<8> opc, SDNode OpNode, ValueType vt, 2680 PatFrag mem_frag, RegisterClass RC, 2681 X86MemOperand x86memop, string asm, 2682 Domain d> { 2683 def rr : PI<opc, MRMSrcReg, 2684 (outs RC:$dst), (ins RC:$src1, RC:$src2), 2685 asm, [(set RC:$dst, 2686 (vt (OpNode RC:$src1, RC:$src2)))], 2687 IIC_SSE_UNPCK, d>, Sched<[WriteFShuffle]>; 2688 def rm : PI<opc, MRMSrcMem, 2689 (outs RC:$dst), (ins RC:$src1, x86memop:$src2), 2690 asm, [(set RC:$dst, 2691 (vt (OpNode RC:$src1, 2692 (mem_frag addr:$src2))))], 2693 IIC_SSE_UNPCK, d>, 2694 Sched<[WriteFShuffleLd, ReadAfterLd]>; 2695} 2696 2697defm VUNPCKHPS: sse12_unpack_interleave<0x15, X86Unpckh, v4f32, loadv4f32, 2698 VR128, f128mem, "unpckhps\t{$src2, $src1, $dst|$dst, $src1, $src2}", 2699 SSEPackedSingle>, PS, VEX_4V; 2700defm VUNPCKHPD: sse12_unpack_interleave<0x15, X86Unpckh, v2f64, loadv2f64, 2701 VR128, f128mem, "unpckhpd\t{$src2, $src1, $dst|$dst, $src1, $src2}", 2702 SSEPackedDouble>, PD, VEX_4V; 2703defm VUNPCKLPS: sse12_unpack_interleave<0x14, X86Unpckl, v4f32, loadv4f32, 2704 VR128, f128mem, "unpcklps\t{$src2, $src1, $dst|$dst, $src1, $src2}", 2705 SSEPackedSingle>, PS, VEX_4V; 2706defm VUNPCKLPD: sse12_unpack_interleave<0x14, X86Unpckl, v2f64, loadv2f64, 2707 VR128, f128mem, "unpcklpd\t{$src2, $src1, $dst|$dst, $src1, $src2}", 2708 SSEPackedDouble>, PD, VEX_4V; 2709 2710defm VUNPCKHPSY: sse12_unpack_interleave<0x15, X86Unpckh, v8f32, loadv8f32, 2711 VR256, f256mem, "unpckhps\t{$src2, $src1, $dst|$dst, $src1, $src2}", 2712 SSEPackedSingle>, PS, VEX_4V, VEX_L; 2713defm VUNPCKHPDY: sse12_unpack_interleave<0x15, X86Unpckh, v4f64, loadv4f64, 2714 VR256, f256mem, "unpckhpd\t{$src2, $src1, $dst|$dst, $src1, $src2}", 2715 SSEPackedDouble>, PD, VEX_4V, VEX_L; 2716defm VUNPCKLPSY: sse12_unpack_interleave<0x14, X86Unpckl, v8f32, loadv8f32, 2717 VR256, f256mem, "unpcklps\t{$src2, $src1, $dst|$dst, $src1, $src2}", 2718 SSEPackedSingle>, PS, VEX_4V, VEX_L; 2719defm VUNPCKLPDY: sse12_unpack_interleave<0x14, X86Unpckl, v4f64, loadv4f64, 2720 VR256, f256mem, "unpcklpd\t{$src2, $src1, $dst|$dst, $src1, $src2}", 2721 SSEPackedDouble>, PD, VEX_4V, VEX_L; 2722 2723let Constraints = "$src1 = $dst" in { 2724 defm UNPCKHPS: sse12_unpack_interleave<0x15, X86Unpckh, v4f32, memopv4f32, 2725 VR128, f128mem, "unpckhps\t{$src2, $dst|$dst, $src2}", 2726 SSEPackedSingle>, PS; 2727 defm UNPCKHPD: sse12_unpack_interleave<0x15, X86Unpckh, v2f64, memopv2f64, 2728 VR128, f128mem, "unpckhpd\t{$src2, $dst|$dst, $src2}", 2729 SSEPackedDouble>, PD; 2730 defm UNPCKLPS: sse12_unpack_interleave<0x14, X86Unpckl, v4f32, memopv4f32, 2731 VR128, f128mem, "unpcklps\t{$src2, $dst|$dst, $src2}", 2732 SSEPackedSingle>, PS; 2733 defm UNPCKLPD: sse12_unpack_interleave<0x14, X86Unpckl, v2f64, memopv2f64, 2734 VR128, f128mem, "unpcklpd\t{$src2, $dst|$dst, $src2}", 2735 SSEPackedDouble>, PD; 2736} // Constraints = "$src1 = $dst" 2737 2738let Predicates = [HasAVX1Only] in { 2739 def : Pat<(v8i32 (X86Unpckl VR256:$src1, (bc_v8i32 (loadv4i64 addr:$src2)))), 2740 (VUNPCKLPSYrm VR256:$src1, addr:$src2)>; 2741 def : Pat<(v8i32 (X86Unpckl VR256:$src1, VR256:$src2)), 2742 (VUNPCKLPSYrr VR256:$src1, VR256:$src2)>; 2743 def : Pat<(v8i32 (X86Unpckh VR256:$src1, (bc_v8i32 (loadv4i64 addr:$src2)))), 2744 (VUNPCKHPSYrm VR256:$src1, addr:$src2)>; 2745 def : Pat<(v8i32 (X86Unpckh VR256:$src1, VR256:$src2)), 2746 (VUNPCKHPSYrr VR256:$src1, VR256:$src2)>; 2747 2748 def : Pat<(v4i64 (X86Unpckl VR256:$src1, (loadv4i64 addr:$src2))), 2749 (VUNPCKLPDYrm VR256:$src1, addr:$src2)>; 2750 def : Pat<(v4i64 (X86Unpckl VR256:$src1, VR256:$src2)), 2751 (VUNPCKLPDYrr VR256:$src1, VR256:$src2)>; 2752 def : Pat<(v4i64 (X86Unpckh VR256:$src1, (loadv4i64 addr:$src2))), 2753 (VUNPCKHPDYrm VR256:$src1, addr:$src2)>; 2754 def : Pat<(v4i64 (X86Unpckh VR256:$src1, VR256:$src2)), 2755 (VUNPCKHPDYrr VR256:$src1, VR256:$src2)>; 2756} 2757 2758//===----------------------------------------------------------------------===// 2759// SSE 1 & 2 - Extract Floating-Point Sign mask 2760//===----------------------------------------------------------------------===// 2761 2762/// sse12_extr_sign_mask - sse 1 & 2 unpack and interleave 2763multiclass sse12_extr_sign_mask<RegisterClass RC, Intrinsic Int, string asm, 2764 Domain d> { 2765 def rr : PI<0x50, MRMSrcReg, (outs GR32orGR64:$dst), (ins RC:$src), 2766 !strconcat(asm, "\t{$src, $dst|$dst, $src}"), 2767 [(set GR32orGR64:$dst, (Int RC:$src))], IIC_SSE_MOVMSK, d>, 2768 Sched<[WriteVecLogic]>; 2769} 2770 2771let Predicates = [HasAVX] in { 2772 defm VMOVMSKPS : sse12_extr_sign_mask<VR128, int_x86_sse_movmsk_ps, 2773 "movmskps", SSEPackedSingle>, PS, VEX; 2774 defm VMOVMSKPD : sse12_extr_sign_mask<VR128, int_x86_sse2_movmsk_pd, 2775 "movmskpd", SSEPackedDouble>, PD, VEX; 2776 defm VMOVMSKPSY : sse12_extr_sign_mask<VR256, int_x86_avx_movmsk_ps_256, 2777 "movmskps", SSEPackedSingle>, PS, 2778 VEX, VEX_L; 2779 defm VMOVMSKPDY : sse12_extr_sign_mask<VR256, int_x86_avx_movmsk_pd_256, 2780 "movmskpd", SSEPackedDouble>, PD, 2781 VEX, VEX_L; 2782 2783 def : Pat<(i32 (X86fgetsign FR32:$src)), 2784 (VMOVMSKPSrr (COPY_TO_REGCLASS FR32:$src, VR128))>; 2785 def : Pat<(i64 (X86fgetsign FR32:$src)), 2786 (SUBREG_TO_REG (i64 0), 2787 (VMOVMSKPSrr (COPY_TO_REGCLASS FR32:$src, VR128)), sub_32bit)>; 2788 def : Pat<(i32 (X86fgetsign FR64:$src)), 2789 (VMOVMSKPDrr (COPY_TO_REGCLASS FR64:$src, VR128))>; 2790 def : Pat<(i64 (X86fgetsign FR64:$src)), 2791 (SUBREG_TO_REG (i64 0), 2792 (VMOVMSKPDrr (COPY_TO_REGCLASS FR64:$src, VR128)), sub_32bit)>; 2793} 2794 2795defm MOVMSKPS : sse12_extr_sign_mask<VR128, int_x86_sse_movmsk_ps, "movmskps", 2796 SSEPackedSingle>, PS; 2797defm MOVMSKPD : sse12_extr_sign_mask<VR128, int_x86_sse2_movmsk_pd, "movmskpd", 2798 SSEPackedDouble>, PD; 2799 2800def : Pat<(i32 (X86fgetsign FR32:$src)), 2801 (MOVMSKPSrr (COPY_TO_REGCLASS FR32:$src, VR128))>, 2802 Requires<[UseSSE1]>; 2803def : Pat<(i64 (X86fgetsign FR32:$src)), 2804 (SUBREG_TO_REG (i64 0), 2805 (MOVMSKPSrr (COPY_TO_REGCLASS FR32:$src, VR128)), sub_32bit)>, 2806 Requires<[UseSSE1]>; 2807def : Pat<(i32 (X86fgetsign FR64:$src)), 2808 (MOVMSKPDrr (COPY_TO_REGCLASS FR64:$src, VR128))>, 2809 Requires<[UseSSE2]>; 2810def : Pat<(i64 (X86fgetsign FR64:$src)), 2811 (SUBREG_TO_REG (i64 0), 2812 (MOVMSKPDrr (COPY_TO_REGCLASS FR64:$src, VR128)), sub_32bit)>, 2813 Requires<[UseSSE2]>; 2814 2815//===---------------------------------------------------------------------===// 2816// SSE2 - Packed Integer Logical Instructions 2817//===---------------------------------------------------------------------===// 2818 2819let ExeDomain = SSEPackedInt in { // SSE integer instructions 2820 2821/// PDI_binop_rm - Simple SSE2 binary operator. 2822multiclass PDI_binop_rm<bits<8> opc, string OpcodeStr, SDNode OpNode, 2823 ValueType OpVT, RegisterClass RC, PatFrag memop_frag, 2824 X86MemOperand x86memop, OpndItins itins, 2825 bit IsCommutable, bit Is2Addr> { 2826 let isCommutable = IsCommutable in 2827 def rr : PDI<opc, MRMSrcReg, (outs RC:$dst), 2828 (ins RC:$src1, RC:$src2), 2829 !if(Is2Addr, 2830 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 2831 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 2832 [(set RC:$dst, (OpVT (OpNode RC:$src1, RC:$src2)))], itins.rr>, 2833 Sched<[itins.Sched]>; 2834 def rm : PDI<opc, MRMSrcMem, (outs RC:$dst), 2835 (ins RC:$src1, x86memop:$src2), 2836 !if(Is2Addr, 2837 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 2838 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 2839 [(set RC:$dst, (OpVT (OpNode RC:$src1, 2840 (bitconvert (memop_frag addr:$src2)))))], 2841 itins.rm>, 2842 Sched<[itins.Sched.Folded, ReadAfterLd]>; 2843} 2844} // ExeDomain = SSEPackedInt 2845 2846multiclass PDI_binop_all<bits<8> opc, string OpcodeStr, SDNode Opcode, 2847 ValueType OpVT128, ValueType OpVT256, 2848 OpndItins itins, bit IsCommutable = 0> { 2849let Predicates = [HasAVX, NoVLX] in 2850 defm V#NAME : PDI_binop_rm<opc, !strconcat("v", OpcodeStr), Opcode, OpVT128, 2851 VR128, loadv2i64, i128mem, itins, IsCommutable, 0>, VEX_4V; 2852 2853let Constraints = "$src1 = $dst" in 2854 defm NAME : PDI_binop_rm<opc, OpcodeStr, Opcode, OpVT128, VR128, 2855 memopv2i64, i128mem, itins, IsCommutable, 1>; 2856 2857let Predicates = [HasAVX2, NoVLX] in 2858 defm V#NAME#Y : PDI_binop_rm<opc, !strconcat("v", OpcodeStr), Opcode, 2859 OpVT256, VR256, loadv4i64, i256mem, itins, 2860 IsCommutable, 0>, VEX_4V, VEX_L; 2861} 2862 2863// These are ordered here for pattern ordering requirements with the fp versions 2864 2865defm PAND : PDI_binop_all<0xDB, "pand", and, v2i64, v4i64, 2866 SSE_VEC_BIT_ITINS_P, 1>; 2867defm POR : PDI_binop_all<0xEB, "por", or, v2i64, v4i64, 2868 SSE_VEC_BIT_ITINS_P, 1>; 2869defm PXOR : PDI_binop_all<0xEF, "pxor", xor, v2i64, v4i64, 2870 SSE_VEC_BIT_ITINS_P, 1>; 2871defm PANDN : PDI_binop_all<0xDF, "pandn", X86andnp, v2i64, v4i64, 2872 SSE_VEC_BIT_ITINS_P, 0>; 2873 2874//===----------------------------------------------------------------------===// 2875// SSE 1 & 2 - Logical Instructions 2876//===----------------------------------------------------------------------===// 2877 2878// Multiclass for scalars using the X86 logical operation aliases for FP. 2879multiclass sse12_fp_packed_scalar_logical_alias< 2880 bits<8> opc, string OpcodeStr, SDNode OpNode, OpndItins itins> { 2881 defm V#NAME#PS : sse12_fp_packed<opc, !strconcat(OpcodeStr, "ps"), OpNode, 2882 FR32, f32, f128mem, loadf32_128, SSEPackedSingle, itins, 0>, 2883 PS, VEX_4V; 2884 2885 defm V#NAME#PD : sse12_fp_packed<opc, !strconcat(OpcodeStr, "pd"), OpNode, 2886 FR64, f64, f128mem, loadf64_128, SSEPackedDouble, itins, 0>, 2887 PD, VEX_4V; 2888 2889 let Constraints = "$src1 = $dst" in { 2890 defm PS : sse12_fp_packed<opc, !strconcat(OpcodeStr, "ps"), OpNode, FR32, 2891 f32, f128mem, memopfsf32_128, SSEPackedSingle, itins>, PS; 2892 2893 defm PD : sse12_fp_packed<opc, !strconcat(OpcodeStr, "pd"), OpNode, FR64, 2894 f64, f128mem, memopfsf64_128, SSEPackedDouble, itins>, PD; 2895 } 2896} 2897 2898let isCodeGenOnly = 1 in { 2899 defm FsAND : sse12_fp_packed_scalar_logical_alias<0x54, "and", X86fand, 2900 SSE_BIT_ITINS_P>; 2901 defm FsOR : sse12_fp_packed_scalar_logical_alias<0x56, "or", X86for, 2902 SSE_BIT_ITINS_P>; 2903 defm FsXOR : sse12_fp_packed_scalar_logical_alias<0x57, "xor", X86fxor, 2904 SSE_BIT_ITINS_P>; 2905 2906 let isCommutable = 0 in 2907 defm FsANDN : sse12_fp_packed_scalar_logical_alias<0x55, "andn", X86fandn, 2908 SSE_BIT_ITINS_P>; 2909} 2910 2911// Multiclass for vectors using the X86 logical operation aliases for FP. 2912multiclass sse12_fp_packed_vector_logical_alias< 2913 bits<8> opc, string OpcodeStr, SDNode OpNode, OpndItins itins> { 2914 let Predicates = [HasAVX, NoVLX] in { 2915 defm V#NAME#PS : sse12_fp_packed<opc, !strconcat(OpcodeStr, "ps"), OpNode, 2916 VR128, v4f32, f128mem, loadv4f32, SSEPackedSingle, itins, 0>, 2917 PS, VEX_4V; 2918 2919 defm V#NAME#PD : sse12_fp_packed<opc, !strconcat(OpcodeStr, "pd"), OpNode, 2920 VR128, v2f64, f128mem, loadv2f64, SSEPackedDouble, itins, 0>, 2921 PD, VEX_4V; 2922 2923 defm V#NAME#PSY : sse12_fp_packed<opc, !strconcat(OpcodeStr, "ps"), OpNode, 2924 VR256, v8f32, f256mem, loadv8f32, SSEPackedSingle, itins, 0>, 2925 PS, VEX_4V, VEX_L; 2926 2927 defm V#NAME#PDY : sse12_fp_packed<opc, !strconcat(OpcodeStr, "pd"), OpNode, 2928 VR256, v4f64, f256mem, loadv4f64, SSEPackedDouble, itins, 0>, 2929 PD, VEX_4V, VEX_L; 2930 } 2931 2932 let Constraints = "$src1 = $dst" in { 2933 defm PS : sse12_fp_packed<opc, !strconcat(OpcodeStr, "ps"), OpNode, VR128, 2934 v4f32, f128mem, memopv4f32, SSEPackedSingle, itins>, 2935 PS; 2936 2937 defm PD : sse12_fp_packed<opc, !strconcat(OpcodeStr, "pd"), OpNode, VR128, 2938 v2f64, f128mem, memopv2f64, SSEPackedDouble, itins>, 2939 PD; 2940 } 2941} 2942 2943let isCodeGenOnly = 1 in { 2944 defm FvAND : sse12_fp_packed_vector_logical_alias<0x54, "and", X86fand, 2945 SSE_BIT_ITINS_P>; 2946 defm FvOR : sse12_fp_packed_vector_logical_alias<0x56, "or", X86for, 2947 SSE_BIT_ITINS_P>; 2948 defm FvXOR : sse12_fp_packed_vector_logical_alias<0x57, "xor", X86fxor, 2949 SSE_BIT_ITINS_P>; 2950 2951 let isCommutable = 0 in 2952 defm FvANDN : sse12_fp_packed_vector_logical_alias<0x55, "andn", X86fandn, 2953 SSE_BIT_ITINS_P>; 2954} 2955 2956/// sse12_fp_packed_logical - SSE 1 & 2 packed FP logical ops 2957/// 2958multiclass sse12_fp_packed_logical<bits<8> opc, string OpcodeStr, 2959 SDNode OpNode> { 2960 let Predicates = [HasAVX, NoVLX] in { 2961 defm V#NAME#PSY : sse12_fp_packed_logical_rm<opc, VR256, SSEPackedSingle, 2962 !strconcat(OpcodeStr, "ps"), f256mem, 2963 [(set VR256:$dst, (v4i64 (OpNode VR256:$src1, VR256:$src2)))], 2964 [(set VR256:$dst, (OpNode (bc_v4i64 (v8f32 VR256:$src1)), 2965 (loadv4i64 addr:$src2)))], 0>, PS, VEX_4V, VEX_L; 2966 2967 defm V#NAME#PDY : sse12_fp_packed_logical_rm<opc, VR256, SSEPackedDouble, 2968 !strconcat(OpcodeStr, "pd"), f256mem, 2969 [(set VR256:$dst, (OpNode (bc_v4i64 (v4f64 VR256:$src1)), 2970 (bc_v4i64 (v4f64 VR256:$src2))))], 2971 [(set VR256:$dst, (OpNode (bc_v4i64 (v4f64 VR256:$src1)), 2972 (loadv4i64 addr:$src2)))], 0>, 2973 PD, VEX_4V, VEX_L; 2974 2975 // In AVX no need to add a pattern for 128-bit logical rr ps, because they 2976 // are all promoted to v2i64, and the patterns are covered by the int 2977 // version. This is needed in SSE only, because v2i64 isn't supported on 2978 // SSE1, but only on SSE2. 2979 defm V#NAME#PS : sse12_fp_packed_logical_rm<opc, VR128, SSEPackedSingle, 2980 !strconcat(OpcodeStr, "ps"), f128mem, [], 2981 [(set VR128:$dst, (OpNode (bc_v2i64 (v4f32 VR128:$src1)), 2982 (loadv2i64 addr:$src2)))], 0>, PS, VEX_4V; 2983 2984 defm V#NAME#PD : sse12_fp_packed_logical_rm<opc, VR128, SSEPackedDouble, 2985 !strconcat(OpcodeStr, "pd"), f128mem, 2986 [(set VR128:$dst, (OpNode (bc_v2i64 (v2f64 VR128:$src1)), 2987 (bc_v2i64 (v2f64 VR128:$src2))))], 2988 [(set VR128:$dst, (OpNode (bc_v2i64 (v2f64 VR128:$src1)), 2989 (loadv2i64 addr:$src2)))], 0>, 2990 PD, VEX_4V; 2991 } 2992 2993 let Constraints = "$src1 = $dst" in { 2994 defm PS : sse12_fp_packed_logical_rm<opc, VR128, SSEPackedSingle, 2995 !strconcat(OpcodeStr, "ps"), f128mem, 2996 [(set VR128:$dst, (v2i64 (OpNode VR128:$src1, VR128:$src2)))], 2997 [(set VR128:$dst, (OpNode (bc_v2i64 (v4f32 VR128:$src1)), 2998 (memopv2i64 addr:$src2)))]>, PS; 2999 3000 defm PD : sse12_fp_packed_logical_rm<opc, VR128, SSEPackedDouble, 3001 !strconcat(OpcodeStr, "pd"), f128mem, 3002 [(set VR128:$dst, (OpNode (bc_v2i64 (v2f64 VR128:$src1)), 3003 (bc_v2i64 (v2f64 VR128:$src2))))], 3004 [(set VR128:$dst, (OpNode (bc_v2i64 (v2f64 VR128:$src1)), 3005 (memopv2i64 addr:$src2)))]>, PD; 3006 } 3007} 3008 3009defm AND : sse12_fp_packed_logical<0x54, "and", and>; 3010defm OR : sse12_fp_packed_logical<0x56, "or", or>; 3011defm XOR : sse12_fp_packed_logical<0x57, "xor", xor>; 3012let isCommutable = 0 in 3013 defm ANDN : sse12_fp_packed_logical<0x55, "andn", X86andnp>; 3014 3015// AVX1 requires type coercions in order to fold loads directly into logical 3016// operations. 3017let Predicates = [HasAVX1Only] in { 3018 def : Pat<(bc_v8f32 (and VR256:$src1, (loadv4i64 addr:$src2))), 3019 (VANDPSYrm VR256:$src1, addr:$src2)>; 3020 def : Pat<(bc_v8f32 (or VR256:$src1, (loadv4i64 addr:$src2))), 3021 (VORPSYrm VR256:$src1, addr:$src2)>; 3022 def : Pat<(bc_v8f32 (xor VR256:$src1, (loadv4i64 addr:$src2))), 3023 (VXORPSYrm VR256:$src1, addr:$src2)>; 3024 def : Pat<(bc_v8f32 (X86andnp VR256:$src1, (loadv4i64 addr:$src2))), 3025 (VANDNPSYrm VR256:$src1, addr:$src2)>; 3026} 3027 3028//===----------------------------------------------------------------------===// 3029// SSE 1 & 2 - Arithmetic Instructions 3030//===----------------------------------------------------------------------===// 3031 3032/// basic_sse12_fp_binop_xxx - SSE 1 & 2 binops come in both scalar and 3033/// vector forms. 3034/// 3035/// In addition, we also have a special variant of the scalar form here to 3036/// represent the associated intrinsic operation. This form is unlike the 3037/// plain scalar form, in that it takes an entire vector (instead of a scalar) 3038/// and leaves the top elements unmodified (therefore these cannot be commuted). 3039/// 3040/// These three forms can each be reg+reg or reg+mem. 3041/// 3042 3043/// FIXME: once all 256-bit intrinsics are matched, cleanup and refactor those 3044/// classes below 3045multiclass basic_sse12_fp_binop_p<bits<8> opc, string OpcodeStr, 3046 SDNode OpNode, SizeItins itins> { 3047 let Predicates = [HasAVX, NoVLX] in { 3048 defm V#NAME#PS : sse12_fp_packed<opc, !strconcat(OpcodeStr, "ps"), OpNode, 3049 VR128, v4f32, f128mem, loadv4f32, 3050 SSEPackedSingle, itins.s, 0>, PS, VEX_4V; 3051 defm V#NAME#PD : sse12_fp_packed<opc, !strconcat(OpcodeStr, "pd"), OpNode, 3052 VR128, v2f64, f128mem, loadv2f64, 3053 SSEPackedDouble, itins.d, 0>, PD, VEX_4V; 3054 3055 defm V#NAME#PSY : sse12_fp_packed<opc, !strconcat(OpcodeStr, "ps"), 3056 OpNode, VR256, v8f32, f256mem, loadv8f32, 3057 SSEPackedSingle, itins.s, 0>, PS, VEX_4V, VEX_L; 3058 defm V#NAME#PDY : sse12_fp_packed<opc, !strconcat(OpcodeStr, "pd"), 3059 OpNode, VR256, v4f64, f256mem, loadv4f64, 3060 SSEPackedDouble, itins.d, 0>, PD, VEX_4V, VEX_L; 3061 } 3062 3063 let Constraints = "$src1 = $dst" in { 3064 defm PS : sse12_fp_packed<opc, !strconcat(OpcodeStr, "ps"), OpNode, VR128, 3065 v4f32, f128mem, memopv4f32, SSEPackedSingle, 3066 itins.s>, PS; 3067 defm PD : sse12_fp_packed<opc, !strconcat(OpcodeStr, "pd"), OpNode, VR128, 3068 v2f64, f128mem, memopv2f64, SSEPackedDouble, 3069 itins.d>, PD; 3070 } 3071} 3072 3073multiclass basic_sse12_fp_binop_s<bits<8> opc, string OpcodeStr, SDNode OpNode, 3074 SizeItins itins> { 3075 defm V#NAME#SS : sse12_fp_scalar<opc, !strconcat(OpcodeStr, "ss"), 3076 OpNode, FR32, f32mem, SSEPackedSingle, itins.s, 0>, 3077 XS, VEX_4V, VEX_LIG; 3078 defm V#NAME#SD : sse12_fp_scalar<opc, !strconcat(OpcodeStr, "sd"), 3079 OpNode, FR64, f64mem, SSEPackedDouble, itins.d, 0>, 3080 XD, VEX_4V, VEX_LIG; 3081 3082 let Constraints = "$src1 = $dst" in { 3083 defm SS : sse12_fp_scalar<opc, !strconcat(OpcodeStr, "ss"), 3084 OpNode, FR32, f32mem, SSEPackedSingle, 3085 itins.s>, XS; 3086 defm SD : sse12_fp_scalar<opc, !strconcat(OpcodeStr, "sd"), 3087 OpNode, FR64, f64mem, SSEPackedDouble, 3088 itins.d>, XD; 3089 } 3090} 3091 3092multiclass basic_sse12_fp_binop_s_int<bits<8> opc, string OpcodeStr, 3093 SizeItins itins> { 3094 defm V#NAME#SS : sse12_fp_scalar_int<opc, OpcodeStr, VR128, 3095 !strconcat(OpcodeStr, "ss"), "", "_ss", ssmem, sse_load_f32, 3096 SSEPackedSingle, itins.s, 0>, XS, VEX_4V, VEX_LIG; 3097 defm V#NAME#SD : sse12_fp_scalar_int<opc, OpcodeStr, VR128, 3098 !strconcat(OpcodeStr, "sd"), "2", "_sd", sdmem, sse_load_f64, 3099 SSEPackedDouble, itins.d, 0>, XD, VEX_4V, VEX_LIG; 3100 3101 let Constraints = "$src1 = $dst" in { 3102 defm SS : sse12_fp_scalar_int<opc, OpcodeStr, VR128, 3103 !strconcat(OpcodeStr, "ss"), "", "_ss", ssmem, sse_load_f32, 3104 SSEPackedSingle, itins.s>, XS; 3105 defm SD : sse12_fp_scalar_int<opc, OpcodeStr, VR128, 3106 !strconcat(OpcodeStr, "sd"), "2", "_sd", sdmem, sse_load_f64, 3107 SSEPackedDouble, itins.d>, XD; 3108 } 3109} 3110 3111// Binary Arithmetic instructions 3112defm ADD : basic_sse12_fp_binop_p<0x58, "add", fadd, SSE_ALU_ITINS_P>, 3113 basic_sse12_fp_binop_s<0x58, "add", fadd, SSE_ALU_ITINS_S>, 3114 basic_sse12_fp_binop_s_int<0x58, "add", SSE_ALU_ITINS_S>; 3115defm MUL : basic_sse12_fp_binop_p<0x59, "mul", fmul, SSE_MUL_ITINS_P>, 3116 basic_sse12_fp_binop_s<0x59, "mul", fmul, SSE_MUL_ITINS_S>, 3117 basic_sse12_fp_binop_s_int<0x59, "mul", SSE_MUL_ITINS_S>; 3118let isCommutable = 0 in { 3119 defm SUB : basic_sse12_fp_binop_p<0x5C, "sub", fsub, SSE_ALU_ITINS_P>, 3120 basic_sse12_fp_binop_s<0x5C, "sub", fsub, SSE_ALU_ITINS_S>, 3121 basic_sse12_fp_binop_s_int<0x5C, "sub", SSE_ALU_ITINS_S>; 3122 defm DIV : basic_sse12_fp_binop_p<0x5E, "div", fdiv, SSE_DIV_ITINS_P>, 3123 basic_sse12_fp_binop_s<0x5E, "div", fdiv, SSE_DIV_ITINS_S>, 3124 basic_sse12_fp_binop_s_int<0x5E, "div", SSE_DIV_ITINS_S>; 3125 defm MAX : basic_sse12_fp_binop_p<0x5F, "max", X86fmax, SSE_ALU_ITINS_P>, 3126 basic_sse12_fp_binop_s<0x5F, "max", X86fmax, SSE_ALU_ITINS_S>, 3127 basic_sse12_fp_binop_s_int<0x5F, "max", SSE_ALU_ITINS_S>; 3128 defm MIN : basic_sse12_fp_binop_p<0x5D, "min", X86fmin, SSE_ALU_ITINS_P>, 3129 basic_sse12_fp_binop_s<0x5D, "min", X86fmin, SSE_ALU_ITINS_S>, 3130 basic_sse12_fp_binop_s_int<0x5D, "min", SSE_ALU_ITINS_S>; 3131} 3132 3133let isCodeGenOnly = 1 in { 3134 defm MAXC: basic_sse12_fp_binop_p<0x5F, "max", X86fmaxc, SSE_ALU_ITINS_P>, 3135 basic_sse12_fp_binop_s<0x5F, "max", X86fmaxc, SSE_ALU_ITINS_S>; 3136 defm MINC: basic_sse12_fp_binop_p<0x5D, "min", X86fminc, SSE_ALU_ITINS_P>, 3137 basic_sse12_fp_binop_s<0x5D, "min", X86fminc, SSE_ALU_ITINS_S>; 3138} 3139 3140// Patterns used to select SSE scalar fp arithmetic instructions from 3141// either: 3142// 3143// (1) a scalar fp operation followed by a blend 3144// 3145// The effect is that the backend no longer emits unnecessary vector 3146// insert instructions immediately after SSE scalar fp instructions 3147// like addss or mulss. 3148// 3149// For example, given the following code: 3150// __m128 foo(__m128 A, __m128 B) { 3151// A[0] += B[0]; 3152// return A; 3153// } 3154// 3155// Previously we generated: 3156// addss %xmm0, %xmm1 3157// movss %xmm1, %xmm0 3158// 3159// We now generate: 3160// addss %xmm1, %xmm0 3161// 3162// (2) a vector packed single/double fp operation followed by a vector insert 3163// 3164// The effect is that the backend converts the packed fp instruction 3165// followed by a vector insert into a single SSE scalar fp instruction. 3166// 3167// For example, given the following code: 3168// __m128 foo(__m128 A, __m128 B) { 3169// __m128 C = A + B; 3170// return (__m128) {c[0], a[1], a[2], a[3]}; 3171// } 3172// 3173// Previously we generated: 3174// addps %xmm0, %xmm1 3175// movss %xmm1, %xmm0 3176// 3177// We now generate: 3178// addss %xmm1, %xmm0 3179 3180// TODO: Some canonicalization in lowering would simplify the number of 3181// patterns we have to try to match. 3182multiclass scalar_math_f32_patterns<SDNode Op, string OpcPrefix> { 3183 let Predicates = [UseSSE1] in { 3184 // extracted scalar math op with insert via movss 3185 def : Pat<(v4f32 (X86Movss (v4f32 VR128:$dst), (v4f32 (scalar_to_vector 3186 (Op (f32 (vector_extract (v4f32 VR128:$dst), (iPTR 0))), 3187 FR32:$src))))), 3188 (!cast<I>(OpcPrefix#SSrr_Int) v4f32:$dst, 3189 (COPY_TO_REGCLASS FR32:$src, VR128))>; 3190 3191 // vector math op with insert via movss 3192 def : Pat<(v4f32 (X86Movss (v4f32 VR128:$dst), 3193 (Op (v4f32 VR128:$dst), (v4f32 VR128:$src)))), 3194 (!cast<I>(OpcPrefix#SSrr_Int) v4f32:$dst, v4f32:$src)>; 3195 } 3196 3197 // With SSE 4.1, blendi is preferred to movsd, so match that too. 3198 let Predicates = [UseSSE41] in { 3199 // extracted scalar math op with insert via blend 3200 def : Pat<(v4f32 (X86Blendi (v4f32 VR128:$dst), (v4f32 (scalar_to_vector 3201 (Op (f32 (vector_extract (v4f32 VR128:$dst), (iPTR 0))), 3202 FR32:$src))), (i8 1))), 3203 (!cast<I>(OpcPrefix#SSrr_Int) v4f32:$dst, 3204 (COPY_TO_REGCLASS FR32:$src, VR128))>; 3205 3206 // vector math op with insert via blend 3207 def : Pat<(v4f32 (X86Blendi (v4f32 VR128:$dst), 3208 (Op (v4f32 VR128:$dst), (v4f32 VR128:$src)), (i8 1))), 3209 (!cast<I>(OpcPrefix#SSrr_Int)v4f32:$dst, v4f32:$src)>; 3210 3211 } 3212 3213 // Repeat everything for AVX, except for the movss + scalar combo... 3214 // because that one shouldn't occur with AVX codegen? 3215 let Predicates = [HasAVX] in { 3216 // extracted scalar math op with insert via blend 3217 def : Pat<(v4f32 (X86Blendi (v4f32 VR128:$dst), (v4f32 (scalar_to_vector 3218 (Op (f32 (vector_extract (v4f32 VR128:$dst), (iPTR 0))), 3219 FR32:$src))), (i8 1))), 3220 (!cast<I>("V"#OpcPrefix#SSrr_Int) v4f32:$dst, 3221 (COPY_TO_REGCLASS FR32:$src, VR128))>; 3222 3223 // vector math op with insert via movss 3224 def : Pat<(v4f32 (X86Movss (v4f32 VR128:$dst), 3225 (Op (v4f32 VR128:$dst), (v4f32 VR128:$src)))), 3226 (!cast<I>("V"#OpcPrefix#SSrr_Int) v4f32:$dst, v4f32:$src)>; 3227 3228 // vector math op with insert via blend 3229 def : Pat<(v4f32 (X86Blendi (v4f32 VR128:$dst), 3230 (Op (v4f32 VR128:$dst), (v4f32 VR128:$src)), (i8 1))), 3231 (!cast<I>("V"#OpcPrefix#SSrr_Int) v4f32:$dst, v4f32:$src)>; 3232 } 3233} 3234 3235defm : scalar_math_f32_patterns<fadd, "ADD">; 3236defm : scalar_math_f32_patterns<fsub, "SUB">; 3237defm : scalar_math_f32_patterns<fmul, "MUL">; 3238defm : scalar_math_f32_patterns<fdiv, "DIV">; 3239 3240multiclass scalar_math_f64_patterns<SDNode Op, string OpcPrefix> { 3241 let Predicates = [UseSSE2] in { 3242 // extracted scalar math op with insert via movsd 3243 def : Pat<(v2f64 (X86Movsd (v2f64 VR128:$dst), (v2f64 (scalar_to_vector 3244 (Op (f64 (vector_extract (v2f64 VR128:$dst), (iPTR 0))), 3245 FR64:$src))))), 3246 (!cast<I>(OpcPrefix#SDrr_Int) v2f64:$dst, 3247 (COPY_TO_REGCLASS FR64:$src, VR128))>; 3248 3249 // vector math op with insert via movsd 3250 def : Pat<(v2f64 (X86Movsd (v2f64 VR128:$dst), 3251 (Op (v2f64 VR128:$dst), (v2f64 VR128:$src)))), 3252 (!cast<I>(OpcPrefix#SDrr_Int) v2f64:$dst, v2f64:$src)>; 3253 } 3254 3255 // With SSE 4.1, blendi is preferred to movsd, so match those too. 3256 let Predicates = [UseSSE41] in { 3257 // extracted scalar math op with insert via blend 3258 def : Pat<(v2f64 (X86Blendi (v2f64 VR128:$dst), (v2f64 (scalar_to_vector 3259 (Op (f64 (vector_extract (v2f64 VR128:$dst), (iPTR 0))), 3260 FR64:$src))), (i8 1))), 3261 (!cast<I>(OpcPrefix#SDrr_Int) v2f64:$dst, 3262 (COPY_TO_REGCLASS FR64:$src, VR128))>; 3263 3264 // vector math op with insert via blend 3265 def : Pat<(v2f64 (X86Blendi (v2f64 VR128:$dst), 3266 (Op (v2f64 VR128:$dst), (v2f64 VR128:$src)), (i8 1))), 3267 (!cast<I>(OpcPrefix#SDrr_Int) v2f64:$dst, v2f64:$src)>; 3268 } 3269 3270 // Repeat everything for AVX. 3271 let Predicates = [HasAVX] in { 3272 // extracted scalar math op with insert via movsd 3273 def : Pat<(v2f64 (X86Movsd (v2f64 VR128:$dst), (v2f64 (scalar_to_vector 3274 (Op (f64 (vector_extract (v2f64 VR128:$dst), (iPTR 0))), 3275 FR64:$src))))), 3276 (!cast<I>("V"#OpcPrefix#SDrr_Int) v2f64:$dst, 3277 (COPY_TO_REGCLASS FR64:$src, VR128))>; 3278 3279 // extracted scalar math op with insert via blend 3280 def : Pat<(v2f64 (X86Blendi (v2f64 VR128:$dst), (v2f64 (scalar_to_vector 3281 (Op (f64 (vector_extract (v2f64 VR128:$dst), (iPTR 0))), 3282 FR64:$src))), (i8 1))), 3283 (!cast<I>("V"#OpcPrefix#SDrr_Int) v2f64:$dst, 3284 (COPY_TO_REGCLASS FR64:$src, VR128))>; 3285 3286 // vector math op with insert via movsd 3287 def : Pat<(v2f64 (X86Movsd (v2f64 VR128:$dst), 3288 (Op (v2f64 VR128:$dst), (v2f64 VR128:$src)))), 3289 (!cast<I>("V"#OpcPrefix#SDrr_Int) v2f64:$dst, v2f64:$src)>; 3290 3291 // vector math op with insert via blend 3292 def : Pat<(v2f64 (X86Blendi (v2f64 VR128:$dst), 3293 (Op (v2f64 VR128:$dst), (v2f64 VR128:$src)), (i8 1))), 3294 (!cast<I>("V"#OpcPrefix#SDrr_Int) v2f64:$dst, v2f64:$src)>; 3295 } 3296} 3297 3298defm : scalar_math_f64_patterns<fadd, "ADD">; 3299defm : scalar_math_f64_patterns<fsub, "SUB">; 3300defm : scalar_math_f64_patterns<fmul, "MUL">; 3301defm : scalar_math_f64_patterns<fdiv, "DIV">; 3302 3303 3304/// Unop Arithmetic 3305/// In addition, we also have a special variant of the scalar form here to 3306/// represent the associated intrinsic operation. This form is unlike the 3307/// plain scalar form, in that it takes an entire vector (instead of a 3308/// scalar) and leaves the top elements undefined. 3309/// 3310/// And, we have a special variant form for a full-vector intrinsic form. 3311 3312let Sched = WriteFSqrt in { 3313def SSE_SQRTPS : OpndItins< 3314 IIC_SSE_SQRTPS_RR, IIC_SSE_SQRTPS_RM 3315>; 3316 3317def SSE_SQRTSS : OpndItins< 3318 IIC_SSE_SQRTSS_RR, IIC_SSE_SQRTSS_RM 3319>; 3320 3321def SSE_SQRTPD : OpndItins< 3322 IIC_SSE_SQRTPD_RR, IIC_SSE_SQRTPD_RM 3323>; 3324 3325def SSE_SQRTSD : OpndItins< 3326 IIC_SSE_SQRTSD_RR, IIC_SSE_SQRTSD_RM 3327>; 3328} 3329 3330let Sched = WriteFRsqrt in { 3331def SSE_RSQRTPS : OpndItins< 3332 IIC_SSE_RSQRTPS_RR, IIC_SSE_RSQRTPS_RM 3333>; 3334 3335def SSE_RSQRTSS : OpndItins< 3336 IIC_SSE_RSQRTSS_RR, IIC_SSE_RSQRTSS_RM 3337>; 3338} 3339 3340let Sched = WriteFRcp in { 3341def SSE_RCPP : OpndItins< 3342 IIC_SSE_RCPP_RR, IIC_SSE_RCPP_RM 3343>; 3344 3345def SSE_RCPS : OpndItins< 3346 IIC_SSE_RCPS_RR, IIC_SSE_RCPS_RM 3347>; 3348} 3349 3350/// sse_fp_unop_s - SSE1 unops in scalar form 3351/// For the non-AVX defs, we need $src1 to be tied to $dst because 3352/// the HW instructions are 2 operand / destructive. 3353multiclass sse_fp_unop_s<bits<8> opc, string OpcodeStr, RegisterClass RC, 3354 ValueType vt, ValueType ScalarVT, 3355 X86MemOperand x86memop, Operand vec_memop, 3356 ComplexPattern mem_cpat, Intrinsic Intr, 3357 SDNode OpNode, Domain d, OpndItins itins, 3358 Predicate target, string Suffix> { 3359 let hasSideEffects = 0 in { 3360 def r : I<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1), 3361 !strconcat(OpcodeStr, "\t{$src1, $dst|$dst, $src1}"), 3362 [(set RC:$dst, (OpNode RC:$src1))], itins.rr, d>, Sched<[itins.Sched]>, 3363 Requires<[target]>; 3364 let mayLoad = 1 in 3365 def m : I<opc, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src1), 3366 !strconcat(OpcodeStr, "\t{$src1, $dst|$dst, $src1}"), 3367 [(set RC:$dst, (OpNode (load addr:$src1)))], itins.rm, d>, 3368 Sched<[itins.Sched.Folded, ReadAfterLd]>, 3369 Requires<[target, OptForSize]>; 3370 3371 let isCodeGenOnly = 1, Constraints = "$src1 = $dst" in { 3372 def r_Int : I<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src1, VR128:$src2), 3373 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 3374 []>, Sched<[itins.Sched.Folded, ReadAfterLd]>; 3375 let mayLoad = 1 in 3376 def m_Int : I<opc, MRMSrcMem, (outs VR128:$dst), (ins VR128:$src1, vec_memop:$src2), 3377 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 3378 []>, Sched<[itins.Sched.Folded, ReadAfterLd]>; 3379 } 3380 } 3381 3382 let Predicates = [target] in { 3383 def : Pat<(vt (OpNode mem_cpat:$src)), 3384 (vt (COPY_TO_REGCLASS (vt (!cast<Instruction>(NAME#Suffix##m_Int) 3385 (vt (IMPLICIT_DEF)), mem_cpat:$src)), RC))>; 3386 // These are unary operations, but they are modeled as having 2 source operands 3387 // because the high elements of the destination are unchanged in SSE. 3388 def : Pat<(Intr VR128:$src), 3389 (!cast<Instruction>(NAME#Suffix##r_Int) VR128:$src, VR128:$src)>; 3390 def : Pat<(Intr (load addr:$src)), 3391 (vt (COPY_TO_REGCLASS(!cast<Instruction>(NAME#Suffix##m) 3392 addr:$src), VR128))>; 3393 def : Pat<(Intr mem_cpat:$src), 3394 (!cast<Instruction>(NAME#Suffix##m_Int) 3395 (vt (IMPLICIT_DEF)), mem_cpat:$src)>; 3396 } 3397} 3398 3399multiclass avx_fp_unop_s<bits<8> opc, string OpcodeStr, RegisterClass RC, 3400 ValueType vt, ValueType ScalarVT, 3401 X86MemOperand x86memop, Operand vec_memop, 3402 ComplexPattern mem_cpat, 3403 Intrinsic Intr, SDNode OpNode, Domain d, 3404 OpndItins itins, string Suffix> { 3405 let hasSideEffects = 0 in { 3406 def r : I<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2), 3407 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 3408 [], itins.rr, d>, Sched<[itins.Sched]>; 3409 let mayLoad = 1 in 3410 def m : I<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2), 3411 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 3412 [], itins.rm, d>, Sched<[itins.Sched.Folded, ReadAfterLd]>; 3413 let isCodeGenOnly = 1 in { 3414 def r_Int : I<opc, MRMSrcReg, (outs VR128:$dst), 3415 (ins VR128:$src1, VR128:$src2), 3416 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 3417 []>, Sched<[itins.Sched.Folded]>; 3418 let mayLoad = 1 in 3419 def m_Int : I<opc, MRMSrcMem, (outs VR128:$dst), 3420 (ins VR128:$src1, vec_memop:$src2), 3421 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 3422 []>, Sched<[itins.Sched.Folded, ReadAfterLd]>; 3423 } 3424 } 3425 3426 let Predicates = [UseAVX] in { 3427 def : Pat<(OpNode RC:$src), (!cast<Instruction>("V"#NAME#Suffix##r) 3428 (ScalarVT (IMPLICIT_DEF)), RC:$src)>; 3429 3430 def : Pat<(vt (OpNode mem_cpat:$src)), 3431 (!cast<Instruction>("V"#NAME#Suffix##m_Int) (vt (IMPLICIT_DEF)), 3432 mem_cpat:$src)>; 3433 3434 } 3435 let Predicates = [HasAVX] in { 3436 def : Pat<(Intr VR128:$src), 3437 (!cast<Instruction>("V"#NAME#Suffix##r_Int) (vt (IMPLICIT_DEF)), 3438 VR128:$src)>; 3439 3440 def : Pat<(Intr mem_cpat:$src), 3441 (!cast<Instruction>("V"#NAME#Suffix##m_Int) 3442 (vt (IMPLICIT_DEF)), mem_cpat:$src)>; 3443 } 3444 let Predicates = [UseAVX, OptForSize] in 3445 def : Pat<(ScalarVT (OpNode (load addr:$src))), 3446 (!cast<Instruction>("V"#NAME#Suffix##m) (ScalarVT (IMPLICIT_DEF)), 3447 addr:$src)>; 3448} 3449 3450/// sse1_fp_unop_p - SSE1 unops in packed form. 3451multiclass sse1_fp_unop_p<bits<8> opc, string OpcodeStr, SDNode OpNode, 3452 OpndItins itins> { 3453let Predicates = [HasAVX] in { 3454 def V#NAME#PSr : PSI<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 3455 !strconcat("v", OpcodeStr, 3456 "ps\t{$src, $dst|$dst, $src}"), 3457 [(set VR128:$dst, (v4f32 (OpNode VR128:$src)))], 3458 itins.rr>, VEX, Sched<[itins.Sched]>; 3459 def V#NAME#PSm : PSI<opc, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src), 3460 !strconcat("v", OpcodeStr, 3461 "ps\t{$src, $dst|$dst, $src}"), 3462 [(set VR128:$dst, (OpNode (loadv4f32 addr:$src)))], 3463 itins.rm>, VEX, Sched<[itins.Sched.Folded]>; 3464 def V#NAME#PSYr : PSI<opc, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src), 3465 !strconcat("v", OpcodeStr, 3466 "ps\t{$src, $dst|$dst, $src}"), 3467 [(set VR256:$dst, (v8f32 (OpNode VR256:$src)))], 3468 itins.rr>, VEX, VEX_L, Sched<[itins.Sched]>; 3469 def V#NAME#PSYm : PSI<opc, MRMSrcMem, (outs VR256:$dst), (ins f256mem:$src), 3470 !strconcat("v", OpcodeStr, 3471 "ps\t{$src, $dst|$dst, $src}"), 3472 [(set VR256:$dst, (OpNode (loadv8f32 addr:$src)))], 3473 itins.rm>, VEX, VEX_L, Sched<[itins.Sched.Folded]>; 3474} 3475 3476 def PSr : PSI<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 3477 !strconcat(OpcodeStr, "ps\t{$src, $dst|$dst, $src}"), 3478 [(set VR128:$dst, (v4f32 (OpNode VR128:$src)))], itins.rr>, 3479 Sched<[itins.Sched]>; 3480 def PSm : PSI<opc, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src), 3481 !strconcat(OpcodeStr, "ps\t{$src, $dst|$dst, $src}"), 3482 [(set VR128:$dst, (OpNode (memopv4f32 addr:$src)))], itins.rm>, 3483 Sched<[itins.Sched.Folded]>; 3484} 3485 3486/// sse2_fp_unop_p - SSE2 unops in vector forms. 3487multiclass sse2_fp_unop_p<bits<8> opc, string OpcodeStr, 3488 SDNode OpNode, OpndItins itins> { 3489let Predicates = [HasAVX] in { 3490 def V#NAME#PDr : PDI<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 3491 !strconcat("v", OpcodeStr, 3492 "pd\t{$src, $dst|$dst, $src}"), 3493 [(set VR128:$dst, (v2f64 (OpNode VR128:$src)))], 3494 itins.rr>, VEX, Sched<[itins.Sched]>; 3495 def V#NAME#PDm : PDI<opc, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src), 3496 !strconcat("v", OpcodeStr, 3497 "pd\t{$src, $dst|$dst, $src}"), 3498 [(set VR128:$dst, (OpNode (loadv2f64 addr:$src)))], 3499 itins.rm>, VEX, Sched<[itins.Sched.Folded]>; 3500 def V#NAME#PDYr : PDI<opc, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src), 3501 !strconcat("v", OpcodeStr, 3502 "pd\t{$src, $dst|$dst, $src}"), 3503 [(set VR256:$dst, (v4f64 (OpNode VR256:$src)))], 3504 itins.rr>, VEX, VEX_L, Sched<[itins.Sched]>; 3505 def V#NAME#PDYm : PDI<opc, MRMSrcMem, (outs VR256:$dst), (ins f256mem:$src), 3506 !strconcat("v", OpcodeStr, 3507 "pd\t{$src, $dst|$dst, $src}"), 3508 [(set VR256:$dst, (OpNode (loadv4f64 addr:$src)))], 3509 itins.rm>, VEX, VEX_L, Sched<[itins.Sched.Folded]>; 3510} 3511 3512 def PDr : PDI<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 3513 !strconcat(OpcodeStr, "pd\t{$src, $dst|$dst, $src}"), 3514 [(set VR128:$dst, (v2f64 (OpNode VR128:$src)))], itins.rr>, 3515 Sched<[itins.Sched]>; 3516 def PDm : PDI<opc, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src), 3517 !strconcat(OpcodeStr, "pd\t{$src, $dst|$dst, $src}"), 3518 [(set VR128:$dst, (OpNode (memopv2f64 addr:$src)))], itins.rm>, 3519 Sched<[itins.Sched.Folded]>; 3520} 3521 3522multiclass sse1_fp_unop_s<bits<8> opc, string OpcodeStr, SDNode OpNode, 3523 OpndItins itins> { 3524 defm SS : sse_fp_unop_s<opc, OpcodeStr##ss, FR32, v4f32, f32, f32mem, 3525 ssmem, sse_load_f32, 3526 !cast<Intrinsic>("int_x86_sse_"##OpcodeStr##_ss), OpNode, 3527 SSEPackedSingle, itins, UseSSE1, "SS">, XS; 3528 defm V#NAME#SS : avx_fp_unop_s<opc, "v"#OpcodeStr##ss, FR32, v4f32, f32, 3529 f32mem, ssmem, sse_load_f32, 3530 !cast<Intrinsic>("int_x86_sse_"##OpcodeStr##_ss), OpNode, 3531 SSEPackedSingle, itins, "SS">, XS, VEX_4V, VEX_LIG; 3532} 3533 3534multiclass sse2_fp_unop_s<bits<8> opc, string OpcodeStr, SDNode OpNode, 3535 OpndItins itins> { 3536 defm SD : sse_fp_unop_s<opc, OpcodeStr##sd, FR64, v2f64, f64, f64mem, 3537 sdmem, sse_load_f64, 3538 !cast<Intrinsic>("int_x86_sse2_"##OpcodeStr##_sd), 3539 OpNode, SSEPackedDouble, itins, UseSSE2, "SD">, XD; 3540 defm V#NAME#SD : avx_fp_unop_s<opc, "v"#OpcodeStr##sd, FR64, v2f64, f64, 3541 f64mem, sdmem, sse_load_f64, 3542 !cast<Intrinsic>("int_x86_sse2_"##OpcodeStr##_sd), 3543 OpNode, SSEPackedDouble, itins, "SD">, 3544 XD, VEX_4V, VEX_LIG; 3545} 3546 3547// Square root. 3548defm SQRT : sse1_fp_unop_s<0x51, "sqrt", fsqrt, SSE_SQRTSS>, 3549 sse1_fp_unop_p<0x51, "sqrt", fsqrt, SSE_SQRTPS>, 3550 sse2_fp_unop_s<0x51, "sqrt", fsqrt, SSE_SQRTSD>, 3551 sse2_fp_unop_p<0x51, "sqrt", fsqrt, SSE_SQRTPD>; 3552 3553// Reciprocal approximations. Note that these typically require refinement 3554// in order to obtain suitable precision. 3555defm RSQRT : sse1_fp_unop_s<0x52, "rsqrt", X86frsqrt, SSE_RSQRTSS>, 3556 sse1_fp_unop_p<0x52, "rsqrt", X86frsqrt, SSE_RSQRTPS>; 3557defm RCP : sse1_fp_unop_s<0x53, "rcp", X86frcp, SSE_RCPS>, 3558 sse1_fp_unop_p<0x53, "rcp", X86frcp, SSE_RCPP>; 3559 3560// There is no f64 version of the reciprocal approximation instructions. 3561 3562// TODO: We should add *scalar* op patterns for these just like we have for 3563// the binops above. If the binop and unop patterns could all be unified 3564// that would be even better. 3565 3566multiclass scalar_unary_math_patterns<Intrinsic Intr, string OpcPrefix, 3567 SDNode Move, ValueType VT, 3568 Predicate BasePredicate> { 3569 let Predicates = [BasePredicate] in { 3570 def : Pat<(VT (Move VT:$dst, (Intr VT:$src))), 3571 (!cast<I>(OpcPrefix#r_Int) VT:$dst, VT:$src)>; 3572 } 3573 3574 // With SSE 4.1, blendi is preferred to movs*, so match that too. 3575 let Predicates = [UseSSE41] in { 3576 def : Pat<(VT (X86Blendi VT:$dst, (Intr VT:$src), (i8 1))), 3577 (!cast<I>(OpcPrefix#r_Int) VT:$dst, VT:$src)>; 3578 } 3579 3580 // Repeat for AVX versions of the instructions. 3581 let Predicates = [HasAVX] in { 3582 def : Pat<(VT (Move VT:$dst, (Intr VT:$src))), 3583 (!cast<I>("V"#OpcPrefix#r_Int) VT:$dst, VT:$src)>; 3584 3585 def : Pat<(VT (X86Blendi VT:$dst, (Intr VT:$src), (i8 1))), 3586 (!cast<I>("V"#OpcPrefix#r_Int) VT:$dst, VT:$src)>; 3587 } 3588} 3589 3590defm : scalar_unary_math_patterns<int_x86_sse_rcp_ss, "RCPSS", X86Movss, 3591 v4f32, UseSSE1>; 3592defm : scalar_unary_math_patterns<int_x86_sse_rsqrt_ss, "RSQRTSS", X86Movss, 3593 v4f32, UseSSE1>; 3594defm : scalar_unary_math_patterns<int_x86_sse_sqrt_ss, "SQRTSS", X86Movss, 3595 v4f32, UseSSE1>; 3596defm : scalar_unary_math_patterns<int_x86_sse2_sqrt_sd, "SQRTSD", X86Movsd, 3597 v2f64, UseSSE2>; 3598 3599 3600//===----------------------------------------------------------------------===// 3601// SSE 1 & 2 - Non-temporal stores 3602//===----------------------------------------------------------------------===// 3603 3604let AddedComplexity = 400 in { // Prefer non-temporal versions 3605let SchedRW = [WriteStore] in { 3606let Predicates = [HasAVX, NoVLX] in { 3607def VMOVNTPSmr : VPSI<0x2B, MRMDestMem, (outs), 3608 (ins f128mem:$dst, VR128:$src), 3609 "movntps\t{$src, $dst|$dst, $src}", 3610 [(alignednontemporalstore (v4f32 VR128:$src), 3611 addr:$dst)], 3612 IIC_SSE_MOVNT>, VEX; 3613def VMOVNTPDmr : VPDI<0x2B, MRMDestMem, (outs), 3614 (ins f128mem:$dst, VR128:$src), 3615 "movntpd\t{$src, $dst|$dst, $src}", 3616 [(alignednontemporalstore (v2f64 VR128:$src), 3617 addr:$dst)], 3618 IIC_SSE_MOVNT>, VEX; 3619 3620let ExeDomain = SSEPackedInt in 3621def VMOVNTDQmr : VPDI<0xE7, MRMDestMem, (outs), 3622 (ins f128mem:$dst, VR128:$src), 3623 "movntdq\t{$src, $dst|$dst, $src}", 3624 [(alignednontemporalstore (v2i64 VR128:$src), 3625 addr:$dst)], 3626 IIC_SSE_MOVNT>, VEX; 3627 3628def VMOVNTPSYmr : VPSI<0x2B, MRMDestMem, (outs), 3629 (ins f256mem:$dst, VR256:$src), 3630 "movntps\t{$src, $dst|$dst, $src}", 3631 [(alignednontemporalstore (v8f32 VR256:$src), 3632 addr:$dst)], 3633 IIC_SSE_MOVNT>, VEX, VEX_L; 3634def VMOVNTPDYmr : VPDI<0x2B, MRMDestMem, (outs), 3635 (ins f256mem:$dst, VR256:$src), 3636 "movntpd\t{$src, $dst|$dst, $src}", 3637 [(alignednontemporalstore (v4f64 VR256:$src), 3638 addr:$dst)], 3639 IIC_SSE_MOVNT>, VEX, VEX_L; 3640let ExeDomain = SSEPackedInt in 3641def VMOVNTDQYmr : VPDI<0xE7, MRMDestMem, (outs), 3642 (ins f256mem:$dst, VR256:$src), 3643 "movntdq\t{$src, $dst|$dst, $src}", 3644 [(alignednontemporalstore (v4i64 VR256:$src), 3645 addr:$dst)], 3646 IIC_SSE_MOVNT>, VEX, VEX_L; 3647} 3648 3649def MOVNTPSmr : PSI<0x2B, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src), 3650 "movntps\t{$src, $dst|$dst, $src}", 3651 [(alignednontemporalstore (v4f32 VR128:$src), addr:$dst)], 3652 IIC_SSE_MOVNT>; 3653def MOVNTPDmr : PDI<0x2B, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src), 3654 "movntpd\t{$src, $dst|$dst, $src}", 3655 [(alignednontemporalstore(v2f64 VR128:$src), addr:$dst)], 3656 IIC_SSE_MOVNT>; 3657 3658let ExeDomain = SSEPackedInt in 3659def MOVNTDQmr : PDI<0xE7, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src), 3660 "movntdq\t{$src, $dst|$dst, $src}", 3661 [(alignednontemporalstore (v2i64 VR128:$src), addr:$dst)], 3662 IIC_SSE_MOVNT>; 3663 3664// There is no AVX form for instructions below this point 3665def MOVNTImr : I<0xC3, MRMDestMem, (outs), (ins i32mem:$dst, GR32:$src), 3666 "movnti{l}\t{$src, $dst|$dst, $src}", 3667 [(nontemporalstore (i32 GR32:$src), addr:$dst)], 3668 IIC_SSE_MOVNT>, 3669 PS, Requires<[HasSSE2]>; 3670def MOVNTI_64mr : RI<0xC3, MRMDestMem, (outs), (ins i64mem:$dst, GR64:$src), 3671 "movnti{q}\t{$src, $dst|$dst, $src}", 3672 [(nontemporalstore (i64 GR64:$src), addr:$dst)], 3673 IIC_SSE_MOVNT>, 3674 PS, Requires<[HasSSE2]>; 3675} // SchedRW = [WriteStore] 3676 3677let Predicates = [HasAVX2, NoVLX] in { 3678 def : Pat<(alignednontemporalstore (v8i32 VR256:$src), addr:$dst), 3679 (VMOVNTDQYmr addr:$dst, VR256:$src)>; 3680 def : Pat<(alignednontemporalstore (v16i16 VR256:$src), addr:$dst), 3681 (VMOVNTDQYmr addr:$dst, VR256:$src)>; 3682 def : Pat<(alignednontemporalstore (v32i8 VR256:$src), addr:$dst), 3683 (VMOVNTDQYmr addr:$dst, VR256:$src)>; 3684} 3685 3686let Predicates = [HasAVX, NoVLX] in { 3687 def : Pat<(alignednontemporalstore (v4i32 VR128:$src), addr:$dst), 3688 (VMOVNTDQmr addr:$dst, VR128:$src)>; 3689 def : Pat<(alignednontemporalstore (v8i16 VR128:$src), addr:$dst), 3690 (VMOVNTDQmr addr:$dst, VR128:$src)>; 3691 def : Pat<(alignednontemporalstore (v16i8 VR128:$src), addr:$dst), 3692 (VMOVNTDQmr addr:$dst, VR128:$src)>; 3693} 3694 3695def : Pat<(alignednontemporalstore (v4i32 VR128:$src), addr:$dst), 3696 (MOVNTDQmr addr:$dst, VR128:$src)>; 3697def : Pat<(alignednontemporalstore (v8i16 VR128:$src), addr:$dst), 3698 (MOVNTDQmr addr:$dst, VR128:$src)>; 3699def : Pat<(alignednontemporalstore (v16i8 VR128:$src), addr:$dst), 3700 (MOVNTDQmr addr:$dst, VR128:$src)>; 3701 3702} // AddedComplexity 3703 3704//===----------------------------------------------------------------------===// 3705// SSE 1 & 2 - Prefetch and memory fence 3706//===----------------------------------------------------------------------===// 3707 3708// Prefetch intrinsic. 3709let Predicates = [HasSSE1], SchedRW = [WriteLoad] in { 3710def PREFETCHT0 : I<0x18, MRM1m, (outs), (ins i8mem:$src), 3711 "prefetcht0\t$src", [(prefetch addr:$src, imm, (i32 3), (i32 1))], 3712 IIC_SSE_PREFETCH>, TB; 3713def PREFETCHT1 : I<0x18, MRM2m, (outs), (ins i8mem:$src), 3714 "prefetcht1\t$src", [(prefetch addr:$src, imm, (i32 2), (i32 1))], 3715 IIC_SSE_PREFETCH>, TB; 3716def PREFETCHT2 : I<0x18, MRM3m, (outs), (ins i8mem:$src), 3717 "prefetcht2\t$src", [(prefetch addr:$src, imm, (i32 1), (i32 1))], 3718 IIC_SSE_PREFETCH>, TB; 3719def PREFETCHNTA : I<0x18, MRM0m, (outs), (ins i8mem:$src), 3720 "prefetchnta\t$src", [(prefetch addr:$src, imm, (i32 0), (i32 1))], 3721 IIC_SSE_PREFETCH>, TB; 3722} 3723 3724// FIXME: How should flush instruction be modeled? 3725let SchedRW = [WriteLoad] in { 3726// Flush cache 3727def CLFLUSH : I<0xAE, MRM7m, (outs), (ins i8mem:$src), 3728 "clflush\t$src", [(int_x86_sse2_clflush addr:$src)], 3729 IIC_SSE_PREFETCH>, PS, Requires<[HasSSE2]>; 3730} 3731 3732let SchedRW = [WriteNop] in { 3733// Pause. This "instruction" is encoded as "rep; nop", so even though it 3734// was introduced with SSE2, it's backward compatible. 3735def PAUSE : I<0x90, RawFrm, (outs), (ins), 3736 "pause", [(int_x86_sse2_pause)], IIC_SSE_PAUSE>, 3737 OBXS, Requires<[HasSSE2]>; 3738} 3739 3740let SchedRW = [WriteFence] in { 3741// Load, store, and memory fence 3742def SFENCE : I<0xAE, MRM_F8, (outs), (ins), 3743 "sfence", [(int_x86_sse_sfence)], IIC_SSE_SFENCE>, 3744 PS, Requires<[HasSSE1]>; 3745def LFENCE : I<0xAE, MRM_E8, (outs), (ins), 3746 "lfence", [(int_x86_sse2_lfence)], IIC_SSE_LFENCE>, 3747 TB, Requires<[HasSSE2]>; 3748def MFENCE : I<0xAE, MRM_F0, (outs), (ins), 3749 "mfence", [(int_x86_sse2_mfence)], IIC_SSE_MFENCE>, 3750 TB, Requires<[HasSSE2]>; 3751} // SchedRW 3752 3753def : Pat<(X86SFence), (SFENCE)>; 3754def : Pat<(X86LFence), (LFENCE)>; 3755def : Pat<(X86MFence), (MFENCE)>; 3756 3757//===----------------------------------------------------------------------===// 3758// SSE 1 & 2 - Load/Store XCSR register 3759//===----------------------------------------------------------------------===// 3760 3761def VLDMXCSR : VPSI<0xAE, MRM2m, (outs), (ins i32mem:$src), 3762 "ldmxcsr\t$src", [(int_x86_sse_ldmxcsr addr:$src)], 3763 IIC_SSE_LDMXCSR>, VEX, Sched<[WriteLoad]>; 3764def VSTMXCSR : VPSI<0xAE, MRM3m, (outs), (ins i32mem:$dst), 3765 "stmxcsr\t$dst", [(int_x86_sse_stmxcsr addr:$dst)], 3766 IIC_SSE_STMXCSR>, VEX, Sched<[WriteStore]>; 3767 3768let Predicates = [UseSSE1] in { 3769def LDMXCSR : I<0xAE, MRM2m, (outs), (ins i32mem:$src), 3770 "ldmxcsr\t$src", [(int_x86_sse_ldmxcsr addr:$src)], 3771 IIC_SSE_LDMXCSR>, TB, Sched<[WriteLoad]>; 3772def STMXCSR : I<0xAE, MRM3m, (outs), (ins i32mem:$dst), 3773 "stmxcsr\t$dst", [(int_x86_sse_stmxcsr addr:$dst)], 3774 IIC_SSE_STMXCSR>, TB, Sched<[WriteStore]>; 3775} 3776 3777//===---------------------------------------------------------------------===// 3778// SSE2 - Move Aligned/Unaligned Packed Integer Instructions 3779//===---------------------------------------------------------------------===// 3780 3781let ExeDomain = SSEPackedInt in { // SSE integer instructions 3782 3783let hasSideEffects = 0, SchedRW = [WriteMove] in { 3784def VMOVDQArr : VPDI<0x6F, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 3785 "movdqa\t{$src, $dst|$dst, $src}", [], IIC_SSE_MOVA_P_RR>, 3786 VEX; 3787def VMOVDQAYrr : VPDI<0x6F, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src), 3788 "movdqa\t{$src, $dst|$dst, $src}", [], IIC_SSE_MOVA_P_RR>, 3789 VEX, VEX_L; 3790def VMOVDQUrr : VSSI<0x6F, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 3791 "movdqu\t{$src, $dst|$dst, $src}", [], IIC_SSE_MOVU_P_RR>, 3792 VEX; 3793def VMOVDQUYrr : VSSI<0x6F, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src), 3794 "movdqu\t{$src, $dst|$dst, $src}", [], IIC_SSE_MOVU_P_RR>, 3795 VEX, VEX_L; 3796} 3797 3798// For Disassembler 3799let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0, 3800 SchedRW = [WriteMove] in { 3801def VMOVDQArr_REV : VPDI<0x7F, MRMDestReg, (outs VR128:$dst), (ins VR128:$src), 3802 "movdqa\t{$src, $dst|$dst, $src}", [], 3803 IIC_SSE_MOVA_P_RR>, 3804 VEX; 3805def VMOVDQAYrr_REV : VPDI<0x7F, MRMDestReg, (outs VR256:$dst), (ins VR256:$src), 3806 "movdqa\t{$src, $dst|$dst, $src}", [], 3807 IIC_SSE_MOVA_P_RR>, VEX, VEX_L; 3808def VMOVDQUrr_REV : VSSI<0x7F, MRMDestReg, (outs VR128:$dst), (ins VR128:$src), 3809 "movdqu\t{$src, $dst|$dst, $src}", [], 3810 IIC_SSE_MOVU_P_RR>, 3811 VEX; 3812def VMOVDQUYrr_REV : VSSI<0x7F, MRMDestReg, (outs VR256:$dst), (ins VR256:$src), 3813 "movdqu\t{$src, $dst|$dst, $src}", [], 3814 IIC_SSE_MOVU_P_RR>, VEX, VEX_L; 3815} 3816 3817let canFoldAsLoad = 1, mayLoad = 1, isReMaterializable = 1, 3818 hasSideEffects = 0, SchedRW = [WriteLoad] in { 3819def VMOVDQArm : VPDI<0x6F, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src), 3820 "movdqa\t{$src, $dst|$dst, $src}", [], IIC_SSE_MOVA_P_RM>, 3821 VEX; 3822def VMOVDQAYrm : VPDI<0x6F, MRMSrcMem, (outs VR256:$dst), (ins i256mem:$src), 3823 "movdqa\t{$src, $dst|$dst, $src}", [], IIC_SSE_MOVA_P_RM>, 3824 VEX, VEX_L; 3825let Predicates = [HasAVX] in { 3826 def VMOVDQUrm : I<0x6F, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src), 3827 "vmovdqu\t{$src, $dst|$dst, $src}",[], IIC_SSE_MOVU_P_RM>, 3828 XS, VEX; 3829 def VMOVDQUYrm : I<0x6F, MRMSrcMem, (outs VR256:$dst), (ins i256mem:$src), 3830 "vmovdqu\t{$src, $dst|$dst, $src}",[], IIC_SSE_MOVU_P_RM>, 3831 XS, VEX, VEX_L; 3832} 3833} 3834 3835let mayStore = 1, hasSideEffects = 0, SchedRW = [WriteStore] in { 3836def VMOVDQAmr : VPDI<0x7F, MRMDestMem, (outs), 3837 (ins i128mem:$dst, VR128:$src), 3838 "movdqa\t{$src, $dst|$dst, $src}", [], IIC_SSE_MOVA_P_MR>, 3839 VEX; 3840def VMOVDQAYmr : VPDI<0x7F, MRMDestMem, (outs), 3841 (ins i256mem:$dst, VR256:$src), 3842 "movdqa\t{$src, $dst|$dst, $src}", [], IIC_SSE_MOVA_P_MR>, 3843 VEX, VEX_L; 3844let Predicates = [HasAVX] in { 3845def VMOVDQUmr : I<0x7F, MRMDestMem, (outs), (ins i128mem:$dst, VR128:$src), 3846 "vmovdqu\t{$src, $dst|$dst, $src}",[], IIC_SSE_MOVU_P_MR>, 3847 XS, VEX; 3848def VMOVDQUYmr : I<0x7F, MRMDestMem, (outs), (ins i256mem:$dst, VR256:$src), 3849 "vmovdqu\t{$src, $dst|$dst, $src}",[], IIC_SSE_MOVU_P_MR>, 3850 XS, VEX, VEX_L; 3851} 3852} 3853 3854let SchedRW = [WriteMove] in { 3855let hasSideEffects = 0 in 3856def MOVDQArr : PDI<0x6F, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 3857 "movdqa\t{$src, $dst|$dst, $src}", [], IIC_SSE_MOVA_P_RR>; 3858 3859def MOVDQUrr : I<0x6F, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 3860 "movdqu\t{$src, $dst|$dst, $src}", 3861 [], IIC_SSE_MOVU_P_RR>, XS, Requires<[UseSSE2]>; 3862 3863// For Disassembler 3864let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0 in { 3865def MOVDQArr_REV : PDI<0x7F, MRMDestReg, (outs VR128:$dst), (ins VR128:$src), 3866 "movdqa\t{$src, $dst|$dst, $src}", [], 3867 IIC_SSE_MOVA_P_RR>; 3868 3869def MOVDQUrr_REV : I<0x7F, MRMDestReg, (outs VR128:$dst), (ins VR128:$src), 3870 "movdqu\t{$src, $dst|$dst, $src}", 3871 [], IIC_SSE_MOVU_P_RR>, XS, Requires<[UseSSE2]>; 3872} 3873} // SchedRW 3874 3875let canFoldAsLoad = 1, mayLoad = 1, isReMaterializable = 1, 3876 hasSideEffects = 0, SchedRW = [WriteLoad] in { 3877def MOVDQArm : PDI<0x6F, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src), 3878 "movdqa\t{$src, $dst|$dst, $src}", 3879 [/*(set VR128:$dst, (alignedloadv2i64 addr:$src))*/], 3880 IIC_SSE_MOVA_P_RM>; 3881def MOVDQUrm : I<0x6F, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src), 3882 "movdqu\t{$src, $dst|$dst, $src}", 3883 [/*(set VR128:$dst, (loadv2i64 addr:$src))*/], 3884 IIC_SSE_MOVU_P_RM>, 3885 XS, Requires<[UseSSE2]>; 3886} 3887 3888let mayStore = 1, hasSideEffects = 0, SchedRW = [WriteStore] in { 3889def MOVDQAmr : PDI<0x7F, MRMDestMem, (outs), (ins i128mem:$dst, VR128:$src), 3890 "movdqa\t{$src, $dst|$dst, $src}", 3891 [/*(alignedstore (v2i64 VR128:$src), addr:$dst)*/], 3892 IIC_SSE_MOVA_P_MR>; 3893def MOVDQUmr : I<0x7F, MRMDestMem, (outs), (ins i128mem:$dst, VR128:$src), 3894 "movdqu\t{$src, $dst|$dst, $src}", 3895 [/*(store (v2i64 VR128:$src), addr:$dst)*/], 3896 IIC_SSE_MOVU_P_MR>, 3897 XS, Requires<[UseSSE2]>; 3898} 3899 3900} // ExeDomain = SSEPackedInt 3901 3902let Predicates = [HasAVX] in { 3903 def : Pat<(int_x86_sse2_storeu_dq addr:$dst, VR128:$src), 3904 (VMOVDQUmr addr:$dst, VR128:$src)>; 3905 def : Pat<(int_x86_avx_storeu_dq_256 addr:$dst, VR256:$src), 3906 (VMOVDQUYmr addr:$dst, VR256:$src)>; 3907} 3908let Predicates = [UseSSE2] in 3909def : Pat<(int_x86_sse2_storeu_dq addr:$dst, VR128:$src), 3910 (MOVDQUmr addr:$dst, VR128:$src)>; 3911 3912//===---------------------------------------------------------------------===// 3913// SSE2 - Packed Integer Arithmetic Instructions 3914//===---------------------------------------------------------------------===// 3915 3916let Sched = WriteVecIMul in 3917def SSE_PMADD : OpndItins< 3918 IIC_SSE_PMADD, IIC_SSE_PMADD 3919>; 3920 3921let ExeDomain = SSEPackedInt in { // SSE integer instructions 3922 3923multiclass PDI_binop_rm_int<bits<8> opc, string OpcodeStr, Intrinsic IntId, 3924 RegisterClass RC, PatFrag memop_frag, 3925 X86MemOperand x86memop, 3926 OpndItins itins, 3927 bit IsCommutable = 0, 3928 bit Is2Addr = 1> { 3929 let isCommutable = IsCommutable in 3930 def rr : PDI<opc, MRMSrcReg, (outs RC:$dst), 3931 (ins RC:$src1, RC:$src2), 3932 !if(Is2Addr, 3933 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 3934 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 3935 [(set RC:$dst, (IntId RC:$src1, RC:$src2))], itins.rr>, 3936 Sched<[itins.Sched]>; 3937 def rm : PDI<opc, MRMSrcMem, (outs RC:$dst), 3938 (ins RC:$src1, x86memop:$src2), 3939 !if(Is2Addr, 3940 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 3941 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 3942 [(set RC:$dst, (IntId RC:$src1, (bitconvert (memop_frag addr:$src2))))], 3943 itins.rm>, Sched<[itins.Sched.Folded, ReadAfterLd]>; 3944} 3945 3946multiclass PDI_binop_all_int<bits<8> opc, string OpcodeStr, Intrinsic IntId128, 3947 Intrinsic IntId256, OpndItins itins, 3948 bit IsCommutable = 0> { 3949let Predicates = [HasAVX] in 3950 defm V#NAME : PDI_binop_rm_int<opc, !strconcat("v", OpcodeStr), IntId128, 3951 VR128, loadv2i64, i128mem, itins, 3952 IsCommutable, 0>, VEX_4V; 3953 3954let Constraints = "$src1 = $dst" in 3955 defm NAME : PDI_binop_rm_int<opc, OpcodeStr, IntId128, VR128, memopv2i64, 3956 i128mem, itins, IsCommutable, 1>; 3957 3958let Predicates = [HasAVX2] in 3959 defm V#NAME#Y : PDI_binop_rm_int<opc, !strconcat("v", OpcodeStr), IntId256, 3960 VR256, loadv4i64, i256mem, itins, 3961 IsCommutable, 0>, VEX_4V, VEX_L; 3962} 3963 3964multiclass PDI_binop_rmi<bits<8> opc, bits<8> opc2, Format ImmForm, 3965 string OpcodeStr, SDNode OpNode, 3966 SDNode OpNode2, RegisterClass RC, 3967 ValueType DstVT, ValueType SrcVT, PatFrag bc_frag, 3968 PatFrag ld_frag, ShiftOpndItins itins, 3969 bit Is2Addr = 1> { 3970 // src2 is always 128-bit 3971 def rr : PDI<opc, MRMSrcReg, (outs RC:$dst), 3972 (ins RC:$src1, VR128:$src2), 3973 !if(Is2Addr, 3974 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 3975 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 3976 [(set RC:$dst, (DstVT (OpNode RC:$src1, (SrcVT VR128:$src2))))], 3977 itins.rr>, Sched<[WriteVecShift]>; 3978 def rm : PDI<opc, MRMSrcMem, (outs RC:$dst), 3979 (ins RC:$src1, i128mem:$src2), 3980 !if(Is2Addr, 3981 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 3982 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 3983 [(set RC:$dst, (DstVT (OpNode RC:$src1, 3984 (bc_frag (ld_frag addr:$src2)))))], itins.rm>, 3985 Sched<[WriteVecShiftLd, ReadAfterLd]>; 3986 def ri : PDIi8<opc2, ImmForm, (outs RC:$dst), 3987 (ins RC:$src1, u8imm:$src2), 3988 !if(Is2Addr, 3989 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 3990 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 3991 [(set RC:$dst, (DstVT (OpNode2 RC:$src1, (i8 imm:$src2))))], itins.ri>, 3992 Sched<[WriteVecShift]>; 3993} 3994 3995/// PDI_binop_rm2 - Simple SSE2 binary operator with different src and dst types 3996multiclass PDI_binop_rm2<bits<8> opc, string OpcodeStr, SDNode OpNode, 3997 ValueType DstVT, ValueType SrcVT, RegisterClass RC, 3998 PatFrag memop_frag, X86MemOperand x86memop, 3999 OpndItins itins, 4000 bit IsCommutable = 0, bit Is2Addr = 1> { 4001 let isCommutable = IsCommutable in 4002 def rr : PDI<opc, MRMSrcReg, (outs RC:$dst), 4003 (ins RC:$src1, RC:$src2), 4004 !if(Is2Addr, 4005 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 4006 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 4007 [(set RC:$dst, (DstVT (OpNode (SrcVT RC:$src1), RC:$src2)))]>, 4008 Sched<[itins.Sched]>; 4009 def rm : PDI<opc, MRMSrcMem, (outs RC:$dst), 4010 (ins RC:$src1, x86memop:$src2), 4011 !if(Is2Addr, 4012 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 4013 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 4014 [(set RC:$dst, (DstVT (OpNode (SrcVT RC:$src1), 4015 (bitconvert (memop_frag addr:$src2)))))]>, 4016 Sched<[itins.Sched.Folded, ReadAfterLd]>; 4017} 4018} // ExeDomain = SSEPackedInt 4019 4020defm PADDB : PDI_binop_all<0xFC, "paddb", add, v16i8, v32i8, 4021 SSE_INTALU_ITINS_P, 1>; 4022defm PADDW : PDI_binop_all<0xFD, "paddw", add, v8i16, v16i16, 4023 SSE_INTALU_ITINS_P, 1>; 4024defm PADDD : PDI_binop_all<0xFE, "paddd", add, v4i32, v8i32, 4025 SSE_INTALU_ITINS_P, 1>; 4026defm PADDQ : PDI_binop_all<0xD4, "paddq", add, v2i64, v4i64, 4027 SSE_INTALUQ_ITINS_P, 1>; 4028defm PMULLW : PDI_binop_all<0xD5, "pmullw", mul, v8i16, v16i16, 4029 SSE_INTMUL_ITINS_P, 1>; 4030defm PMULHUW : PDI_binop_all<0xE4, "pmulhuw", mulhu, v8i16, v16i16, 4031 SSE_INTMUL_ITINS_P, 1>; 4032defm PMULHW : PDI_binop_all<0xE5, "pmulhw", mulhs, v8i16, v16i16, 4033 SSE_INTMUL_ITINS_P, 1>; 4034defm PSUBB : PDI_binop_all<0xF8, "psubb", sub, v16i8, v32i8, 4035 SSE_INTALU_ITINS_P, 0>; 4036defm PSUBW : PDI_binop_all<0xF9, "psubw", sub, v8i16, v16i16, 4037 SSE_INTALU_ITINS_P, 0>; 4038defm PSUBD : PDI_binop_all<0xFA, "psubd", sub, v4i32, v8i32, 4039 SSE_INTALU_ITINS_P, 0>; 4040defm PSUBQ : PDI_binop_all<0xFB, "psubq", sub, v2i64, v4i64, 4041 SSE_INTALUQ_ITINS_P, 0>; 4042defm PSUBUSB : PDI_binop_all<0xD8, "psubusb", X86subus, v16i8, v32i8, 4043 SSE_INTALU_ITINS_P, 0>; 4044defm PSUBUSW : PDI_binop_all<0xD9, "psubusw", X86subus, v8i16, v16i16, 4045 SSE_INTALU_ITINS_P, 0>; 4046defm PMINUB : PDI_binop_all<0xDA, "pminub", umin, v16i8, v32i8, 4047 SSE_INTALU_ITINS_P, 1>; 4048defm PMINSW : PDI_binop_all<0xEA, "pminsw", smin, v8i16, v16i16, 4049 SSE_INTALU_ITINS_P, 1>; 4050defm PMAXUB : PDI_binop_all<0xDE, "pmaxub", umax, v16i8, v32i8, 4051 SSE_INTALU_ITINS_P, 1>; 4052defm PMAXSW : PDI_binop_all<0xEE, "pmaxsw", smax, v8i16, v16i16, 4053 SSE_INTALU_ITINS_P, 1>; 4054 4055// Intrinsic forms 4056defm PSUBSB : PDI_binop_all_int<0xE8, "psubsb", int_x86_sse2_psubs_b, 4057 int_x86_avx2_psubs_b, SSE_INTALU_ITINS_P, 0>; 4058defm PSUBSW : PDI_binop_all_int<0xE9, "psubsw" , int_x86_sse2_psubs_w, 4059 int_x86_avx2_psubs_w, SSE_INTALU_ITINS_P, 0>; 4060defm PADDSB : PDI_binop_all_int<0xEC, "paddsb" , int_x86_sse2_padds_b, 4061 int_x86_avx2_padds_b, SSE_INTALU_ITINS_P, 1>; 4062defm PADDSW : PDI_binop_all_int<0xED, "paddsw" , int_x86_sse2_padds_w, 4063 int_x86_avx2_padds_w, SSE_INTALU_ITINS_P, 1>; 4064defm PADDUSB : PDI_binop_all_int<0xDC, "paddusb", int_x86_sse2_paddus_b, 4065 int_x86_avx2_paddus_b, SSE_INTALU_ITINS_P, 1>; 4066defm PADDUSW : PDI_binop_all_int<0xDD, "paddusw", int_x86_sse2_paddus_w, 4067 int_x86_avx2_paddus_w, SSE_INTALU_ITINS_P, 1>; 4068defm PMADDWD : PDI_binop_all_int<0xF5, "pmaddwd", int_x86_sse2_pmadd_wd, 4069 int_x86_avx2_pmadd_wd, SSE_PMADD, 1>; 4070defm PAVGB : PDI_binop_all_int<0xE0, "pavgb", int_x86_sse2_pavg_b, 4071 int_x86_avx2_pavg_b, SSE_INTALU_ITINS_P, 1>; 4072defm PAVGW : PDI_binop_all_int<0xE3, "pavgw", int_x86_sse2_pavg_w, 4073 int_x86_avx2_pavg_w, SSE_INTALU_ITINS_P, 1>; 4074defm PSADBW : PDI_binop_all_int<0xF6, "psadbw", int_x86_sse2_psad_bw, 4075 int_x86_avx2_psad_bw, SSE_PMADD, 1>; 4076 4077let Predicates = [HasAVX2] in 4078 def : Pat<(v32i8 (X86psadbw (v32i8 VR256:$src1), 4079 (v32i8 VR256:$src2))), 4080 (VPSADBWYrr VR256:$src2, VR256:$src1)>; 4081 4082let Predicates = [HasAVX] in 4083 def : Pat<(v16i8 (X86psadbw (v16i8 VR128:$src1), 4084 (v16i8 VR128:$src2))), 4085 (VPSADBWrr VR128:$src2, VR128:$src1)>; 4086 4087def : Pat<(v16i8 (X86psadbw (v16i8 VR128:$src1), 4088 (v16i8 VR128:$src2))), 4089 (PSADBWrr VR128:$src2, VR128:$src1)>; 4090 4091let Predicates = [HasAVX] in 4092defm VPMULUDQ : PDI_binop_rm2<0xF4, "vpmuludq", X86pmuludq, v2i64, v4i32, VR128, 4093 loadv2i64, i128mem, SSE_INTMUL_ITINS_P, 1, 0>, 4094 VEX_4V; 4095let Predicates = [HasAVX2] in 4096defm VPMULUDQY : PDI_binop_rm2<0xF4, "vpmuludq", X86pmuludq, v4i64, v8i32, 4097 VR256, loadv4i64, i256mem, 4098 SSE_INTMUL_ITINS_P, 1, 0>, VEX_4V, VEX_L; 4099let Constraints = "$src1 = $dst" in 4100defm PMULUDQ : PDI_binop_rm2<0xF4, "pmuludq", X86pmuludq, v2i64, v4i32, VR128, 4101 memopv2i64, i128mem, SSE_INTMUL_ITINS_P, 1>; 4102 4103//===---------------------------------------------------------------------===// 4104// SSE2 - Packed Integer Logical Instructions 4105//===---------------------------------------------------------------------===// 4106 4107let Predicates = [HasAVX, NoVLX] in { 4108defm VPSLLW : PDI_binop_rmi<0xF1, 0x71, MRM6r, "vpsllw", X86vshl, X86vshli, 4109 VR128, v8i16, v8i16, bc_v8i16, loadv2i64, 4110 SSE_INTSHIFT_ITINS_P, 0>, VEX_4V; 4111defm VPSLLD : PDI_binop_rmi<0xF2, 0x72, MRM6r, "vpslld", X86vshl, X86vshli, 4112 VR128, v4i32, v4i32, bc_v4i32, loadv2i64, 4113 SSE_INTSHIFT_ITINS_P, 0>, VEX_4V; 4114defm VPSLLQ : PDI_binop_rmi<0xF3, 0x73, MRM6r, "vpsllq", X86vshl, X86vshli, 4115 VR128, v2i64, v2i64, bc_v2i64, loadv2i64, 4116 SSE_INTSHIFT_ITINS_P, 0>, VEX_4V; 4117 4118defm VPSRLW : PDI_binop_rmi<0xD1, 0x71, MRM2r, "vpsrlw", X86vsrl, X86vsrli, 4119 VR128, v8i16, v8i16, bc_v8i16, loadv2i64, 4120 SSE_INTSHIFT_ITINS_P, 0>, VEX_4V; 4121defm VPSRLD : PDI_binop_rmi<0xD2, 0x72, MRM2r, "vpsrld", X86vsrl, X86vsrli, 4122 VR128, v4i32, v4i32, bc_v4i32, loadv2i64, 4123 SSE_INTSHIFT_ITINS_P, 0>, VEX_4V; 4124defm VPSRLQ : PDI_binop_rmi<0xD3, 0x73, MRM2r, "vpsrlq", X86vsrl, X86vsrli, 4125 VR128, v2i64, v2i64, bc_v2i64, loadv2i64, 4126 SSE_INTSHIFT_ITINS_P, 0>, VEX_4V; 4127 4128defm VPSRAW : PDI_binop_rmi<0xE1, 0x71, MRM4r, "vpsraw", X86vsra, X86vsrai, 4129 VR128, v8i16, v8i16, bc_v8i16, loadv2i64, 4130 SSE_INTSHIFT_ITINS_P, 0>, VEX_4V; 4131defm VPSRAD : PDI_binop_rmi<0xE2, 0x72, MRM4r, "vpsrad", X86vsra, X86vsrai, 4132 VR128, v4i32, v4i32, bc_v4i32, loadv2i64, 4133 SSE_INTSHIFT_ITINS_P, 0>, VEX_4V; 4134 4135let ExeDomain = SSEPackedInt, SchedRW = [WriteVecShift] in { 4136 // 128-bit logical shifts. 4137 def VPSLLDQri : PDIi8<0x73, MRM7r, 4138 (outs VR128:$dst), (ins VR128:$src1, u8imm:$src2), 4139 "vpslldq\t{$src2, $src1, $dst|$dst, $src1, $src2}", 4140 [(set VR128:$dst, 4141 (v2i64 (X86vshldq VR128:$src1, (i8 imm:$src2))))]>, 4142 VEX_4V; 4143 def VPSRLDQri : PDIi8<0x73, MRM3r, 4144 (outs VR128:$dst), (ins VR128:$src1, u8imm:$src2), 4145 "vpsrldq\t{$src2, $src1, $dst|$dst, $src1, $src2}", 4146 [(set VR128:$dst, 4147 (v2i64 (X86vshrdq VR128:$src1, (i8 imm:$src2))))]>, 4148 VEX_4V; 4149 // PSRADQri doesn't exist in SSE[1-3]. 4150} 4151} // Predicates = [HasAVX] 4152 4153let Predicates = [HasAVX2, NoVLX] in { 4154defm VPSLLWY : PDI_binop_rmi<0xF1, 0x71, MRM6r, "vpsllw", X86vshl, X86vshli, 4155 VR256, v16i16, v8i16, bc_v8i16, loadv2i64, 4156 SSE_INTSHIFT_ITINS_P, 0>, VEX_4V, VEX_L; 4157defm VPSLLDY : PDI_binop_rmi<0xF2, 0x72, MRM6r, "vpslld", X86vshl, X86vshli, 4158 VR256, v8i32, v4i32, bc_v4i32, loadv2i64, 4159 SSE_INTSHIFT_ITINS_P, 0>, VEX_4V, VEX_L; 4160defm VPSLLQY : PDI_binop_rmi<0xF3, 0x73, MRM6r, "vpsllq", X86vshl, X86vshli, 4161 VR256, v4i64, v2i64, bc_v2i64, loadv2i64, 4162 SSE_INTSHIFT_ITINS_P, 0>, VEX_4V, VEX_L; 4163 4164defm VPSRLWY : PDI_binop_rmi<0xD1, 0x71, MRM2r, "vpsrlw", X86vsrl, X86vsrli, 4165 VR256, v16i16, v8i16, bc_v8i16, loadv2i64, 4166 SSE_INTSHIFT_ITINS_P, 0>, VEX_4V, VEX_L; 4167defm VPSRLDY : PDI_binop_rmi<0xD2, 0x72, MRM2r, "vpsrld", X86vsrl, X86vsrli, 4168 VR256, v8i32, v4i32, bc_v4i32, loadv2i64, 4169 SSE_INTSHIFT_ITINS_P, 0>, VEX_4V, VEX_L; 4170defm VPSRLQY : PDI_binop_rmi<0xD3, 0x73, MRM2r, "vpsrlq", X86vsrl, X86vsrli, 4171 VR256, v4i64, v2i64, bc_v2i64, loadv2i64, 4172 SSE_INTSHIFT_ITINS_P, 0>, VEX_4V, VEX_L; 4173 4174defm VPSRAWY : PDI_binop_rmi<0xE1, 0x71, MRM4r, "vpsraw", X86vsra, X86vsrai, 4175 VR256, v16i16, v8i16, bc_v8i16, loadv2i64, 4176 SSE_INTSHIFT_ITINS_P, 0>, VEX_4V, VEX_L; 4177defm VPSRADY : PDI_binop_rmi<0xE2, 0x72, MRM4r, "vpsrad", X86vsra, X86vsrai, 4178 VR256, v8i32, v4i32, bc_v4i32, loadv2i64, 4179 SSE_INTSHIFT_ITINS_P, 0>, VEX_4V, VEX_L; 4180 4181let ExeDomain = SSEPackedInt, SchedRW = [WriteVecShift], hasSideEffects = 0 in { 4182 // 256-bit logical shifts. 4183 def VPSLLDQYri : PDIi8<0x73, MRM7r, 4184 (outs VR256:$dst), (ins VR256:$src1, u8imm:$src2), 4185 "vpslldq\t{$src2, $src1, $dst|$dst, $src1, $src2}", 4186 [(set VR256:$dst, 4187 (v4i64 (X86vshldq VR256:$src1, (i8 imm:$src2))))]>, 4188 VEX_4V, VEX_L; 4189 def VPSRLDQYri : PDIi8<0x73, MRM3r, 4190 (outs VR256:$dst), (ins VR256:$src1, u8imm:$src2), 4191 "vpsrldq\t{$src2, $src1, $dst|$dst, $src1, $src2}", 4192 [(set VR256:$dst, 4193 (v4i64 (X86vshrdq VR256:$src1, (i8 imm:$src2))))]>, 4194 VEX_4V, VEX_L; 4195 // PSRADQYri doesn't exist in SSE[1-3]. 4196} 4197} // Predicates = [HasAVX2] 4198 4199let Constraints = "$src1 = $dst" in { 4200defm PSLLW : PDI_binop_rmi<0xF1, 0x71, MRM6r, "psllw", X86vshl, X86vshli, 4201 VR128, v8i16, v8i16, bc_v8i16, memopv2i64, 4202 SSE_INTSHIFT_ITINS_P>; 4203defm PSLLD : PDI_binop_rmi<0xF2, 0x72, MRM6r, "pslld", X86vshl, X86vshli, 4204 VR128, v4i32, v4i32, bc_v4i32, memopv2i64, 4205 SSE_INTSHIFT_ITINS_P>; 4206defm PSLLQ : PDI_binop_rmi<0xF3, 0x73, MRM6r, "psllq", X86vshl, X86vshli, 4207 VR128, v2i64, v2i64, bc_v2i64, memopv2i64, 4208 SSE_INTSHIFT_ITINS_P>; 4209 4210defm PSRLW : PDI_binop_rmi<0xD1, 0x71, MRM2r, "psrlw", X86vsrl, X86vsrli, 4211 VR128, v8i16, v8i16, bc_v8i16, memopv2i64, 4212 SSE_INTSHIFT_ITINS_P>; 4213defm PSRLD : PDI_binop_rmi<0xD2, 0x72, MRM2r, "psrld", X86vsrl, X86vsrli, 4214 VR128, v4i32, v4i32, bc_v4i32, memopv2i64, 4215 SSE_INTSHIFT_ITINS_P>; 4216defm PSRLQ : PDI_binop_rmi<0xD3, 0x73, MRM2r, "psrlq", X86vsrl, X86vsrli, 4217 VR128, v2i64, v2i64, bc_v2i64, memopv2i64, 4218 SSE_INTSHIFT_ITINS_P>; 4219 4220defm PSRAW : PDI_binop_rmi<0xE1, 0x71, MRM4r, "psraw", X86vsra, X86vsrai, 4221 VR128, v8i16, v8i16, bc_v8i16, memopv2i64, 4222 SSE_INTSHIFT_ITINS_P>; 4223defm PSRAD : PDI_binop_rmi<0xE2, 0x72, MRM4r, "psrad", X86vsra, X86vsrai, 4224 VR128, v4i32, v4i32, bc_v4i32, memopv2i64, 4225 SSE_INTSHIFT_ITINS_P>; 4226 4227let ExeDomain = SSEPackedInt, SchedRW = [WriteVecShift], hasSideEffects = 0 in { 4228 // 128-bit logical shifts. 4229 def PSLLDQri : PDIi8<0x73, MRM7r, 4230 (outs VR128:$dst), (ins VR128:$src1, u8imm:$src2), 4231 "pslldq\t{$src2, $dst|$dst, $src2}", 4232 [(set VR128:$dst, 4233 (v2i64 (X86vshldq VR128:$src1, (i8 imm:$src2))))], 4234 IIC_SSE_INTSHDQ_P_RI>; 4235 def PSRLDQri : PDIi8<0x73, MRM3r, 4236 (outs VR128:$dst), (ins VR128:$src1, u8imm:$src2), 4237 "psrldq\t{$src2, $dst|$dst, $src2}", 4238 [(set VR128:$dst, 4239 (v2i64 (X86vshrdq VR128:$src1, (i8 imm:$src2))))], 4240 IIC_SSE_INTSHDQ_P_RI>; 4241 // PSRADQri doesn't exist in SSE[1-3]. 4242} 4243} // Constraints = "$src1 = $dst" 4244 4245//===---------------------------------------------------------------------===// 4246// SSE2 - Packed Integer Comparison Instructions 4247//===---------------------------------------------------------------------===// 4248 4249defm PCMPEQB : PDI_binop_all<0x74, "pcmpeqb", X86pcmpeq, v16i8, v32i8, 4250 SSE_INTALU_ITINS_P, 1>; 4251defm PCMPEQW : PDI_binop_all<0x75, "pcmpeqw", X86pcmpeq, v8i16, v16i16, 4252 SSE_INTALU_ITINS_P, 1>; 4253defm PCMPEQD : PDI_binop_all<0x76, "pcmpeqd", X86pcmpeq, v4i32, v8i32, 4254 SSE_INTALU_ITINS_P, 1>; 4255defm PCMPGTB : PDI_binop_all<0x64, "pcmpgtb", X86pcmpgt, v16i8, v32i8, 4256 SSE_INTALU_ITINS_P, 0>; 4257defm PCMPGTW : PDI_binop_all<0x65, "pcmpgtw", X86pcmpgt, v8i16, v16i16, 4258 SSE_INTALU_ITINS_P, 0>; 4259defm PCMPGTD : PDI_binop_all<0x66, "pcmpgtd", X86pcmpgt, v4i32, v8i32, 4260 SSE_INTALU_ITINS_P, 0>; 4261 4262//===---------------------------------------------------------------------===// 4263// SSE2 - Packed Integer Shuffle Instructions 4264//===---------------------------------------------------------------------===// 4265 4266let ExeDomain = SSEPackedInt in { 4267multiclass sse2_pshuffle<string OpcodeStr, ValueType vt128, ValueType vt256, 4268 SDNode OpNode> { 4269let Predicates = [HasAVX] in { 4270 def V#NAME#ri : Ii8<0x70, MRMSrcReg, (outs VR128:$dst), 4271 (ins VR128:$src1, u8imm:$src2), 4272 !strconcat("v", OpcodeStr, 4273 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 4274 [(set VR128:$dst, 4275 (vt128 (OpNode VR128:$src1, (i8 imm:$src2))))], 4276 IIC_SSE_PSHUF_RI>, VEX, Sched<[WriteShuffle]>; 4277 def V#NAME#mi : Ii8<0x70, MRMSrcMem, (outs VR128:$dst), 4278 (ins i128mem:$src1, u8imm:$src2), 4279 !strconcat("v", OpcodeStr, 4280 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 4281 [(set VR128:$dst, 4282 (vt128 (OpNode (bitconvert (loadv2i64 addr:$src1)), 4283 (i8 imm:$src2))))], IIC_SSE_PSHUF_MI>, VEX, 4284 Sched<[WriteShuffleLd]>; 4285} 4286 4287let Predicates = [HasAVX2] in { 4288 def V#NAME#Yri : Ii8<0x70, MRMSrcReg, (outs VR256:$dst), 4289 (ins VR256:$src1, u8imm:$src2), 4290 !strconcat("v", OpcodeStr, 4291 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 4292 [(set VR256:$dst, 4293 (vt256 (OpNode VR256:$src1, (i8 imm:$src2))))], 4294 IIC_SSE_PSHUF_RI>, VEX, VEX_L, Sched<[WriteShuffle]>; 4295 def V#NAME#Ymi : Ii8<0x70, MRMSrcMem, (outs VR256:$dst), 4296 (ins i256mem:$src1, u8imm:$src2), 4297 !strconcat("v", OpcodeStr, 4298 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 4299 [(set VR256:$dst, 4300 (vt256 (OpNode (bitconvert (loadv4i64 addr:$src1)), 4301 (i8 imm:$src2))))], IIC_SSE_PSHUF_MI>, VEX, VEX_L, 4302 Sched<[WriteShuffleLd]>; 4303} 4304 4305let Predicates = [UseSSE2] in { 4306 def ri : Ii8<0x70, MRMSrcReg, 4307 (outs VR128:$dst), (ins VR128:$src1, u8imm:$src2), 4308 !strconcat(OpcodeStr, 4309 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 4310 [(set VR128:$dst, 4311 (vt128 (OpNode VR128:$src1, (i8 imm:$src2))))], 4312 IIC_SSE_PSHUF_RI>, Sched<[WriteShuffle]>; 4313 def mi : Ii8<0x70, MRMSrcMem, 4314 (outs VR128:$dst), (ins i128mem:$src1, u8imm:$src2), 4315 !strconcat(OpcodeStr, 4316 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 4317 [(set VR128:$dst, 4318 (vt128 (OpNode (bitconvert (memopv2i64 addr:$src1)), 4319 (i8 imm:$src2))))], IIC_SSE_PSHUF_MI>, 4320 Sched<[WriteShuffleLd, ReadAfterLd]>; 4321} 4322} 4323} // ExeDomain = SSEPackedInt 4324 4325defm PSHUFD : sse2_pshuffle<"pshufd", v4i32, v8i32, X86PShufd>, PD; 4326defm PSHUFHW : sse2_pshuffle<"pshufhw", v8i16, v16i16, X86PShufhw>, XS; 4327defm PSHUFLW : sse2_pshuffle<"pshuflw", v8i16, v16i16, X86PShuflw>, XD; 4328 4329let Predicates = [HasAVX] in { 4330 def : Pat<(v4f32 (X86PShufd (loadv4f32 addr:$src1), (i8 imm:$imm))), 4331 (VPSHUFDmi addr:$src1, imm:$imm)>; 4332 def : Pat<(v4f32 (X86PShufd VR128:$src1, (i8 imm:$imm))), 4333 (VPSHUFDri VR128:$src1, imm:$imm)>; 4334} 4335 4336let Predicates = [UseSSE2] in { 4337 def : Pat<(v4f32 (X86PShufd (memopv4f32 addr:$src1), (i8 imm:$imm))), 4338 (PSHUFDmi addr:$src1, imm:$imm)>; 4339 def : Pat<(v4f32 (X86PShufd VR128:$src1, (i8 imm:$imm))), 4340 (PSHUFDri VR128:$src1, imm:$imm)>; 4341} 4342 4343//===---------------------------------------------------------------------===// 4344// Packed Integer Pack Instructions (SSE & AVX) 4345//===---------------------------------------------------------------------===// 4346 4347let ExeDomain = SSEPackedInt in { 4348multiclass sse2_pack<bits<8> opc, string OpcodeStr, ValueType OutVT, 4349 ValueType ArgVT, SDNode OpNode, PatFrag bc_frag, 4350 PatFrag ld_frag, bit Is2Addr = 1> { 4351 def rr : PDI<opc, MRMSrcReg, 4352 (outs VR128:$dst), (ins VR128:$src1, VR128:$src2), 4353 !if(Is2Addr, 4354 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 4355 !strconcat(OpcodeStr, 4356 "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 4357 [(set VR128:$dst, 4358 (OutVT (OpNode (ArgVT VR128:$src1), VR128:$src2)))]>, 4359 Sched<[WriteShuffle]>; 4360 def rm : PDI<opc, MRMSrcMem, 4361 (outs VR128:$dst), (ins VR128:$src1, i128mem:$src2), 4362 !if(Is2Addr, 4363 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 4364 !strconcat(OpcodeStr, 4365 "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 4366 [(set VR128:$dst, 4367 (OutVT (OpNode VR128:$src1, 4368 (bc_frag (ld_frag addr:$src2)))))]>, 4369 Sched<[WriteShuffleLd, ReadAfterLd]>; 4370} 4371 4372multiclass sse2_pack_y<bits<8> opc, string OpcodeStr, ValueType OutVT, 4373 ValueType ArgVT, SDNode OpNode, PatFrag bc_frag> { 4374 def Yrr : PDI<opc, MRMSrcReg, 4375 (outs VR256:$dst), (ins VR256:$src1, VR256:$src2), 4376 !strconcat(OpcodeStr, 4377 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 4378 [(set VR256:$dst, 4379 (OutVT (OpNode (ArgVT VR256:$src1), VR256:$src2)))]>, 4380 Sched<[WriteShuffle]>; 4381 def Yrm : PDI<opc, MRMSrcMem, 4382 (outs VR256:$dst), (ins VR256:$src1, i256mem:$src2), 4383 !strconcat(OpcodeStr, 4384 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 4385 [(set VR256:$dst, 4386 (OutVT (OpNode VR256:$src1, 4387 (bc_frag (loadv4i64 addr:$src2)))))]>, 4388 Sched<[WriteShuffleLd, ReadAfterLd]>; 4389} 4390 4391multiclass sse4_pack<bits<8> opc, string OpcodeStr, ValueType OutVT, 4392 ValueType ArgVT, SDNode OpNode, PatFrag bc_frag, 4393 PatFrag ld_frag, bit Is2Addr = 1> { 4394 def rr : SS48I<opc, MRMSrcReg, 4395 (outs VR128:$dst), (ins VR128:$src1, VR128:$src2), 4396 !if(Is2Addr, 4397 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 4398 !strconcat(OpcodeStr, 4399 "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 4400 [(set VR128:$dst, 4401 (OutVT (OpNode (ArgVT VR128:$src1), VR128:$src2)))]>, 4402 Sched<[WriteShuffle]>; 4403 def rm : SS48I<opc, MRMSrcMem, 4404 (outs VR128:$dst), (ins VR128:$src1, i128mem:$src2), 4405 !if(Is2Addr, 4406 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 4407 !strconcat(OpcodeStr, 4408 "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 4409 [(set VR128:$dst, 4410 (OutVT (OpNode VR128:$src1, 4411 (bc_frag (ld_frag addr:$src2)))))]>, 4412 Sched<[WriteShuffleLd, ReadAfterLd]>; 4413} 4414 4415multiclass sse4_pack_y<bits<8> opc, string OpcodeStr, ValueType OutVT, 4416 ValueType ArgVT, SDNode OpNode, PatFrag bc_frag> { 4417 def Yrr : SS48I<opc, MRMSrcReg, 4418 (outs VR256:$dst), (ins VR256:$src1, VR256:$src2), 4419 !strconcat(OpcodeStr, 4420 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 4421 [(set VR256:$dst, 4422 (OutVT (OpNode (ArgVT VR256:$src1), VR256:$src2)))]>, 4423 Sched<[WriteShuffle]>; 4424 def Yrm : SS48I<opc, MRMSrcMem, 4425 (outs VR256:$dst), (ins VR256:$src1, i256mem:$src2), 4426 !strconcat(OpcodeStr, 4427 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 4428 [(set VR256:$dst, 4429 (OutVT (OpNode VR256:$src1, 4430 (bc_frag (loadv4i64 addr:$src2)))))]>, 4431 Sched<[WriteShuffleLd, ReadAfterLd]>; 4432} 4433 4434let Predicates = [HasAVX] in { 4435 defm VPACKSSWB : sse2_pack<0x63, "vpacksswb", v16i8, v8i16, X86Packss, 4436 bc_v8i16, loadv2i64, 0>, VEX_4V; 4437 defm VPACKSSDW : sse2_pack<0x6B, "vpackssdw", v8i16, v4i32, X86Packss, 4438 bc_v4i32, loadv2i64, 0>, VEX_4V; 4439 4440 defm VPACKUSWB : sse2_pack<0x67, "vpackuswb", v16i8, v8i16, X86Packus, 4441 bc_v8i16, loadv2i64, 0>, VEX_4V; 4442 defm VPACKUSDW : sse4_pack<0x2B, "vpackusdw", v8i16, v4i32, X86Packus, 4443 bc_v4i32, loadv2i64, 0>, VEX_4V; 4444} 4445 4446let Predicates = [HasAVX2] in { 4447 defm VPACKSSWB : sse2_pack_y<0x63, "vpacksswb", v32i8, v16i16, X86Packss, 4448 bc_v16i16>, VEX_4V, VEX_L; 4449 defm VPACKSSDW : sse2_pack_y<0x6B, "vpackssdw", v16i16, v8i32, X86Packss, 4450 bc_v8i32>, VEX_4V, VEX_L; 4451 4452 defm VPACKUSWB : sse2_pack_y<0x67, "vpackuswb", v32i8, v16i16, X86Packus, 4453 bc_v16i16>, VEX_4V, VEX_L; 4454 defm VPACKUSDW : sse4_pack_y<0x2B, "vpackusdw", v16i16, v8i32, X86Packus, 4455 bc_v8i32>, VEX_4V, VEX_L; 4456} 4457 4458let Constraints = "$src1 = $dst" in { 4459 defm PACKSSWB : sse2_pack<0x63, "packsswb", v16i8, v8i16, X86Packss, 4460 bc_v8i16, memopv2i64>; 4461 defm PACKSSDW : sse2_pack<0x6B, "packssdw", v8i16, v4i32, X86Packss, 4462 bc_v4i32, memopv2i64>; 4463 4464 defm PACKUSWB : sse2_pack<0x67, "packuswb", v16i8, v8i16, X86Packus, 4465 bc_v8i16, memopv2i64>; 4466 4467 let Predicates = [HasSSE41] in 4468 defm PACKUSDW : sse4_pack<0x2B, "packusdw", v8i16, v4i32, X86Packus, 4469 bc_v4i32, memopv2i64>; 4470} 4471} // ExeDomain = SSEPackedInt 4472 4473//===---------------------------------------------------------------------===// 4474// SSE2 - Packed Integer Unpack Instructions 4475//===---------------------------------------------------------------------===// 4476 4477let ExeDomain = SSEPackedInt in { 4478multiclass sse2_unpack<bits<8> opc, string OpcodeStr, ValueType vt, 4479 SDNode OpNode, PatFrag bc_frag, PatFrag ld_frag, 4480 bit Is2Addr = 1> { 4481 def rr : PDI<opc, MRMSrcReg, 4482 (outs VR128:$dst), (ins VR128:$src1, VR128:$src2), 4483 !if(Is2Addr, 4484 !strconcat(OpcodeStr,"\t{$src2, $dst|$dst, $src2}"), 4485 !strconcat(OpcodeStr,"\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 4486 [(set VR128:$dst, (vt (OpNode VR128:$src1, VR128:$src2)))], 4487 IIC_SSE_UNPCK>, Sched<[WriteShuffle]>; 4488 def rm : PDI<opc, MRMSrcMem, 4489 (outs VR128:$dst), (ins VR128:$src1, i128mem:$src2), 4490 !if(Is2Addr, 4491 !strconcat(OpcodeStr,"\t{$src2, $dst|$dst, $src2}"), 4492 !strconcat(OpcodeStr,"\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 4493 [(set VR128:$dst, (OpNode VR128:$src1, 4494 (bc_frag (ld_frag addr:$src2))))], 4495 IIC_SSE_UNPCK>, 4496 Sched<[WriteShuffleLd, ReadAfterLd]>; 4497} 4498 4499multiclass sse2_unpack_y<bits<8> opc, string OpcodeStr, ValueType vt, 4500 SDNode OpNode, PatFrag bc_frag> { 4501 def Yrr : PDI<opc, MRMSrcReg, 4502 (outs VR256:$dst), (ins VR256:$src1, VR256:$src2), 4503 !strconcat(OpcodeStr,"\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 4504 [(set VR256:$dst, (vt (OpNode VR256:$src1, VR256:$src2)))]>, 4505 Sched<[WriteShuffle]>; 4506 def Yrm : PDI<opc, MRMSrcMem, 4507 (outs VR256:$dst), (ins VR256:$src1, i256mem:$src2), 4508 !strconcat(OpcodeStr,"\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 4509 [(set VR256:$dst, (OpNode VR256:$src1, 4510 (bc_frag (loadv4i64 addr:$src2))))]>, 4511 Sched<[WriteShuffleLd, ReadAfterLd]>; 4512} 4513 4514let Predicates = [HasAVX] in { 4515 defm VPUNPCKLBW : sse2_unpack<0x60, "vpunpcklbw", v16i8, X86Unpckl, 4516 bc_v16i8, loadv2i64, 0>, VEX_4V; 4517 defm VPUNPCKLWD : sse2_unpack<0x61, "vpunpcklwd", v8i16, X86Unpckl, 4518 bc_v8i16, loadv2i64, 0>, VEX_4V; 4519 defm VPUNPCKLDQ : sse2_unpack<0x62, "vpunpckldq", v4i32, X86Unpckl, 4520 bc_v4i32, loadv2i64, 0>, VEX_4V; 4521 defm VPUNPCKLQDQ : sse2_unpack<0x6C, "vpunpcklqdq", v2i64, X86Unpckl, 4522 bc_v2i64, loadv2i64, 0>, VEX_4V; 4523 4524 defm VPUNPCKHBW : sse2_unpack<0x68, "vpunpckhbw", v16i8, X86Unpckh, 4525 bc_v16i8, loadv2i64, 0>, VEX_4V; 4526 defm VPUNPCKHWD : sse2_unpack<0x69, "vpunpckhwd", v8i16, X86Unpckh, 4527 bc_v8i16, loadv2i64, 0>, VEX_4V; 4528 defm VPUNPCKHDQ : sse2_unpack<0x6A, "vpunpckhdq", v4i32, X86Unpckh, 4529 bc_v4i32, loadv2i64, 0>, VEX_4V; 4530 defm VPUNPCKHQDQ : sse2_unpack<0x6D, "vpunpckhqdq", v2i64, X86Unpckh, 4531 bc_v2i64, loadv2i64, 0>, VEX_4V; 4532} 4533 4534let Predicates = [HasAVX2] in { 4535 defm VPUNPCKLBW : sse2_unpack_y<0x60, "vpunpcklbw", v32i8, X86Unpckl, 4536 bc_v32i8>, VEX_4V, VEX_L; 4537 defm VPUNPCKLWD : sse2_unpack_y<0x61, "vpunpcklwd", v16i16, X86Unpckl, 4538 bc_v16i16>, VEX_4V, VEX_L; 4539 defm VPUNPCKLDQ : sse2_unpack_y<0x62, "vpunpckldq", v8i32, X86Unpckl, 4540 bc_v8i32>, VEX_4V, VEX_L; 4541 defm VPUNPCKLQDQ : sse2_unpack_y<0x6C, "vpunpcklqdq", v4i64, X86Unpckl, 4542 bc_v4i64>, VEX_4V, VEX_L; 4543 4544 defm VPUNPCKHBW : sse2_unpack_y<0x68, "vpunpckhbw", v32i8, X86Unpckh, 4545 bc_v32i8>, VEX_4V, VEX_L; 4546 defm VPUNPCKHWD : sse2_unpack_y<0x69, "vpunpckhwd", v16i16, X86Unpckh, 4547 bc_v16i16>, VEX_4V, VEX_L; 4548 defm VPUNPCKHDQ : sse2_unpack_y<0x6A, "vpunpckhdq", v8i32, X86Unpckh, 4549 bc_v8i32>, VEX_4V, VEX_L; 4550 defm VPUNPCKHQDQ : sse2_unpack_y<0x6D, "vpunpckhqdq", v4i64, X86Unpckh, 4551 bc_v4i64>, VEX_4V, VEX_L; 4552} 4553 4554let Constraints = "$src1 = $dst" in { 4555 defm PUNPCKLBW : sse2_unpack<0x60, "punpcklbw", v16i8, X86Unpckl, 4556 bc_v16i8, memopv2i64>; 4557 defm PUNPCKLWD : sse2_unpack<0x61, "punpcklwd", v8i16, X86Unpckl, 4558 bc_v8i16, memopv2i64>; 4559 defm PUNPCKLDQ : sse2_unpack<0x62, "punpckldq", v4i32, X86Unpckl, 4560 bc_v4i32, memopv2i64>; 4561 defm PUNPCKLQDQ : sse2_unpack<0x6C, "punpcklqdq", v2i64, X86Unpckl, 4562 bc_v2i64, memopv2i64>; 4563 4564 defm PUNPCKHBW : sse2_unpack<0x68, "punpckhbw", v16i8, X86Unpckh, 4565 bc_v16i8, memopv2i64>; 4566 defm PUNPCKHWD : sse2_unpack<0x69, "punpckhwd", v8i16, X86Unpckh, 4567 bc_v8i16, memopv2i64>; 4568 defm PUNPCKHDQ : sse2_unpack<0x6A, "punpckhdq", v4i32, X86Unpckh, 4569 bc_v4i32, memopv2i64>; 4570 defm PUNPCKHQDQ : sse2_unpack<0x6D, "punpckhqdq", v2i64, X86Unpckh, 4571 bc_v2i64, memopv2i64>; 4572} 4573} // ExeDomain = SSEPackedInt 4574 4575//===---------------------------------------------------------------------===// 4576// SSE2 - Packed Integer Extract and Insert 4577//===---------------------------------------------------------------------===// 4578 4579let ExeDomain = SSEPackedInt in { 4580multiclass sse2_pinsrw<bit Is2Addr = 1> { 4581 def rri : Ii8<0xC4, MRMSrcReg, 4582 (outs VR128:$dst), (ins VR128:$src1, 4583 GR32orGR64:$src2, u8imm:$src3), 4584 !if(Is2Addr, 4585 "pinsrw\t{$src3, $src2, $dst|$dst, $src2, $src3}", 4586 "vpinsrw\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), 4587 [(set VR128:$dst, 4588 (X86pinsrw VR128:$src1, GR32orGR64:$src2, imm:$src3))], 4589 IIC_SSE_PINSRW>, Sched<[WriteShuffle]>; 4590 def rmi : Ii8<0xC4, MRMSrcMem, 4591 (outs VR128:$dst), (ins VR128:$src1, 4592 i16mem:$src2, u8imm:$src3), 4593 !if(Is2Addr, 4594 "pinsrw\t{$src3, $src2, $dst|$dst, $src2, $src3}", 4595 "vpinsrw\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), 4596 [(set VR128:$dst, 4597 (X86pinsrw VR128:$src1, (extloadi16 addr:$src2), 4598 imm:$src3))], IIC_SSE_PINSRW>, 4599 Sched<[WriteShuffleLd, ReadAfterLd]>; 4600} 4601 4602// Extract 4603let Predicates = [HasAVX] in 4604def VPEXTRWri : Ii8<0xC5, MRMSrcReg, 4605 (outs GR32orGR64:$dst), (ins VR128:$src1, u8imm:$src2), 4606 "vpextrw\t{$src2, $src1, $dst|$dst, $src1, $src2}", 4607 [(set GR32orGR64:$dst, (X86pextrw (v8i16 VR128:$src1), 4608 imm:$src2))]>, PD, VEX, 4609 Sched<[WriteShuffle]>; 4610def PEXTRWri : PDIi8<0xC5, MRMSrcReg, 4611 (outs GR32orGR64:$dst), (ins VR128:$src1, u8imm:$src2), 4612 "pextrw\t{$src2, $src1, $dst|$dst, $src1, $src2}", 4613 [(set GR32orGR64:$dst, (X86pextrw (v8i16 VR128:$src1), 4614 imm:$src2))], IIC_SSE_PEXTRW>, 4615 Sched<[WriteShuffleLd, ReadAfterLd]>; 4616 4617// Insert 4618let Predicates = [HasAVX] in 4619defm VPINSRW : sse2_pinsrw<0>, PD, VEX_4V; 4620 4621let Predicates = [UseSSE2], Constraints = "$src1 = $dst" in 4622defm PINSRW : sse2_pinsrw, PD; 4623 4624} // ExeDomain = SSEPackedInt 4625 4626//===---------------------------------------------------------------------===// 4627// SSE2 - Packed Mask Creation 4628//===---------------------------------------------------------------------===// 4629 4630let ExeDomain = SSEPackedInt, SchedRW = [WriteVecLogic] in { 4631 4632def VPMOVMSKBrr : VPDI<0xD7, MRMSrcReg, (outs GR32orGR64:$dst), 4633 (ins VR128:$src), 4634 "pmovmskb\t{$src, $dst|$dst, $src}", 4635 [(set GR32orGR64:$dst, (int_x86_sse2_pmovmskb_128 VR128:$src))], 4636 IIC_SSE_MOVMSK>, VEX; 4637 4638let Predicates = [HasAVX2] in { 4639def VPMOVMSKBYrr : VPDI<0xD7, MRMSrcReg, (outs GR32orGR64:$dst), 4640 (ins VR256:$src), 4641 "pmovmskb\t{$src, $dst|$dst, $src}", 4642 [(set GR32orGR64:$dst, (int_x86_avx2_pmovmskb VR256:$src))]>, 4643 VEX, VEX_L; 4644} 4645 4646def PMOVMSKBrr : PDI<0xD7, MRMSrcReg, (outs GR32orGR64:$dst), (ins VR128:$src), 4647 "pmovmskb\t{$src, $dst|$dst, $src}", 4648 [(set GR32orGR64:$dst, (int_x86_sse2_pmovmskb_128 VR128:$src))], 4649 IIC_SSE_MOVMSK>; 4650 4651} // ExeDomain = SSEPackedInt 4652 4653//===---------------------------------------------------------------------===// 4654// SSE2 - Conditional Store 4655//===---------------------------------------------------------------------===// 4656 4657let ExeDomain = SSEPackedInt, SchedRW = [WriteStore] in { 4658 4659let Uses = [EDI], Predicates = [HasAVX,Not64BitMode] in 4660def VMASKMOVDQU : VPDI<0xF7, MRMSrcReg, (outs), 4661 (ins VR128:$src, VR128:$mask), 4662 "maskmovdqu\t{$mask, $src|$src, $mask}", 4663 [(int_x86_sse2_maskmov_dqu VR128:$src, VR128:$mask, EDI)], 4664 IIC_SSE_MASKMOV>, VEX; 4665let Uses = [RDI], Predicates = [HasAVX,In64BitMode] in 4666def VMASKMOVDQU64 : VPDI<0xF7, MRMSrcReg, (outs), 4667 (ins VR128:$src, VR128:$mask), 4668 "maskmovdqu\t{$mask, $src|$src, $mask}", 4669 [(int_x86_sse2_maskmov_dqu VR128:$src, VR128:$mask, RDI)], 4670 IIC_SSE_MASKMOV>, VEX; 4671 4672let Uses = [EDI], Predicates = [UseSSE2,Not64BitMode] in 4673def MASKMOVDQU : PDI<0xF7, MRMSrcReg, (outs), (ins VR128:$src, VR128:$mask), 4674 "maskmovdqu\t{$mask, $src|$src, $mask}", 4675 [(int_x86_sse2_maskmov_dqu VR128:$src, VR128:$mask, EDI)], 4676 IIC_SSE_MASKMOV>; 4677let Uses = [RDI], Predicates = [UseSSE2,In64BitMode] in 4678def MASKMOVDQU64 : PDI<0xF7, MRMSrcReg, (outs), (ins VR128:$src, VR128:$mask), 4679 "maskmovdqu\t{$mask, $src|$src, $mask}", 4680 [(int_x86_sse2_maskmov_dqu VR128:$src, VR128:$mask, RDI)], 4681 IIC_SSE_MASKMOV>; 4682 4683} // ExeDomain = SSEPackedInt 4684 4685//===---------------------------------------------------------------------===// 4686// SSE2 - Move Doubleword 4687//===---------------------------------------------------------------------===// 4688 4689//===---------------------------------------------------------------------===// 4690// Move Int Doubleword to Packed Double Int 4691// 4692def VMOVDI2PDIrr : VS2I<0x6E, MRMSrcReg, (outs VR128:$dst), (ins GR32:$src), 4693 "movd\t{$src, $dst|$dst, $src}", 4694 [(set VR128:$dst, 4695 (v4i32 (scalar_to_vector GR32:$src)))], IIC_SSE_MOVDQ>, 4696 VEX, Sched<[WriteMove]>; 4697def VMOVDI2PDIrm : VS2I<0x6E, MRMSrcMem, (outs VR128:$dst), (ins i32mem:$src), 4698 "movd\t{$src, $dst|$dst, $src}", 4699 [(set VR128:$dst, 4700 (v4i32 (scalar_to_vector (loadi32 addr:$src))))], 4701 IIC_SSE_MOVDQ>, 4702 VEX, Sched<[WriteLoad]>; 4703def VMOV64toPQIrr : VRS2I<0x6E, MRMSrcReg, (outs VR128:$dst), (ins GR64:$src), 4704 "movq\t{$src, $dst|$dst, $src}", 4705 [(set VR128:$dst, 4706 (v2i64 (scalar_to_vector GR64:$src)))], 4707 IIC_SSE_MOVDQ>, VEX, Sched<[WriteMove]>; 4708let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0, mayLoad = 1 in 4709def VMOV64toPQIrm : VRS2I<0x6E, MRMSrcMem, (outs VR128:$dst), (ins i64mem:$src), 4710 "movq\t{$src, $dst|$dst, $src}", 4711 [], IIC_SSE_MOVDQ>, VEX, Sched<[WriteLoad]>; 4712let isCodeGenOnly = 1 in 4713def VMOV64toSDrr : VRS2I<0x6E, MRMSrcReg, (outs FR64:$dst), (ins GR64:$src), 4714 "movq\t{$src, $dst|$dst, $src}", 4715 [(set FR64:$dst, (bitconvert GR64:$src))], 4716 IIC_SSE_MOVDQ>, VEX, Sched<[WriteMove]>; 4717 4718def MOVDI2PDIrr : S2I<0x6E, MRMSrcReg, (outs VR128:$dst), (ins GR32:$src), 4719 "movd\t{$src, $dst|$dst, $src}", 4720 [(set VR128:$dst, 4721 (v4i32 (scalar_to_vector GR32:$src)))], IIC_SSE_MOVDQ>, 4722 Sched<[WriteMove]>; 4723def MOVDI2PDIrm : S2I<0x6E, MRMSrcMem, (outs VR128:$dst), (ins i32mem:$src), 4724 "movd\t{$src, $dst|$dst, $src}", 4725 [(set VR128:$dst, 4726 (v4i32 (scalar_to_vector (loadi32 addr:$src))))], 4727 IIC_SSE_MOVDQ>, Sched<[WriteLoad]>; 4728def MOV64toPQIrr : RS2I<0x6E, MRMSrcReg, (outs VR128:$dst), (ins GR64:$src), 4729 "mov{d|q}\t{$src, $dst|$dst, $src}", 4730 [(set VR128:$dst, 4731 (v2i64 (scalar_to_vector GR64:$src)))], 4732 IIC_SSE_MOVDQ>, Sched<[WriteMove]>; 4733let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0, mayLoad = 1 in 4734def MOV64toPQIrm : RS2I<0x6E, MRMSrcMem, (outs VR128:$dst), (ins i64mem:$src), 4735 "mov{d|q}\t{$src, $dst|$dst, $src}", 4736 [], IIC_SSE_MOVDQ>, Sched<[WriteLoad]>; 4737let isCodeGenOnly = 1 in 4738def MOV64toSDrr : RS2I<0x6E, MRMSrcReg, (outs FR64:$dst), (ins GR64:$src), 4739 "mov{d|q}\t{$src, $dst|$dst, $src}", 4740 [(set FR64:$dst, (bitconvert GR64:$src))], 4741 IIC_SSE_MOVDQ>, Sched<[WriteMove]>; 4742 4743//===---------------------------------------------------------------------===// 4744// Move Int Doubleword to Single Scalar 4745// 4746let isCodeGenOnly = 1 in { 4747 def VMOVDI2SSrr : VS2I<0x6E, MRMSrcReg, (outs FR32:$dst), (ins GR32:$src), 4748 "movd\t{$src, $dst|$dst, $src}", 4749 [(set FR32:$dst, (bitconvert GR32:$src))], 4750 IIC_SSE_MOVDQ>, VEX, Sched<[WriteMove]>; 4751 4752 def VMOVDI2SSrm : VS2I<0x6E, MRMSrcMem, (outs FR32:$dst), (ins i32mem:$src), 4753 "movd\t{$src, $dst|$dst, $src}", 4754 [(set FR32:$dst, (bitconvert (loadi32 addr:$src)))], 4755 IIC_SSE_MOVDQ>, 4756 VEX, Sched<[WriteLoad]>; 4757 def MOVDI2SSrr : S2I<0x6E, MRMSrcReg, (outs FR32:$dst), (ins GR32:$src), 4758 "movd\t{$src, $dst|$dst, $src}", 4759 [(set FR32:$dst, (bitconvert GR32:$src))], 4760 IIC_SSE_MOVDQ>, Sched<[WriteMove]>; 4761 4762 def MOVDI2SSrm : S2I<0x6E, MRMSrcMem, (outs FR32:$dst), (ins i32mem:$src), 4763 "movd\t{$src, $dst|$dst, $src}", 4764 [(set FR32:$dst, (bitconvert (loadi32 addr:$src)))], 4765 IIC_SSE_MOVDQ>, Sched<[WriteLoad]>; 4766} 4767 4768//===---------------------------------------------------------------------===// 4769// Move Packed Doubleword Int to Packed Double Int 4770// 4771def VMOVPDI2DIrr : VS2I<0x7E, MRMDestReg, (outs GR32:$dst), (ins VR128:$src), 4772 "movd\t{$src, $dst|$dst, $src}", 4773 [(set GR32:$dst, (vector_extract (v4i32 VR128:$src), 4774 (iPTR 0)))], IIC_SSE_MOVD_ToGP>, VEX, 4775 Sched<[WriteMove]>; 4776def VMOVPDI2DImr : VS2I<0x7E, MRMDestMem, (outs), 4777 (ins i32mem:$dst, VR128:$src), 4778 "movd\t{$src, $dst|$dst, $src}", 4779 [(store (i32 (vector_extract (v4i32 VR128:$src), 4780 (iPTR 0))), addr:$dst)], IIC_SSE_MOVDQ>, 4781 VEX, Sched<[WriteStore]>; 4782def MOVPDI2DIrr : S2I<0x7E, MRMDestReg, (outs GR32:$dst), (ins VR128:$src), 4783 "movd\t{$src, $dst|$dst, $src}", 4784 [(set GR32:$dst, (vector_extract (v4i32 VR128:$src), 4785 (iPTR 0)))], IIC_SSE_MOVD_ToGP>, 4786 Sched<[WriteMove]>; 4787def MOVPDI2DImr : S2I<0x7E, MRMDestMem, (outs), (ins i32mem:$dst, VR128:$src), 4788 "movd\t{$src, $dst|$dst, $src}", 4789 [(store (i32 (vector_extract (v4i32 VR128:$src), 4790 (iPTR 0))), addr:$dst)], 4791 IIC_SSE_MOVDQ>, Sched<[WriteStore]>; 4792 4793def : Pat<(v8i32 (X86Vinsert (v8i32 immAllZerosV), GR32:$src2, (iPTR 0))), 4794 (SUBREG_TO_REG (i32 0), (VMOVDI2PDIrr GR32:$src2), sub_xmm)>; 4795 4796def : Pat<(v4i64 (X86Vinsert (bc_v4i64 (v8i32 immAllZerosV)), GR64:$src2, (iPTR 0))), 4797 (SUBREG_TO_REG (i32 0), (VMOV64toPQIrr GR64:$src2), sub_xmm)>; 4798 4799def : Pat<(v8i32 (X86Vinsert undef, GR32:$src2, (iPTR 0))), 4800 (SUBREG_TO_REG (i32 0), (VMOVDI2PDIrr GR32:$src2), sub_xmm)>; 4801 4802def : Pat<(v4i64 (X86Vinsert undef, GR64:$src2, (iPTR 0))), 4803 (SUBREG_TO_REG (i32 0), (VMOV64toPQIrr GR64:$src2), sub_xmm)>; 4804 4805//===---------------------------------------------------------------------===// 4806// Move Packed Doubleword Int first element to Doubleword Int 4807// 4808let SchedRW = [WriteMove] in { 4809def VMOVPQIto64rr : VRS2I<0x7E, MRMDestReg, (outs GR64:$dst), (ins VR128:$src), 4810 "movq\t{$src, $dst|$dst, $src}", 4811 [(set GR64:$dst, (vector_extract (v2i64 VR128:$src), 4812 (iPTR 0)))], 4813 IIC_SSE_MOVD_ToGP>, 4814 VEX; 4815 4816def MOVPQIto64rr : RS2I<0x7E, MRMDestReg, (outs GR64:$dst), (ins VR128:$src), 4817 "mov{d|q}\t{$src, $dst|$dst, $src}", 4818 [(set GR64:$dst, (vector_extract (v2i64 VR128:$src), 4819 (iPTR 0)))], 4820 IIC_SSE_MOVD_ToGP>; 4821} //SchedRW 4822 4823let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0, mayStore = 1 in 4824def VMOVPQIto64rm : VRS2I<0x7E, MRMDestMem, (outs i64mem:$dst), 4825 (ins VR128:$src), "movq\t{$src, $dst|$dst, $src}", 4826 [], IIC_SSE_MOVDQ>, VEX, Sched<[WriteStore]>; 4827let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0, mayStore = 1 in 4828def MOVPQIto64rm : RS2I<0x7E, MRMDestMem, (outs i64mem:$dst), (ins VR128:$src), 4829 "mov{d|q}\t{$src, $dst|$dst, $src}", 4830 [], IIC_SSE_MOVDQ>, Sched<[WriteStore]>; 4831 4832//===---------------------------------------------------------------------===// 4833// Bitcast FR64 <-> GR64 4834// 4835let isCodeGenOnly = 1 in { 4836 let Predicates = [UseAVX] in 4837 def VMOV64toSDrm : VS2SI<0x7E, MRMSrcMem, (outs FR64:$dst), (ins i64mem:$src), 4838 "movq\t{$src, $dst|$dst, $src}", 4839 [(set FR64:$dst, (bitconvert (loadi64 addr:$src)))]>, 4840 VEX, Sched<[WriteLoad]>; 4841 def VMOVSDto64rr : VRS2I<0x7E, MRMDestReg, (outs GR64:$dst), (ins FR64:$src), 4842 "movq\t{$src, $dst|$dst, $src}", 4843 [(set GR64:$dst, (bitconvert FR64:$src))], 4844 IIC_SSE_MOVDQ>, VEX, Sched<[WriteMove]>; 4845 def VMOVSDto64mr : VRS2I<0x7E, MRMDestMem, (outs), (ins i64mem:$dst, FR64:$src), 4846 "movq\t{$src, $dst|$dst, $src}", 4847 [(store (i64 (bitconvert FR64:$src)), addr:$dst)], 4848 IIC_SSE_MOVDQ>, VEX, Sched<[WriteStore]>; 4849 4850 def MOV64toSDrm : S2SI<0x7E, MRMSrcMem, (outs FR64:$dst), (ins i64mem:$src), 4851 "movq\t{$src, $dst|$dst, $src}", 4852 [(set FR64:$dst, (bitconvert (loadi64 addr:$src)))], 4853 IIC_SSE_MOVDQ>, Sched<[WriteLoad]>; 4854 def MOVSDto64rr : RS2I<0x7E, MRMDestReg, (outs GR64:$dst), (ins FR64:$src), 4855 "mov{d|q}\t{$src, $dst|$dst, $src}", 4856 [(set GR64:$dst, (bitconvert FR64:$src))], 4857 IIC_SSE_MOVD_ToGP>, Sched<[WriteMove]>; 4858 def MOVSDto64mr : RS2I<0x7E, MRMDestMem, (outs), (ins i64mem:$dst, FR64:$src), 4859 "movq\t{$src, $dst|$dst, $src}", 4860 [(store (i64 (bitconvert FR64:$src)), addr:$dst)], 4861 IIC_SSE_MOVDQ>, Sched<[WriteStore]>; 4862} 4863 4864//===---------------------------------------------------------------------===// 4865// Move Scalar Single to Double Int 4866// 4867let isCodeGenOnly = 1 in { 4868 def VMOVSS2DIrr : VS2I<0x7E, MRMDestReg, (outs GR32:$dst), (ins FR32:$src), 4869 "movd\t{$src, $dst|$dst, $src}", 4870 [(set GR32:$dst, (bitconvert FR32:$src))], 4871 IIC_SSE_MOVD_ToGP>, VEX, Sched<[WriteMove]>; 4872 def VMOVSS2DImr : VS2I<0x7E, MRMDestMem, (outs), (ins i32mem:$dst, FR32:$src), 4873 "movd\t{$src, $dst|$dst, $src}", 4874 [(store (i32 (bitconvert FR32:$src)), addr:$dst)], 4875 IIC_SSE_MOVDQ>, VEX, Sched<[WriteStore]>; 4876 def MOVSS2DIrr : S2I<0x7E, MRMDestReg, (outs GR32:$dst), (ins FR32:$src), 4877 "movd\t{$src, $dst|$dst, $src}", 4878 [(set GR32:$dst, (bitconvert FR32:$src))], 4879 IIC_SSE_MOVD_ToGP>, Sched<[WriteMove]>; 4880 def MOVSS2DImr : S2I<0x7E, MRMDestMem, (outs), (ins i32mem:$dst, FR32:$src), 4881 "movd\t{$src, $dst|$dst, $src}", 4882 [(store (i32 (bitconvert FR32:$src)), addr:$dst)], 4883 IIC_SSE_MOVDQ>, Sched<[WriteStore]>; 4884} 4885 4886//===---------------------------------------------------------------------===// 4887// Patterns and instructions to describe movd/movq to XMM register zero-extends 4888// 4889let isCodeGenOnly = 1, SchedRW = [WriteMove] in { 4890let AddedComplexity = 15 in { 4891def VMOVZQI2PQIrr : VS2I<0x6E, MRMSrcReg, (outs VR128:$dst), (ins GR64:$src), 4892 "movq\t{$src, $dst|$dst, $src}", // X86-64 only 4893 [(set VR128:$dst, (v2i64 (X86vzmovl 4894 (v2i64 (scalar_to_vector GR64:$src)))))], 4895 IIC_SSE_MOVDQ>, 4896 VEX, VEX_W; 4897def MOVZQI2PQIrr : RS2I<0x6E, MRMSrcReg, (outs VR128:$dst), (ins GR64:$src), 4898 "mov{d|q}\t{$src, $dst|$dst, $src}", // X86-64 only 4899 [(set VR128:$dst, (v2i64 (X86vzmovl 4900 (v2i64 (scalar_to_vector GR64:$src)))))], 4901 IIC_SSE_MOVDQ>; 4902} 4903} // isCodeGenOnly, SchedRW 4904 4905let Predicates = [UseAVX] in { 4906 let AddedComplexity = 15 in 4907 def : Pat<(v4i32 (X86vzmovl (v4i32 (scalar_to_vector GR32:$src)))), 4908 (VMOVDI2PDIrr GR32:$src)>; 4909 4910 // AVX 128-bit movd/movq instructions write zeros in the high 128-bit part. 4911 // These instructions also write zeros in the high part of a 256-bit register. 4912 let AddedComplexity = 20 in { 4913 def : Pat<(v4i32 (X86vzmovl (v4i32 (scalar_to_vector (loadi32 addr:$src))))), 4914 (VMOVDI2PDIrm addr:$src)>; 4915 def : Pat<(v4i32 (X86vzmovl (bc_v4i32 (loadv4f32 addr:$src)))), 4916 (VMOVDI2PDIrm addr:$src)>; 4917 def : Pat<(v4i32 (X86vzmovl (bc_v4i32 (loadv2i64 addr:$src)))), 4918 (VMOVDI2PDIrm addr:$src)>; 4919 def : Pat<(v8i32 (X86vzmovl (insert_subvector undef, 4920 (v4i32 (scalar_to_vector (loadi32 addr:$src))), (iPTR 0)))), 4921 (SUBREG_TO_REG (i32 0), (VMOVDI2PDIrm addr:$src), sub_xmm)>; 4922 } 4923 // Use regular 128-bit instructions to match 256-bit scalar_to_vec+zext. 4924 def : Pat<(v8i32 (X86vzmovl (insert_subvector undef, 4925 (v4i32 (scalar_to_vector GR32:$src)),(iPTR 0)))), 4926 (SUBREG_TO_REG (i32 0), (VMOVDI2PDIrr GR32:$src), sub_xmm)>; 4927 def : Pat<(v4i64 (X86vzmovl (insert_subvector undef, 4928 (v2i64 (scalar_to_vector GR64:$src)),(iPTR 0)))), 4929 (SUBREG_TO_REG (i64 0), (VMOVZQI2PQIrr GR64:$src), sub_xmm)>; 4930} 4931 4932let Predicates = [UseSSE2] in { 4933 let AddedComplexity = 15 in 4934 def : Pat<(v4i32 (X86vzmovl (v4i32 (scalar_to_vector GR32:$src)))), 4935 (MOVDI2PDIrr GR32:$src)>; 4936 4937 let AddedComplexity = 20 in { 4938 def : Pat<(v4i32 (X86vzmovl (v4i32 (scalar_to_vector (loadi32 addr:$src))))), 4939 (MOVDI2PDIrm addr:$src)>; 4940 def : Pat<(v4i32 (X86vzmovl (bc_v4i32 (loadv4f32 addr:$src)))), 4941 (MOVDI2PDIrm addr:$src)>; 4942 def : Pat<(v4i32 (X86vzmovl (bc_v4i32 (loadv2i64 addr:$src)))), 4943 (MOVDI2PDIrm addr:$src)>; 4944 } 4945} 4946 4947// These are the correct encodings of the instructions so that we know how to 4948// read correct assembly, even though we continue to emit the wrong ones for 4949// compatibility with Darwin's buggy assembler. 4950def : InstAlias<"movq\t{$src, $dst|$dst, $src}", 4951 (MOV64toPQIrr VR128:$dst, GR64:$src), 0>; 4952def : InstAlias<"movq\t{$src, $dst|$dst, $src}", 4953 (MOVPQIto64rr GR64:$dst, VR128:$src), 0>; 4954// Allow "vmovd" but print "vmovq" since we don't need compatibility for AVX. 4955def : InstAlias<"vmovd\t{$src, $dst|$dst, $src}", 4956 (VMOV64toPQIrr VR128:$dst, GR64:$src), 0>; 4957def : InstAlias<"vmovd\t{$src, $dst|$dst, $src}", 4958 (VMOVPQIto64rr GR64:$dst, VR128:$src), 0>; 4959 4960//===---------------------------------------------------------------------===// 4961// SSE2 - Move Quadword 4962//===---------------------------------------------------------------------===// 4963 4964//===---------------------------------------------------------------------===// 4965// Move Quadword Int to Packed Quadword Int 4966// 4967 4968let ExeDomain = SSEPackedInt, SchedRW = [WriteLoad] in { 4969def VMOVQI2PQIrm : I<0x7E, MRMSrcMem, (outs VR128:$dst), (ins i64mem:$src), 4970 "vmovq\t{$src, $dst|$dst, $src}", 4971 [(set VR128:$dst, 4972 (v2i64 (scalar_to_vector (loadi64 addr:$src))))]>, XS, 4973 VEX, Requires<[UseAVX]>; 4974def MOVQI2PQIrm : I<0x7E, MRMSrcMem, (outs VR128:$dst), (ins i64mem:$src), 4975 "movq\t{$src, $dst|$dst, $src}", 4976 [(set VR128:$dst, 4977 (v2i64 (scalar_to_vector (loadi64 addr:$src))))], 4978 IIC_SSE_MOVDQ>, XS, 4979 Requires<[UseSSE2]>; // SSE2 instruction with XS Prefix 4980} // ExeDomain, SchedRW 4981 4982//===---------------------------------------------------------------------===// 4983// Move Packed Quadword Int to Quadword Int 4984// 4985let ExeDomain = SSEPackedInt, SchedRW = [WriteStore] in { 4986def VMOVPQI2QImr : VS2I<0xD6, MRMDestMem, (outs), (ins i64mem:$dst, VR128:$src), 4987 "movq\t{$src, $dst|$dst, $src}", 4988 [(store (i64 (vector_extract (v2i64 VR128:$src), 4989 (iPTR 0))), addr:$dst)], 4990 IIC_SSE_MOVDQ>, VEX; 4991def MOVPQI2QImr : S2I<0xD6, MRMDestMem, (outs), (ins i64mem:$dst, VR128:$src), 4992 "movq\t{$src, $dst|$dst, $src}", 4993 [(store (i64 (vector_extract (v2i64 VR128:$src), 4994 (iPTR 0))), addr:$dst)], 4995 IIC_SSE_MOVDQ>; 4996} // ExeDomain, SchedRW 4997 4998// For disassembler only 4999let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0, 5000 SchedRW = [WriteVecLogic] in { 5001def VMOVPQI2QIrr : VS2I<0xD6, MRMDestReg, (outs VR128:$dst), (ins VR128:$src), 5002 "movq\t{$src, $dst|$dst, $src}", [], IIC_SSE_MOVQ_RR>, VEX; 5003def MOVPQI2QIrr : S2I<0xD6, MRMDestReg, (outs VR128:$dst), (ins VR128:$src), 5004 "movq\t{$src, $dst|$dst, $src}", [], IIC_SSE_MOVQ_RR>; 5005} 5006 5007//===---------------------------------------------------------------------===// 5008// Store / copy lower 64-bits of a XMM register. 5009// 5010let Predicates = [HasAVX] in 5011def : Pat<(int_x86_sse2_storel_dq addr:$dst, VR128:$src), 5012 (VMOVPQI2QImr addr:$dst, VR128:$src)>; 5013let Predicates = [UseSSE2] in 5014def : Pat<(int_x86_sse2_storel_dq addr:$dst, VR128:$src), 5015 (MOVPQI2QImr addr:$dst, VR128:$src)>; 5016 5017let ExeDomain = SSEPackedInt, isCodeGenOnly = 1, AddedComplexity = 20 in { 5018def VMOVZQI2PQIrm : I<0x7E, MRMSrcMem, (outs VR128:$dst), (ins i64mem:$src), 5019 "vmovq\t{$src, $dst|$dst, $src}", 5020 [(set VR128:$dst, 5021 (v2i64 (X86vzmovl (v2i64 (scalar_to_vector 5022 (loadi64 addr:$src))))))], 5023 IIC_SSE_MOVDQ>, 5024 XS, VEX, Requires<[UseAVX]>, Sched<[WriteLoad]>; 5025 5026def MOVZQI2PQIrm : I<0x7E, MRMSrcMem, (outs VR128:$dst), (ins i64mem:$src), 5027 "movq\t{$src, $dst|$dst, $src}", 5028 [(set VR128:$dst, 5029 (v2i64 (X86vzmovl (v2i64 (scalar_to_vector 5030 (loadi64 addr:$src))))))], 5031 IIC_SSE_MOVDQ>, 5032 XS, Requires<[UseSSE2]>, Sched<[WriteLoad]>; 5033} // ExeDomain, isCodeGenOnly, AddedComplexity 5034 5035let Predicates = [UseAVX], AddedComplexity = 20 in { 5036 def : Pat<(v2i64 (X86vzmovl (bc_v2i64 (loadv4f32 addr:$src)))), 5037 (VMOVZQI2PQIrm addr:$src)>; 5038 def : Pat<(v2i64 (X86vzload addr:$src)), 5039 (VMOVZQI2PQIrm addr:$src)>; 5040 def : Pat<(v4i64 (X86vzmovl (insert_subvector undef, 5041 (v2i64 (scalar_to_vector (loadi64 addr:$src))), (iPTR 0)))), 5042 (SUBREG_TO_REG (i64 0), (VMOVZQI2PQIrm addr:$src), sub_xmm)>; 5043} 5044 5045let Predicates = [UseSSE2], AddedComplexity = 20 in { 5046 def : Pat<(v2i64 (X86vzmovl (bc_v2i64 (loadv4f32 addr:$src)))), 5047 (MOVZQI2PQIrm addr:$src)>; 5048 def : Pat<(v2i64 (X86vzload addr:$src)), (MOVZQI2PQIrm addr:$src)>; 5049} 5050 5051let Predicates = [HasAVX] in { 5052def : Pat<(v4i64 (alignedX86vzload addr:$src)), 5053 (SUBREG_TO_REG (i32 0), (VMOVAPSrm addr:$src), sub_xmm)>; 5054def : Pat<(v4i64 (X86vzload addr:$src)), 5055 (SUBREG_TO_REG (i32 0), (VMOVUPSrm addr:$src), sub_xmm)>; 5056} 5057 5058//===---------------------------------------------------------------------===// 5059// Moving from XMM to XMM and clear upper 64 bits. Note, there is a bug in 5060// IA32 document. movq xmm1, xmm2 does clear the high bits. 5061// 5062let ExeDomain = SSEPackedInt, SchedRW = [WriteVecLogic] in { 5063let AddedComplexity = 15 in 5064def VMOVZPQILo2PQIrr : I<0x7E, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 5065 "vmovq\t{$src, $dst|$dst, $src}", 5066 [(set VR128:$dst, (v2i64 (X86vzmovl (v2i64 VR128:$src))))], 5067 IIC_SSE_MOVQ_RR>, 5068 XS, VEX, Requires<[UseAVX]>; 5069let AddedComplexity = 15 in 5070def MOVZPQILo2PQIrr : I<0x7E, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 5071 "movq\t{$src, $dst|$dst, $src}", 5072 [(set VR128:$dst, (v2i64 (X86vzmovl (v2i64 VR128:$src))))], 5073 IIC_SSE_MOVQ_RR>, 5074 XS, Requires<[UseSSE2]>; 5075} // ExeDomain, SchedRW 5076 5077let ExeDomain = SSEPackedInt, isCodeGenOnly = 1, SchedRW = [WriteVecLogicLd] in { 5078let AddedComplexity = 20 in 5079def VMOVZPQILo2PQIrm : I<0x7E, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src), 5080 "vmovq\t{$src, $dst|$dst, $src}", 5081 [(set VR128:$dst, (v2i64 (X86vzmovl 5082 (loadv2i64 addr:$src))))], 5083 IIC_SSE_MOVDQ>, 5084 XS, VEX, Requires<[UseAVX]>; 5085let AddedComplexity = 20 in { 5086def MOVZPQILo2PQIrm : I<0x7E, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src), 5087 "movq\t{$src, $dst|$dst, $src}", 5088 [(set VR128:$dst, (v2i64 (X86vzmovl 5089 (loadv2i64 addr:$src))))], 5090 IIC_SSE_MOVDQ>, 5091 XS, Requires<[UseSSE2]>; 5092} 5093} // ExeDomain, isCodeGenOnly, SchedRW 5094 5095let AddedComplexity = 20 in { 5096 let Predicates = [UseAVX] in { 5097 def : Pat<(v2f64 (X86vzmovl (v2f64 VR128:$src))), 5098 (VMOVZPQILo2PQIrr VR128:$src)>; 5099 } 5100 let Predicates = [UseSSE2] in { 5101 def : Pat<(v2f64 (X86vzmovl (v2f64 VR128:$src))), 5102 (MOVZPQILo2PQIrr VR128:$src)>; 5103 } 5104} 5105 5106//===---------------------------------------------------------------------===// 5107// SSE3 - Replicate Single FP - MOVSHDUP and MOVSLDUP 5108//===---------------------------------------------------------------------===// 5109multiclass sse3_replicate_sfp<bits<8> op, SDNode OpNode, string OpcodeStr, 5110 ValueType vt, RegisterClass RC, PatFrag mem_frag, 5111 X86MemOperand x86memop> { 5112def rr : S3SI<op, MRMSrcReg, (outs RC:$dst), (ins RC:$src), 5113 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 5114 [(set RC:$dst, (vt (OpNode RC:$src)))], 5115 IIC_SSE_MOV_LH>, Sched<[WriteFShuffle]>; 5116def rm : S3SI<op, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src), 5117 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 5118 [(set RC:$dst, (OpNode (mem_frag addr:$src)))], 5119 IIC_SSE_MOV_LH>, Sched<[WriteLoad]>; 5120} 5121 5122let Predicates = [HasAVX] in { 5123 defm VMOVSHDUP : sse3_replicate_sfp<0x16, X86Movshdup, "vmovshdup", 5124 v4f32, VR128, loadv4f32, f128mem>, VEX; 5125 defm VMOVSLDUP : sse3_replicate_sfp<0x12, X86Movsldup, "vmovsldup", 5126 v4f32, VR128, loadv4f32, f128mem>, VEX; 5127 defm VMOVSHDUPY : sse3_replicate_sfp<0x16, X86Movshdup, "vmovshdup", 5128 v8f32, VR256, loadv8f32, f256mem>, VEX, VEX_L; 5129 defm VMOVSLDUPY : sse3_replicate_sfp<0x12, X86Movsldup, "vmovsldup", 5130 v8f32, VR256, loadv8f32, f256mem>, VEX, VEX_L; 5131} 5132defm MOVSHDUP : sse3_replicate_sfp<0x16, X86Movshdup, "movshdup", v4f32, VR128, 5133 memopv4f32, f128mem>; 5134defm MOVSLDUP : sse3_replicate_sfp<0x12, X86Movsldup, "movsldup", v4f32, VR128, 5135 memopv4f32, f128mem>; 5136 5137let Predicates = [HasAVX] in { 5138 def : Pat<(v4i32 (X86Movshdup VR128:$src)), 5139 (VMOVSHDUPrr VR128:$src)>; 5140 def : Pat<(v4i32 (X86Movshdup (bc_v4i32 (loadv2i64 addr:$src)))), 5141 (VMOVSHDUPrm addr:$src)>; 5142 def : Pat<(v4i32 (X86Movsldup VR128:$src)), 5143 (VMOVSLDUPrr VR128:$src)>; 5144 def : Pat<(v4i32 (X86Movsldup (bc_v4i32 (loadv2i64 addr:$src)))), 5145 (VMOVSLDUPrm addr:$src)>; 5146 def : Pat<(v8i32 (X86Movshdup VR256:$src)), 5147 (VMOVSHDUPYrr VR256:$src)>; 5148 def : Pat<(v8i32 (X86Movshdup (bc_v8i32 (loadv4i64 addr:$src)))), 5149 (VMOVSHDUPYrm addr:$src)>; 5150 def : Pat<(v8i32 (X86Movsldup VR256:$src)), 5151 (VMOVSLDUPYrr VR256:$src)>; 5152 def : Pat<(v8i32 (X86Movsldup (bc_v8i32 (loadv4i64 addr:$src)))), 5153 (VMOVSLDUPYrm addr:$src)>; 5154} 5155 5156let Predicates = [UseSSE3] in { 5157 def : Pat<(v4i32 (X86Movshdup VR128:$src)), 5158 (MOVSHDUPrr VR128:$src)>; 5159 def : Pat<(v4i32 (X86Movshdup (bc_v4i32 (memopv2i64 addr:$src)))), 5160 (MOVSHDUPrm addr:$src)>; 5161 def : Pat<(v4i32 (X86Movsldup VR128:$src)), 5162 (MOVSLDUPrr VR128:$src)>; 5163 def : Pat<(v4i32 (X86Movsldup (bc_v4i32 (memopv2i64 addr:$src)))), 5164 (MOVSLDUPrm addr:$src)>; 5165} 5166 5167//===---------------------------------------------------------------------===// 5168// SSE3 - Replicate Double FP - MOVDDUP 5169//===---------------------------------------------------------------------===// 5170 5171multiclass sse3_replicate_dfp<string OpcodeStr> { 5172def rr : S3DI<0x12, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 5173 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 5174 [(set VR128:$dst, (v2f64 (X86Movddup VR128:$src)))], 5175 IIC_SSE_MOV_LH>, Sched<[WriteFShuffle]>; 5176def rm : S3DI<0x12, MRMSrcMem, (outs VR128:$dst), (ins f64mem:$src), 5177 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 5178 [(set VR128:$dst, 5179 (v2f64 (X86Movddup 5180 (scalar_to_vector (loadf64 addr:$src)))))], 5181 IIC_SSE_MOV_LH>, Sched<[WriteLoad]>; 5182} 5183 5184// FIXME: Merge with above classe when there're patterns for the ymm version 5185multiclass sse3_replicate_dfp_y<string OpcodeStr> { 5186def rr : S3DI<0x12, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src), 5187 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 5188 [(set VR256:$dst, (v4f64 (X86Movddup VR256:$src)))]>, 5189 Sched<[WriteFShuffle]>; 5190def rm : S3DI<0x12, MRMSrcMem, (outs VR256:$dst), (ins f256mem:$src), 5191 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 5192 [(set VR256:$dst, 5193 (v4f64 (X86Movddup 5194 (scalar_to_vector (loadf64 addr:$src)))))]>, 5195 Sched<[WriteLoad]>; 5196} 5197 5198let Predicates = [HasAVX] in { 5199 defm VMOVDDUP : sse3_replicate_dfp<"vmovddup">, VEX; 5200 defm VMOVDDUPY : sse3_replicate_dfp_y<"vmovddup">, VEX, VEX_L; 5201} 5202 5203defm MOVDDUP : sse3_replicate_dfp<"movddup">; 5204 5205let Predicates = [HasAVX] in { 5206 def : Pat<(X86Movddup (loadv2f64 addr:$src)), 5207 (VMOVDDUPrm addr:$src)>, Requires<[HasAVX]>; 5208 def : Pat<(X86Movddup (bc_v2f64 (loadv4f32 addr:$src))), 5209 (VMOVDDUPrm addr:$src)>, Requires<[HasAVX]>; 5210 def : Pat<(X86Movddup (bc_v2f64 (loadv2i64 addr:$src))), 5211 (VMOVDDUPrm addr:$src)>, Requires<[HasAVX]>; 5212 def : Pat<(X86Movddup (bc_v2f64 5213 (v2i64 (scalar_to_vector (loadi64 addr:$src))))), 5214 (VMOVDDUPrm addr:$src)>, Requires<[HasAVX]>; 5215 5216 // 256-bit version 5217 def : Pat<(X86Movddup (loadv4f64 addr:$src)), 5218 (VMOVDDUPYrm addr:$src)>; 5219 def : Pat<(X86Movddup (loadv4i64 addr:$src)), 5220 (VMOVDDUPYrm addr:$src)>; 5221 def : Pat<(X86Movddup (v4i64 (scalar_to_vector (loadi64 addr:$src)))), 5222 (VMOVDDUPYrm addr:$src)>; 5223 def : Pat<(X86Movddup (v4i64 VR256:$src)), 5224 (VMOVDDUPYrr VR256:$src)>; 5225} 5226 5227let Predicates = [UseAVX, OptForSize] in { 5228 def : Pat<(v2f64 (X86VBroadcast (loadf64 addr:$src))), 5229 (VMOVDDUPrm addr:$src)>; 5230 def : Pat<(v2i64 (X86VBroadcast (loadi64 addr:$src))), 5231 (VMOVDDUPrm addr:$src)>; 5232} 5233 5234let Predicates = [UseSSE3] in { 5235 def : Pat<(X86Movddup (memopv2f64 addr:$src)), 5236 (MOVDDUPrm addr:$src)>; 5237 def : Pat<(X86Movddup (bc_v2f64 (memopv4f32 addr:$src))), 5238 (MOVDDUPrm addr:$src)>; 5239 def : Pat<(X86Movddup (bc_v2f64 (memopv2i64 addr:$src))), 5240 (MOVDDUPrm addr:$src)>; 5241 def : Pat<(X86Movddup (bc_v2f64 5242 (v2i64 (scalar_to_vector (loadi64 addr:$src))))), 5243 (MOVDDUPrm addr:$src)>; 5244} 5245 5246//===---------------------------------------------------------------------===// 5247// SSE3 - Move Unaligned Integer 5248//===---------------------------------------------------------------------===// 5249 5250let SchedRW = [WriteLoad] in { 5251let Predicates = [HasAVX] in { 5252 def VLDDQUrm : S3DI<0xF0, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src), 5253 "vlddqu\t{$src, $dst|$dst, $src}", 5254 [(set VR128:$dst, (int_x86_sse3_ldu_dq addr:$src))]>, VEX; 5255 def VLDDQUYrm : S3DI<0xF0, MRMSrcMem, (outs VR256:$dst), (ins i256mem:$src), 5256 "vlddqu\t{$src, $dst|$dst, $src}", 5257 [(set VR256:$dst, (int_x86_avx_ldu_dq_256 addr:$src))]>, 5258 VEX, VEX_L; 5259} 5260def LDDQUrm : S3DI<0xF0, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src), 5261 "lddqu\t{$src, $dst|$dst, $src}", 5262 [(set VR128:$dst, (int_x86_sse3_ldu_dq addr:$src))], 5263 IIC_SSE_LDDQU>; 5264} 5265 5266//===---------------------------------------------------------------------===// 5267// SSE3 - Arithmetic 5268//===---------------------------------------------------------------------===// 5269 5270multiclass sse3_addsub<Intrinsic Int, string OpcodeStr, RegisterClass RC, 5271 X86MemOperand x86memop, OpndItins itins, 5272 PatFrag ld_frag, bit Is2Addr = 1> { 5273 def rr : I<0xD0, MRMSrcReg, 5274 (outs RC:$dst), (ins RC:$src1, RC:$src2), 5275 !if(Is2Addr, 5276 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 5277 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 5278 [(set RC:$dst, (Int RC:$src1, RC:$src2))], itins.rr>, 5279 Sched<[itins.Sched]>; 5280 def rm : I<0xD0, MRMSrcMem, 5281 (outs RC:$dst), (ins RC:$src1, x86memop:$src2), 5282 !if(Is2Addr, 5283 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 5284 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 5285 [(set RC:$dst, (Int RC:$src1, (ld_frag addr:$src2)))], itins.rr>, 5286 Sched<[itins.Sched.Folded, ReadAfterLd]>; 5287} 5288 5289let Predicates = [HasAVX] in { 5290 let ExeDomain = SSEPackedSingle in { 5291 defm VADDSUBPS : sse3_addsub<int_x86_sse3_addsub_ps, "vaddsubps", VR128, 5292 f128mem, SSE_ALU_F32P, loadv4f32, 0>, XD, VEX_4V; 5293 defm VADDSUBPSY : sse3_addsub<int_x86_avx_addsub_ps_256, "vaddsubps", VR256, 5294 f256mem, SSE_ALU_F32P, loadv8f32, 0>, XD, VEX_4V, VEX_L; 5295 } 5296 let ExeDomain = SSEPackedDouble in { 5297 defm VADDSUBPD : sse3_addsub<int_x86_sse3_addsub_pd, "vaddsubpd", VR128, 5298 f128mem, SSE_ALU_F64P, loadv2f64, 0>, PD, VEX_4V; 5299 defm VADDSUBPDY : sse3_addsub<int_x86_avx_addsub_pd_256, "vaddsubpd", VR256, 5300 f256mem, SSE_ALU_F64P, loadv4f64, 0>, PD, VEX_4V, VEX_L; 5301 } 5302} 5303let Constraints = "$src1 = $dst", Predicates = [UseSSE3] in { 5304 let ExeDomain = SSEPackedSingle in 5305 defm ADDSUBPS : sse3_addsub<int_x86_sse3_addsub_ps, "addsubps", VR128, 5306 f128mem, SSE_ALU_F32P, memopv4f32>, XD; 5307 let ExeDomain = SSEPackedDouble in 5308 defm ADDSUBPD : sse3_addsub<int_x86_sse3_addsub_pd, "addsubpd", VR128, 5309 f128mem, SSE_ALU_F64P, memopv2f64>, PD; 5310} 5311 5312// Patterns used to select 'addsub' instructions. 5313let Predicates = [HasAVX] in { 5314 def : Pat<(v4f32 (X86Addsub (v4f32 VR128:$lhs), (v4f32 VR128:$rhs))), 5315 (VADDSUBPSrr VR128:$lhs, VR128:$rhs)>; 5316 def : Pat<(v4f32 (X86Addsub (v4f32 VR128:$lhs), (loadv4f32 addr:$rhs))), 5317 (VADDSUBPSrm VR128:$lhs, f128mem:$rhs)>; 5318 def : Pat<(v2f64 (X86Addsub (v2f64 VR128:$lhs), (v2f64 VR128:$rhs))), 5319 (VADDSUBPDrr VR128:$lhs, VR128:$rhs)>; 5320 def : Pat<(v2f64 (X86Addsub (v2f64 VR128:$lhs), (loadv2f64 addr:$rhs))), 5321 (VADDSUBPDrm VR128:$lhs, f128mem:$rhs)>; 5322 5323 def : Pat<(v8f32 (X86Addsub (v8f32 VR256:$lhs), (v8f32 VR256:$rhs))), 5324 (VADDSUBPSYrr VR256:$lhs, VR256:$rhs)>; 5325 def : Pat<(v8f32 (X86Addsub (v8f32 VR256:$lhs), (loadv8f32 addr:$rhs))), 5326 (VADDSUBPSYrm VR256:$lhs, f256mem:$rhs)>; 5327 def : Pat<(v4f64 (X86Addsub (v4f64 VR256:$lhs), (v4f64 VR256:$rhs))), 5328 (VADDSUBPDYrr VR256:$lhs, VR256:$rhs)>; 5329 def : Pat<(v4f64 (X86Addsub (v4f64 VR256:$lhs), (loadv4f64 addr:$rhs))), 5330 (VADDSUBPDYrm VR256:$lhs, f256mem:$rhs)>; 5331} 5332 5333let Predicates = [UseSSE3] in { 5334 def : Pat<(v4f32 (X86Addsub (v4f32 VR128:$lhs), (v4f32 VR128:$rhs))), 5335 (ADDSUBPSrr VR128:$lhs, VR128:$rhs)>; 5336 def : Pat<(v4f32 (X86Addsub (v4f32 VR128:$lhs), (memopv4f32 addr:$rhs))), 5337 (ADDSUBPSrm VR128:$lhs, f128mem:$rhs)>; 5338 def : Pat<(v2f64 (X86Addsub (v2f64 VR128:$lhs), (v2f64 VR128:$rhs))), 5339 (ADDSUBPDrr VR128:$lhs, VR128:$rhs)>; 5340 def : Pat<(v2f64 (X86Addsub (v2f64 VR128:$lhs), (memopv2f64 addr:$rhs))), 5341 (ADDSUBPDrm VR128:$lhs, f128mem:$rhs)>; 5342} 5343 5344//===---------------------------------------------------------------------===// 5345// SSE3 Instructions 5346//===---------------------------------------------------------------------===// 5347 5348// Horizontal ops 5349multiclass S3D_Int<bits<8> o, string OpcodeStr, ValueType vt, RegisterClass RC, 5350 X86MemOperand x86memop, SDNode OpNode, PatFrag ld_frag, 5351 bit Is2Addr = 1> { 5352 def rr : S3DI<o, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2), 5353 !if(Is2Addr, 5354 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 5355 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 5356 [(set RC:$dst, (vt (OpNode RC:$src1, RC:$src2)))], IIC_SSE_HADDSUB_RR>, 5357 Sched<[WriteFAdd]>; 5358 5359 def rm : S3DI<o, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2), 5360 !if(Is2Addr, 5361 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 5362 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 5363 [(set RC:$dst, (vt (OpNode RC:$src1, (ld_frag addr:$src2))))], 5364 IIC_SSE_HADDSUB_RM>, Sched<[WriteFAddLd, ReadAfterLd]>; 5365} 5366multiclass S3_Int<bits<8> o, string OpcodeStr, ValueType vt, RegisterClass RC, 5367 X86MemOperand x86memop, SDNode OpNode, PatFrag ld_frag, 5368 bit Is2Addr = 1> { 5369 def rr : S3I<o, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2), 5370 !if(Is2Addr, 5371 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 5372 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 5373 [(set RC:$dst, (vt (OpNode RC:$src1, RC:$src2)))], IIC_SSE_HADDSUB_RR>, 5374 Sched<[WriteFAdd]>; 5375 5376 def rm : S3I<o, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2), 5377 !if(Is2Addr, 5378 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 5379 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 5380 [(set RC:$dst, (vt (OpNode RC:$src1, (ld_frag addr:$src2))))], 5381 IIC_SSE_HADDSUB_RM>, Sched<[WriteFAddLd, ReadAfterLd]>; 5382} 5383 5384let Predicates = [HasAVX] in { 5385 let ExeDomain = SSEPackedSingle in { 5386 defm VHADDPS : S3D_Int<0x7C, "vhaddps", v4f32, VR128, f128mem, 5387 X86fhadd, loadv4f32, 0>, VEX_4V; 5388 defm VHSUBPS : S3D_Int<0x7D, "vhsubps", v4f32, VR128, f128mem, 5389 X86fhsub, loadv4f32, 0>, VEX_4V; 5390 defm VHADDPSY : S3D_Int<0x7C, "vhaddps", v8f32, VR256, f256mem, 5391 X86fhadd, loadv8f32, 0>, VEX_4V, VEX_L; 5392 defm VHSUBPSY : S3D_Int<0x7D, "vhsubps", v8f32, VR256, f256mem, 5393 X86fhsub, loadv8f32, 0>, VEX_4V, VEX_L; 5394 } 5395 let ExeDomain = SSEPackedDouble in { 5396 defm VHADDPD : S3_Int <0x7C, "vhaddpd", v2f64, VR128, f128mem, 5397 X86fhadd, loadv2f64, 0>, VEX_4V; 5398 defm VHSUBPD : S3_Int <0x7D, "vhsubpd", v2f64, VR128, f128mem, 5399 X86fhsub, loadv2f64, 0>, VEX_4V; 5400 defm VHADDPDY : S3_Int <0x7C, "vhaddpd", v4f64, VR256, f256mem, 5401 X86fhadd, loadv4f64, 0>, VEX_4V, VEX_L; 5402 defm VHSUBPDY : S3_Int <0x7D, "vhsubpd", v4f64, VR256, f256mem, 5403 X86fhsub, loadv4f64, 0>, VEX_4V, VEX_L; 5404 } 5405} 5406 5407let Constraints = "$src1 = $dst" in { 5408 let ExeDomain = SSEPackedSingle in { 5409 defm HADDPS : S3D_Int<0x7C, "haddps", v4f32, VR128, f128mem, X86fhadd, 5410 memopv4f32>; 5411 defm HSUBPS : S3D_Int<0x7D, "hsubps", v4f32, VR128, f128mem, X86fhsub, 5412 memopv4f32>; 5413 } 5414 let ExeDomain = SSEPackedDouble in { 5415 defm HADDPD : S3_Int<0x7C, "haddpd", v2f64, VR128, f128mem, X86fhadd, 5416 memopv2f64>; 5417 defm HSUBPD : S3_Int<0x7D, "hsubpd", v2f64, VR128, f128mem, X86fhsub, 5418 memopv2f64>; 5419 } 5420} 5421 5422//===---------------------------------------------------------------------===// 5423// SSSE3 - Packed Absolute Instructions 5424//===---------------------------------------------------------------------===// 5425 5426 5427/// SS3I_unop_rm_int - Simple SSSE3 unary op whose type can be v*{i8,i16,i32}. 5428multiclass SS3I_unop_rm_int<bits<8> opc, string OpcodeStr, Intrinsic IntId128, 5429 PatFrag ld_frag> { 5430 def rr128 : SS38I<opc, MRMSrcReg, (outs VR128:$dst), 5431 (ins VR128:$src), 5432 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 5433 [(set VR128:$dst, (IntId128 VR128:$src))], IIC_SSE_PABS_RR>, 5434 Sched<[WriteVecALU]>; 5435 5436 def rm128 : SS38I<opc, MRMSrcMem, (outs VR128:$dst), 5437 (ins i128mem:$src), 5438 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 5439 [(set VR128:$dst, 5440 (IntId128 5441 (bitconvert (ld_frag addr:$src))))], IIC_SSE_PABS_RM>, 5442 Sched<[WriteVecALULd]>; 5443} 5444 5445/// SS3I_unop_rm_int_y - Simple SSSE3 unary op whose type can be v*{i8,i16,i32}. 5446multiclass SS3I_unop_rm_int_y<bits<8> opc, string OpcodeStr, 5447 Intrinsic IntId256> { 5448 def rr256 : SS38I<opc, MRMSrcReg, (outs VR256:$dst), 5449 (ins VR256:$src), 5450 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 5451 [(set VR256:$dst, (IntId256 VR256:$src))]>, 5452 Sched<[WriteVecALU]>; 5453 5454 def rm256 : SS38I<opc, MRMSrcMem, (outs VR256:$dst), 5455 (ins i256mem:$src), 5456 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 5457 [(set VR256:$dst, 5458 (IntId256 5459 (bitconvert (loadv4i64 addr:$src))))]>, 5460 Sched<[WriteVecALULd]>; 5461} 5462 5463// Helper fragments to match sext vXi1 to vXiY. 5464def v16i1sextv16i8 : PatLeaf<(v16i8 (X86pcmpgt (bc_v16i8 (v4i32 immAllZerosV)), 5465 VR128:$src))>; 5466def v8i1sextv8i16 : PatLeaf<(v8i16 (X86vsrai VR128:$src, (i8 15)))>; 5467def v4i1sextv4i32 : PatLeaf<(v4i32 (X86vsrai VR128:$src, (i8 31)))>; 5468def v32i1sextv32i8 : PatLeaf<(v32i8 (X86pcmpgt (bc_v32i8 (v8i32 immAllZerosV)), 5469 VR256:$src))>; 5470def v16i1sextv16i16: PatLeaf<(v16i16 (X86vsrai VR256:$src, (i8 15)))>; 5471def v8i1sextv8i32 : PatLeaf<(v8i32 (X86vsrai VR256:$src, (i8 31)))>; 5472 5473let Predicates = [HasAVX] in { 5474 defm VPABSB : SS3I_unop_rm_int<0x1C, "vpabsb", int_x86_ssse3_pabs_b_128, 5475 loadv2i64>, VEX; 5476 defm VPABSW : SS3I_unop_rm_int<0x1D, "vpabsw", int_x86_ssse3_pabs_w_128, 5477 loadv2i64>, VEX; 5478 defm VPABSD : SS3I_unop_rm_int<0x1E, "vpabsd", int_x86_ssse3_pabs_d_128, 5479 loadv2i64>, VEX; 5480 5481 def : Pat<(xor 5482 (bc_v2i64 (v16i1sextv16i8)), 5483 (bc_v2i64 (add (v16i8 VR128:$src), (v16i1sextv16i8)))), 5484 (VPABSBrr128 VR128:$src)>; 5485 def : Pat<(xor 5486 (bc_v2i64 (v8i1sextv8i16)), 5487 (bc_v2i64 (add (v8i16 VR128:$src), (v8i1sextv8i16)))), 5488 (VPABSWrr128 VR128:$src)>; 5489 def : Pat<(xor 5490 (bc_v2i64 (v4i1sextv4i32)), 5491 (bc_v2i64 (add (v4i32 VR128:$src), (v4i1sextv4i32)))), 5492 (VPABSDrr128 VR128:$src)>; 5493} 5494 5495let Predicates = [HasAVX2] in { 5496 defm VPABSB : SS3I_unop_rm_int_y<0x1C, "vpabsb", 5497 int_x86_avx2_pabs_b>, VEX, VEX_L; 5498 defm VPABSW : SS3I_unop_rm_int_y<0x1D, "vpabsw", 5499 int_x86_avx2_pabs_w>, VEX, VEX_L; 5500 defm VPABSD : SS3I_unop_rm_int_y<0x1E, "vpabsd", 5501 int_x86_avx2_pabs_d>, VEX, VEX_L; 5502 5503 def : Pat<(xor 5504 (bc_v4i64 (v32i1sextv32i8)), 5505 (bc_v4i64 (add (v32i8 VR256:$src), (v32i1sextv32i8)))), 5506 (VPABSBrr256 VR256:$src)>; 5507 def : Pat<(xor 5508 (bc_v4i64 (v16i1sextv16i16)), 5509 (bc_v4i64 (add (v16i16 VR256:$src), (v16i1sextv16i16)))), 5510 (VPABSWrr256 VR256:$src)>; 5511 def : Pat<(xor 5512 (bc_v4i64 (v8i1sextv8i32)), 5513 (bc_v4i64 (add (v8i32 VR256:$src), (v8i1sextv8i32)))), 5514 (VPABSDrr256 VR256:$src)>; 5515} 5516 5517defm PABSB : SS3I_unop_rm_int<0x1C, "pabsb", int_x86_ssse3_pabs_b_128, 5518 memopv2i64>; 5519defm PABSW : SS3I_unop_rm_int<0x1D, "pabsw", int_x86_ssse3_pabs_w_128, 5520 memopv2i64>; 5521defm PABSD : SS3I_unop_rm_int<0x1E, "pabsd", int_x86_ssse3_pabs_d_128, 5522 memopv2i64>; 5523 5524let Predicates = [HasSSSE3] in { 5525 def : Pat<(xor 5526 (bc_v2i64 (v16i1sextv16i8)), 5527 (bc_v2i64 (add (v16i8 VR128:$src), (v16i1sextv16i8)))), 5528 (PABSBrr128 VR128:$src)>; 5529 def : Pat<(xor 5530 (bc_v2i64 (v8i1sextv8i16)), 5531 (bc_v2i64 (add (v8i16 VR128:$src), (v8i1sextv8i16)))), 5532 (PABSWrr128 VR128:$src)>; 5533 def : Pat<(xor 5534 (bc_v2i64 (v4i1sextv4i32)), 5535 (bc_v2i64 (add (v4i32 VR128:$src), (v4i1sextv4i32)))), 5536 (PABSDrr128 VR128:$src)>; 5537} 5538 5539//===---------------------------------------------------------------------===// 5540// SSSE3 - Packed Binary Operator Instructions 5541//===---------------------------------------------------------------------===// 5542 5543let Sched = WriteVecALU in { 5544def SSE_PHADDSUBD : OpndItins< 5545 IIC_SSE_PHADDSUBD_RR, IIC_SSE_PHADDSUBD_RM 5546>; 5547def SSE_PHADDSUBSW : OpndItins< 5548 IIC_SSE_PHADDSUBSW_RR, IIC_SSE_PHADDSUBSW_RM 5549>; 5550def SSE_PHADDSUBW : OpndItins< 5551 IIC_SSE_PHADDSUBW_RR, IIC_SSE_PHADDSUBW_RM 5552>; 5553} 5554let Sched = WriteShuffle in 5555def SSE_PSHUFB : OpndItins< 5556 IIC_SSE_PSHUFB_RR, IIC_SSE_PSHUFB_RM 5557>; 5558let Sched = WriteVecALU in 5559def SSE_PSIGN : OpndItins< 5560 IIC_SSE_PSIGN_RR, IIC_SSE_PSIGN_RM 5561>; 5562let Sched = WriteVecIMul in 5563def SSE_PMULHRSW : OpndItins< 5564 IIC_SSE_PMULHRSW, IIC_SSE_PMULHRSW 5565>; 5566 5567/// SS3I_binop_rm - Simple SSSE3 bin op 5568multiclass SS3I_binop_rm<bits<8> opc, string OpcodeStr, SDNode OpNode, 5569 ValueType OpVT, RegisterClass RC, PatFrag memop_frag, 5570 X86MemOperand x86memop, OpndItins itins, 5571 bit Is2Addr = 1> { 5572 let isCommutable = 1 in 5573 def rr : SS38I<opc, MRMSrcReg, (outs RC:$dst), 5574 (ins RC:$src1, RC:$src2), 5575 !if(Is2Addr, 5576 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 5577 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 5578 [(set RC:$dst, (OpVT (OpNode RC:$src1, RC:$src2)))], itins.rr>, 5579 Sched<[itins.Sched]>; 5580 def rm : SS38I<opc, MRMSrcMem, (outs RC:$dst), 5581 (ins RC:$src1, x86memop:$src2), 5582 !if(Is2Addr, 5583 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 5584 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 5585 [(set RC:$dst, 5586 (OpVT (OpNode RC:$src1, 5587 (bitconvert (memop_frag addr:$src2)))))], itins.rm>, 5588 Sched<[itins.Sched.Folded, ReadAfterLd]>; 5589} 5590 5591/// SS3I_binop_rm_int - Simple SSSE3 bin op whose type can be v*{i8,i16,i32}. 5592multiclass SS3I_binop_rm_int<bits<8> opc, string OpcodeStr, 5593 Intrinsic IntId128, OpndItins itins, 5594 PatFrag ld_frag, bit Is2Addr = 1> { 5595 let isCommutable = 1 in 5596 def rr128 : SS38I<opc, MRMSrcReg, (outs VR128:$dst), 5597 (ins VR128:$src1, VR128:$src2), 5598 !if(Is2Addr, 5599 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 5600 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 5601 [(set VR128:$dst, (IntId128 VR128:$src1, VR128:$src2))]>, 5602 Sched<[itins.Sched]>; 5603 def rm128 : SS38I<opc, MRMSrcMem, (outs VR128:$dst), 5604 (ins VR128:$src1, i128mem:$src2), 5605 !if(Is2Addr, 5606 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 5607 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 5608 [(set VR128:$dst, 5609 (IntId128 VR128:$src1, 5610 (bitconvert (ld_frag addr:$src2))))]>, 5611 Sched<[itins.Sched.Folded, ReadAfterLd]>; 5612} 5613 5614multiclass SS3I_binop_rm_int_y<bits<8> opc, string OpcodeStr, 5615 Intrinsic IntId256, 5616 X86FoldableSchedWrite Sched> { 5617 let isCommutable = 1 in 5618 def rr256 : SS38I<opc, MRMSrcReg, (outs VR256:$dst), 5619 (ins VR256:$src1, VR256:$src2), 5620 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 5621 [(set VR256:$dst, (IntId256 VR256:$src1, VR256:$src2))]>, 5622 Sched<[Sched]>; 5623 def rm256 : SS38I<opc, MRMSrcMem, (outs VR256:$dst), 5624 (ins VR256:$src1, i256mem:$src2), 5625 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 5626 [(set VR256:$dst, 5627 (IntId256 VR256:$src1, (bitconvert (loadv4i64 addr:$src2))))]>, 5628 Sched<[Sched.Folded, ReadAfterLd]>; 5629} 5630 5631let ImmT = NoImm, Predicates = [HasAVX] in { 5632let isCommutable = 0 in { 5633 defm VPHADDW : SS3I_binop_rm<0x01, "vphaddw", X86hadd, v8i16, VR128, 5634 loadv2i64, i128mem, 5635 SSE_PHADDSUBW, 0>, VEX_4V; 5636 defm VPHADDD : SS3I_binop_rm<0x02, "vphaddd", X86hadd, v4i32, VR128, 5637 loadv2i64, i128mem, 5638 SSE_PHADDSUBD, 0>, VEX_4V; 5639 defm VPHSUBW : SS3I_binop_rm<0x05, "vphsubw", X86hsub, v8i16, VR128, 5640 loadv2i64, i128mem, 5641 SSE_PHADDSUBW, 0>, VEX_4V; 5642 defm VPHSUBD : SS3I_binop_rm<0x06, "vphsubd", X86hsub, v4i32, VR128, 5643 loadv2i64, i128mem, 5644 SSE_PHADDSUBD, 0>, VEX_4V; 5645 defm VPSIGNB : SS3I_binop_rm<0x08, "vpsignb", X86psign, v16i8, VR128, 5646 loadv2i64, i128mem, 5647 SSE_PSIGN, 0>, VEX_4V; 5648 defm VPSIGNW : SS3I_binop_rm<0x09, "vpsignw", X86psign, v8i16, VR128, 5649 loadv2i64, i128mem, 5650 SSE_PSIGN, 0>, VEX_4V; 5651 defm VPSIGND : SS3I_binop_rm<0x0A, "vpsignd", X86psign, v4i32, VR128, 5652 loadv2i64, i128mem, 5653 SSE_PSIGN, 0>, VEX_4V; 5654 defm VPSHUFB : SS3I_binop_rm<0x00, "vpshufb", X86pshufb, v16i8, VR128, 5655 loadv2i64, i128mem, 5656 SSE_PSHUFB, 0>, VEX_4V; 5657 defm VPHADDSW : SS3I_binop_rm_int<0x03, "vphaddsw", 5658 int_x86_ssse3_phadd_sw_128, 5659 SSE_PHADDSUBSW, loadv2i64, 0>, VEX_4V; 5660 defm VPHSUBSW : SS3I_binop_rm_int<0x07, "vphsubsw", 5661 int_x86_ssse3_phsub_sw_128, 5662 SSE_PHADDSUBSW, loadv2i64, 0>, VEX_4V; 5663 defm VPMADDUBSW : SS3I_binop_rm_int<0x04, "vpmaddubsw", 5664 int_x86_ssse3_pmadd_ub_sw_128, 5665 SSE_PMADD, loadv2i64, 0>, VEX_4V; 5666} 5667defm VPMULHRSW : SS3I_binop_rm_int<0x0B, "vpmulhrsw", 5668 int_x86_ssse3_pmul_hr_sw_128, 5669 SSE_PMULHRSW, loadv2i64, 0>, VEX_4V; 5670} 5671 5672let ImmT = NoImm, Predicates = [HasAVX2] in { 5673let isCommutable = 0 in { 5674 defm VPHADDWY : SS3I_binop_rm<0x01, "vphaddw", X86hadd, v16i16, VR256, 5675 loadv4i64, i256mem, 5676 SSE_PHADDSUBW, 0>, VEX_4V, VEX_L; 5677 defm VPHADDDY : SS3I_binop_rm<0x02, "vphaddd", X86hadd, v8i32, VR256, 5678 loadv4i64, i256mem, 5679 SSE_PHADDSUBW, 0>, VEX_4V, VEX_L; 5680 defm VPHSUBWY : SS3I_binop_rm<0x05, "vphsubw", X86hsub, v16i16, VR256, 5681 loadv4i64, i256mem, 5682 SSE_PHADDSUBW, 0>, VEX_4V, VEX_L; 5683 defm VPHSUBDY : SS3I_binop_rm<0x06, "vphsubd", X86hsub, v8i32, VR256, 5684 loadv4i64, i256mem, 5685 SSE_PHADDSUBW, 0>, VEX_4V, VEX_L; 5686 defm VPSIGNBY : SS3I_binop_rm<0x08, "vpsignb", X86psign, v32i8, VR256, 5687 loadv4i64, i256mem, 5688 SSE_PHADDSUBW, 0>, VEX_4V, VEX_L; 5689 defm VPSIGNWY : SS3I_binop_rm<0x09, "vpsignw", X86psign, v16i16, VR256, 5690 loadv4i64, i256mem, 5691 SSE_PHADDSUBW, 0>, VEX_4V, VEX_L; 5692 defm VPSIGNDY : SS3I_binop_rm<0x0A, "vpsignd", X86psign, v8i32, VR256, 5693 loadv4i64, i256mem, 5694 SSE_PHADDSUBW, 0>, VEX_4V, VEX_L; 5695 defm VPSHUFBY : SS3I_binop_rm<0x00, "vpshufb", X86pshufb, v32i8, VR256, 5696 loadv4i64, i256mem, 5697 SSE_PSHUFB, 0>, VEX_4V, VEX_L; 5698 defm VPHADDSW : SS3I_binop_rm_int_y<0x03, "vphaddsw", 5699 int_x86_avx2_phadd_sw, 5700 WriteVecALU>, VEX_4V, VEX_L; 5701 defm VPHSUBSW : SS3I_binop_rm_int_y<0x07, "vphsubsw", 5702 int_x86_avx2_phsub_sw, 5703 WriteVecALU>, VEX_4V, VEX_L; 5704 defm VPMADDUBSW : SS3I_binop_rm_int_y<0x04, "vpmaddubsw", 5705 int_x86_avx2_pmadd_ub_sw, 5706 WriteVecIMul>, VEX_4V, VEX_L; 5707} 5708defm VPMULHRSW : SS3I_binop_rm_int_y<0x0B, "vpmulhrsw", 5709 int_x86_avx2_pmul_hr_sw, 5710 WriteVecIMul>, VEX_4V, VEX_L; 5711} 5712 5713// None of these have i8 immediate fields. 5714let ImmT = NoImm, Constraints = "$src1 = $dst" in { 5715let isCommutable = 0 in { 5716 defm PHADDW : SS3I_binop_rm<0x01, "phaddw", X86hadd, v8i16, VR128, 5717 memopv2i64, i128mem, SSE_PHADDSUBW>; 5718 defm PHADDD : SS3I_binop_rm<0x02, "phaddd", X86hadd, v4i32, VR128, 5719 memopv2i64, i128mem, SSE_PHADDSUBD>; 5720 defm PHSUBW : SS3I_binop_rm<0x05, "phsubw", X86hsub, v8i16, VR128, 5721 memopv2i64, i128mem, SSE_PHADDSUBW>; 5722 defm PHSUBD : SS3I_binop_rm<0x06, "phsubd", X86hsub, v4i32, VR128, 5723 memopv2i64, i128mem, SSE_PHADDSUBD>; 5724 defm PSIGNB : SS3I_binop_rm<0x08, "psignb", X86psign, v16i8, VR128, 5725 memopv2i64, i128mem, SSE_PSIGN>; 5726 defm PSIGNW : SS3I_binop_rm<0x09, "psignw", X86psign, v8i16, VR128, 5727 memopv2i64, i128mem, SSE_PSIGN>; 5728 defm PSIGND : SS3I_binop_rm<0x0A, "psignd", X86psign, v4i32, VR128, 5729 memopv2i64, i128mem, SSE_PSIGN>; 5730 defm PSHUFB : SS3I_binop_rm<0x00, "pshufb", X86pshufb, v16i8, VR128, 5731 memopv2i64, i128mem, SSE_PSHUFB>; 5732 defm PHADDSW : SS3I_binop_rm_int<0x03, "phaddsw", 5733 int_x86_ssse3_phadd_sw_128, 5734 SSE_PHADDSUBSW, memopv2i64>; 5735 defm PHSUBSW : SS3I_binop_rm_int<0x07, "phsubsw", 5736 int_x86_ssse3_phsub_sw_128, 5737 SSE_PHADDSUBSW, memopv2i64>; 5738 defm PMADDUBSW : SS3I_binop_rm_int<0x04, "pmaddubsw", 5739 int_x86_ssse3_pmadd_ub_sw_128, 5740 SSE_PMADD, memopv2i64>; 5741} 5742defm PMULHRSW : SS3I_binop_rm_int<0x0B, "pmulhrsw", 5743 int_x86_ssse3_pmul_hr_sw_128, 5744 SSE_PMULHRSW, memopv2i64>; 5745} 5746 5747//===---------------------------------------------------------------------===// 5748// SSSE3 - Packed Align Instruction Patterns 5749//===---------------------------------------------------------------------===// 5750 5751multiclass ssse3_palignr<string asm, bit Is2Addr = 1> { 5752 let hasSideEffects = 0 in { 5753 def R128rr : SS3AI<0x0F, MRMSrcReg, (outs VR128:$dst), 5754 (ins VR128:$src1, VR128:$src2, u8imm:$src3), 5755 !if(Is2Addr, 5756 !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 5757 !strconcat(asm, 5758 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 5759 [], IIC_SSE_PALIGNRR>, Sched<[WriteShuffle]>; 5760 let mayLoad = 1 in 5761 def R128rm : SS3AI<0x0F, MRMSrcMem, (outs VR128:$dst), 5762 (ins VR128:$src1, i128mem:$src2, u8imm:$src3), 5763 !if(Is2Addr, 5764 !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 5765 !strconcat(asm, 5766 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 5767 [], IIC_SSE_PALIGNRM>, Sched<[WriteShuffleLd, ReadAfterLd]>; 5768 } 5769} 5770 5771multiclass ssse3_palignr_y<string asm, bit Is2Addr = 1> { 5772 let hasSideEffects = 0 in { 5773 def R256rr : SS3AI<0x0F, MRMSrcReg, (outs VR256:$dst), 5774 (ins VR256:$src1, VR256:$src2, u8imm:$src3), 5775 !strconcat(asm, 5776 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), 5777 []>, Sched<[WriteShuffle]>; 5778 let mayLoad = 1 in 5779 def R256rm : SS3AI<0x0F, MRMSrcMem, (outs VR256:$dst), 5780 (ins VR256:$src1, i256mem:$src2, u8imm:$src3), 5781 !strconcat(asm, 5782 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), 5783 []>, Sched<[WriteShuffleLd, ReadAfterLd]>; 5784 } 5785} 5786 5787let Predicates = [HasAVX] in 5788 defm VPALIGN : ssse3_palignr<"vpalignr", 0>, VEX_4V; 5789let Predicates = [HasAVX2] in 5790 defm VPALIGN : ssse3_palignr_y<"vpalignr", 0>, VEX_4V, VEX_L; 5791let Constraints = "$src1 = $dst", Predicates = [UseSSSE3] in 5792 defm PALIGN : ssse3_palignr<"palignr">; 5793 5794let Predicates = [HasAVX2] in { 5795def : Pat<(v8i32 (X86PAlignr VR256:$src1, VR256:$src2, (i8 imm:$imm))), 5796 (VPALIGNR256rr VR256:$src2, VR256:$src1, imm:$imm)>; 5797def : Pat<(v8f32 (X86PAlignr VR256:$src1, VR256:$src2, (i8 imm:$imm))), 5798 (VPALIGNR256rr VR256:$src2, VR256:$src1, imm:$imm)>; 5799def : Pat<(v16i16 (X86PAlignr VR256:$src1, VR256:$src2, (i8 imm:$imm))), 5800 (VPALIGNR256rr VR256:$src2, VR256:$src1, imm:$imm)>; 5801def : Pat<(v32i8 (X86PAlignr VR256:$src1, VR256:$src2, (i8 imm:$imm))), 5802 (VPALIGNR256rr VR256:$src2, VR256:$src1, imm:$imm)>; 5803} 5804 5805let Predicates = [HasAVX] in { 5806def : Pat<(v4i32 (X86PAlignr VR128:$src1, VR128:$src2, (i8 imm:$imm))), 5807 (VPALIGNR128rr VR128:$src2, VR128:$src1, imm:$imm)>; 5808def : Pat<(v4f32 (X86PAlignr VR128:$src1, VR128:$src2, (i8 imm:$imm))), 5809 (VPALIGNR128rr VR128:$src2, VR128:$src1, imm:$imm)>; 5810def : Pat<(v8i16 (X86PAlignr VR128:$src1, VR128:$src2, (i8 imm:$imm))), 5811 (VPALIGNR128rr VR128:$src2, VR128:$src1, imm:$imm)>; 5812def : Pat<(v16i8 (X86PAlignr VR128:$src1, VR128:$src2, (i8 imm:$imm))), 5813 (VPALIGNR128rr VR128:$src2, VR128:$src1, imm:$imm)>; 5814} 5815 5816let Predicates = [UseSSSE3] in { 5817def : Pat<(v4i32 (X86PAlignr VR128:$src1, VR128:$src2, (i8 imm:$imm))), 5818 (PALIGNR128rr VR128:$src2, VR128:$src1, imm:$imm)>; 5819def : Pat<(v4f32 (X86PAlignr VR128:$src1, VR128:$src2, (i8 imm:$imm))), 5820 (PALIGNR128rr VR128:$src2, VR128:$src1, imm:$imm)>; 5821def : Pat<(v8i16 (X86PAlignr VR128:$src1, VR128:$src2, (i8 imm:$imm))), 5822 (PALIGNR128rr VR128:$src2, VR128:$src1, imm:$imm)>; 5823def : Pat<(v16i8 (X86PAlignr VR128:$src1, VR128:$src2, (i8 imm:$imm))), 5824 (PALIGNR128rr VR128:$src2, VR128:$src1, imm:$imm)>; 5825} 5826 5827//===---------------------------------------------------------------------===// 5828// SSSE3 - Thread synchronization 5829//===---------------------------------------------------------------------===// 5830 5831let SchedRW = [WriteSystem] in { 5832let usesCustomInserter = 1 in { 5833def MONITOR : PseudoI<(outs), (ins i32mem:$src1, GR32:$src2, GR32:$src3), 5834 [(int_x86_sse3_monitor addr:$src1, GR32:$src2, GR32:$src3)]>, 5835 Requires<[HasSSE3]>; 5836} 5837 5838let Uses = [EAX, ECX, EDX] in 5839def MONITORrrr : I<0x01, MRM_C8, (outs), (ins), "monitor", [], IIC_SSE_MONITOR>, 5840 TB, Requires<[HasSSE3]>; 5841let Uses = [ECX, EAX] in 5842def MWAITrr : I<0x01, MRM_C9, (outs), (ins), "mwait", 5843 [(int_x86_sse3_mwait ECX, EAX)], IIC_SSE_MWAIT>, 5844 TB, Requires<[HasSSE3]>; 5845} // SchedRW 5846 5847def : InstAlias<"mwait\t{%eax, %ecx|ecx, eax}", (MWAITrr)>, Requires<[Not64BitMode]>; 5848def : InstAlias<"mwait\t{%rax, %rcx|rcx, rax}", (MWAITrr)>, Requires<[In64BitMode]>; 5849 5850def : InstAlias<"monitor\t{%eax, %ecx, %edx|edx, ecx, eax}", (MONITORrrr)>, 5851 Requires<[Not64BitMode]>; 5852def : InstAlias<"monitor\t{%rax, %rcx, %rdx|rdx, rcx, rax}", (MONITORrrr)>, 5853 Requires<[In64BitMode]>; 5854 5855//===----------------------------------------------------------------------===// 5856// SSE4.1 - Packed Move with Sign/Zero Extend 5857//===----------------------------------------------------------------------===// 5858 5859multiclass SS41I_pmovx_rrrm<bits<8> opc, string OpcodeStr, X86MemOperand MemOp, 5860 RegisterClass OutRC, RegisterClass InRC, 5861 OpndItins itins> { 5862 def rr : SS48I<opc, MRMSrcReg, (outs OutRC:$dst), (ins InRC:$src), 5863 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 5864 [], itins.rr>, 5865 Sched<[itins.Sched]>; 5866 5867 def rm : SS48I<opc, MRMSrcMem, (outs OutRC:$dst), (ins MemOp:$src), 5868 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 5869 [], 5870 itins.rm>, Sched<[itins.Sched.Folded]>; 5871} 5872 5873multiclass SS41I_pmovx_rm_all<bits<8> opc, string OpcodeStr, 5874 X86MemOperand MemOp, X86MemOperand MemYOp, 5875 OpndItins SSEItins, OpndItins AVXItins, 5876 OpndItins AVX2Itins> { 5877 defm NAME : SS41I_pmovx_rrrm<opc, OpcodeStr, MemOp, VR128, VR128, SSEItins>; 5878 let Predicates = [HasAVX, NoVLX] in 5879 defm V#NAME : SS41I_pmovx_rrrm<opc, !strconcat("v", OpcodeStr), MemOp, 5880 VR128, VR128, AVXItins>, VEX; 5881 let Predicates = [HasAVX2, NoVLX] in 5882 defm V#NAME#Y : SS41I_pmovx_rrrm<opc, !strconcat("v", OpcodeStr), MemYOp, 5883 VR256, VR128, AVX2Itins>, VEX, VEX_L; 5884} 5885 5886multiclass SS41I_pmovx_rm<bits<8> opc, string OpcodeStr, 5887 X86MemOperand MemOp, X86MemOperand MemYOp> { 5888 defm PMOVSX#NAME : SS41I_pmovx_rm_all<opc, !strconcat("pmovsx", OpcodeStr), 5889 MemOp, MemYOp, 5890 SSE_INTALU_ITINS_SHUFF_P, 5891 DEFAULT_ITINS_SHUFFLESCHED, 5892 DEFAULT_ITINS_SHUFFLESCHED>; 5893 defm PMOVZX#NAME : SS41I_pmovx_rm_all<!add(opc, 0x10), 5894 !strconcat("pmovzx", OpcodeStr), 5895 MemOp, MemYOp, 5896 SSE_INTALU_ITINS_SHUFF_P, 5897 DEFAULT_ITINS_SHUFFLESCHED, 5898 DEFAULT_ITINS_SHUFFLESCHED>; 5899} 5900 5901defm BW : SS41I_pmovx_rm<0x20, "bw", i64mem, i128mem>; 5902defm WD : SS41I_pmovx_rm<0x23, "wd", i64mem, i128mem>; 5903defm DQ : SS41I_pmovx_rm<0x25, "dq", i64mem, i128mem>; 5904 5905defm BD : SS41I_pmovx_rm<0x21, "bd", i32mem, i64mem>; 5906defm WQ : SS41I_pmovx_rm<0x24, "wq", i32mem, i64mem>; 5907 5908defm BQ : SS41I_pmovx_rm<0x22, "bq", i16mem, i32mem>; 5909 5910// AVX2 Patterns 5911multiclass SS41I_pmovx_avx2_patterns<string OpcPrefix, string ExtTy, SDNode ExtOp> { 5912 // Register-Register patterns 5913 def : Pat<(v16i16 (ExtOp (v16i8 VR128:$src))), 5914 (!cast<I>(OpcPrefix#BWYrr) VR128:$src)>; 5915 def : Pat<(v8i32 (ExtOp (v16i8 VR128:$src))), 5916 (!cast<I>(OpcPrefix#BDYrr) VR128:$src)>; 5917 def : Pat<(v4i64 (ExtOp (v16i8 VR128:$src))), 5918 (!cast<I>(OpcPrefix#BQYrr) VR128:$src)>; 5919 5920 def : Pat<(v8i32 (ExtOp (v8i16 VR128:$src))), 5921 (!cast<I>(OpcPrefix#WDYrr) VR128:$src)>; 5922 def : Pat<(v4i64 (ExtOp (v8i16 VR128:$src))), 5923 (!cast<I>(OpcPrefix#WQYrr) VR128:$src)>; 5924 5925 def : Pat<(v4i64 (ExtOp (v4i32 VR128:$src))), 5926 (!cast<I>(OpcPrefix#DQYrr) VR128:$src)>; 5927 5928 // On AVX2, we also support 256bit inputs. 5929 def : Pat<(v16i16 (ExtOp (v32i8 VR256:$src))), 5930 (!cast<I>(OpcPrefix#BWYrr) (EXTRACT_SUBREG VR256:$src, sub_xmm))>; 5931 def : Pat<(v8i32 (ExtOp (v32i8 VR256:$src))), 5932 (!cast<I>(OpcPrefix#BDYrr) (EXTRACT_SUBREG VR256:$src, sub_xmm))>; 5933 def : Pat<(v4i64 (ExtOp (v32i8 VR256:$src))), 5934 (!cast<I>(OpcPrefix#BQYrr) (EXTRACT_SUBREG VR256:$src, sub_xmm))>; 5935 5936 def : Pat<(v8i32 (ExtOp (v16i16 VR256:$src))), 5937 (!cast<I>(OpcPrefix#WDYrr) (EXTRACT_SUBREG VR256:$src, sub_xmm))>; 5938 def : Pat<(v4i64 (ExtOp (v16i16 VR256:$src))), 5939 (!cast<I>(OpcPrefix#WQYrr) (EXTRACT_SUBREG VR256:$src, sub_xmm))>; 5940 5941 def : Pat<(v4i64 (ExtOp (v8i32 VR256:$src))), 5942 (!cast<I>(OpcPrefix#DQYrr) (EXTRACT_SUBREG VR256:$src, sub_xmm))>; 5943 5944 // Simple Register-Memory patterns 5945 def : Pat<(v16i16 (!cast<PatFrag>(ExtTy#"extloadvi8") addr:$src)), 5946 (!cast<I>(OpcPrefix#BWYrm) addr:$src)>; 5947 def : Pat<(v8i32 (!cast<PatFrag>(ExtTy#"extloadvi8") addr:$src)), 5948 (!cast<I>(OpcPrefix#BDYrm) addr:$src)>; 5949 def : Pat<(v4i64 (!cast<PatFrag>(ExtTy#"extloadvi8") addr:$src)), 5950 (!cast<I>(OpcPrefix#BQYrm) addr:$src)>; 5951 5952 def : Pat<(v8i32 (!cast<PatFrag>(ExtTy#"extloadvi16") addr:$src)), 5953 (!cast<I>(OpcPrefix#WDYrm) addr:$src)>; 5954 def : Pat<(v4i64 (!cast<PatFrag>(ExtTy#"extloadvi16") addr:$src)), 5955 (!cast<I>(OpcPrefix#WQYrm) addr:$src)>; 5956 5957 def : Pat<(v4i64 (!cast<PatFrag>(ExtTy#"extloadvi32") addr:$src)), 5958 (!cast<I>(OpcPrefix#DQYrm) addr:$src)>; 5959 5960 // AVX2 Register-Memory patterns 5961 def : Pat<(v16i16 (ExtOp (bc_v16i8 (loadv2i64 addr:$src)))), 5962 (!cast<I>(OpcPrefix#BWYrm) addr:$src)>; 5963 def : Pat<(v16i16 (ExtOp (v16i8 (vzmovl_v2i64 addr:$src)))), 5964 (!cast<I>(OpcPrefix#BWYrm) addr:$src)>; 5965 def : Pat<(v16i16 (ExtOp (v16i8 (vzload_v2i64 addr:$src)))), 5966 (!cast<I>(OpcPrefix#BWYrm) addr:$src)>; 5967 def : Pat<(v16i16 (ExtOp (bc_v16i8 (loadv2i64 addr:$src)))), 5968 (!cast<I>(OpcPrefix#BWYrm) addr:$src)>; 5969 5970 def : Pat<(v8i32 (ExtOp (bc_v16i8 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))), 5971 (!cast<I>(OpcPrefix#BDYrm) addr:$src)>; 5972 def : Pat<(v8i32 (ExtOp (v16i8 (vzmovl_v2i64 addr:$src)))), 5973 (!cast<I>(OpcPrefix#BDYrm) addr:$src)>; 5974 def : Pat<(v8i32 (ExtOp (v16i8 (vzload_v2i64 addr:$src)))), 5975 (!cast<I>(OpcPrefix#BDYrm) addr:$src)>; 5976 def : Pat<(v8i32 (ExtOp (bc_v16i8 (loadv2i64 addr:$src)))), 5977 (!cast<I>(OpcPrefix#BDYrm) addr:$src)>; 5978 5979 def : Pat<(v4i64 (ExtOp (bc_v16i8 (v4i32 (scalar_to_vector (loadi32 addr:$src)))))), 5980 (!cast<I>(OpcPrefix#BQYrm) addr:$src)>; 5981 def : Pat<(v4i64 (ExtOp (v16i8 (vzmovl_v4i32 addr:$src)))), 5982 (!cast<I>(OpcPrefix#BQYrm) addr:$src)>; 5983 def : Pat<(v4i64 (ExtOp (v16i8 (vzload_v2i64 addr:$src)))), 5984 (!cast<I>(OpcPrefix#BQYrm) addr:$src)>; 5985 def : Pat<(v4i64 (ExtOp (bc_v16i8 (loadv2i64 addr:$src)))), 5986 (!cast<I>(OpcPrefix#BQYrm) addr:$src)>; 5987 5988 def : Pat<(v8i32 (ExtOp (bc_v8i16 (loadv2i64 addr:$src)))), 5989 (!cast<I>(OpcPrefix#WDYrm) addr:$src)>; 5990 def : Pat<(v8i32 (ExtOp (v8i16 (vzmovl_v2i64 addr:$src)))), 5991 (!cast<I>(OpcPrefix#WDYrm) addr:$src)>; 5992 def : Pat<(v8i32 (ExtOp (v8i16 (vzload_v2i64 addr:$src)))), 5993 (!cast<I>(OpcPrefix#WDYrm) addr:$src)>; 5994 def : Pat<(v8i32 (ExtOp (bc_v8i16 (loadv2i64 addr:$src)))), 5995 (!cast<I>(OpcPrefix#WDYrm) addr:$src)>; 5996 5997 def : Pat<(v4i64 (ExtOp (bc_v8i16 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))), 5998 (!cast<I>(OpcPrefix#WQYrm) addr:$src)>; 5999 def : Pat<(v4i64 (ExtOp (v8i16 (vzmovl_v2i64 addr:$src)))), 6000 (!cast<I>(OpcPrefix#WQYrm) addr:$src)>; 6001 def : Pat<(v4i64 (ExtOp (v8i16 (vzload_v2i64 addr:$src)))), 6002 (!cast<I>(OpcPrefix#WQYrm) addr:$src)>; 6003 def : Pat<(v4i64 (ExtOp (bc_v8i16 (loadv2i64 addr:$src)))), 6004 (!cast<I>(OpcPrefix#WQYrm) addr:$src)>; 6005 6006 def : Pat<(v4i64 (ExtOp (bc_v4i32 (loadv2i64 addr:$src)))), 6007 (!cast<I>(OpcPrefix#DQYrm) addr:$src)>; 6008 def : Pat<(v4i64 (ExtOp (v4i32 (vzmovl_v2i64 addr:$src)))), 6009 (!cast<I>(OpcPrefix#DQYrm) addr:$src)>; 6010 def : Pat<(v4i64 (ExtOp (v4i32 (vzload_v2i64 addr:$src)))), 6011 (!cast<I>(OpcPrefix#DQYrm) addr:$src)>; 6012 def : Pat<(v4i64 (ExtOp (bc_v4i32 (loadv2i64 addr:$src)))), 6013 (!cast<I>(OpcPrefix#DQYrm) addr:$src)>; 6014} 6015 6016let Predicates = [HasAVX2, NoVLX] in { 6017 defm : SS41I_pmovx_avx2_patterns<"VPMOVSX", "s", X86vsext>; 6018 defm : SS41I_pmovx_avx2_patterns<"VPMOVZX", "z", X86vzext>; 6019} 6020 6021// SSE4.1/AVX patterns. 6022multiclass SS41I_pmovx_patterns<string OpcPrefix, string ExtTy, 6023 SDNode ExtOp, PatFrag ExtLoad16> { 6024 def : Pat<(v8i16 (ExtOp (v16i8 VR128:$src))), 6025 (!cast<I>(OpcPrefix#BWrr) VR128:$src)>; 6026 def : Pat<(v4i32 (ExtOp (v16i8 VR128:$src))), 6027 (!cast<I>(OpcPrefix#BDrr) VR128:$src)>; 6028 def : Pat<(v2i64 (ExtOp (v16i8 VR128:$src))), 6029 (!cast<I>(OpcPrefix#BQrr) VR128:$src)>; 6030 6031 def : Pat<(v4i32 (ExtOp (v8i16 VR128:$src))), 6032 (!cast<I>(OpcPrefix#WDrr) VR128:$src)>; 6033 def : Pat<(v2i64 (ExtOp (v8i16 VR128:$src))), 6034 (!cast<I>(OpcPrefix#WQrr) VR128:$src)>; 6035 6036 def : Pat<(v2i64 (ExtOp (v4i32 VR128:$src))), 6037 (!cast<I>(OpcPrefix#DQrr) VR128:$src)>; 6038 6039 def : Pat<(v8i16 (!cast<PatFrag>(ExtTy#"extloadvi8") addr:$src)), 6040 (!cast<I>(OpcPrefix#BWrm) addr:$src)>; 6041 def : Pat<(v4i32 (!cast<PatFrag>(ExtTy#"extloadvi8") addr:$src)), 6042 (!cast<I>(OpcPrefix#BDrm) addr:$src)>; 6043 def : Pat<(v2i64 (!cast<PatFrag>(ExtTy#"extloadvi8") addr:$src)), 6044 (!cast<I>(OpcPrefix#BQrm) addr:$src)>; 6045 6046 def : Pat<(v4i32 (!cast<PatFrag>(ExtTy#"extloadvi16") addr:$src)), 6047 (!cast<I>(OpcPrefix#WDrm) addr:$src)>; 6048 def : Pat<(v2i64 (!cast<PatFrag>(ExtTy#"extloadvi16") addr:$src)), 6049 (!cast<I>(OpcPrefix#WQrm) addr:$src)>; 6050 6051 def : Pat<(v2i64 (!cast<PatFrag>(ExtTy#"extloadvi32") addr:$src)), 6052 (!cast<I>(OpcPrefix#DQrm) addr:$src)>; 6053 6054 def : Pat<(v8i16 (ExtOp (bc_v16i8 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))), 6055 (!cast<I>(OpcPrefix#BWrm) addr:$src)>; 6056 def : Pat<(v8i16 (ExtOp (bc_v16i8 (v2f64 (scalar_to_vector (loadf64 addr:$src)))))), 6057 (!cast<I>(OpcPrefix#BWrm) addr:$src)>; 6058 def : Pat<(v8i16 (ExtOp (v16i8 (vzmovl_v2i64 addr:$src)))), 6059 (!cast<I>(OpcPrefix#BWrm) addr:$src)>; 6060 def : Pat<(v8i16 (ExtOp (v16i8 (vzload_v2i64 addr:$src)))), 6061 (!cast<I>(OpcPrefix#BWrm) addr:$src)>; 6062 def : Pat<(v8i16 (ExtOp (bc_v16i8 (loadv2i64 addr:$src)))), 6063 (!cast<I>(OpcPrefix#BWrm) addr:$src)>; 6064 6065 def : Pat<(v4i32 (ExtOp (bc_v16i8 (v4i32 (scalar_to_vector (loadi32 addr:$src)))))), 6066 (!cast<I>(OpcPrefix#BDrm) addr:$src)>; 6067 def : Pat<(v4i32 (ExtOp (v16i8 (vzmovl_v4i32 addr:$src)))), 6068 (!cast<I>(OpcPrefix#BDrm) addr:$src)>; 6069 def : Pat<(v4i32 (ExtOp (v16i8 (vzload_v2i64 addr:$src)))), 6070 (!cast<I>(OpcPrefix#BDrm) addr:$src)>; 6071 def : Pat<(v4i32 (ExtOp (bc_v16i8 (loadv2i64 addr:$src)))), 6072 (!cast<I>(OpcPrefix#BDrm) addr:$src)>; 6073 6074 def : Pat<(v2i64 (ExtOp (bc_v16i8 (v4i32 (scalar_to_vector (ExtLoad16 addr:$src)))))), 6075 (!cast<I>(OpcPrefix#BQrm) addr:$src)>; 6076 def : Pat<(v2i64 (ExtOp (v16i8 (vzmovl_v4i32 addr:$src)))), 6077 (!cast<I>(OpcPrefix#BQrm) addr:$src)>; 6078 def : Pat<(v2i64 (ExtOp (v16i8 (vzload_v2i64 addr:$src)))), 6079 (!cast<I>(OpcPrefix#BQrm) addr:$src)>; 6080 def : Pat<(v2i64 (ExtOp (bc_v16i8 (loadv2i64 addr:$src)))), 6081 (!cast<I>(OpcPrefix#BQrm) addr:$src)>; 6082 6083 def : Pat<(v4i32 (ExtOp (bc_v8i16 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))), 6084 (!cast<I>(OpcPrefix#WDrm) addr:$src)>; 6085 def : Pat<(v4i32 (ExtOp (bc_v8i16 (v2f64 (scalar_to_vector (loadf64 addr:$src)))))), 6086 (!cast<I>(OpcPrefix#WDrm) addr:$src)>; 6087 def : Pat<(v4i32 (ExtOp (v8i16 (vzmovl_v2i64 addr:$src)))), 6088 (!cast<I>(OpcPrefix#WDrm) addr:$src)>; 6089 def : Pat<(v4i32 (ExtOp (v8i16 (vzload_v2i64 addr:$src)))), 6090 (!cast<I>(OpcPrefix#WDrm) addr:$src)>; 6091 def : Pat<(v4i32 (ExtOp (bc_v8i16 (loadv2i64 addr:$src)))), 6092 (!cast<I>(OpcPrefix#WDrm) addr:$src)>; 6093 6094 def : Pat<(v2i64 (ExtOp (bc_v8i16 (v4i32 (scalar_to_vector (loadi32 addr:$src)))))), 6095 (!cast<I>(OpcPrefix#WQrm) addr:$src)>; 6096 def : Pat<(v2i64 (ExtOp (v8i16 (vzmovl_v4i32 addr:$src)))), 6097 (!cast<I>(OpcPrefix#WQrm) addr:$src)>; 6098 def : Pat<(v2i64 (ExtOp (v8i16 (vzload_v2i64 addr:$src)))), 6099 (!cast<I>(OpcPrefix#WQrm) addr:$src)>; 6100 def : Pat<(v2i64 (ExtOp (bc_v8i16 (loadv2i64 addr:$src)))), 6101 (!cast<I>(OpcPrefix#WQrm) addr:$src)>; 6102 6103 def : Pat<(v2i64 (ExtOp (bc_v4i32 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))), 6104 (!cast<I>(OpcPrefix#DQrm) addr:$src)>; 6105 def : Pat<(v2i64 (ExtOp (bc_v4i32 (v2f64 (scalar_to_vector (loadf64 addr:$src)))))), 6106 (!cast<I>(OpcPrefix#DQrm) addr:$src)>; 6107 def : Pat<(v2i64 (ExtOp (v4i32 (vzmovl_v2i64 addr:$src)))), 6108 (!cast<I>(OpcPrefix#DQrm) addr:$src)>; 6109 def : Pat<(v2i64 (ExtOp (v4i32 (vzload_v2i64 addr:$src)))), 6110 (!cast<I>(OpcPrefix#DQrm) addr:$src)>; 6111 def : Pat<(v2i64 (ExtOp (bc_v4i32 (loadv2i64 addr:$src)))), 6112 (!cast<I>(OpcPrefix#DQrm) addr:$src)>; 6113} 6114 6115let Predicates = [HasAVX, NoVLX] in { 6116 defm : SS41I_pmovx_patterns<"VPMOVSX", "s", X86vsext, extloadi32i16>; 6117 defm : SS41I_pmovx_patterns<"VPMOVZX", "z", X86vzext, loadi16_anyext>; 6118} 6119 6120let Predicates = [UseSSE41] in { 6121 defm : SS41I_pmovx_patterns<"PMOVSX", "s", X86vsext, extloadi32i16>; 6122 defm : SS41I_pmovx_patterns<"PMOVZX", "z", X86vzext, loadi16_anyext>; 6123} 6124 6125//===----------------------------------------------------------------------===// 6126// SSE4.1 - Extract Instructions 6127//===----------------------------------------------------------------------===// 6128 6129/// SS41I_binop_ext8 - SSE 4.1 extract 8 bits to 32 bit reg or 8 bit mem 6130multiclass SS41I_extract8<bits<8> opc, string OpcodeStr> { 6131 def rr : SS4AIi8<opc, MRMDestReg, (outs GR32orGR64:$dst), 6132 (ins VR128:$src1, u8imm:$src2), 6133 !strconcat(OpcodeStr, 6134 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 6135 [(set GR32orGR64:$dst, (X86pextrb (v16i8 VR128:$src1), 6136 imm:$src2))]>, 6137 Sched<[WriteShuffle]>; 6138 let hasSideEffects = 0, mayStore = 1, 6139 SchedRW = [WriteShuffleLd, WriteRMW] in 6140 def mr : SS4AIi8<opc, MRMDestMem, (outs), 6141 (ins i8mem:$dst, VR128:$src1, u8imm:$src2), 6142 !strconcat(OpcodeStr, 6143 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 6144 [(store (i8 (trunc (assertzext (X86pextrb (v16i8 VR128:$src1), 6145 imm:$src2)))), addr:$dst)]>; 6146} 6147 6148let Predicates = [HasAVX] in 6149 defm VPEXTRB : SS41I_extract8<0x14, "vpextrb">, VEX; 6150 6151defm PEXTRB : SS41I_extract8<0x14, "pextrb">; 6152 6153 6154/// SS41I_extract16 - SSE 4.1 extract 16 bits to memory destination 6155multiclass SS41I_extract16<bits<8> opc, string OpcodeStr> { 6156 let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0 in 6157 def rr_REV : SS4AIi8<opc, MRMDestReg, (outs GR32orGR64:$dst), 6158 (ins VR128:$src1, u8imm:$src2), 6159 !strconcat(OpcodeStr, 6160 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 6161 []>, Sched<[WriteShuffle]>; 6162 6163 let hasSideEffects = 0, mayStore = 1, 6164 SchedRW = [WriteShuffleLd, WriteRMW] in 6165 def mr : SS4AIi8<opc, MRMDestMem, (outs), 6166 (ins i16mem:$dst, VR128:$src1, u8imm:$src2), 6167 !strconcat(OpcodeStr, 6168 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 6169 [(store (i16 (trunc (assertzext (X86pextrw (v8i16 VR128:$src1), 6170 imm:$src2)))), addr:$dst)]>; 6171} 6172 6173let Predicates = [HasAVX] in 6174 defm VPEXTRW : SS41I_extract16<0x15, "vpextrw">, VEX; 6175 6176defm PEXTRW : SS41I_extract16<0x15, "pextrw">; 6177 6178 6179/// SS41I_extract32 - SSE 4.1 extract 32 bits to int reg or memory destination 6180multiclass SS41I_extract32<bits<8> opc, string OpcodeStr> { 6181 def rr : SS4AIi8<opc, MRMDestReg, (outs GR32:$dst), 6182 (ins VR128:$src1, u8imm:$src2), 6183 !strconcat(OpcodeStr, 6184 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 6185 [(set GR32:$dst, 6186 (extractelt (v4i32 VR128:$src1), imm:$src2))]>, 6187 Sched<[WriteShuffle]>; 6188 let SchedRW = [WriteShuffleLd, WriteRMW] in 6189 def mr : SS4AIi8<opc, MRMDestMem, (outs), 6190 (ins i32mem:$dst, VR128:$src1, u8imm:$src2), 6191 !strconcat(OpcodeStr, 6192 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 6193 [(store (extractelt (v4i32 VR128:$src1), imm:$src2), 6194 addr:$dst)]>; 6195} 6196 6197let Predicates = [HasAVX] in 6198 defm VPEXTRD : SS41I_extract32<0x16, "vpextrd">, VEX; 6199 6200defm PEXTRD : SS41I_extract32<0x16, "pextrd">; 6201 6202/// SS41I_extract32 - SSE 4.1 extract 32 bits to int reg or memory destination 6203multiclass SS41I_extract64<bits<8> opc, string OpcodeStr> { 6204 def rr : SS4AIi8<opc, MRMDestReg, (outs GR64:$dst), 6205 (ins VR128:$src1, u8imm:$src2), 6206 !strconcat(OpcodeStr, 6207 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 6208 [(set GR64:$dst, 6209 (extractelt (v2i64 VR128:$src1), imm:$src2))]>, 6210 Sched<[WriteShuffle]>, REX_W; 6211 let SchedRW = [WriteShuffleLd, WriteRMW] in 6212 def mr : SS4AIi8<opc, MRMDestMem, (outs), 6213 (ins i64mem:$dst, VR128:$src1, u8imm:$src2), 6214 !strconcat(OpcodeStr, 6215 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 6216 [(store (extractelt (v2i64 VR128:$src1), imm:$src2), 6217 addr:$dst)]>, REX_W; 6218} 6219 6220let Predicates = [HasAVX] in 6221 defm VPEXTRQ : SS41I_extract64<0x16, "vpextrq">, VEX, VEX_W; 6222 6223defm PEXTRQ : SS41I_extract64<0x16, "pextrq">; 6224 6225/// SS41I_extractf32 - SSE 4.1 extract 32 bits fp value to int reg or memory 6226/// destination 6227multiclass SS41I_extractf32<bits<8> opc, string OpcodeStr, 6228 OpndItins itins = DEFAULT_ITINS> { 6229 def rr : SS4AIi8<opc, MRMDestReg, (outs GR32orGR64:$dst), 6230 (ins VR128:$src1, u8imm:$src2), 6231 !strconcat(OpcodeStr, 6232 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 6233 [(set GR32orGR64:$dst, 6234 (extractelt (bc_v4i32 (v4f32 VR128:$src1)), imm:$src2))], 6235 itins.rr>, Sched<[WriteFBlend]>; 6236 let SchedRW = [WriteFBlendLd, WriteRMW] in 6237 def mr : SS4AIi8<opc, MRMDestMem, (outs), 6238 (ins f32mem:$dst, VR128:$src1, u8imm:$src2), 6239 !strconcat(OpcodeStr, 6240 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 6241 [(store (extractelt (bc_v4i32 (v4f32 VR128:$src1)), imm:$src2), 6242 addr:$dst)], itins.rm>; 6243} 6244 6245let ExeDomain = SSEPackedSingle in { 6246 let Predicates = [UseAVX] in 6247 defm VEXTRACTPS : SS41I_extractf32<0x17, "vextractps">, VEX; 6248 defm EXTRACTPS : SS41I_extractf32<0x17, "extractps", SSE_EXTRACT_ITINS>; 6249} 6250 6251// Also match an EXTRACTPS store when the store is done as f32 instead of i32. 6252def : Pat<(store (f32 (bitconvert (extractelt (bc_v4i32 (v4f32 VR128:$src1)), 6253 imm:$src2))), 6254 addr:$dst), 6255 (VEXTRACTPSmr addr:$dst, VR128:$src1, imm:$src2)>, 6256 Requires<[HasAVX]>; 6257def : Pat<(store (f32 (bitconvert (extractelt (bc_v4i32 (v4f32 VR128:$src1)), 6258 imm:$src2))), 6259 addr:$dst), 6260 (EXTRACTPSmr addr:$dst, VR128:$src1, imm:$src2)>, 6261 Requires<[UseSSE41]>; 6262 6263//===----------------------------------------------------------------------===// 6264// SSE4.1 - Insert Instructions 6265//===----------------------------------------------------------------------===// 6266 6267multiclass SS41I_insert8<bits<8> opc, string asm, bit Is2Addr = 1> { 6268 def rr : SS4AIi8<opc, MRMSrcReg, (outs VR128:$dst), 6269 (ins VR128:$src1, GR32orGR64:$src2, u8imm:$src3), 6270 !if(Is2Addr, 6271 !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 6272 !strconcat(asm, 6273 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 6274 [(set VR128:$dst, 6275 (X86pinsrb VR128:$src1, GR32orGR64:$src2, imm:$src3))]>, 6276 Sched<[WriteShuffle]>; 6277 def rm : SS4AIi8<opc, MRMSrcMem, (outs VR128:$dst), 6278 (ins VR128:$src1, i8mem:$src2, u8imm:$src3), 6279 !if(Is2Addr, 6280 !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 6281 !strconcat(asm, 6282 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 6283 [(set VR128:$dst, 6284 (X86pinsrb VR128:$src1, (extloadi8 addr:$src2), 6285 imm:$src3))]>, Sched<[WriteShuffleLd, ReadAfterLd]>; 6286} 6287 6288let Predicates = [HasAVX] in 6289 defm VPINSRB : SS41I_insert8<0x20, "vpinsrb", 0>, VEX_4V; 6290let Constraints = "$src1 = $dst" in 6291 defm PINSRB : SS41I_insert8<0x20, "pinsrb">; 6292 6293multiclass SS41I_insert32<bits<8> opc, string asm, bit Is2Addr = 1> { 6294 def rr : SS4AIi8<opc, MRMSrcReg, (outs VR128:$dst), 6295 (ins VR128:$src1, GR32:$src2, u8imm:$src3), 6296 !if(Is2Addr, 6297 !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 6298 !strconcat(asm, 6299 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 6300 [(set VR128:$dst, 6301 (v4i32 (insertelt VR128:$src1, GR32:$src2, imm:$src3)))]>, 6302 Sched<[WriteShuffle]>; 6303 def rm : SS4AIi8<opc, MRMSrcMem, (outs VR128:$dst), 6304 (ins VR128:$src1, i32mem:$src2, u8imm:$src3), 6305 !if(Is2Addr, 6306 !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 6307 !strconcat(asm, 6308 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 6309 [(set VR128:$dst, 6310 (v4i32 (insertelt VR128:$src1, (loadi32 addr:$src2), 6311 imm:$src3)))]>, Sched<[WriteShuffleLd, ReadAfterLd]>; 6312} 6313 6314let Predicates = [HasAVX] in 6315 defm VPINSRD : SS41I_insert32<0x22, "vpinsrd", 0>, VEX_4V; 6316let Constraints = "$src1 = $dst" in 6317 defm PINSRD : SS41I_insert32<0x22, "pinsrd">; 6318 6319multiclass SS41I_insert64<bits<8> opc, string asm, bit Is2Addr = 1> { 6320 def rr : SS4AIi8<opc, MRMSrcReg, (outs VR128:$dst), 6321 (ins VR128:$src1, GR64:$src2, u8imm:$src3), 6322 !if(Is2Addr, 6323 !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 6324 !strconcat(asm, 6325 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 6326 [(set VR128:$dst, 6327 (v2i64 (insertelt VR128:$src1, GR64:$src2, imm:$src3)))]>, 6328 Sched<[WriteShuffle]>; 6329 def rm : SS4AIi8<opc, MRMSrcMem, (outs VR128:$dst), 6330 (ins VR128:$src1, i64mem:$src2, u8imm:$src3), 6331 !if(Is2Addr, 6332 !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 6333 !strconcat(asm, 6334 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 6335 [(set VR128:$dst, 6336 (v2i64 (insertelt VR128:$src1, (loadi64 addr:$src2), 6337 imm:$src3)))]>, Sched<[WriteShuffleLd, ReadAfterLd]>; 6338} 6339 6340let Predicates = [HasAVX] in 6341 defm VPINSRQ : SS41I_insert64<0x22, "vpinsrq", 0>, VEX_4V, VEX_W; 6342let Constraints = "$src1 = $dst" in 6343 defm PINSRQ : SS41I_insert64<0x22, "pinsrq">, REX_W; 6344 6345// insertps has a few different modes, there's the first two here below which 6346// are optimized inserts that won't zero arbitrary elements in the destination 6347// vector. The next one matches the intrinsic and could zero arbitrary elements 6348// in the target vector. 6349multiclass SS41I_insertf32<bits<8> opc, string asm, bit Is2Addr = 1, 6350 OpndItins itins = DEFAULT_ITINS> { 6351 def rr : SS4AIi8<opc, MRMSrcReg, (outs VR128:$dst), 6352 (ins VR128:$src1, VR128:$src2, u8imm:$src3), 6353 !if(Is2Addr, 6354 !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 6355 !strconcat(asm, 6356 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 6357 [(set VR128:$dst, 6358 (X86insertps VR128:$src1, VR128:$src2, imm:$src3))], itins.rr>, 6359 Sched<[WriteFShuffle]>; 6360 def rm : SS4AIi8<opc, MRMSrcMem, (outs VR128:$dst), 6361 (ins VR128:$src1, f32mem:$src2, u8imm:$src3), 6362 !if(Is2Addr, 6363 !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 6364 !strconcat(asm, 6365 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 6366 [(set VR128:$dst, 6367 (X86insertps VR128:$src1, 6368 (v4f32 (scalar_to_vector (loadf32 addr:$src2))), 6369 imm:$src3))], itins.rm>, 6370 Sched<[WriteFShuffleLd, ReadAfterLd]>; 6371} 6372 6373let ExeDomain = SSEPackedSingle in { 6374 let Predicates = [UseAVX] in 6375 defm VINSERTPS : SS41I_insertf32<0x21, "vinsertps", 0>, VEX_4V; 6376 let Constraints = "$src1 = $dst" in 6377 defm INSERTPS : SS41I_insertf32<0x21, "insertps", 1, SSE_INSERT_ITINS>; 6378} 6379 6380let Predicates = [UseSSE41] in { 6381 // If we're inserting an element from a load or a null pshuf of a load, 6382 // fold the load into the insertps instruction. 6383 def : Pat<(v4f32 (X86insertps (v4f32 VR128:$src1), (X86PShufd (v4f32 6384 (scalar_to_vector (loadf32 addr:$src2))), (i8 0)), 6385 imm:$src3)), 6386 (INSERTPSrm VR128:$src1, addr:$src2, imm:$src3)>; 6387 def : Pat<(v4f32 (X86insertps (v4f32 VR128:$src1), (X86PShufd 6388 (loadv4f32 addr:$src2), (i8 0)), imm:$src3)), 6389 (INSERTPSrm VR128:$src1, addr:$src2, imm:$src3)>; 6390} 6391 6392let Predicates = [UseAVX] in { 6393 // If we're inserting an element from a vbroadcast of a load, fold the 6394 // load into the X86insertps instruction. 6395 def : Pat<(v4f32 (X86insertps (v4f32 VR128:$src1), 6396 (X86VBroadcast (loadf32 addr:$src2)), imm:$src3)), 6397 (VINSERTPSrm VR128:$src1, addr:$src2, imm:$src3)>; 6398 def : Pat<(v4f32 (X86insertps (v4f32 VR128:$src1), 6399 (X86VBroadcast (loadv4f32 addr:$src2)), imm:$src3)), 6400 (VINSERTPSrm VR128:$src1, addr:$src2, imm:$src3)>; 6401} 6402 6403//===----------------------------------------------------------------------===// 6404// SSE4.1 - Round Instructions 6405//===----------------------------------------------------------------------===// 6406 6407multiclass sse41_fp_unop_rm<bits<8> opcps, bits<8> opcpd, string OpcodeStr, 6408 X86MemOperand x86memop, RegisterClass RC, 6409 PatFrag mem_frag32, PatFrag mem_frag64, 6410 Intrinsic V4F32Int, Intrinsic V2F64Int> { 6411let ExeDomain = SSEPackedSingle in { 6412 // Intrinsic operation, reg. 6413 // Vector intrinsic operation, reg 6414 def PSr : SS4AIi8<opcps, MRMSrcReg, 6415 (outs RC:$dst), (ins RC:$src1, i32u8imm:$src2), 6416 !strconcat(OpcodeStr, 6417 "ps\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 6418 [(set RC:$dst, (V4F32Int RC:$src1, imm:$src2))], 6419 IIC_SSE_ROUNDPS_REG>, Sched<[WriteFAdd]>; 6420 6421 // Vector intrinsic operation, mem 6422 def PSm : SS4AIi8<opcps, MRMSrcMem, 6423 (outs RC:$dst), (ins x86memop:$src1, i32u8imm:$src2), 6424 !strconcat(OpcodeStr, 6425 "ps\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 6426 [(set RC:$dst, 6427 (V4F32Int (mem_frag32 addr:$src1),imm:$src2))], 6428 IIC_SSE_ROUNDPS_MEM>, Sched<[WriteFAddLd]>; 6429} // ExeDomain = SSEPackedSingle 6430 6431let ExeDomain = SSEPackedDouble in { 6432 // Vector intrinsic operation, reg 6433 def PDr : SS4AIi8<opcpd, MRMSrcReg, 6434 (outs RC:$dst), (ins RC:$src1, i32u8imm:$src2), 6435 !strconcat(OpcodeStr, 6436 "pd\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 6437 [(set RC:$dst, (V2F64Int RC:$src1, imm:$src2))], 6438 IIC_SSE_ROUNDPS_REG>, Sched<[WriteFAdd]>; 6439 6440 // Vector intrinsic operation, mem 6441 def PDm : SS4AIi8<opcpd, MRMSrcMem, 6442 (outs RC:$dst), (ins x86memop:$src1, i32u8imm:$src2), 6443 !strconcat(OpcodeStr, 6444 "pd\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 6445 [(set RC:$dst, 6446 (V2F64Int (mem_frag64 addr:$src1),imm:$src2))], 6447 IIC_SSE_ROUNDPS_REG>, Sched<[WriteFAddLd]>; 6448} // ExeDomain = SSEPackedDouble 6449} 6450 6451multiclass sse41_fp_binop_rm<bits<8> opcss, bits<8> opcsd, 6452 string OpcodeStr, 6453 Intrinsic F32Int, 6454 Intrinsic F64Int, bit Is2Addr = 1> { 6455let ExeDomain = GenericDomain in { 6456 // Operation, reg. 6457 let hasSideEffects = 0 in 6458 def SSr : SS4AIi8<opcss, MRMSrcReg, 6459 (outs FR32:$dst), (ins FR32:$src1, FR32:$src2, i32u8imm:$src3), 6460 !if(Is2Addr, 6461 !strconcat(OpcodeStr, 6462 "ss\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 6463 !strconcat(OpcodeStr, 6464 "ss\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 6465 []>, Sched<[WriteFAdd]>; 6466 6467 // Intrinsic operation, reg. 6468 let isCodeGenOnly = 1 in 6469 def SSr_Int : SS4AIi8<opcss, MRMSrcReg, 6470 (outs VR128:$dst), (ins VR128:$src1, VR128:$src2, i32u8imm:$src3), 6471 !if(Is2Addr, 6472 !strconcat(OpcodeStr, 6473 "ss\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 6474 !strconcat(OpcodeStr, 6475 "ss\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 6476 [(set VR128:$dst, (F32Int VR128:$src1, VR128:$src2, imm:$src3))]>, 6477 Sched<[WriteFAdd]>; 6478 6479 // Intrinsic operation, mem. 6480 def SSm : SS4AIi8<opcss, MRMSrcMem, 6481 (outs VR128:$dst), (ins VR128:$src1, ssmem:$src2, i32u8imm:$src3), 6482 !if(Is2Addr, 6483 !strconcat(OpcodeStr, 6484 "ss\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 6485 !strconcat(OpcodeStr, 6486 "ss\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 6487 [(set VR128:$dst, 6488 (F32Int VR128:$src1, sse_load_f32:$src2, imm:$src3))]>, 6489 Sched<[WriteFAddLd, ReadAfterLd]>; 6490 6491 // Operation, reg. 6492 let hasSideEffects = 0 in 6493 def SDr : SS4AIi8<opcsd, MRMSrcReg, 6494 (outs FR64:$dst), (ins FR64:$src1, FR64:$src2, i32u8imm:$src3), 6495 !if(Is2Addr, 6496 !strconcat(OpcodeStr, 6497 "sd\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 6498 !strconcat(OpcodeStr, 6499 "sd\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 6500 []>, Sched<[WriteFAdd]>; 6501 6502 // Intrinsic operation, reg. 6503 let isCodeGenOnly = 1 in 6504 def SDr_Int : SS4AIi8<opcsd, MRMSrcReg, 6505 (outs VR128:$dst), (ins VR128:$src1, VR128:$src2, i32u8imm:$src3), 6506 !if(Is2Addr, 6507 !strconcat(OpcodeStr, 6508 "sd\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 6509 !strconcat(OpcodeStr, 6510 "sd\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 6511 [(set VR128:$dst, (F64Int VR128:$src1, VR128:$src2, imm:$src3))]>, 6512 Sched<[WriteFAdd]>; 6513 6514 // Intrinsic operation, mem. 6515 def SDm : SS4AIi8<opcsd, MRMSrcMem, 6516 (outs VR128:$dst), (ins VR128:$src1, sdmem:$src2, i32u8imm:$src3), 6517 !if(Is2Addr, 6518 !strconcat(OpcodeStr, 6519 "sd\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 6520 !strconcat(OpcodeStr, 6521 "sd\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 6522 [(set VR128:$dst, 6523 (F64Int VR128:$src1, sse_load_f64:$src2, imm:$src3))]>, 6524 Sched<[WriteFAddLd, ReadAfterLd]>; 6525} // ExeDomain = GenericDomain 6526} 6527 6528// FP round - roundss, roundps, roundsd, roundpd 6529let Predicates = [HasAVX] in { 6530 // Intrinsic form 6531 defm VROUND : sse41_fp_unop_rm<0x08, 0x09, "vround", f128mem, VR128, 6532 loadv4f32, loadv2f64, 6533 int_x86_sse41_round_ps, 6534 int_x86_sse41_round_pd>, VEX; 6535 defm VROUNDY : sse41_fp_unop_rm<0x08, 0x09, "vround", f256mem, VR256, 6536 loadv8f32, loadv4f64, 6537 int_x86_avx_round_ps_256, 6538 int_x86_avx_round_pd_256>, VEX, VEX_L; 6539 defm VROUND : sse41_fp_binop_rm<0x0A, 0x0B, "vround", 6540 int_x86_sse41_round_ss, 6541 int_x86_sse41_round_sd, 0>, VEX_4V, VEX_LIG; 6542} 6543 6544let Predicates = [UseAVX] in { 6545 def : Pat<(ffloor FR32:$src), 6546 (VROUNDSSr (f32 (IMPLICIT_DEF)), FR32:$src, (i32 0x1))>; 6547 def : Pat<(f64 (ffloor FR64:$src)), 6548 (VROUNDSDr (f64 (IMPLICIT_DEF)), FR64:$src, (i32 0x1))>; 6549 def : Pat<(f32 (fnearbyint FR32:$src)), 6550 (VROUNDSSr (f32 (IMPLICIT_DEF)), FR32:$src, (i32 0xC))>; 6551 def : Pat<(f64 (fnearbyint FR64:$src)), 6552 (VROUNDSDr (f64 (IMPLICIT_DEF)), FR64:$src, (i32 0xC))>; 6553 def : Pat<(f32 (fceil FR32:$src)), 6554 (VROUNDSSr (f32 (IMPLICIT_DEF)), FR32:$src, (i32 0x2))>; 6555 def : Pat<(f64 (fceil FR64:$src)), 6556 (VROUNDSDr (f64 (IMPLICIT_DEF)), FR64:$src, (i32 0x2))>; 6557 def : Pat<(f32 (frint FR32:$src)), 6558 (VROUNDSSr (f32 (IMPLICIT_DEF)), FR32:$src, (i32 0x4))>; 6559 def : Pat<(f64 (frint FR64:$src)), 6560 (VROUNDSDr (f64 (IMPLICIT_DEF)), FR64:$src, (i32 0x4))>; 6561 def : Pat<(f32 (ftrunc FR32:$src)), 6562 (VROUNDSSr (f32 (IMPLICIT_DEF)), FR32:$src, (i32 0x3))>; 6563 def : Pat<(f64 (ftrunc FR64:$src)), 6564 (VROUNDSDr (f64 (IMPLICIT_DEF)), FR64:$src, (i32 0x3))>; 6565} 6566 6567let Predicates = [HasAVX] in { 6568 def : Pat<(v4f32 (ffloor VR128:$src)), 6569 (VROUNDPSr VR128:$src, (i32 0x1))>; 6570 def : Pat<(v4f32 (fnearbyint VR128:$src)), 6571 (VROUNDPSr VR128:$src, (i32 0xC))>; 6572 def : Pat<(v4f32 (fceil VR128:$src)), 6573 (VROUNDPSr VR128:$src, (i32 0x2))>; 6574 def : Pat<(v4f32 (frint VR128:$src)), 6575 (VROUNDPSr VR128:$src, (i32 0x4))>; 6576 def : Pat<(v4f32 (ftrunc VR128:$src)), 6577 (VROUNDPSr VR128:$src, (i32 0x3))>; 6578 6579 def : Pat<(v2f64 (ffloor VR128:$src)), 6580 (VROUNDPDr VR128:$src, (i32 0x1))>; 6581 def : Pat<(v2f64 (fnearbyint VR128:$src)), 6582 (VROUNDPDr VR128:$src, (i32 0xC))>; 6583 def : Pat<(v2f64 (fceil VR128:$src)), 6584 (VROUNDPDr VR128:$src, (i32 0x2))>; 6585 def : Pat<(v2f64 (frint VR128:$src)), 6586 (VROUNDPDr VR128:$src, (i32 0x4))>; 6587 def : Pat<(v2f64 (ftrunc VR128:$src)), 6588 (VROUNDPDr VR128:$src, (i32 0x3))>; 6589 6590 def : Pat<(v8f32 (ffloor VR256:$src)), 6591 (VROUNDYPSr VR256:$src, (i32 0x1))>; 6592 def : Pat<(v8f32 (fnearbyint VR256:$src)), 6593 (VROUNDYPSr VR256:$src, (i32 0xC))>; 6594 def : Pat<(v8f32 (fceil VR256:$src)), 6595 (VROUNDYPSr VR256:$src, (i32 0x2))>; 6596 def : Pat<(v8f32 (frint VR256:$src)), 6597 (VROUNDYPSr VR256:$src, (i32 0x4))>; 6598 def : Pat<(v8f32 (ftrunc VR256:$src)), 6599 (VROUNDYPSr VR256:$src, (i32 0x3))>; 6600 6601 def : Pat<(v4f64 (ffloor VR256:$src)), 6602 (VROUNDYPDr VR256:$src, (i32 0x1))>; 6603 def : Pat<(v4f64 (fnearbyint VR256:$src)), 6604 (VROUNDYPDr VR256:$src, (i32 0xC))>; 6605 def : Pat<(v4f64 (fceil VR256:$src)), 6606 (VROUNDYPDr VR256:$src, (i32 0x2))>; 6607 def : Pat<(v4f64 (frint VR256:$src)), 6608 (VROUNDYPDr VR256:$src, (i32 0x4))>; 6609 def : Pat<(v4f64 (ftrunc VR256:$src)), 6610 (VROUNDYPDr VR256:$src, (i32 0x3))>; 6611} 6612 6613defm ROUND : sse41_fp_unop_rm<0x08, 0x09, "round", f128mem, VR128, 6614 memopv4f32, memopv2f64, 6615 int_x86_sse41_round_ps, int_x86_sse41_round_pd>; 6616let Constraints = "$src1 = $dst" in 6617defm ROUND : sse41_fp_binop_rm<0x0A, 0x0B, "round", 6618 int_x86_sse41_round_ss, int_x86_sse41_round_sd>; 6619 6620let Predicates = [UseSSE41] in { 6621 def : Pat<(ffloor FR32:$src), 6622 (ROUNDSSr (f32 (IMPLICIT_DEF)), FR32:$src, (i32 0x1))>; 6623 def : Pat<(f64 (ffloor FR64:$src)), 6624 (ROUNDSDr (f64 (IMPLICIT_DEF)), FR64:$src, (i32 0x1))>; 6625 def : Pat<(f32 (fnearbyint FR32:$src)), 6626 (ROUNDSSr (f32 (IMPLICIT_DEF)), FR32:$src, (i32 0xC))>; 6627 def : Pat<(f64 (fnearbyint FR64:$src)), 6628 (ROUNDSDr (f64 (IMPLICIT_DEF)), FR64:$src, (i32 0xC))>; 6629 def : Pat<(f32 (fceil FR32:$src)), 6630 (ROUNDSSr (f32 (IMPLICIT_DEF)), FR32:$src, (i32 0x2))>; 6631 def : Pat<(f64 (fceil FR64:$src)), 6632 (ROUNDSDr (f64 (IMPLICIT_DEF)), FR64:$src, (i32 0x2))>; 6633 def : Pat<(f32 (frint FR32:$src)), 6634 (ROUNDSSr (f32 (IMPLICIT_DEF)), FR32:$src, (i32 0x4))>; 6635 def : Pat<(f64 (frint FR64:$src)), 6636 (ROUNDSDr (f64 (IMPLICIT_DEF)), FR64:$src, (i32 0x4))>; 6637 def : Pat<(f32 (ftrunc FR32:$src)), 6638 (ROUNDSSr (f32 (IMPLICIT_DEF)), FR32:$src, (i32 0x3))>; 6639 def : Pat<(f64 (ftrunc FR64:$src)), 6640 (ROUNDSDr (f64 (IMPLICIT_DEF)), FR64:$src, (i32 0x3))>; 6641 6642 def : Pat<(v4f32 (ffloor VR128:$src)), 6643 (ROUNDPSr VR128:$src, (i32 0x1))>; 6644 def : Pat<(v4f32 (fnearbyint VR128:$src)), 6645 (ROUNDPSr VR128:$src, (i32 0xC))>; 6646 def : Pat<(v4f32 (fceil VR128:$src)), 6647 (ROUNDPSr VR128:$src, (i32 0x2))>; 6648 def : Pat<(v4f32 (frint VR128:$src)), 6649 (ROUNDPSr VR128:$src, (i32 0x4))>; 6650 def : Pat<(v4f32 (ftrunc VR128:$src)), 6651 (ROUNDPSr VR128:$src, (i32 0x3))>; 6652 6653 def : Pat<(v2f64 (ffloor VR128:$src)), 6654 (ROUNDPDr VR128:$src, (i32 0x1))>; 6655 def : Pat<(v2f64 (fnearbyint VR128:$src)), 6656 (ROUNDPDr VR128:$src, (i32 0xC))>; 6657 def : Pat<(v2f64 (fceil VR128:$src)), 6658 (ROUNDPDr VR128:$src, (i32 0x2))>; 6659 def : Pat<(v2f64 (frint VR128:$src)), 6660 (ROUNDPDr VR128:$src, (i32 0x4))>; 6661 def : Pat<(v2f64 (ftrunc VR128:$src)), 6662 (ROUNDPDr VR128:$src, (i32 0x3))>; 6663} 6664 6665//===----------------------------------------------------------------------===// 6666// SSE4.1 - Packed Bit Test 6667//===----------------------------------------------------------------------===// 6668 6669// ptest instruction we'll lower to this in X86ISelLowering primarily from 6670// the intel intrinsic that corresponds to this. 6671let Defs = [EFLAGS], Predicates = [HasAVX] in { 6672def VPTESTrr : SS48I<0x17, MRMSrcReg, (outs), (ins VR128:$src1, VR128:$src2), 6673 "vptest\t{$src2, $src1|$src1, $src2}", 6674 [(set EFLAGS, (X86ptest VR128:$src1, (v2i64 VR128:$src2)))]>, 6675 Sched<[WriteVecLogic]>, VEX; 6676def VPTESTrm : SS48I<0x17, MRMSrcMem, (outs), (ins VR128:$src1, f128mem:$src2), 6677 "vptest\t{$src2, $src1|$src1, $src2}", 6678 [(set EFLAGS,(X86ptest VR128:$src1, (loadv2i64 addr:$src2)))]>, 6679 Sched<[WriteVecLogicLd, ReadAfterLd]>, VEX; 6680 6681def VPTESTYrr : SS48I<0x17, MRMSrcReg, (outs), (ins VR256:$src1, VR256:$src2), 6682 "vptest\t{$src2, $src1|$src1, $src2}", 6683 [(set EFLAGS, (X86ptest VR256:$src1, (v4i64 VR256:$src2)))]>, 6684 Sched<[WriteVecLogic]>, VEX, VEX_L; 6685def VPTESTYrm : SS48I<0x17, MRMSrcMem, (outs), (ins VR256:$src1, i256mem:$src2), 6686 "vptest\t{$src2, $src1|$src1, $src2}", 6687 [(set EFLAGS,(X86ptest VR256:$src1, (loadv4i64 addr:$src2)))]>, 6688 Sched<[WriteVecLogicLd, ReadAfterLd]>, VEX, VEX_L; 6689} 6690 6691let Defs = [EFLAGS] in { 6692def PTESTrr : SS48I<0x17, MRMSrcReg, (outs), (ins VR128:$src1, VR128:$src2), 6693 "ptest\t{$src2, $src1|$src1, $src2}", 6694 [(set EFLAGS, (X86ptest VR128:$src1, (v2i64 VR128:$src2)))]>, 6695 Sched<[WriteVecLogic]>; 6696def PTESTrm : SS48I<0x17, MRMSrcMem, (outs), (ins VR128:$src1, f128mem:$src2), 6697 "ptest\t{$src2, $src1|$src1, $src2}", 6698 [(set EFLAGS, (X86ptest VR128:$src1, (memopv2i64 addr:$src2)))]>, 6699 Sched<[WriteVecLogicLd, ReadAfterLd]>; 6700} 6701 6702// The bit test instructions below are AVX only 6703multiclass avx_bittest<bits<8> opc, string OpcodeStr, RegisterClass RC, 6704 X86MemOperand x86memop, PatFrag mem_frag, ValueType vt> { 6705 def rr : SS48I<opc, MRMSrcReg, (outs), (ins RC:$src1, RC:$src2), 6706 !strconcat(OpcodeStr, "\t{$src2, $src1|$src1, $src2}"), 6707 [(set EFLAGS, (X86testp RC:$src1, (vt RC:$src2)))]>, 6708 Sched<[WriteVecLogic]>, VEX; 6709 def rm : SS48I<opc, MRMSrcMem, (outs), (ins RC:$src1, x86memop:$src2), 6710 !strconcat(OpcodeStr, "\t{$src2, $src1|$src1, $src2}"), 6711 [(set EFLAGS, (X86testp RC:$src1, (mem_frag addr:$src2)))]>, 6712 Sched<[WriteVecLogicLd, ReadAfterLd]>, VEX; 6713} 6714 6715let Defs = [EFLAGS], Predicates = [HasAVX] in { 6716let ExeDomain = SSEPackedSingle in { 6717defm VTESTPS : avx_bittest<0x0E, "vtestps", VR128, f128mem, loadv4f32, v4f32>; 6718defm VTESTPSY : avx_bittest<0x0E, "vtestps", VR256, f256mem, loadv8f32, v8f32>, 6719 VEX_L; 6720} 6721let ExeDomain = SSEPackedDouble in { 6722defm VTESTPD : avx_bittest<0x0F, "vtestpd", VR128, f128mem, loadv2f64, v2f64>; 6723defm VTESTPDY : avx_bittest<0x0F, "vtestpd", VR256, f256mem, loadv4f64, v4f64>, 6724 VEX_L; 6725} 6726} 6727 6728//===----------------------------------------------------------------------===// 6729// SSE4.1 - Misc Instructions 6730//===----------------------------------------------------------------------===// 6731 6732let Defs = [EFLAGS], Predicates = [HasPOPCNT] in { 6733 def POPCNT16rr : I<0xB8, MRMSrcReg, (outs GR16:$dst), (ins GR16:$src), 6734 "popcnt{w}\t{$src, $dst|$dst, $src}", 6735 [(set GR16:$dst, (ctpop GR16:$src)), (implicit EFLAGS)], 6736 IIC_SSE_POPCNT_RR>, Sched<[WriteFAdd]>, 6737 OpSize16, XS; 6738 def POPCNT16rm : I<0xB8, MRMSrcMem, (outs GR16:$dst), (ins i16mem:$src), 6739 "popcnt{w}\t{$src, $dst|$dst, $src}", 6740 [(set GR16:$dst, (ctpop (loadi16 addr:$src))), 6741 (implicit EFLAGS)], IIC_SSE_POPCNT_RM>, 6742 Sched<[WriteFAddLd]>, OpSize16, XS; 6743 6744 def POPCNT32rr : I<0xB8, MRMSrcReg, (outs GR32:$dst), (ins GR32:$src), 6745 "popcnt{l}\t{$src, $dst|$dst, $src}", 6746 [(set GR32:$dst, (ctpop GR32:$src)), (implicit EFLAGS)], 6747 IIC_SSE_POPCNT_RR>, Sched<[WriteFAdd]>, 6748 OpSize32, XS; 6749 6750 def POPCNT32rm : I<0xB8, MRMSrcMem, (outs GR32:$dst), (ins i32mem:$src), 6751 "popcnt{l}\t{$src, $dst|$dst, $src}", 6752 [(set GR32:$dst, (ctpop (loadi32 addr:$src))), 6753 (implicit EFLAGS)], IIC_SSE_POPCNT_RM>, 6754 Sched<[WriteFAddLd]>, OpSize32, XS; 6755 6756 def POPCNT64rr : RI<0xB8, MRMSrcReg, (outs GR64:$dst), (ins GR64:$src), 6757 "popcnt{q}\t{$src, $dst|$dst, $src}", 6758 [(set GR64:$dst, (ctpop GR64:$src)), (implicit EFLAGS)], 6759 IIC_SSE_POPCNT_RR>, Sched<[WriteFAdd]>, XS; 6760 def POPCNT64rm : RI<0xB8, MRMSrcMem, (outs GR64:$dst), (ins i64mem:$src), 6761 "popcnt{q}\t{$src, $dst|$dst, $src}", 6762 [(set GR64:$dst, (ctpop (loadi64 addr:$src))), 6763 (implicit EFLAGS)], IIC_SSE_POPCNT_RM>, 6764 Sched<[WriteFAddLd]>, XS; 6765} 6766 6767 6768 6769// SS41I_unop_rm_int_v16 - SSE 4.1 unary operator whose type is v8i16. 6770multiclass SS41I_unop_rm_int_v16<bits<8> opc, string OpcodeStr, 6771 Intrinsic IntId128, PatFrag ld_frag, 6772 X86FoldableSchedWrite Sched> { 6773 def rr128 : SS48I<opc, MRMSrcReg, (outs VR128:$dst), 6774 (ins VR128:$src), 6775 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 6776 [(set VR128:$dst, (IntId128 VR128:$src))]>, 6777 Sched<[Sched]>; 6778 def rm128 : SS48I<opc, MRMSrcMem, (outs VR128:$dst), 6779 (ins i128mem:$src), 6780 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 6781 [(set VR128:$dst, 6782 (IntId128 (bitconvert (ld_frag addr:$src))))]>, 6783 Sched<[Sched.Folded]>; 6784} 6785 6786// PHMIN has the same profile as PSAD, thus we use the same scheduling 6787// model, although the naming is misleading. 6788let Predicates = [HasAVX] in 6789defm VPHMINPOSUW : SS41I_unop_rm_int_v16 <0x41, "vphminposuw", 6790 int_x86_sse41_phminposuw, loadv2i64, 6791 WriteVecIMul>, VEX; 6792defm PHMINPOSUW : SS41I_unop_rm_int_v16 <0x41, "phminposuw", 6793 int_x86_sse41_phminposuw, memopv2i64, 6794 WriteVecIMul>; 6795 6796/// SS48I_binop_rm - Simple SSE41 binary operator. 6797multiclass SS48I_binop_rm<bits<8> opc, string OpcodeStr, SDNode OpNode, 6798 ValueType OpVT, RegisterClass RC, PatFrag memop_frag, 6799 X86MemOperand x86memop, bit Is2Addr = 1, 6800 OpndItins itins = SSE_INTALU_ITINS_P> { 6801 let isCommutable = 1 in 6802 def rr : SS48I<opc, MRMSrcReg, (outs RC:$dst), 6803 (ins RC:$src1, RC:$src2), 6804 !if(Is2Addr, 6805 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 6806 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 6807 [(set RC:$dst, (OpVT (OpNode RC:$src1, RC:$src2)))]>, 6808 Sched<[itins.Sched]>; 6809 def rm : SS48I<opc, MRMSrcMem, (outs RC:$dst), 6810 (ins RC:$src1, x86memop:$src2), 6811 !if(Is2Addr, 6812 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 6813 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 6814 [(set RC:$dst, 6815 (OpVT (OpNode RC:$src1, (bitconvert (memop_frag addr:$src2)))))]>, 6816 Sched<[itins.Sched.Folded, ReadAfterLd]>; 6817} 6818 6819/// SS48I_binop_rm2 - Simple SSE41 binary operator with different src and dst 6820/// types. 6821multiclass SS48I_binop_rm2<bits<8> opc, string OpcodeStr, SDNode OpNode, 6822 ValueType DstVT, ValueType SrcVT, RegisterClass RC, 6823 PatFrag memop_frag, X86MemOperand x86memop, 6824 OpndItins itins, 6825 bit IsCommutable = 0, bit Is2Addr = 1> { 6826 let isCommutable = IsCommutable in 6827 def rr : SS48I<opc, MRMSrcReg, (outs RC:$dst), 6828 (ins RC:$src1, RC:$src2), 6829 !if(Is2Addr, 6830 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 6831 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 6832 [(set RC:$dst, (DstVT (OpNode (SrcVT RC:$src1), RC:$src2)))]>, 6833 Sched<[itins.Sched]>; 6834 def rm : SS48I<opc, MRMSrcMem, (outs RC:$dst), 6835 (ins RC:$src1, x86memop:$src2), 6836 !if(Is2Addr, 6837 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 6838 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 6839 [(set RC:$dst, (DstVT (OpNode (SrcVT RC:$src1), 6840 (bitconvert (memop_frag addr:$src2)))))]>, 6841 Sched<[itins.Sched.Folded, ReadAfterLd]>; 6842} 6843 6844let Predicates = [HasAVX, NoVLX] in { 6845 defm VPMINSB : SS48I_binop_rm<0x38, "vpminsb", smin, v16i8, VR128, 6846 loadv2i64, i128mem, 0, SSE_INTALU_ITINS_P>, 6847 VEX_4V; 6848 defm VPMINSD : SS48I_binop_rm<0x39, "vpminsd", smin, v4i32, VR128, 6849 loadv2i64, i128mem, 0, SSE_INTALU_ITINS_P>, 6850 VEX_4V; 6851 defm VPMINUD : SS48I_binop_rm<0x3B, "vpminud", umin, v4i32, VR128, 6852 loadv2i64, i128mem, 0, SSE_INTALU_ITINS_P>, 6853 VEX_4V; 6854 defm VPMINUW : SS48I_binop_rm<0x3A, "vpminuw", umin, v8i16, VR128, 6855 loadv2i64, i128mem, 0, SSE_INTALU_ITINS_P>, 6856 VEX_4V; 6857 defm VPMAXSB : SS48I_binop_rm<0x3C, "vpmaxsb", smax, v16i8, VR128, 6858 loadv2i64, i128mem, 0, SSE_INTALU_ITINS_P>, 6859 VEX_4V; 6860 defm VPMAXSD : SS48I_binop_rm<0x3D, "vpmaxsd", smax, v4i32, VR128, 6861 loadv2i64, i128mem, 0, SSE_INTALU_ITINS_P>, 6862 VEX_4V; 6863 defm VPMAXUD : SS48I_binop_rm<0x3F, "vpmaxud", umax, v4i32, VR128, 6864 loadv2i64, i128mem, 0, SSE_INTALU_ITINS_P>, 6865 VEX_4V; 6866 defm VPMAXUW : SS48I_binop_rm<0x3E, "vpmaxuw", umax, v8i16, VR128, 6867 loadv2i64, i128mem, 0, SSE_INTALU_ITINS_P>, 6868 VEX_4V; 6869 defm VPMULDQ : SS48I_binop_rm2<0x28, "vpmuldq", X86pmuldq, v2i64, v4i32, 6870 VR128, loadv2i64, i128mem, 6871 SSE_INTMUL_ITINS_P, 1, 0>, VEX_4V; 6872} 6873 6874let Predicates = [HasAVX2, NoVLX] in { 6875 defm VPMINSBY : SS48I_binop_rm<0x38, "vpminsb", smin, v32i8, VR256, 6876 loadv4i64, i256mem, 0, SSE_INTALU_ITINS_P>, 6877 VEX_4V, VEX_L; 6878 defm VPMINSDY : SS48I_binop_rm<0x39, "vpminsd", smin, v8i32, VR256, 6879 loadv4i64, i256mem, 0, SSE_INTALU_ITINS_P>, 6880 VEX_4V, VEX_L; 6881 defm VPMINUDY : SS48I_binop_rm<0x3B, "vpminud", umin, v8i32, VR256, 6882 loadv4i64, i256mem, 0, SSE_INTALU_ITINS_P>, 6883 VEX_4V, VEX_L; 6884 defm VPMINUWY : SS48I_binop_rm<0x3A, "vpminuw", umin, v16i16, VR256, 6885 loadv4i64, i256mem, 0, SSE_INTALU_ITINS_P>, 6886 VEX_4V, VEX_L; 6887 defm VPMAXSBY : SS48I_binop_rm<0x3C, "vpmaxsb", smax, v32i8, VR256, 6888 loadv4i64, i256mem, 0, SSE_INTALU_ITINS_P>, 6889 VEX_4V, VEX_L; 6890 defm VPMAXSDY : SS48I_binop_rm<0x3D, "vpmaxsd", smax, v8i32, VR256, 6891 loadv4i64, i256mem, 0, SSE_INTALU_ITINS_P>, 6892 VEX_4V, VEX_L; 6893 defm VPMAXUDY : SS48I_binop_rm<0x3F, "vpmaxud", umax, v8i32, VR256, 6894 loadv4i64, i256mem, 0, SSE_INTALU_ITINS_P>, 6895 VEX_4V, VEX_L; 6896 defm VPMAXUWY : SS48I_binop_rm<0x3E, "vpmaxuw", umax, v16i16, VR256, 6897 loadv4i64, i256mem, 0, SSE_INTALU_ITINS_P>, 6898 VEX_4V, VEX_L; 6899 defm VPMULDQY : SS48I_binop_rm2<0x28, "vpmuldq", X86pmuldq, v4i64, v8i32, 6900 VR256, loadv4i64, i256mem, 6901 SSE_INTMUL_ITINS_P, 1, 0>, VEX_4V, VEX_L; 6902} 6903 6904let Constraints = "$src1 = $dst" in { 6905 defm PMINSB : SS48I_binop_rm<0x38, "pminsb", smin, v16i8, VR128, 6906 memopv2i64, i128mem, 1, SSE_INTALU_ITINS_P>; 6907 defm PMINSD : SS48I_binop_rm<0x39, "pminsd", smin, v4i32, VR128, 6908 memopv2i64, i128mem, 1, SSE_INTALU_ITINS_P>; 6909 defm PMINUD : SS48I_binop_rm<0x3B, "pminud", umin, v4i32, VR128, 6910 memopv2i64, i128mem, 1, SSE_INTALU_ITINS_P>; 6911 defm PMINUW : SS48I_binop_rm<0x3A, "pminuw", umin, v8i16, VR128, 6912 memopv2i64, i128mem, 1, SSE_INTALU_ITINS_P>; 6913 defm PMAXSB : SS48I_binop_rm<0x3C, "pmaxsb", smax, v16i8, VR128, 6914 memopv2i64, i128mem, 1, SSE_INTALU_ITINS_P>; 6915 defm PMAXSD : SS48I_binop_rm<0x3D, "pmaxsd", smax, v4i32, VR128, 6916 memopv2i64, i128mem, 1, SSE_INTALU_ITINS_P>; 6917 defm PMAXUD : SS48I_binop_rm<0x3F, "pmaxud", umax, v4i32, VR128, 6918 memopv2i64, i128mem, 1, SSE_INTALU_ITINS_P>; 6919 defm PMAXUW : SS48I_binop_rm<0x3E, "pmaxuw", umax, v8i16, VR128, 6920 memopv2i64, i128mem, 1, SSE_INTALU_ITINS_P>; 6921 defm PMULDQ : SS48I_binop_rm2<0x28, "pmuldq", X86pmuldq, v2i64, v4i32, 6922 VR128, memopv2i64, i128mem, 6923 SSE_INTMUL_ITINS_P, 1>; 6924} 6925 6926let Predicates = [HasAVX, NoVLX] in { 6927 defm VPMULLD : SS48I_binop_rm<0x40, "vpmulld", mul, v4i32, VR128, 6928 memopv2i64, i128mem, 0, SSE_PMULLD_ITINS>, 6929 VEX_4V; 6930 defm VPCMPEQQ : SS48I_binop_rm<0x29, "vpcmpeqq", X86pcmpeq, v2i64, VR128, 6931 memopv2i64, i128mem, 0, SSE_INTALU_ITINS_P>, 6932 VEX_4V; 6933} 6934let Predicates = [HasAVX2] in { 6935 defm VPMULLDY : SS48I_binop_rm<0x40, "vpmulld", mul, v8i32, VR256, 6936 loadv4i64, i256mem, 0, SSE_PMULLD_ITINS>, 6937 VEX_4V, VEX_L; 6938 defm VPCMPEQQY : SS48I_binop_rm<0x29, "vpcmpeqq", X86pcmpeq, v4i64, VR256, 6939 loadv4i64, i256mem, 0, SSE_INTALU_ITINS_P>, 6940 VEX_4V, VEX_L; 6941} 6942 6943let Constraints = "$src1 = $dst" in { 6944 defm PMULLD : SS48I_binop_rm<0x40, "pmulld", mul, v4i32, VR128, 6945 memopv2i64, i128mem, 1, SSE_PMULLD_ITINS>; 6946 defm PCMPEQQ : SS48I_binop_rm<0x29, "pcmpeqq", X86pcmpeq, v2i64, VR128, 6947 memopv2i64, i128mem, 1, SSE_INTALUQ_ITINS_P>; 6948} 6949 6950/// SS41I_binop_rmi_int - SSE 4.1 binary operator with 8-bit immediate 6951multiclass SS41I_binop_rmi_int<bits<8> opc, string OpcodeStr, 6952 Intrinsic IntId, RegisterClass RC, PatFrag memop_frag, 6953 X86MemOperand x86memop, bit Is2Addr = 1, 6954 OpndItins itins = DEFAULT_ITINS> { 6955 let isCommutable = 1 in 6956 def rri : SS4AIi8<opc, MRMSrcReg, (outs RC:$dst), 6957 (ins RC:$src1, RC:$src2, u8imm:$src3), 6958 !if(Is2Addr, 6959 !strconcat(OpcodeStr, 6960 "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 6961 !strconcat(OpcodeStr, 6962 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 6963 [(set RC:$dst, (IntId RC:$src1, RC:$src2, imm:$src3))], itins.rr>, 6964 Sched<[itins.Sched]>; 6965 def rmi : SS4AIi8<opc, MRMSrcMem, (outs RC:$dst), 6966 (ins RC:$src1, x86memop:$src2, u8imm:$src3), 6967 !if(Is2Addr, 6968 !strconcat(OpcodeStr, 6969 "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 6970 !strconcat(OpcodeStr, 6971 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 6972 [(set RC:$dst, 6973 (IntId RC:$src1, 6974 (bitconvert (memop_frag addr:$src2)), imm:$src3))], itins.rm>, 6975 Sched<[itins.Sched.Folded, ReadAfterLd]>; 6976} 6977 6978/// SS41I_binop_rmi - SSE 4.1 binary operator with 8-bit immediate 6979multiclass SS41I_binop_rmi<bits<8> opc, string OpcodeStr, SDNode OpNode, 6980 ValueType OpVT, RegisterClass RC, PatFrag memop_frag, 6981 X86MemOperand x86memop, bit Is2Addr = 1, 6982 OpndItins itins = DEFAULT_ITINS> { 6983 let isCommutable = 1 in 6984 def rri : SS4AIi8<opc, MRMSrcReg, (outs RC:$dst), 6985 (ins RC:$src1, RC:$src2, u8imm:$src3), 6986 !if(Is2Addr, 6987 !strconcat(OpcodeStr, 6988 "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 6989 !strconcat(OpcodeStr, 6990 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 6991 [(set RC:$dst, (OpVT (OpNode RC:$src1, RC:$src2, imm:$src3)))], 6992 itins.rr>, Sched<[itins.Sched]>; 6993 def rmi : SS4AIi8<opc, MRMSrcMem, (outs RC:$dst), 6994 (ins RC:$src1, x86memop:$src2, u8imm:$src3), 6995 !if(Is2Addr, 6996 !strconcat(OpcodeStr, 6997 "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 6998 !strconcat(OpcodeStr, 6999 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 7000 [(set RC:$dst, 7001 (OpVT (OpNode RC:$src1, 7002 (bitconvert (memop_frag addr:$src2)), imm:$src3)))], itins.rm>, 7003 Sched<[itins.Sched.Folded, ReadAfterLd]>; 7004} 7005 7006let Predicates = [HasAVX] in { 7007 let isCommutable = 0 in { 7008 defm VMPSADBW : SS41I_binop_rmi_int<0x42, "vmpsadbw", int_x86_sse41_mpsadbw, 7009 VR128, loadv2i64, i128mem, 0, 7010 DEFAULT_ITINS_MPSADSCHED>, VEX_4V; 7011 } 7012 7013 let ExeDomain = SSEPackedSingle in { 7014 defm VBLENDPS : SS41I_binop_rmi<0x0C, "vblendps", X86Blendi, v4f32, 7015 VR128, loadv4f32, f128mem, 0, 7016 DEFAULT_ITINS_FBLENDSCHED>, VEX_4V; 7017 defm VBLENDPSY : SS41I_binop_rmi<0x0C, "vblendps", X86Blendi, v8f32, 7018 VR256, loadv8f32, f256mem, 0, 7019 DEFAULT_ITINS_FBLENDSCHED>, VEX_4V, VEX_L; 7020 } 7021 let ExeDomain = SSEPackedDouble in { 7022 defm VBLENDPD : SS41I_binop_rmi<0x0D, "vblendpd", X86Blendi, v2f64, 7023 VR128, loadv2f64, f128mem, 0, 7024 DEFAULT_ITINS_FBLENDSCHED>, VEX_4V; 7025 defm VBLENDPDY : SS41I_binop_rmi<0x0D, "vblendpd", X86Blendi, v4f64, 7026 VR256, loadv4f64, f256mem, 0, 7027 DEFAULT_ITINS_FBLENDSCHED>, VEX_4V, VEX_L; 7028 } 7029 defm VPBLENDW : SS41I_binop_rmi<0x0E, "vpblendw", X86Blendi, v8i16, 7030 VR128, loadv2i64, i128mem, 0, 7031 DEFAULT_ITINS_BLENDSCHED>, VEX_4V; 7032 7033 let ExeDomain = SSEPackedSingle in 7034 defm VDPPS : SS41I_binop_rmi_int<0x40, "vdpps", int_x86_sse41_dpps, 7035 VR128, loadv4f32, f128mem, 0, 7036 SSE_DPPS_ITINS>, VEX_4V; 7037 let ExeDomain = SSEPackedDouble in 7038 defm VDPPD : SS41I_binop_rmi_int<0x41, "vdppd", int_x86_sse41_dppd, 7039 VR128, loadv2f64, f128mem, 0, 7040 SSE_DPPS_ITINS>, VEX_4V; 7041 let ExeDomain = SSEPackedSingle in 7042 defm VDPPSY : SS41I_binop_rmi_int<0x40, "vdpps", int_x86_avx_dp_ps_256, 7043 VR256, loadv8f32, i256mem, 0, 7044 SSE_DPPS_ITINS>, VEX_4V, VEX_L; 7045} 7046 7047let Predicates = [HasAVX2] in { 7048 let isCommutable = 0 in { 7049 defm VMPSADBWY : SS41I_binop_rmi_int<0x42, "vmpsadbw", int_x86_avx2_mpsadbw, 7050 VR256, loadv4i64, i256mem, 0, 7051 DEFAULT_ITINS_MPSADSCHED>, VEX_4V, VEX_L; 7052 } 7053 defm VPBLENDWY : SS41I_binop_rmi<0x0E, "vpblendw", X86Blendi, v16i16, 7054 VR256, loadv4i64, i256mem, 0, 7055 DEFAULT_ITINS_BLENDSCHED>, VEX_4V, VEX_L; 7056} 7057 7058let Constraints = "$src1 = $dst" in { 7059 let isCommutable = 0 in { 7060 defm MPSADBW : SS41I_binop_rmi_int<0x42, "mpsadbw", int_x86_sse41_mpsadbw, 7061 VR128, memopv2i64, i128mem, 7062 1, SSE_MPSADBW_ITINS>; 7063 } 7064 let ExeDomain = SSEPackedSingle in 7065 defm BLENDPS : SS41I_binop_rmi<0x0C, "blendps", X86Blendi, v4f32, 7066 VR128, memopv4f32, f128mem, 7067 1, SSE_INTALU_ITINS_FBLEND_P>; 7068 let ExeDomain = SSEPackedDouble in 7069 defm BLENDPD : SS41I_binop_rmi<0x0D, "blendpd", X86Blendi, v2f64, 7070 VR128, memopv2f64, f128mem, 7071 1, SSE_INTALU_ITINS_FBLEND_P>; 7072 defm PBLENDW : SS41I_binop_rmi<0x0E, "pblendw", X86Blendi, v8i16, 7073 VR128, memopv2i64, i128mem, 7074 1, SSE_INTALU_ITINS_BLEND_P>; 7075 let ExeDomain = SSEPackedSingle in 7076 defm DPPS : SS41I_binop_rmi_int<0x40, "dpps", int_x86_sse41_dpps, 7077 VR128, memopv4f32, f128mem, 1, 7078 SSE_DPPS_ITINS>; 7079 let ExeDomain = SSEPackedDouble in 7080 defm DPPD : SS41I_binop_rmi_int<0x41, "dppd", int_x86_sse41_dppd, 7081 VR128, memopv2f64, f128mem, 1, 7082 SSE_DPPD_ITINS>; 7083} 7084 7085/// SS41I_quaternary_int_avx - AVX SSE 4.1 with 4 operators 7086multiclass SS41I_quaternary_int_avx<bits<8> opc, string OpcodeStr, 7087 RegisterClass RC, X86MemOperand x86memop, 7088 PatFrag mem_frag, Intrinsic IntId, 7089 X86FoldableSchedWrite Sched> { 7090 def rr : Ii8<opc, MRMSrcReg, (outs RC:$dst), 7091 (ins RC:$src1, RC:$src2, RC:$src3), 7092 !strconcat(OpcodeStr, 7093 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), 7094 [(set RC:$dst, (IntId RC:$src1, RC:$src2, RC:$src3))], 7095 NoItinerary, SSEPackedInt>, TAPD, VEX_4V, VEX_I8IMM, 7096 Sched<[Sched]>; 7097 7098 def rm : Ii8<opc, MRMSrcMem, (outs RC:$dst), 7099 (ins RC:$src1, x86memop:$src2, RC:$src3), 7100 !strconcat(OpcodeStr, 7101 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), 7102 [(set RC:$dst, 7103 (IntId RC:$src1, (bitconvert (mem_frag addr:$src2)), 7104 RC:$src3))], 7105 NoItinerary, SSEPackedInt>, TAPD, VEX_4V, VEX_I8IMM, 7106 Sched<[Sched.Folded, ReadAfterLd]>; 7107} 7108 7109let Predicates = [HasAVX] in { 7110let ExeDomain = SSEPackedDouble in { 7111defm VBLENDVPD : SS41I_quaternary_int_avx<0x4B, "vblendvpd", VR128, f128mem, 7112 loadv2f64, int_x86_sse41_blendvpd, 7113 WriteFVarBlend>; 7114defm VBLENDVPDY : SS41I_quaternary_int_avx<0x4B, "vblendvpd", VR256, f256mem, 7115 loadv4f64, int_x86_avx_blendv_pd_256, 7116 WriteFVarBlend>, VEX_L; 7117} // ExeDomain = SSEPackedDouble 7118let ExeDomain = SSEPackedSingle in { 7119defm VBLENDVPS : SS41I_quaternary_int_avx<0x4A, "vblendvps", VR128, f128mem, 7120 loadv4f32, int_x86_sse41_blendvps, 7121 WriteFVarBlend>; 7122defm VBLENDVPSY : SS41I_quaternary_int_avx<0x4A, "vblendvps", VR256, f256mem, 7123 loadv8f32, int_x86_avx_blendv_ps_256, 7124 WriteFVarBlend>, VEX_L; 7125} // ExeDomain = SSEPackedSingle 7126defm VPBLENDVB : SS41I_quaternary_int_avx<0x4C, "vpblendvb", VR128, i128mem, 7127 loadv2i64, int_x86_sse41_pblendvb, 7128 WriteVarBlend>; 7129} 7130 7131let Predicates = [HasAVX2] in { 7132defm VPBLENDVBY : SS41I_quaternary_int_avx<0x4C, "vpblendvb", VR256, i256mem, 7133 loadv4i64, int_x86_avx2_pblendvb, 7134 WriteVarBlend>, VEX_L; 7135} 7136 7137let Predicates = [HasAVX] in { 7138 def : Pat<(v16i8 (vselect (v16i8 VR128:$mask), (v16i8 VR128:$src1), 7139 (v16i8 VR128:$src2))), 7140 (VPBLENDVBrr VR128:$src2, VR128:$src1, VR128:$mask)>; 7141 def : Pat<(v4i32 (vselect (v4i32 VR128:$mask), (v4i32 VR128:$src1), 7142 (v4i32 VR128:$src2))), 7143 (VBLENDVPSrr VR128:$src2, VR128:$src1, VR128:$mask)>; 7144 def : Pat<(v4f32 (vselect (v4i32 VR128:$mask), (v4f32 VR128:$src1), 7145 (v4f32 VR128:$src2))), 7146 (VBLENDVPSrr VR128:$src2, VR128:$src1, VR128:$mask)>; 7147 def : Pat<(v2i64 (vselect (v2i64 VR128:$mask), (v2i64 VR128:$src1), 7148 (v2i64 VR128:$src2))), 7149 (VBLENDVPDrr VR128:$src2, VR128:$src1, VR128:$mask)>; 7150 def : Pat<(v2f64 (vselect (v2i64 VR128:$mask), (v2f64 VR128:$src1), 7151 (v2f64 VR128:$src2))), 7152 (VBLENDVPDrr VR128:$src2, VR128:$src1, VR128:$mask)>; 7153 def : Pat<(v8i32 (vselect (v8i32 VR256:$mask), (v8i32 VR256:$src1), 7154 (v8i32 VR256:$src2))), 7155 (VBLENDVPSYrr VR256:$src2, VR256:$src1, VR256:$mask)>; 7156 def : Pat<(v8f32 (vselect (v8i32 VR256:$mask), (v8f32 VR256:$src1), 7157 (v8f32 VR256:$src2))), 7158 (VBLENDVPSYrr VR256:$src2, VR256:$src1, VR256:$mask)>; 7159 def : Pat<(v4i64 (vselect (v4i64 VR256:$mask), (v4i64 VR256:$src1), 7160 (v4i64 VR256:$src2))), 7161 (VBLENDVPDYrr VR256:$src2, VR256:$src1, VR256:$mask)>; 7162 def : Pat<(v4f64 (vselect (v4i64 VR256:$mask), (v4f64 VR256:$src1), 7163 (v4f64 VR256:$src2))), 7164 (VBLENDVPDYrr VR256:$src2, VR256:$src1, VR256:$mask)>; 7165} 7166 7167let Predicates = [HasAVX2] in { 7168 def : Pat<(v32i8 (vselect (v32i8 VR256:$mask), (v32i8 VR256:$src1), 7169 (v32i8 VR256:$src2))), 7170 (VPBLENDVBYrr VR256:$src2, VR256:$src1, VR256:$mask)>; 7171} 7172 7173// Patterns 7174// FIXME: Prefer a movss or movsd over a blendps when optimizing for size or 7175// on targets where they have equal performance. These were changed to use 7176// blends because blends have better throughput on SandyBridge and Haswell, but 7177// movs[s/d] are 1-2 byte shorter instructions. 7178let Predicates = [UseAVX] in { 7179 let AddedComplexity = 15 in { 7180 // Move scalar to XMM zero-extended, zeroing a VR128 then do a 7181 // MOVS{S,D} to the lower bits. 7182 def : Pat<(v4f32 (X86vzmovl (v4f32 (scalar_to_vector FR32:$src)))), 7183 (VMOVSSrr (v4f32 (V_SET0)), FR32:$src)>; 7184 def : Pat<(v4f32 (X86vzmovl (v4f32 VR128:$src))), 7185 (VBLENDPSrri (v4f32 (V_SET0)), VR128:$src, (i8 1))>; 7186 def : Pat<(v4i32 (X86vzmovl (v4i32 VR128:$src))), 7187 (VPBLENDWrri (v4i32 (V_SET0)), VR128:$src, (i8 3))>; 7188 def : Pat<(v2f64 (X86vzmovl (v2f64 (scalar_to_vector FR64:$src)))), 7189 (VMOVSDrr (v2f64 (V_SET0)), FR64:$src)>; 7190 7191 // Move low f32 and clear high bits. 7192 def : Pat<(v8f32 (X86vzmovl (v8f32 VR256:$src))), 7193 (VBLENDPSYrri (v8f32 (AVX_SET0)), VR256:$src, (i8 1))>; 7194 7195 // Move low f64 and clear high bits. 7196 def : Pat<(v4f64 (X86vzmovl (v4f64 VR256:$src))), 7197 (VBLENDPDYrri (v4f64 (AVX_SET0)), VR256:$src, (i8 1))>; 7198 } 7199 7200 def : Pat<(v8f32 (X86vzmovl (insert_subvector undef, 7201 (v4f32 (scalar_to_vector FR32:$src)), (iPTR 0)))), 7202 (SUBREG_TO_REG (i32 0), 7203 (v4f32 (VMOVSSrr (v4f32 (V_SET0)), FR32:$src)), 7204 sub_xmm)>; 7205 def : Pat<(v4f64 (X86vzmovl (insert_subvector undef, 7206 (v2f64 (scalar_to_vector FR64:$src)), (iPTR 0)))), 7207 (SUBREG_TO_REG (i64 0), 7208 (v2f64 (VMOVSDrr (v2f64 (V_SET0)), FR64:$src)), 7209 sub_xmm)>; 7210 7211 // These will incur an FP/int domain crossing penalty, but it may be the only 7212 // way without AVX2. Do not add any complexity because we may be able to match 7213 // more optimal patterns defined earlier in this file. 7214 def : Pat<(v8i32 (X86vzmovl (v8i32 VR256:$src))), 7215 (VBLENDPSYrri (v8i32 (AVX_SET0)), VR256:$src, (i8 1))>; 7216 def : Pat<(v4i64 (X86vzmovl (v4i64 VR256:$src))), 7217 (VBLENDPDYrri (v4i64 (AVX_SET0)), VR256:$src, (i8 1))>; 7218} 7219 7220// FIXME: Prefer a movss or movsd over a blendps when optimizing for size or 7221// on targets where they have equal performance. These were changed to use 7222// blends because blends have better throughput on SandyBridge and Haswell, but 7223// movs[s/d] are 1-2 byte shorter instructions. 7224let Predicates = [UseSSE41] in { 7225 // With SSE41 we can use blends for these patterns. 7226 def : Pat<(v4f32 (X86vzmovl (v4f32 VR128:$src))), 7227 (BLENDPSrri (v4f32 (V_SET0)), VR128:$src, (i8 1))>; 7228 def : Pat<(v4i32 (X86vzmovl (v4i32 VR128:$src))), 7229 (PBLENDWrri (v4i32 (V_SET0)), VR128:$src, (i8 3))>; 7230 def : Pat<(v2f64 (X86vzmovl (v2f64 VR128:$src))), 7231 (BLENDPDrri (v2f64 (V_SET0)), VR128:$src, (i8 1))>; 7232} 7233 7234 7235/// SS41I_ternary_int - SSE 4.1 ternary operator 7236let Uses = [XMM0], Constraints = "$src1 = $dst" in { 7237 multiclass SS41I_ternary_int<bits<8> opc, string OpcodeStr, PatFrag mem_frag, 7238 X86MemOperand x86memop, Intrinsic IntId, 7239 OpndItins itins = DEFAULT_ITINS> { 7240 def rr0 : SS48I<opc, MRMSrcReg, (outs VR128:$dst), 7241 (ins VR128:$src1, VR128:$src2), 7242 !strconcat(OpcodeStr, 7243 "\t{$src2, $dst|$dst, $src2}"), 7244 [(set VR128:$dst, (IntId VR128:$src1, VR128:$src2, XMM0))], 7245 itins.rr>, Sched<[itins.Sched]>; 7246 7247 def rm0 : SS48I<opc, MRMSrcMem, (outs VR128:$dst), 7248 (ins VR128:$src1, x86memop:$src2), 7249 !strconcat(OpcodeStr, 7250 "\t{$src2, $dst|$dst, $src2}"), 7251 [(set VR128:$dst, 7252 (IntId VR128:$src1, 7253 (bitconvert (mem_frag addr:$src2)), XMM0))], 7254 itins.rm>, Sched<[itins.Sched.Folded, ReadAfterLd]>; 7255 } 7256} 7257 7258let ExeDomain = SSEPackedDouble in 7259defm BLENDVPD : SS41I_ternary_int<0x15, "blendvpd", memopv2f64, f128mem, 7260 int_x86_sse41_blendvpd, 7261 DEFAULT_ITINS_FBLENDSCHED>; 7262let ExeDomain = SSEPackedSingle in 7263defm BLENDVPS : SS41I_ternary_int<0x14, "blendvps", memopv4f32, f128mem, 7264 int_x86_sse41_blendvps, 7265 DEFAULT_ITINS_FBLENDSCHED>; 7266defm PBLENDVB : SS41I_ternary_int<0x10, "pblendvb", memopv2i64, i128mem, 7267 int_x86_sse41_pblendvb, 7268 DEFAULT_ITINS_VARBLENDSCHED>; 7269 7270// Aliases with the implicit xmm0 argument 7271def : InstAlias<"blendvpd\t{%xmm0, $src2, $dst|$dst, $src2, xmm0}", 7272 (BLENDVPDrr0 VR128:$dst, VR128:$src2)>; 7273def : InstAlias<"blendvpd\t{%xmm0, $src2, $dst|$dst, $src2, xmm0}", 7274 (BLENDVPDrm0 VR128:$dst, f128mem:$src2)>; 7275def : InstAlias<"blendvps\t{%xmm0, $src2, $dst|$dst, $src2, xmm0}", 7276 (BLENDVPSrr0 VR128:$dst, VR128:$src2)>; 7277def : InstAlias<"blendvps\t{%xmm0, $src2, $dst|$dst, $src2, xmm0}", 7278 (BLENDVPSrm0 VR128:$dst, f128mem:$src2)>; 7279def : InstAlias<"pblendvb\t{%xmm0, $src2, $dst|$dst, $src2, xmm0}", 7280 (PBLENDVBrr0 VR128:$dst, VR128:$src2)>; 7281def : InstAlias<"pblendvb\t{%xmm0, $src2, $dst|$dst, $src2, xmm0}", 7282 (PBLENDVBrm0 VR128:$dst, i128mem:$src2)>; 7283 7284let Predicates = [UseSSE41] in { 7285 def : Pat<(v16i8 (vselect (v16i8 XMM0), (v16i8 VR128:$src1), 7286 (v16i8 VR128:$src2))), 7287 (PBLENDVBrr0 VR128:$src2, VR128:$src1)>; 7288 def : Pat<(v4i32 (vselect (v4i32 XMM0), (v4i32 VR128:$src1), 7289 (v4i32 VR128:$src2))), 7290 (BLENDVPSrr0 VR128:$src2, VR128:$src1)>; 7291 def : Pat<(v4f32 (vselect (v4i32 XMM0), (v4f32 VR128:$src1), 7292 (v4f32 VR128:$src2))), 7293 (BLENDVPSrr0 VR128:$src2, VR128:$src1)>; 7294 def : Pat<(v2i64 (vselect (v2i64 XMM0), (v2i64 VR128:$src1), 7295 (v2i64 VR128:$src2))), 7296 (BLENDVPDrr0 VR128:$src2, VR128:$src1)>; 7297 def : Pat<(v2f64 (vselect (v2i64 XMM0), (v2f64 VR128:$src1), 7298 (v2f64 VR128:$src2))), 7299 (BLENDVPDrr0 VR128:$src2, VR128:$src1)>; 7300} 7301 7302let SchedRW = [WriteLoad] in { 7303let Predicates = [HasAVX] in 7304def VMOVNTDQArm : SS48I<0x2A, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src), 7305 "vmovntdqa\t{$src, $dst|$dst, $src}", 7306 [(set VR128:$dst, (int_x86_sse41_movntdqa addr:$src))]>, 7307 VEX; 7308let Predicates = [HasAVX2] in 7309def VMOVNTDQAYrm : SS48I<0x2A, MRMSrcMem, (outs VR256:$dst), (ins i256mem:$src), 7310 "vmovntdqa\t{$src, $dst|$dst, $src}", 7311 [(set VR256:$dst, (int_x86_avx2_movntdqa addr:$src))]>, 7312 VEX, VEX_L; 7313def MOVNTDQArm : SS48I<0x2A, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src), 7314 "movntdqa\t{$src, $dst|$dst, $src}", 7315 [(set VR128:$dst, (int_x86_sse41_movntdqa addr:$src))]>; 7316} // SchedRW 7317 7318//===----------------------------------------------------------------------===// 7319// SSE4.2 - Compare Instructions 7320//===----------------------------------------------------------------------===// 7321 7322/// SS42I_binop_rm - Simple SSE 4.2 binary operator 7323multiclass SS42I_binop_rm<bits<8> opc, string OpcodeStr, SDNode OpNode, 7324 ValueType OpVT, RegisterClass RC, PatFrag memop_frag, 7325 X86MemOperand x86memop, bit Is2Addr = 1> { 7326 def rr : SS428I<opc, MRMSrcReg, (outs RC:$dst), 7327 (ins RC:$src1, RC:$src2), 7328 !if(Is2Addr, 7329 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 7330 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 7331 [(set RC:$dst, (OpVT (OpNode RC:$src1, RC:$src2)))]>; 7332 def rm : SS428I<opc, MRMSrcMem, (outs RC:$dst), 7333 (ins RC:$src1, x86memop:$src2), 7334 !if(Is2Addr, 7335 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 7336 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 7337 [(set RC:$dst, 7338 (OpVT (OpNode RC:$src1, (memop_frag addr:$src2))))]>; 7339} 7340 7341let Predicates = [HasAVX] in 7342 defm VPCMPGTQ : SS42I_binop_rm<0x37, "vpcmpgtq", X86pcmpgt, v2i64, VR128, 7343 loadv2i64, i128mem, 0>, VEX_4V; 7344 7345let Predicates = [HasAVX2] in 7346 defm VPCMPGTQY : SS42I_binop_rm<0x37, "vpcmpgtq", X86pcmpgt, v4i64, VR256, 7347 loadv4i64, i256mem, 0>, VEX_4V, VEX_L; 7348 7349let Constraints = "$src1 = $dst" in 7350 defm PCMPGTQ : SS42I_binop_rm<0x37, "pcmpgtq", X86pcmpgt, v2i64, VR128, 7351 memopv2i64, i128mem>; 7352 7353//===----------------------------------------------------------------------===// 7354// SSE4.2 - String/text Processing Instructions 7355//===----------------------------------------------------------------------===// 7356 7357// Packed Compare Implicit Length Strings, Return Mask 7358multiclass pseudo_pcmpistrm<string asm, PatFrag ld_frag> { 7359 def REG : PseudoI<(outs VR128:$dst), 7360 (ins VR128:$src1, VR128:$src2, u8imm:$src3), 7361 [(set VR128:$dst, (int_x86_sse42_pcmpistrm128 VR128:$src1, VR128:$src2, 7362 imm:$src3))]>; 7363 def MEM : PseudoI<(outs VR128:$dst), 7364 (ins VR128:$src1, i128mem:$src2, u8imm:$src3), 7365 [(set VR128:$dst, (int_x86_sse42_pcmpistrm128 VR128:$src1, 7366 (bc_v16i8 (ld_frag addr:$src2)), imm:$src3))]>; 7367} 7368 7369let Defs = [EFLAGS], usesCustomInserter = 1 in { 7370 defm VPCMPISTRM128 : pseudo_pcmpistrm<"#VPCMPISTRM128", loadv2i64>, 7371 Requires<[HasAVX]>; 7372 defm PCMPISTRM128 : pseudo_pcmpistrm<"#PCMPISTRM128", memopv2i64>, 7373 Requires<[UseSSE42]>; 7374} 7375 7376multiclass pcmpistrm_SS42AI<string asm> { 7377 def rr : SS42AI<0x62, MRMSrcReg, (outs), 7378 (ins VR128:$src1, VR128:$src2, u8imm:$src3), 7379 !strconcat(asm, "\t{$src3, $src2, $src1|$src1, $src2, $src3}"), 7380 []>, Sched<[WritePCmpIStrM]>; 7381 let mayLoad = 1 in 7382 def rm :SS42AI<0x62, MRMSrcMem, (outs), 7383 (ins VR128:$src1, i128mem:$src2, u8imm:$src3), 7384 !strconcat(asm, "\t{$src3, $src2, $src1|$src1, $src2, $src3}"), 7385 []>, Sched<[WritePCmpIStrMLd, ReadAfterLd]>; 7386} 7387 7388let Defs = [XMM0, EFLAGS], hasSideEffects = 0 in { 7389 let Predicates = [HasAVX] in 7390 defm VPCMPISTRM128 : pcmpistrm_SS42AI<"vpcmpistrm">, VEX; 7391 defm PCMPISTRM128 : pcmpistrm_SS42AI<"pcmpistrm"> ; 7392} 7393 7394// Packed Compare Explicit Length Strings, Return Mask 7395multiclass pseudo_pcmpestrm<string asm, PatFrag ld_frag> { 7396 def REG : PseudoI<(outs VR128:$dst), 7397 (ins VR128:$src1, VR128:$src3, u8imm:$src5), 7398 [(set VR128:$dst, (int_x86_sse42_pcmpestrm128 7399 VR128:$src1, EAX, VR128:$src3, EDX, imm:$src5))]>; 7400 def MEM : PseudoI<(outs VR128:$dst), 7401 (ins VR128:$src1, i128mem:$src3, u8imm:$src5), 7402 [(set VR128:$dst, (int_x86_sse42_pcmpestrm128 VR128:$src1, EAX, 7403 (bc_v16i8 (ld_frag addr:$src3)), EDX, imm:$src5))]>; 7404} 7405 7406let Defs = [EFLAGS], Uses = [EAX, EDX], usesCustomInserter = 1 in { 7407 defm VPCMPESTRM128 : pseudo_pcmpestrm<"#VPCMPESTRM128", loadv2i64>, 7408 Requires<[HasAVX]>; 7409 defm PCMPESTRM128 : pseudo_pcmpestrm<"#PCMPESTRM128", memopv2i64>, 7410 Requires<[UseSSE42]>; 7411} 7412 7413multiclass SS42AI_pcmpestrm<string asm> { 7414 def rr : SS42AI<0x60, MRMSrcReg, (outs), 7415 (ins VR128:$src1, VR128:$src3, u8imm:$src5), 7416 !strconcat(asm, "\t{$src5, $src3, $src1|$src1, $src3, $src5}"), 7417 []>, Sched<[WritePCmpEStrM]>; 7418 let mayLoad = 1 in 7419 def rm : SS42AI<0x60, MRMSrcMem, (outs), 7420 (ins VR128:$src1, i128mem:$src3, u8imm:$src5), 7421 !strconcat(asm, "\t{$src5, $src3, $src1|$src1, $src3, $src5}"), 7422 []>, Sched<[WritePCmpEStrMLd, ReadAfterLd]>; 7423} 7424 7425let Defs = [XMM0, EFLAGS], Uses = [EAX, EDX], hasSideEffects = 0 in { 7426 let Predicates = [HasAVX] in 7427 defm VPCMPESTRM128 : SS42AI_pcmpestrm<"vpcmpestrm">, VEX; 7428 defm PCMPESTRM128 : SS42AI_pcmpestrm<"pcmpestrm">; 7429} 7430 7431// Packed Compare Implicit Length Strings, Return Index 7432multiclass pseudo_pcmpistri<string asm, PatFrag ld_frag> { 7433 def REG : PseudoI<(outs GR32:$dst), 7434 (ins VR128:$src1, VR128:$src2, u8imm:$src3), 7435 [(set GR32:$dst, EFLAGS, 7436 (X86pcmpistri VR128:$src1, VR128:$src2, imm:$src3))]>; 7437 def MEM : PseudoI<(outs GR32:$dst), 7438 (ins VR128:$src1, i128mem:$src2, u8imm:$src3), 7439 [(set GR32:$dst, EFLAGS, (X86pcmpistri VR128:$src1, 7440 (bc_v16i8 (ld_frag addr:$src2)), imm:$src3))]>; 7441} 7442 7443let Defs = [EFLAGS], usesCustomInserter = 1 in { 7444 defm VPCMPISTRI : pseudo_pcmpistri<"#VPCMPISTRI", loadv2i64>, 7445 Requires<[HasAVX]>; 7446 defm PCMPISTRI : pseudo_pcmpistri<"#PCMPISTRI", memopv2i64>, 7447 Requires<[UseSSE42]>; 7448} 7449 7450multiclass SS42AI_pcmpistri<string asm> { 7451 def rr : SS42AI<0x63, MRMSrcReg, (outs), 7452 (ins VR128:$src1, VR128:$src2, u8imm:$src3), 7453 !strconcat(asm, "\t{$src3, $src2, $src1|$src1, $src2, $src3}"), 7454 []>, Sched<[WritePCmpIStrI]>; 7455 let mayLoad = 1 in 7456 def rm : SS42AI<0x63, MRMSrcMem, (outs), 7457 (ins VR128:$src1, i128mem:$src2, u8imm:$src3), 7458 !strconcat(asm, "\t{$src3, $src2, $src1|$src1, $src2, $src3}"), 7459 []>, Sched<[WritePCmpIStrILd, ReadAfterLd]>; 7460} 7461 7462let Defs = [ECX, EFLAGS], hasSideEffects = 0 in { 7463 let Predicates = [HasAVX] in 7464 defm VPCMPISTRI : SS42AI_pcmpistri<"vpcmpistri">, VEX; 7465 defm PCMPISTRI : SS42AI_pcmpistri<"pcmpistri">; 7466} 7467 7468// Packed Compare Explicit Length Strings, Return Index 7469multiclass pseudo_pcmpestri<string asm, PatFrag ld_frag> { 7470 def REG : PseudoI<(outs GR32:$dst), 7471 (ins VR128:$src1, VR128:$src3, u8imm:$src5), 7472 [(set GR32:$dst, EFLAGS, 7473 (X86pcmpestri VR128:$src1, EAX, VR128:$src3, EDX, imm:$src5))]>; 7474 def MEM : PseudoI<(outs GR32:$dst), 7475 (ins VR128:$src1, i128mem:$src3, u8imm:$src5), 7476 [(set GR32:$dst, EFLAGS, 7477 (X86pcmpestri VR128:$src1, EAX, (bc_v16i8 (ld_frag addr:$src3)), EDX, 7478 imm:$src5))]>; 7479} 7480 7481let Defs = [EFLAGS], Uses = [EAX, EDX], usesCustomInserter = 1 in { 7482 defm VPCMPESTRI : pseudo_pcmpestri<"#VPCMPESTRI", loadv2i64>, 7483 Requires<[HasAVX]>; 7484 defm PCMPESTRI : pseudo_pcmpestri<"#PCMPESTRI", memopv2i64>, 7485 Requires<[UseSSE42]>; 7486} 7487 7488multiclass SS42AI_pcmpestri<string asm> { 7489 def rr : SS42AI<0x61, MRMSrcReg, (outs), 7490 (ins VR128:$src1, VR128:$src3, u8imm:$src5), 7491 !strconcat(asm, "\t{$src5, $src3, $src1|$src1, $src3, $src5}"), 7492 []>, Sched<[WritePCmpEStrI]>; 7493 let mayLoad = 1 in 7494 def rm : SS42AI<0x61, MRMSrcMem, (outs), 7495 (ins VR128:$src1, i128mem:$src3, u8imm:$src5), 7496 !strconcat(asm, "\t{$src5, $src3, $src1|$src1, $src3, $src5}"), 7497 []>, Sched<[WritePCmpEStrILd, ReadAfterLd]>; 7498} 7499 7500let Defs = [ECX, EFLAGS], Uses = [EAX, EDX], hasSideEffects = 0 in { 7501 let Predicates = [HasAVX] in 7502 defm VPCMPESTRI : SS42AI_pcmpestri<"vpcmpestri">, VEX; 7503 defm PCMPESTRI : SS42AI_pcmpestri<"pcmpestri">; 7504} 7505 7506//===----------------------------------------------------------------------===// 7507// SSE4.2 - CRC Instructions 7508//===----------------------------------------------------------------------===// 7509 7510// No CRC instructions have AVX equivalents 7511 7512// crc intrinsic instruction 7513// This set of instructions are only rm, the only difference is the size 7514// of r and m. 7515class SS42I_crc32r<bits<8> opc, string asm, RegisterClass RCOut, 7516 RegisterClass RCIn, SDPatternOperator Int> : 7517 SS42FI<opc, MRMSrcReg, (outs RCOut:$dst), (ins RCOut:$src1, RCIn:$src2), 7518 !strconcat(asm, "\t{$src2, $src1|$src1, $src2}"), 7519 [(set RCOut:$dst, (Int RCOut:$src1, RCIn:$src2))], IIC_CRC32_REG>, 7520 Sched<[WriteFAdd]>; 7521 7522class SS42I_crc32m<bits<8> opc, string asm, RegisterClass RCOut, 7523 X86MemOperand x86memop, SDPatternOperator Int> : 7524 SS42FI<opc, MRMSrcMem, (outs RCOut:$dst), (ins RCOut:$src1, x86memop:$src2), 7525 !strconcat(asm, "\t{$src2, $src1|$src1, $src2}"), 7526 [(set RCOut:$dst, (Int RCOut:$src1, (load addr:$src2)))], 7527 IIC_CRC32_MEM>, Sched<[WriteFAddLd, ReadAfterLd]>; 7528 7529let Constraints = "$src1 = $dst" in { 7530 def CRC32r32m8 : SS42I_crc32m<0xF0, "crc32{b}", GR32, i8mem, 7531 int_x86_sse42_crc32_32_8>; 7532 def CRC32r32r8 : SS42I_crc32r<0xF0, "crc32{b}", GR32, GR8, 7533 int_x86_sse42_crc32_32_8>; 7534 def CRC32r32m16 : SS42I_crc32m<0xF1, "crc32{w}", GR32, i16mem, 7535 int_x86_sse42_crc32_32_16>, OpSize16; 7536 def CRC32r32r16 : SS42I_crc32r<0xF1, "crc32{w}", GR32, GR16, 7537 int_x86_sse42_crc32_32_16>, OpSize16; 7538 def CRC32r32m32 : SS42I_crc32m<0xF1, "crc32{l}", GR32, i32mem, 7539 int_x86_sse42_crc32_32_32>, OpSize32; 7540 def CRC32r32r32 : SS42I_crc32r<0xF1, "crc32{l}", GR32, GR32, 7541 int_x86_sse42_crc32_32_32>, OpSize32; 7542 def CRC32r64m64 : SS42I_crc32m<0xF1, "crc32{q}", GR64, i64mem, 7543 int_x86_sse42_crc32_64_64>, REX_W; 7544 def CRC32r64r64 : SS42I_crc32r<0xF1, "crc32{q}", GR64, GR64, 7545 int_x86_sse42_crc32_64_64>, REX_W; 7546 let hasSideEffects = 0 in { 7547 let mayLoad = 1 in 7548 def CRC32r64m8 : SS42I_crc32m<0xF0, "crc32{b}", GR64, i8mem, 7549 null_frag>, REX_W; 7550 def CRC32r64r8 : SS42I_crc32r<0xF0, "crc32{b}", GR64, GR8, 7551 null_frag>, REX_W; 7552 } 7553} 7554 7555//===----------------------------------------------------------------------===// 7556// SHA-NI Instructions 7557//===----------------------------------------------------------------------===// 7558 7559multiclass SHAI_binop<bits<8> Opc, string OpcodeStr, Intrinsic IntId, 7560 bit UsesXMM0 = 0> { 7561 def rr : I<Opc, MRMSrcReg, (outs VR128:$dst), 7562 (ins VR128:$src1, VR128:$src2), 7563 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 7564 [!if(UsesXMM0, 7565 (set VR128:$dst, (IntId VR128:$src1, VR128:$src2, XMM0)), 7566 (set VR128:$dst, (IntId VR128:$src1, VR128:$src2)))]>, T8; 7567 7568 def rm : I<Opc, MRMSrcMem, (outs VR128:$dst), 7569 (ins VR128:$src1, i128mem:$src2), 7570 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 7571 [!if(UsesXMM0, 7572 (set VR128:$dst, (IntId VR128:$src1, 7573 (bc_v4i32 (memopv2i64 addr:$src2)), XMM0)), 7574 (set VR128:$dst, (IntId VR128:$src1, 7575 (bc_v4i32 (memopv2i64 addr:$src2)))))]>, T8; 7576} 7577 7578let Constraints = "$src1 = $dst", Predicates = [HasSHA] in { 7579 def SHA1RNDS4rri : Ii8<0xCC, MRMSrcReg, (outs VR128:$dst), 7580 (ins VR128:$src1, VR128:$src2, u8imm:$src3), 7581 "sha1rnds4\t{$src3, $src2, $dst|$dst, $src2, $src3}", 7582 [(set VR128:$dst, 7583 (int_x86_sha1rnds4 VR128:$src1, VR128:$src2, 7584 (i8 imm:$src3)))]>, TA; 7585 def SHA1RNDS4rmi : Ii8<0xCC, MRMSrcMem, (outs VR128:$dst), 7586 (ins VR128:$src1, i128mem:$src2, u8imm:$src3), 7587 "sha1rnds4\t{$src3, $src2, $dst|$dst, $src2, $src3}", 7588 [(set VR128:$dst, 7589 (int_x86_sha1rnds4 VR128:$src1, 7590 (bc_v4i32 (memopv2i64 addr:$src2)), 7591 (i8 imm:$src3)))]>, TA; 7592 7593 defm SHA1NEXTE : SHAI_binop<0xC8, "sha1nexte", int_x86_sha1nexte>; 7594 defm SHA1MSG1 : SHAI_binop<0xC9, "sha1msg1", int_x86_sha1msg1>; 7595 defm SHA1MSG2 : SHAI_binop<0xCA, "sha1msg2", int_x86_sha1msg2>; 7596 7597 let Uses=[XMM0] in 7598 defm SHA256RNDS2 : SHAI_binop<0xCB, "sha256rnds2", int_x86_sha256rnds2, 1>; 7599 7600 defm SHA256MSG1 : SHAI_binop<0xCC, "sha256msg1", int_x86_sha256msg1>; 7601 defm SHA256MSG2 : SHAI_binop<0xCD, "sha256msg2", int_x86_sha256msg2>; 7602} 7603 7604// Aliases with explicit %xmm0 7605def : InstAlias<"sha256rnds2\t{%xmm0, $src2, $dst|$dst, $src2, xmm0}", 7606 (SHA256RNDS2rr VR128:$dst, VR128:$src2)>; 7607def : InstAlias<"sha256rnds2\t{%xmm0, $src2, $dst|$dst, $src2, xmm0}", 7608 (SHA256RNDS2rm VR128:$dst, i128mem:$src2)>; 7609 7610//===----------------------------------------------------------------------===// 7611// AES-NI Instructions 7612//===----------------------------------------------------------------------===// 7613 7614multiclass AESI_binop_rm_int<bits<8> opc, string OpcodeStr, Intrinsic IntId128, 7615 PatFrag ld_frag, bit Is2Addr = 1> { 7616 def rr : AES8I<opc, MRMSrcReg, (outs VR128:$dst), 7617 (ins VR128:$src1, VR128:$src2), 7618 !if(Is2Addr, 7619 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 7620 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 7621 [(set VR128:$dst, (IntId128 VR128:$src1, VR128:$src2))]>, 7622 Sched<[WriteAESDecEnc]>; 7623 def rm : AES8I<opc, MRMSrcMem, (outs VR128:$dst), 7624 (ins VR128:$src1, i128mem:$src2), 7625 !if(Is2Addr, 7626 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 7627 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 7628 [(set VR128:$dst, 7629 (IntId128 VR128:$src1, (ld_frag addr:$src2)))]>, 7630 Sched<[WriteAESDecEncLd, ReadAfterLd]>; 7631} 7632 7633// Perform One Round of an AES Encryption/Decryption Flow 7634let Predicates = [HasAVX, HasAES] in { 7635 defm VAESENC : AESI_binop_rm_int<0xDC, "vaesenc", 7636 int_x86_aesni_aesenc, loadv2i64, 0>, VEX_4V; 7637 defm VAESENCLAST : AESI_binop_rm_int<0xDD, "vaesenclast", 7638 int_x86_aesni_aesenclast, loadv2i64, 0>, VEX_4V; 7639 defm VAESDEC : AESI_binop_rm_int<0xDE, "vaesdec", 7640 int_x86_aesni_aesdec, loadv2i64, 0>, VEX_4V; 7641 defm VAESDECLAST : AESI_binop_rm_int<0xDF, "vaesdeclast", 7642 int_x86_aesni_aesdeclast, loadv2i64, 0>, VEX_4V; 7643} 7644 7645let Constraints = "$src1 = $dst" in { 7646 defm AESENC : AESI_binop_rm_int<0xDC, "aesenc", 7647 int_x86_aesni_aesenc, memopv2i64>; 7648 defm AESENCLAST : AESI_binop_rm_int<0xDD, "aesenclast", 7649 int_x86_aesni_aesenclast, memopv2i64>; 7650 defm AESDEC : AESI_binop_rm_int<0xDE, "aesdec", 7651 int_x86_aesni_aesdec, memopv2i64>; 7652 defm AESDECLAST : AESI_binop_rm_int<0xDF, "aesdeclast", 7653 int_x86_aesni_aesdeclast, memopv2i64>; 7654} 7655 7656// Perform the AES InvMixColumn Transformation 7657let Predicates = [HasAVX, HasAES] in { 7658 def VAESIMCrr : AES8I<0xDB, MRMSrcReg, (outs VR128:$dst), 7659 (ins VR128:$src1), 7660 "vaesimc\t{$src1, $dst|$dst, $src1}", 7661 [(set VR128:$dst, 7662 (int_x86_aesni_aesimc VR128:$src1))]>, Sched<[WriteAESIMC]>, 7663 VEX; 7664 def VAESIMCrm : AES8I<0xDB, MRMSrcMem, (outs VR128:$dst), 7665 (ins i128mem:$src1), 7666 "vaesimc\t{$src1, $dst|$dst, $src1}", 7667 [(set VR128:$dst, (int_x86_aesni_aesimc (loadv2i64 addr:$src1)))]>, 7668 Sched<[WriteAESIMCLd]>, VEX; 7669} 7670def AESIMCrr : AES8I<0xDB, MRMSrcReg, (outs VR128:$dst), 7671 (ins VR128:$src1), 7672 "aesimc\t{$src1, $dst|$dst, $src1}", 7673 [(set VR128:$dst, 7674 (int_x86_aesni_aesimc VR128:$src1))]>, Sched<[WriteAESIMC]>; 7675def AESIMCrm : AES8I<0xDB, MRMSrcMem, (outs VR128:$dst), 7676 (ins i128mem:$src1), 7677 "aesimc\t{$src1, $dst|$dst, $src1}", 7678 [(set VR128:$dst, (int_x86_aesni_aesimc (memopv2i64 addr:$src1)))]>, 7679 Sched<[WriteAESIMCLd]>; 7680 7681// AES Round Key Generation Assist 7682let Predicates = [HasAVX, HasAES] in { 7683 def VAESKEYGENASSIST128rr : AESAI<0xDF, MRMSrcReg, (outs VR128:$dst), 7684 (ins VR128:$src1, u8imm:$src2), 7685 "vaeskeygenassist\t{$src2, $src1, $dst|$dst, $src1, $src2}", 7686 [(set VR128:$dst, 7687 (int_x86_aesni_aeskeygenassist VR128:$src1, imm:$src2))]>, 7688 Sched<[WriteAESKeyGen]>, VEX; 7689 def VAESKEYGENASSIST128rm : AESAI<0xDF, MRMSrcMem, (outs VR128:$dst), 7690 (ins i128mem:$src1, u8imm:$src2), 7691 "vaeskeygenassist\t{$src2, $src1, $dst|$dst, $src1, $src2}", 7692 [(set VR128:$dst, 7693 (int_x86_aesni_aeskeygenassist (loadv2i64 addr:$src1), imm:$src2))]>, 7694 Sched<[WriteAESKeyGenLd]>, VEX; 7695} 7696def AESKEYGENASSIST128rr : AESAI<0xDF, MRMSrcReg, (outs VR128:$dst), 7697 (ins VR128:$src1, u8imm:$src2), 7698 "aeskeygenassist\t{$src2, $src1, $dst|$dst, $src1, $src2}", 7699 [(set VR128:$dst, 7700 (int_x86_aesni_aeskeygenassist VR128:$src1, imm:$src2))]>, 7701 Sched<[WriteAESKeyGen]>; 7702def AESKEYGENASSIST128rm : AESAI<0xDF, MRMSrcMem, (outs VR128:$dst), 7703 (ins i128mem:$src1, u8imm:$src2), 7704 "aeskeygenassist\t{$src2, $src1, $dst|$dst, $src1, $src2}", 7705 [(set VR128:$dst, 7706 (int_x86_aesni_aeskeygenassist (memopv2i64 addr:$src1), imm:$src2))]>, 7707 Sched<[WriteAESKeyGenLd]>; 7708 7709//===----------------------------------------------------------------------===// 7710// PCLMUL Instructions 7711//===----------------------------------------------------------------------===// 7712 7713// AVX carry-less Multiplication instructions 7714let isCommutable = 1 in 7715def VPCLMULQDQrr : AVXPCLMULIi8<0x44, MRMSrcReg, (outs VR128:$dst), 7716 (ins VR128:$src1, VR128:$src2, u8imm:$src3), 7717 "vpclmulqdq\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", 7718 [(set VR128:$dst, 7719 (int_x86_pclmulqdq VR128:$src1, VR128:$src2, imm:$src3))]>, 7720 Sched<[WriteCLMul]>; 7721 7722def VPCLMULQDQrm : AVXPCLMULIi8<0x44, MRMSrcMem, (outs VR128:$dst), 7723 (ins VR128:$src1, i128mem:$src2, u8imm:$src3), 7724 "vpclmulqdq\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", 7725 [(set VR128:$dst, (int_x86_pclmulqdq VR128:$src1, 7726 (loadv2i64 addr:$src2), imm:$src3))]>, 7727 Sched<[WriteCLMulLd, ReadAfterLd]>; 7728 7729// Carry-less Multiplication instructions 7730let Constraints = "$src1 = $dst" in { 7731let isCommutable = 1 in 7732def PCLMULQDQrr : PCLMULIi8<0x44, MRMSrcReg, (outs VR128:$dst), 7733 (ins VR128:$src1, VR128:$src2, u8imm:$src3), 7734 "pclmulqdq\t{$src3, $src2, $dst|$dst, $src2, $src3}", 7735 [(set VR128:$dst, 7736 (int_x86_pclmulqdq VR128:$src1, VR128:$src2, imm:$src3))], 7737 IIC_SSE_PCLMULQDQ_RR>, Sched<[WriteCLMul]>; 7738 7739def PCLMULQDQrm : PCLMULIi8<0x44, MRMSrcMem, (outs VR128:$dst), 7740 (ins VR128:$src1, i128mem:$src2, u8imm:$src3), 7741 "pclmulqdq\t{$src3, $src2, $dst|$dst, $src2, $src3}", 7742 [(set VR128:$dst, (int_x86_pclmulqdq VR128:$src1, 7743 (memopv2i64 addr:$src2), imm:$src3))], 7744 IIC_SSE_PCLMULQDQ_RM>, 7745 Sched<[WriteCLMulLd, ReadAfterLd]>; 7746} // Constraints = "$src1 = $dst" 7747 7748 7749multiclass pclmul_alias<string asm, int immop> { 7750 def : InstAlias<!strconcat("pclmul", asm, "dq {$src, $dst|$dst, $src}"), 7751 (PCLMULQDQrr VR128:$dst, VR128:$src, immop), 0>; 7752 7753 def : InstAlias<!strconcat("pclmul", asm, "dq {$src, $dst|$dst, $src}"), 7754 (PCLMULQDQrm VR128:$dst, i128mem:$src, immop), 0>; 7755 7756 def : InstAlias<!strconcat("vpclmul", asm, 7757 "dq {$src2, $src1, $dst|$dst, $src1, $src2}"), 7758 (VPCLMULQDQrr VR128:$dst, VR128:$src1, VR128:$src2, immop), 7759 0>; 7760 7761 def : InstAlias<!strconcat("vpclmul", asm, 7762 "dq {$src2, $src1, $dst|$dst, $src1, $src2}"), 7763 (VPCLMULQDQrm VR128:$dst, VR128:$src1, i128mem:$src2, immop), 7764 0>; 7765} 7766defm : pclmul_alias<"hqhq", 0x11>; 7767defm : pclmul_alias<"hqlq", 0x01>; 7768defm : pclmul_alias<"lqhq", 0x10>; 7769defm : pclmul_alias<"lqlq", 0x00>; 7770 7771//===----------------------------------------------------------------------===// 7772// SSE4A Instructions 7773//===----------------------------------------------------------------------===// 7774 7775let Predicates = [HasSSE4A] in { 7776 7777let Constraints = "$src = $dst" in { 7778def EXTRQI : Ii8<0x78, MRMXr, (outs VR128:$dst), 7779 (ins VR128:$src, u8imm:$len, u8imm:$idx), 7780 "extrq\t{$idx, $len, $src|$src, $len, $idx}", 7781 [(set VR128:$dst, (X86extrqi VR128:$src, imm:$len, 7782 imm:$idx))]>, PD; 7783def EXTRQ : I<0x79, MRMSrcReg, (outs VR128:$dst), 7784 (ins VR128:$src, VR128:$mask), 7785 "extrq\t{$mask, $src|$src, $mask}", 7786 [(set VR128:$dst, (int_x86_sse4a_extrq VR128:$src, 7787 VR128:$mask))]>, PD; 7788 7789def INSERTQI : Ii8<0x78, MRMSrcReg, (outs VR128:$dst), 7790 (ins VR128:$src, VR128:$src2, u8imm:$len, u8imm:$idx), 7791 "insertq\t{$idx, $len, $src2, $src|$src, $src2, $len, $idx}", 7792 [(set VR128:$dst, (X86insertqi VR128:$src, VR128:$src2, 7793 imm:$len, imm:$idx))]>, XD; 7794def INSERTQ : I<0x79, MRMSrcReg, (outs VR128:$dst), 7795 (ins VR128:$src, VR128:$mask), 7796 "insertq\t{$mask, $src|$src, $mask}", 7797 [(set VR128:$dst, (int_x86_sse4a_insertq VR128:$src, 7798 VR128:$mask))]>, XD; 7799} 7800 7801def MOVNTSS : I<0x2B, MRMDestMem, (outs), (ins f32mem:$dst, VR128:$src), 7802 "movntss\t{$src, $dst|$dst, $src}", 7803 [(int_x86_sse4a_movnt_ss addr:$dst, VR128:$src)]>, XS; 7804 7805def MOVNTSD : I<0x2B, MRMDestMem, (outs), (ins f64mem:$dst, VR128:$src), 7806 "movntsd\t{$src, $dst|$dst, $src}", 7807 [(int_x86_sse4a_movnt_sd addr:$dst, VR128:$src)]>, XD; 7808} 7809 7810//===----------------------------------------------------------------------===// 7811// AVX Instructions 7812//===----------------------------------------------------------------------===// 7813 7814//===----------------------------------------------------------------------===// 7815// VBROADCAST - Load from memory and broadcast to all elements of the 7816// destination operand 7817// 7818class avx_broadcast<bits<8> opc, string OpcodeStr, RegisterClass RC, 7819 X86MemOperand x86memop, Intrinsic Int, SchedWrite Sched> : 7820 AVX8I<opc, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src), 7821 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 7822 [(set RC:$dst, (Int addr:$src))]>, Sched<[Sched]>, VEX; 7823 7824class avx_broadcast_no_int<bits<8> opc, string OpcodeStr, RegisterClass RC, 7825 X86MemOperand x86memop, ValueType VT, 7826 PatFrag ld_frag, SchedWrite Sched> : 7827 AVX8I<opc, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src), 7828 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 7829 [(set RC:$dst, (VT (X86VBroadcast (ld_frag addr:$src))))]>, 7830 Sched<[Sched]>, VEX { 7831 let mayLoad = 1; 7832} 7833 7834// AVX2 adds register forms 7835class avx2_broadcast_reg<bits<8> opc, string OpcodeStr, RegisterClass RC, 7836 Intrinsic Int, SchedWrite Sched> : 7837 AVX28I<opc, MRMSrcReg, (outs RC:$dst), (ins VR128:$src), 7838 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 7839 [(set RC:$dst, (Int VR128:$src))]>, Sched<[Sched]>, VEX; 7840 7841let ExeDomain = SSEPackedSingle in { 7842 def VBROADCASTSSrm : avx_broadcast_no_int<0x18, "vbroadcastss", VR128, 7843 f32mem, v4f32, loadf32, WriteLoad>; 7844 def VBROADCASTSSYrm : avx_broadcast_no_int<0x18, "vbroadcastss", VR256, 7845 f32mem, v8f32, loadf32, 7846 WriteFShuffleLd>, VEX_L; 7847} 7848let ExeDomain = SSEPackedDouble in 7849def VBROADCASTSDYrm : avx_broadcast_no_int<0x19, "vbroadcastsd", VR256, f64mem, 7850 v4f64, loadf64, WriteFShuffleLd>, VEX_L; 7851def VBROADCASTF128 : avx_broadcast<0x1A, "vbroadcastf128", VR256, f128mem, 7852 int_x86_avx_vbroadcastf128_pd_256, 7853 WriteFShuffleLd>, VEX_L; 7854 7855let ExeDomain = SSEPackedSingle in { 7856 def VBROADCASTSSrr : avx2_broadcast_reg<0x18, "vbroadcastss", VR128, 7857 int_x86_avx2_vbroadcast_ss_ps, 7858 WriteFShuffle>; 7859 def VBROADCASTSSYrr : avx2_broadcast_reg<0x18, "vbroadcastss", VR256, 7860 int_x86_avx2_vbroadcast_ss_ps_256, 7861 WriteFShuffle256>, VEX_L; 7862} 7863let ExeDomain = SSEPackedDouble in 7864def VBROADCASTSDYrr : avx2_broadcast_reg<0x19, "vbroadcastsd", VR256, 7865 int_x86_avx2_vbroadcast_sd_pd_256, 7866 WriteFShuffle256>, VEX_L; 7867 7868let mayLoad = 1, Predicates = [HasAVX2] in 7869def VBROADCASTI128 : AVX8I<0x5A, MRMSrcMem, (outs VR256:$dst), 7870 (ins i128mem:$src), 7871 "vbroadcasti128\t{$src, $dst|$dst, $src}", []>, 7872 Sched<[WriteLoad]>, VEX, VEX_L; 7873 7874let Predicates = [HasAVX] in 7875def : Pat<(int_x86_avx_vbroadcastf128_ps_256 addr:$src), 7876 (VBROADCASTF128 addr:$src)>; 7877 7878 7879//===----------------------------------------------------------------------===// 7880// VINSERTF128 - Insert packed floating-point values 7881// 7882let hasSideEffects = 0, ExeDomain = SSEPackedSingle in { 7883def VINSERTF128rr : AVXAIi8<0x18, MRMSrcReg, (outs VR256:$dst), 7884 (ins VR256:$src1, VR128:$src2, u8imm:$src3), 7885 "vinsertf128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", 7886 []>, Sched<[WriteFShuffle]>, VEX_4V, VEX_L; 7887let mayLoad = 1 in 7888def VINSERTF128rm : AVXAIi8<0x18, MRMSrcMem, (outs VR256:$dst), 7889 (ins VR256:$src1, f128mem:$src2, u8imm:$src3), 7890 "vinsertf128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", 7891 []>, Sched<[WriteFShuffleLd, ReadAfterLd]>, VEX_4V, VEX_L; 7892} 7893 7894let Predicates = [HasAVX] in { 7895def : Pat<(vinsert128_insert:$ins (v8f32 VR256:$src1), (v4f32 VR128:$src2), 7896 (iPTR imm)), 7897 (VINSERTF128rr VR256:$src1, VR128:$src2, 7898 (INSERT_get_vinsert128_imm VR256:$ins))>; 7899def : Pat<(vinsert128_insert:$ins (v4f64 VR256:$src1), (v2f64 VR128:$src2), 7900 (iPTR imm)), 7901 (VINSERTF128rr VR256:$src1, VR128:$src2, 7902 (INSERT_get_vinsert128_imm VR256:$ins))>; 7903 7904def : Pat<(vinsert128_insert:$ins (v8f32 VR256:$src1), (loadv4f32 addr:$src2), 7905 (iPTR imm)), 7906 (VINSERTF128rm VR256:$src1, addr:$src2, 7907 (INSERT_get_vinsert128_imm VR256:$ins))>; 7908def : Pat<(vinsert128_insert:$ins (v4f64 VR256:$src1), (loadv2f64 addr:$src2), 7909 (iPTR imm)), 7910 (VINSERTF128rm VR256:$src1, addr:$src2, 7911 (INSERT_get_vinsert128_imm VR256:$ins))>; 7912} 7913 7914let Predicates = [HasAVX1Only] in { 7915def : Pat<(vinsert128_insert:$ins (v4i64 VR256:$src1), (v2i64 VR128:$src2), 7916 (iPTR imm)), 7917 (VINSERTF128rr VR256:$src1, VR128:$src2, 7918 (INSERT_get_vinsert128_imm VR256:$ins))>; 7919def : Pat<(vinsert128_insert:$ins (v8i32 VR256:$src1), (v4i32 VR128:$src2), 7920 (iPTR imm)), 7921 (VINSERTF128rr VR256:$src1, VR128:$src2, 7922 (INSERT_get_vinsert128_imm VR256:$ins))>; 7923def : Pat<(vinsert128_insert:$ins (v32i8 VR256:$src1), (v16i8 VR128:$src2), 7924 (iPTR imm)), 7925 (VINSERTF128rr VR256:$src1, VR128:$src2, 7926 (INSERT_get_vinsert128_imm VR256:$ins))>; 7927def : Pat<(vinsert128_insert:$ins (v16i16 VR256:$src1), (v8i16 VR128:$src2), 7928 (iPTR imm)), 7929 (VINSERTF128rr VR256:$src1, VR128:$src2, 7930 (INSERT_get_vinsert128_imm VR256:$ins))>; 7931 7932def : Pat<(vinsert128_insert:$ins (v4i64 VR256:$src1), (loadv2i64 addr:$src2), 7933 (iPTR imm)), 7934 (VINSERTF128rm VR256:$src1, addr:$src2, 7935 (INSERT_get_vinsert128_imm VR256:$ins))>; 7936def : Pat<(vinsert128_insert:$ins (v8i32 VR256:$src1), 7937 (bc_v4i32 (loadv2i64 addr:$src2)), 7938 (iPTR imm)), 7939 (VINSERTF128rm VR256:$src1, addr:$src2, 7940 (INSERT_get_vinsert128_imm VR256:$ins))>; 7941def : Pat<(vinsert128_insert:$ins (v32i8 VR256:$src1), 7942 (bc_v16i8 (loadv2i64 addr:$src2)), 7943 (iPTR imm)), 7944 (VINSERTF128rm VR256:$src1, addr:$src2, 7945 (INSERT_get_vinsert128_imm VR256:$ins))>; 7946def : Pat<(vinsert128_insert:$ins (v16i16 VR256:$src1), 7947 (bc_v8i16 (loadv2i64 addr:$src2)), 7948 (iPTR imm)), 7949 (VINSERTF128rm VR256:$src1, addr:$src2, 7950 (INSERT_get_vinsert128_imm VR256:$ins))>; 7951} 7952 7953//===----------------------------------------------------------------------===// 7954// VEXTRACTF128 - Extract packed floating-point values 7955// 7956let hasSideEffects = 0, ExeDomain = SSEPackedSingle in { 7957def VEXTRACTF128rr : AVXAIi8<0x19, MRMDestReg, (outs VR128:$dst), 7958 (ins VR256:$src1, u8imm:$src2), 7959 "vextractf128\t{$src2, $src1, $dst|$dst, $src1, $src2}", 7960 []>, Sched<[WriteFShuffle]>, VEX, VEX_L; 7961let mayStore = 1 in 7962def VEXTRACTF128mr : AVXAIi8<0x19, MRMDestMem, (outs), 7963 (ins f128mem:$dst, VR256:$src1, u8imm:$src2), 7964 "vextractf128\t{$src2, $src1, $dst|$dst, $src1, $src2}", 7965 []>, Sched<[WriteStore]>, VEX, VEX_L; 7966} 7967 7968// AVX1 patterns 7969let Predicates = [HasAVX] in { 7970def : Pat<(vextract128_extract:$ext VR256:$src1, (iPTR imm)), 7971 (v4f32 (VEXTRACTF128rr 7972 (v8f32 VR256:$src1), 7973 (EXTRACT_get_vextract128_imm VR128:$ext)))>; 7974def : Pat<(vextract128_extract:$ext VR256:$src1, (iPTR imm)), 7975 (v2f64 (VEXTRACTF128rr 7976 (v4f64 VR256:$src1), 7977 (EXTRACT_get_vextract128_imm VR128:$ext)))>; 7978 7979def : Pat<(store (v4f32 (vextract128_extract:$ext (v8f32 VR256:$src1), 7980 (iPTR imm))), addr:$dst), 7981 (VEXTRACTF128mr addr:$dst, VR256:$src1, 7982 (EXTRACT_get_vextract128_imm VR128:$ext))>; 7983def : Pat<(store (v2f64 (vextract128_extract:$ext (v4f64 VR256:$src1), 7984 (iPTR imm))), addr:$dst), 7985 (VEXTRACTF128mr addr:$dst, VR256:$src1, 7986 (EXTRACT_get_vextract128_imm VR128:$ext))>; 7987} 7988 7989let Predicates = [HasAVX1Only] in { 7990def : Pat<(vextract128_extract:$ext VR256:$src1, (iPTR imm)), 7991 (v2i64 (VEXTRACTF128rr 7992 (v4i64 VR256:$src1), 7993 (EXTRACT_get_vextract128_imm VR128:$ext)))>; 7994def : Pat<(vextract128_extract:$ext VR256:$src1, (iPTR imm)), 7995 (v4i32 (VEXTRACTF128rr 7996 (v8i32 VR256:$src1), 7997 (EXTRACT_get_vextract128_imm VR128:$ext)))>; 7998def : Pat<(vextract128_extract:$ext VR256:$src1, (iPTR imm)), 7999 (v8i16 (VEXTRACTF128rr 8000 (v16i16 VR256:$src1), 8001 (EXTRACT_get_vextract128_imm VR128:$ext)))>; 8002def : Pat<(vextract128_extract:$ext VR256:$src1, (iPTR imm)), 8003 (v16i8 (VEXTRACTF128rr 8004 (v32i8 VR256:$src1), 8005 (EXTRACT_get_vextract128_imm VR128:$ext)))>; 8006 8007def : Pat<(alignedstore (v2i64 (vextract128_extract:$ext (v4i64 VR256:$src1), 8008 (iPTR imm))), addr:$dst), 8009 (VEXTRACTF128mr addr:$dst, VR256:$src1, 8010 (EXTRACT_get_vextract128_imm VR128:$ext))>; 8011def : Pat<(alignedstore (v4i32 (vextract128_extract:$ext (v8i32 VR256:$src1), 8012 (iPTR imm))), addr:$dst), 8013 (VEXTRACTF128mr addr:$dst, VR256:$src1, 8014 (EXTRACT_get_vextract128_imm VR128:$ext))>; 8015def : Pat<(alignedstore (v8i16 (vextract128_extract:$ext (v16i16 VR256:$src1), 8016 (iPTR imm))), addr:$dst), 8017 (VEXTRACTF128mr addr:$dst, VR256:$src1, 8018 (EXTRACT_get_vextract128_imm VR128:$ext))>; 8019def : Pat<(alignedstore (v16i8 (vextract128_extract:$ext (v32i8 VR256:$src1), 8020 (iPTR imm))), addr:$dst), 8021 (VEXTRACTF128mr addr:$dst, VR256:$src1, 8022 (EXTRACT_get_vextract128_imm VR128:$ext))>; 8023} 8024 8025//===----------------------------------------------------------------------===// 8026// VMASKMOV - Conditional SIMD Packed Loads and Stores 8027// 8028multiclass avx_movmask_rm<bits<8> opc_rm, bits<8> opc_mr, string OpcodeStr, 8029 Intrinsic IntLd, Intrinsic IntLd256, 8030 Intrinsic IntSt, Intrinsic IntSt256> { 8031 def rm : AVX8I<opc_rm, MRMSrcMem, (outs VR128:$dst), 8032 (ins VR128:$src1, f128mem:$src2), 8033 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 8034 [(set VR128:$dst, (IntLd addr:$src2, VR128:$src1))]>, 8035 VEX_4V; 8036 def Yrm : AVX8I<opc_rm, MRMSrcMem, (outs VR256:$dst), 8037 (ins VR256:$src1, f256mem:$src2), 8038 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 8039 [(set VR256:$dst, (IntLd256 addr:$src2, VR256:$src1))]>, 8040 VEX_4V, VEX_L; 8041 def mr : AVX8I<opc_mr, MRMDestMem, (outs), 8042 (ins f128mem:$dst, VR128:$src1, VR128:$src2), 8043 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 8044 [(IntSt addr:$dst, VR128:$src1, VR128:$src2)]>, VEX_4V; 8045 def Ymr : AVX8I<opc_mr, MRMDestMem, (outs), 8046 (ins f256mem:$dst, VR256:$src1, VR256:$src2), 8047 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 8048 [(IntSt256 addr:$dst, VR256:$src1, VR256:$src2)]>, VEX_4V, VEX_L; 8049} 8050 8051let ExeDomain = SSEPackedSingle in 8052defm VMASKMOVPS : avx_movmask_rm<0x2C, 0x2E, "vmaskmovps", 8053 int_x86_avx_maskload_ps, 8054 int_x86_avx_maskload_ps_256, 8055 int_x86_avx_maskstore_ps, 8056 int_x86_avx_maskstore_ps_256>; 8057let ExeDomain = SSEPackedDouble in 8058defm VMASKMOVPD : avx_movmask_rm<0x2D, 0x2F, "vmaskmovpd", 8059 int_x86_avx_maskload_pd, 8060 int_x86_avx_maskload_pd_256, 8061 int_x86_avx_maskstore_pd, 8062 int_x86_avx_maskstore_pd_256>; 8063 8064//===----------------------------------------------------------------------===// 8065// VPERMIL - Permute Single and Double Floating-Point Values 8066// 8067multiclass avx_permil<bits<8> opc_rm, bits<8> opc_rmi, string OpcodeStr, 8068 RegisterClass RC, X86MemOperand x86memop_f, 8069 X86MemOperand x86memop_i, PatFrag i_frag, 8070 Intrinsic IntVar, ValueType vt> { 8071 def rr : AVX8I<opc_rm, MRMSrcReg, (outs RC:$dst), 8072 (ins RC:$src1, RC:$src2), 8073 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 8074 [(set RC:$dst, (IntVar RC:$src1, RC:$src2))]>, VEX_4V, 8075 Sched<[WriteFShuffle]>; 8076 def rm : AVX8I<opc_rm, MRMSrcMem, (outs RC:$dst), 8077 (ins RC:$src1, x86memop_i:$src2), 8078 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 8079 [(set RC:$dst, (IntVar RC:$src1, 8080 (bitconvert (i_frag addr:$src2))))]>, VEX_4V, 8081 Sched<[WriteFShuffleLd, ReadAfterLd]>; 8082 8083 def ri : AVXAIi8<opc_rmi, MRMSrcReg, (outs RC:$dst), 8084 (ins RC:$src1, u8imm:$src2), 8085 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 8086 [(set RC:$dst, (vt (X86VPermilpi RC:$src1, (i8 imm:$src2))))]>, VEX, 8087 Sched<[WriteFShuffle]>; 8088 def mi : AVXAIi8<opc_rmi, MRMSrcMem, (outs RC:$dst), 8089 (ins x86memop_f:$src1, u8imm:$src2), 8090 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 8091 [(set RC:$dst, 8092 (vt (X86VPermilpi (load addr:$src1), (i8 imm:$src2))))]>, VEX, 8093 Sched<[WriteFShuffleLd]>; 8094} 8095 8096let ExeDomain = SSEPackedSingle in { 8097 defm VPERMILPS : avx_permil<0x0C, 0x04, "vpermilps", VR128, f128mem, i128mem, 8098 loadv2i64, int_x86_avx_vpermilvar_ps, v4f32>; 8099 defm VPERMILPSY : avx_permil<0x0C, 0x04, "vpermilps", VR256, f256mem, i256mem, 8100 loadv4i64, int_x86_avx_vpermilvar_ps_256, v8f32>, VEX_L; 8101} 8102let ExeDomain = SSEPackedDouble in { 8103 defm VPERMILPD : avx_permil<0x0D, 0x05, "vpermilpd", VR128, f128mem, i128mem, 8104 loadv2i64, int_x86_avx_vpermilvar_pd, v2f64>; 8105 defm VPERMILPDY : avx_permil<0x0D, 0x05, "vpermilpd", VR256, f256mem, i256mem, 8106 loadv4i64, int_x86_avx_vpermilvar_pd_256, v4f64>, VEX_L; 8107} 8108 8109let Predicates = [HasAVX] in { 8110def : Pat<(v8f32 (X86VPermilpv VR256:$src1, (v8i32 VR256:$src2))), 8111 (VPERMILPSYrr VR256:$src1, VR256:$src2)>; 8112def : Pat<(v8f32 (X86VPermilpv VR256:$src1, (bc_v8i32 (loadv4i64 addr:$src2)))), 8113 (VPERMILPSYrm VR256:$src1, addr:$src2)>; 8114def : Pat<(v4f64 (X86VPermilpv VR256:$src1, (v4i64 VR256:$src2))), 8115 (VPERMILPDYrr VR256:$src1, VR256:$src2)>; 8116def : Pat<(v4f64 (X86VPermilpv VR256:$src1, (loadv4i64 addr:$src2))), 8117 (VPERMILPDYrm VR256:$src1, addr:$src2)>; 8118 8119def : Pat<(v8i32 (X86VPermilpi VR256:$src1, (i8 imm:$imm))), 8120 (VPERMILPSYri VR256:$src1, imm:$imm)>; 8121def : Pat<(v4i64 (X86VPermilpi VR256:$src1, (i8 imm:$imm))), 8122 (VPERMILPDYri VR256:$src1, imm:$imm)>; 8123def : Pat<(v8i32 (X86VPermilpi (bc_v8i32 (loadv4i64 addr:$src1)), 8124 (i8 imm:$imm))), 8125 (VPERMILPSYmi addr:$src1, imm:$imm)>; 8126def : Pat<(v4i64 (X86VPermilpi (loadv4i64 addr:$src1), (i8 imm:$imm))), 8127 (VPERMILPDYmi addr:$src1, imm:$imm)>; 8128 8129def : Pat<(v4f32 (X86VPermilpv VR128:$src1, (v4i32 VR128:$src2))), 8130 (VPERMILPSrr VR128:$src1, VR128:$src2)>; 8131def : Pat<(v4f32 (X86VPermilpv VR128:$src1, (bc_v4i32 (loadv2i64 addr:$src2)))), 8132 (VPERMILPSrm VR128:$src1, addr:$src2)>; 8133def : Pat<(v2f64 (X86VPermilpv VR128:$src1, (v2i64 VR128:$src2))), 8134 (VPERMILPDrr VR128:$src1, VR128:$src2)>; 8135def : Pat<(v2f64 (X86VPermilpv VR128:$src1, (loadv2i64 addr:$src2))), 8136 (VPERMILPDrm VR128:$src1, addr:$src2)>; 8137 8138def : Pat<(v2i64 (X86VPermilpi VR128:$src1, (i8 imm:$imm))), 8139 (VPERMILPDri VR128:$src1, imm:$imm)>; 8140def : Pat<(v2i64 (X86VPermilpi (loadv2i64 addr:$src1), (i8 imm:$imm))), 8141 (VPERMILPDmi addr:$src1, imm:$imm)>; 8142} 8143 8144//===----------------------------------------------------------------------===// 8145// VPERM2F128 - Permute Floating-Point Values in 128-bit chunks 8146// 8147let ExeDomain = SSEPackedSingle in { 8148def VPERM2F128rr : AVXAIi8<0x06, MRMSrcReg, (outs VR256:$dst), 8149 (ins VR256:$src1, VR256:$src2, u8imm:$src3), 8150 "vperm2f128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", 8151 [(set VR256:$dst, (v8f32 (X86VPerm2x128 VR256:$src1, VR256:$src2, 8152 (i8 imm:$src3))))]>, VEX_4V, VEX_L, 8153 Sched<[WriteFShuffle]>; 8154def VPERM2F128rm : AVXAIi8<0x06, MRMSrcMem, (outs VR256:$dst), 8155 (ins VR256:$src1, f256mem:$src2, u8imm:$src3), 8156 "vperm2f128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", 8157 [(set VR256:$dst, (X86VPerm2x128 VR256:$src1, (loadv8f32 addr:$src2), 8158 (i8 imm:$src3)))]>, VEX_4V, VEX_L, 8159 Sched<[WriteFShuffleLd, ReadAfterLd]>; 8160} 8161 8162let Predicates = [HasAVX] in { 8163def : Pat<(v4f64 (X86VPerm2x128 VR256:$src1, VR256:$src2, (i8 imm:$imm))), 8164 (VPERM2F128rr VR256:$src1, VR256:$src2, imm:$imm)>; 8165def : Pat<(v4f64 (X86VPerm2x128 VR256:$src1, 8166 (loadv4f64 addr:$src2), (i8 imm:$imm))), 8167 (VPERM2F128rm VR256:$src1, addr:$src2, imm:$imm)>; 8168} 8169 8170let Predicates = [HasAVX1Only] in { 8171def : Pat<(v8i32 (X86VPerm2x128 VR256:$src1, VR256:$src2, (i8 imm:$imm))), 8172 (VPERM2F128rr VR256:$src1, VR256:$src2, imm:$imm)>; 8173def : Pat<(v4i64 (X86VPerm2x128 VR256:$src1, VR256:$src2, (i8 imm:$imm))), 8174 (VPERM2F128rr VR256:$src1, VR256:$src2, imm:$imm)>; 8175def : Pat<(v32i8 (X86VPerm2x128 VR256:$src1, VR256:$src2, (i8 imm:$imm))), 8176 (VPERM2F128rr VR256:$src1, VR256:$src2, imm:$imm)>; 8177def : Pat<(v16i16 (X86VPerm2x128 VR256:$src1, VR256:$src2, (i8 imm:$imm))), 8178 (VPERM2F128rr VR256:$src1, VR256:$src2, imm:$imm)>; 8179 8180def : Pat<(v8i32 (X86VPerm2x128 VR256:$src1, 8181 (bc_v8i32 (loadv4i64 addr:$src2)), (i8 imm:$imm))), 8182 (VPERM2F128rm VR256:$src1, addr:$src2, imm:$imm)>; 8183def : Pat<(v4i64 (X86VPerm2x128 VR256:$src1, 8184 (loadv4i64 addr:$src2), (i8 imm:$imm))), 8185 (VPERM2F128rm VR256:$src1, addr:$src2, imm:$imm)>; 8186def : Pat<(v32i8 (X86VPerm2x128 VR256:$src1, 8187 (bc_v32i8 (loadv4i64 addr:$src2)), (i8 imm:$imm))), 8188 (VPERM2F128rm VR256:$src1, addr:$src2, imm:$imm)>; 8189def : Pat<(v16i16 (X86VPerm2x128 VR256:$src1, 8190 (bc_v16i16 (loadv4i64 addr:$src2)), (i8 imm:$imm))), 8191 (VPERM2F128rm VR256:$src1, addr:$src2, imm:$imm)>; 8192} 8193 8194//===----------------------------------------------------------------------===// 8195// VZERO - Zero YMM registers 8196// 8197let Defs = [YMM0, YMM1, YMM2, YMM3, YMM4, YMM5, YMM6, YMM7, 8198 YMM8, YMM9, YMM10, YMM11, YMM12, YMM13, YMM14, YMM15] in { 8199 // Zero All YMM registers 8200 def VZEROALL : I<0x77, RawFrm, (outs), (ins), "vzeroall", 8201 [(int_x86_avx_vzeroall)]>, PS, VEX, VEX_L, Requires<[HasAVX]>; 8202 8203 // Zero Upper bits of YMM registers 8204 def VZEROUPPER : I<0x77, RawFrm, (outs), (ins), "vzeroupper", 8205 [(int_x86_avx_vzeroupper)]>, PS, VEX, Requires<[HasAVX]>; 8206} 8207 8208//===----------------------------------------------------------------------===// 8209// Half precision conversion instructions 8210//===----------------------------------------------------------------------===// 8211multiclass f16c_ph2ps<RegisterClass RC, X86MemOperand x86memop, Intrinsic Int> { 8212 def rr : I<0x13, MRMSrcReg, (outs RC:$dst), (ins VR128:$src), 8213 "vcvtph2ps\t{$src, $dst|$dst, $src}", 8214 [(set RC:$dst, (Int VR128:$src))]>, 8215 T8PD, VEX, Sched<[WriteCvtF2F]>; 8216 let hasSideEffects = 0, mayLoad = 1 in 8217 def rm : I<0x13, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src), 8218 "vcvtph2ps\t{$src, $dst|$dst, $src}", []>, T8PD, VEX, 8219 Sched<[WriteCvtF2FLd]>; 8220} 8221 8222multiclass f16c_ps2ph<RegisterClass RC, X86MemOperand x86memop, Intrinsic Int> { 8223 def rr : Ii8<0x1D, MRMDestReg, (outs VR128:$dst), 8224 (ins RC:$src1, i32u8imm:$src2), 8225 "vcvtps2ph\t{$src2, $src1, $dst|$dst, $src1, $src2}", 8226 [(set VR128:$dst, (Int RC:$src1, imm:$src2))]>, 8227 TAPD, VEX, Sched<[WriteCvtF2F]>; 8228 let hasSideEffects = 0, mayStore = 1, 8229 SchedRW = [WriteCvtF2FLd, WriteRMW] in 8230 def mr : Ii8<0x1D, MRMDestMem, (outs), 8231 (ins x86memop:$dst, RC:$src1, i32u8imm:$src2), 8232 "vcvtps2ph\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>, 8233 TAPD, VEX; 8234} 8235 8236let Predicates = [HasF16C] in { 8237 defm VCVTPH2PS : f16c_ph2ps<VR128, f64mem, int_x86_vcvtph2ps_128>; 8238 defm VCVTPH2PSY : f16c_ph2ps<VR256, f128mem, int_x86_vcvtph2ps_256>, VEX_L; 8239 defm VCVTPS2PH : f16c_ps2ph<VR128, f64mem, int_x86_vcvtps2ph_128>; 8240 defm VCVTPS2PHY : f16c_ps2ph<VR256, f128mem, int_x86_vcvtps2ph_256>, VEX_L; 8241 8242 // Pattern match vcvtph2ps of a scalar i64 load. 8243 def : Pat<(int_x86_vcvtph2ps_128 (vzmovl_v2i64 addr:$src)), 8244 (VCVTPH2PSrm addr:$src)>; 8245 def : Pat<(int_x86_vcvtph2ps_128 (vzload_v2i64 addr:$src)), 8246 (VCVTPH2PSrm addr:$src)>; 8247 8248 def : Pat<(store (f64 (vector_extract (bc_v2f64 (v8i16 8249 (int_x86_vcvtps2ph_128 VR128:$src1, i32:$src2))), (iPTR 0))), 8250 addr:$dst), 8251 (VCVTPS2PHmr addr:$dst, VR128:$src1, imm:$src2)>; 8252 def : Pat<(store (i64 (vector_extract (bc_v2i64 (v8i16 8253 (int_x86_vcvtps2ph_128 VR128:$src1, i32:$src2))), (iPTR 0))), 8254 addr:$dst), 8255 (VCVTPS2PHmr addr:$dst, VR128:$src1, imm:$src2)>; 8256 def : Pat<(store (v8i16 (int_x86_vcvtps2ph_256 VR256:$src1, i32:$src2)), 8257 addr:$dst), 8258 (VCVTPS2PHYmr addr:$dst, VR256:$src1, imm:$src2)>; 8259} 8260 8261// Patterns for matching conversions from float to half-float and vice versa. 8262let Predicates = [HasF16C] in { 8263 def : Pat<(fp_to_f16 FR32:$src), 8264 (i16 (EXTRACT_SUBREG (VMOVPDI2DIrr (VCVTPS2PHrr 8265 (COPY_TO_REGCLASS FR32:$src, VR128), 0)), sub_16bit))>; 8266 8267 def : Pat<(f16_to_fp GR16:$src), 8268 (f32 (COPY_TO_REGCLASS (VCVTPH2PSrr 8269 (COPY_TO_REGCLASS (MOVSX32rr16 GR16:$src), VR128)), FR32)) >; 8270 8271 def : Pat<(f16_to_fp (i16 (fp_to_f16 FR32:$src))), 8272 (f32 (COPY_TO_REGCLASS (VCVTPH2PSrr 8273 (VCVTPS2PHrr (COPY_TO_REGCLASS FR32:$src, VR128), 0)), FR32)) >; 8274} 8275 8276//===----------------------------------------------------------------------===// 8277// AVX2 Instructions 8278//===----------------------------------------------------------------------===// 8279 8280/// AVX2_binop_rmi - AVX2 binary operator with 8-bit immediate 8281multiclass AVX2_binop_rmi<bits<8> opc, string OpcodeStr, SDNode OpNode, 8282 ValueType OpVT, RegisterClass RC, PatFrag memop_frag, 8283 X86MemOperand x86memop> { 8284 let isCommutable = 1 in 8285 def rri : AVX2AIi8<opc, MRMSrcReg, (outs RC:$dst), 8286 (ins RC:$src1, RC:$src2, u8imm:$src3), 8287 !strconcat(OpcodeStr, 8288 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), 8289 [(set RC:$dst, (OpVT (OpNode RC:$src1, RC:$src2, imm:$src3)))]>, 8290 Sched<[WriteBlend]>, VEX_4V; 8291 def rmi : AVX2AIi8<opc, MRMSrcMem, (outs RC:$dst), 8292 (ins RC:$src1, x86memop:$src2, u8imm:$src3), 8293 !strconcat(OpcodeStr, 8294 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), 8295 [(set RC:$dst, 8296 (OpVT (OpNode RC:$src1, 8297 (bitconvert (memop_frag addr:$src2)), imm:$src3)))]>, 8298 Sched<[WriteBlendLd, ReadAfterLd]>, VEX_4V; 8299} 8300 8301defm VPBLENDD : AVX2_binop_rmi<0x02, "vpblendd", X86Blendi, v4i32, 8302 VR128, loadv2i64, i128mem>; 8303defm VPBLENDDY : AVX2_binop_rmi<0x02, "vpblendd", X86Blendi, v8i32, 8304 VR256, loadv4i64, i256mem>, VEX_L; 8305 8306//===----------------------------------------------------------------------===// 8307// VPBROADCAST - Load from memory and broadcast to all elements of the 8308// destination operand 8309// 8310multiclass avx2_broadcast<bits<8> opc, string OpcodeStr, 8311 X86MemOperand x86memop, PatFrag ld_frag, 8312 Intrinsic Int128, Intrinsic Int256> { 8313 def rr : AVX28I<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 8314 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 8315 [(set VR128:$dst, (Int128 VR128:$src))]>, 8316 Sched<[WriteShuffle]>, VEX; 8317 def rm : AVX28I<opc, MRMSrcMem, (outs VR128:$dst), (ins x86memop:$src), 8318 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 8319 [(set VR128:$dst, 8320 (Int128 (scalar_to_vector (ld_frag addr:$src))))]>, 8321 Sched<[WriteLoad]>, VEX; 8322 def Yrr : AVX28I<opc, MRMSrcReg, (outs VR256:$dst), (ins VR128:$src), 8323 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 8324 [(set VR256:$dst, (Int256 VR128:$src))]>, 8325 Sched<[WriteShuffle256]>, VEX, VEX_L; 8326 def Yrm : AVX28I<opc, MRMSrcMem, (outs VR256:$dst), (ins x86memop:$src), 8327 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 8328 [(set VR256:$dst, 8329 (Int256 (scalar_to_vector (ld_frag addr:$src))))]>, 8330 Sched<[WriteLoad]>, VEX, VEX_L; 8331} 8332 8333defm VPBROADCASTB : avx2_broadcast<0x78, "vpbroadcastb", i8mem, loadi8, 8334 int_x86_avx2_pbroadcastb_128, 8335 int_x86_avx2_pbroadcastb_256>; 8336defm VPBROADCASTW : avx2_broadcast<0x79, "vpbroadcastw", i16mem, loadi16, 8337 int_x86_avx2_pbroadcastw_128, 8338 int_x86_avx2_pbroadcastw_256>; 8339defm VPBROADCASTD : avx2_broadcast<0x58, "vpbroadcastd", i32mem, loadi32, 8340 int_x86_avx2_pbroadcastd_128, 8341 int_x86_avx2_pbroadcastd_256>; 8342defm VPBROADCASTQ : avx2_broadcast<0x59, "vpbroadcastq", i64mem, loadi64, 8343 int_x86_avx2_pbroadcastq_128, 8344 int_x86_avx2_pbroadcastq_256>; 8345 8346let Predicates = [HasAVX2] in { 8347 def : Pat<(v16i8 (X86VBroadcast (loadi8 addr:$src))), 8348 (VPBROADCASTBrm addr:$src)>; 8349 def : Pat<(v32i8 (X86VBroadcast (loadi8 addr:$src))), 8350 (VPBROADCASTBYrm addr:$src)>; 8351 def : Pat<(v8i16 (X86VBroadcast (loadi16 addr:$src))), 8352 (VPBROADCASTWrm addr:$src)>; 8353 def : Pat<(v16i16 (X86VBroadcast (loadi16 addr:$src))), 8354 (VPBROADCASTWYrm addr:$src)>; 8355 def : Pat<(v4i32 (X86VBroadcast (loadi32 addr:$src))), 8356 (VPBROADCASTDrm addr:$src)>; 8357 def : Pat<(v8i32 (X86VBroadcast (loadi32 addr:$src))), 8358 (VPBROADCASTDYrm addr:$src)>; 8359 def : Pat<(v2i64 (X86VBroadcast (loadi64 addr:$src))), 8360 (VPBROADCASTQrm addr:$src)>; 8361 def : Pat<(v4i64 (X86VBroadcast (loadi64 addr:$src))), 8362 (VPBROADCASTQYrm addr:$src)>; 8363 8364 def : Pat<(v16i8 (X86VBroadcast (v16i8 VR128:$src))), 8365 (VPBROADCASTBrr VR128:$src)>; 8366 def : Pat<(v32i8 (X86VBroadcast (v16i8 VR128:$src))), 8367 (VPBROADCASTBYrr VR128:$src)>; 8368 def : Pat<(v8i16 (X86VBroadcast (v8i16 VR128:$src))), 8369 (VPBROADCASTWrr VR128:$src)>; 8370 def : Pat<(v16i16 (X86VBroadcast (v8i16 VR128:$src))), 8371 (VPBROADCASTWYrr VR128:$src)>; 8372 def : Pat<(v4i32 (X86VBroadcast (v4i32 VR128:$src))), 8373 (VPBROADCASTDrr VR128:$src)>; 8374 def : Pat<(v8i32 (X86VBroadcast (v4i32 VR128:$src))), 8375 (VPBROADCASTDYrr VR128:$src)>; 8376 def : Pat<(v2i64 (X86VBroadcast (v2i64 VR128:$src))), 8377 (VPBROADCASTQrr VR128:$src)>; 8378 def : Pat<(v4i64 (X86VBroadcast (v2i64 VR128:$src))), 8379 (VPBROADCASTQYrr VR128:$src)>; 8380 def : Pat<(v4f32 (X86VBroadcast (v4f32 VR128:$src))), 8381 (VBROADCASTSSrr VR128:$src)>; 8382 def : Pat<(v8f32 (X86VBroadcast (v4f32 VR128:$src))), 8383 (VBROADCASTSSYrr VR128:$src)>; 8384 def : Pat<(v2f64 (X86VBroadcast (v2f64 VR128:$src))), 8385 (VPBROADCASTQrr VR128:$src)>; 8386 def : Pat<(v4f64 (X86VBroadcast (v2f64 VR128:$src))), 8387 (VBROADCASTSDYrr VR128:$src)>; 8388 8389 // Provide aliases for broadcast from the same register class that 8390 // automatically does the extract. 8391 def : Pat<(v32i8 (X86VBroadcast (v32i8 VR256:$src))), 8392 (VPBROADCASTBYrr (v16i8 (EXTRACT_SUBREG (v32i8 VR256:$src), 8393 sub_xmm)))>; 8394 def : Pat<(v16i16 (X86VBroadcast (v16i16 VR256:$src))), 8395 (VPBROADCASTWYrr (v8i16 (EXTRACT_SUBREG (v16i16 VR256:$src), 8396 sub_xmm)))>; 8397 def : Pat<(v8i32 (X86VBroadcast (v8i32 VR256:$src))), 8398 (VPBROADCASTDYrr (v4i32 (EXTRACT_SUBREG (v8i32 VR256:$src), 8399 sub_xmm)))>; 8400 def : Pat<(v4i64 (X86VBroadcast (v4i64 VR256:$src))), 8401 (VPBROADCASTQYrr (v2i64 (EXTRACT_SUBREG (v4i64 VR256:$src), 8402 sub_xmm)))>; 8403 def : Pat<(v8f32 (X86VBroadcast (v8f32 VR256:$src))), 8404 (VBROADCASTSSYrr (v4f32 (EXTRACT_SUBREG (v8f32 VR256:$src), 8405 sub_xmm)))>; 8406 def : Pat<(v4f64 (X86VBroadcast (v4f64 VR256:$src))), 8407 (VBROADCASTSDYrr (v2f64 (EXTRACT_SUBREG (v4f64 VR256:$src), 8408 sub_xmm)))>; 8409 8410 // Provide fallback in case the load node that is used in the patterns above 8411 // is used by additional users, which prevents the pattern selection. 8412 let AddedComplexity = 20 in { 8413 def : Pat<(v4f32 (X86VBroadcast FR32:$src)), 8414 (VBROADCASTSSrr (COPY_TO_REGCLASS FR32:$src, VR128))>; 8415 def : Pat<(v8f32 (X86VBroadcast FR32:$src)), 8416 (VBROADCASTSSYrr (COPY_TO_REGCLASS FR32:$src, VR128))>; 8417 def : Pat<(v4f64 (X86VBroadcast FR64:$src)), 8418 (VBROADCASTSDYrr (COPY_TO_REGCLASS FR64:$src, VR128))>; 8419 8420 def : Pat<(v4i32 (X86VBroadcast GR32:$src)), 8421 (VBROADCASTSSrr (COPY_TO_REGCLASS GR32:$src, VR128))>; 8422 def : Pat<(v8i32 (X86VBroadcast GR32:$src)), 8423 (VBROADCASTSSYrr (COPY_TO_REGCLASS GR32:$src, VR128))>; 8424 def : Pat<(v4i64 (X86VBroadcast GR64:$src)), 8425 (VBROADCASTSDYrr (COPY_TO_REGCLASS GR64:$src, VR128))>; 8426 8427 def : Pat<(v16i8 (X86VBroadcast GR8:$src)), 8428 (VPBROADCASTBrr (COPY_TO_REGCLASS 8429 (i32 (SUBREG_TO_REG (i32 0), GR8:$src, sub_8bit)), 8430 VR128))>; 8431 def : Pat<(v32i8 (X86VBroadcast GR8:$src)), 8432 (VPBROADCASTBYrr (COPY_TO_REGCLASS 8433 (i32 (SUBREG_TO_REG (i32 0), GR8:$src, sub_8bit)), 8434 VR128))>; 8435 8436 def : Pat<(v8i16 (X86VBroadcast GR16:$src)), 8437 (VPBROADCASTWrr (COPY_TO_REGCLASS 8438 (i32 (SUBREG_TO_REG (i32 0), GR16:$src, sub_16bit)), 8439 VR128))>; 8440 def : Pat<(v16i16 (X86VBroadcast GR16:$src)), 8441 (VPBROADCASTWYrr (COPY_TO_REGCLASS 8442 (i32 (SUBREG_TO_REG (i32 0), GR16:$src, sub_16bit)), 8443 VR128))>; 8444 8445 // The patterns for VPBROADCASTD are not needed because they would match 8446 // the exact same thing as VBROADCASTSS patterns. 8447 8448 def : Pat<(v2i64 (X86VBroadcast GR64:$src)), 8449 (VPBROADCASTQrr (COPY_TO_REGCLASS GR64:$src, VR128))>; 8450 // The v4i64 pattern is not needed because VBROADCASTSDYrr already match. 8451 } 8452} 8453 8454// AVX1 broadcast patterns 8455let Predicates = [HasAVX1Only] in { 8456def : Pat<(v8i32 (X86VBroadcast (loadi32 addr:$src))), 8457 (VBROADCASTSSYrm addr:$src)>; 8458def : Pat<(v4i64 (X86VBroadcast (loadi64 addr:$src))), 8459 (VBROADCASTSDYrm addr:$src)>; 8460def : Pat<(v4i32 (X86VBroadcast (loadi32 addr:$src))), 8461 (VBROADCASTSSrm addr:$src)>; 8462} 8463 8464let Predicates = [HasAVX] in { 8465 // Provide fallback in case the load node that is used in the patterns above 8466 // is used by additional users, which prevents the pattern selection. 8467 let AddedComplexity = 20 in { 8468 // 128bit broadcasts: 8469 def : Pat<(v4f32 (X86VBroadcast FR32:$src)), 8470 (VPSHUFDri (COPY_TO_REGCLASS FR32:$src, VR128), 0)>; 8471 def : Pat<(v8f32 (X86VBroadcast FR32:$src)), 8472 (VINSERTF128rr (INSERT_SUBREG (v8f32 (IMPLICIT_DEF)), 8473 (VPSHUFDri (COPY_TO_REGCLASS FR32:$src, VR128), 0), sub_xmm), 8474 (VPSHUFDri (COPY_TO_REGCLASS FR32:$src, VR128), 0), 1)>; 8475 def : Pat<(v4f64 (X86VBroadcast FR64:$src)), 8476 (VINSERTF128rr (INSERT_SUBREG (v4f64 (IMPLICIT_DEF)), 8477 (VPSHUFDri (COPY_TO_REGCLASS FR64:$src, VR128), 0x44), sub_xmm), 8478 (VPSHUFDri (COPY_TO_REGCLASS FR64:$src, VR128), 0x44), 1)>; 8479 8480 def : Pat<(v4i32 (X86VBroadcast GR32:$src)), 8481 (VPSHUFDri (COPY_TO_REGCLASS GR32:$src, VR128), 0)>; 8482 def : Pat<(v8i32 (X86VBroadcast GR32:$src)), 8483 (VINSERTF128rr (INSERT_SUBREG (v8i32 (IMPLICIT_DEF)), 8484 (VPSHUFDri (COPY_TO_REGCLASS GR32:$src, VR128), 0), sub_xmm), 8485 (VPSHUFDri (COPY_TO_REGCLASS GR32:$src, VR128), 0), 1)>; 8486 def : Pat<(v4i64 (X86VBroadcast GR64:$src)), 8487 (VINSERTF128rr (INSERT_SUBREG (v4i64 (IMPLICIT_DEF)), 8488 (VPSHUFDri (COPY_TO_REGCLASS GR64:$src, VR128), 0x44), sub_xmm), 8489 (VPSHUFDri (COPY_TO_REGCLASS GR64:$src, VR128), 0x44), 1)>; 8490 } 8491 8492 def : Pat<(v2f64 (X86VBroadcast f64:$src)), 8493 (VMOVDDUPrr (COPY_TO_REGCLASS FR64:$src, VR128))>; 8494 def : Pat<(v2i64 (X86VBroadcast i64:$src)), 8495 (VMOVDDUPrr (COPY_TO_REGCLASS GR64:$src, VR128))>; 8496} 8497 8498//===----------------------------------------------------------------------===// 8499// VPERM - Permute instructions 8500// 8501 8502multiclass avx2_perm<bits<8> opc, string OpcodeStr, PatFrag mem_frag, 8503 ValueType OpVT, X86FoldableSchedWrite Sched> { 8504 def Yrr : AVX28I<opc, MRMSrcReg, (outs VR256:$dst), 8505 (ins VR256:$src1, VR256:$src2), 8506 !strconcat(OpcodeStr, 8507 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 8508 [(set VR256:$dst, 8509 (OpVT (X86VPermv VR256:$src1, VR256:$src2)))]>, 8510 Sched<[Sched]>, VEX_4V, VEX_L; 8511 def Yrm : AVX28I<opc, MRMSrcMem, (outs VR256:$dst), 8512 (ins VR256:$src1, i256mem:$src2), 8513 !strconcat(OpcodeStr, 8514 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 8515 [(set VR256:$dst, 8516 (OpVT (X86VPermv VR256:$src1, 8517 (bitconvert (mem_frag addr:$src2)))))]>, 8518 Sched<[Sched.Folded, ReadAfterLd]>, VEX_4V, VEX_L; 8519} 8520 8521defm VPERMD : avx2_perm<0x36, "vpermd", loadv4i64, v8i32, WriteShuffle256>; 8522let ExeDomain = SSEPackedSingle in 8523defm VPERMPS : avx2_perm<0x16, "vpermps", loadv8f32, v8f32, WriteFShuffle256>; 8524 8525multiclass avx2_perm_imm<bits<8> opc, string OpcodeStr, PatFrag mem_frag, 8526 ValueType OpVT, X86FoldableSchedWrite Sched> { 8527 def Yri : AVX2AIi8<opc, MRMSrcReg, (outs VR256:$dst), 8528 (ins VR256:$src1, u8imm:$src2), 8529 !strconcat(OpcodeStr, 8530 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 8531 [(set VR256:$dst, 8532 (OpVT (X86VPermi VR256:$src1, (i8 imm:$src2))))]>, 8533 Sched<[Sched]>, VEX, VEX_L; 8534 def Ymi : AVX2AIi8<opc, MRMSrcMem, (outs VR256:$dst), 8535 (ins i256mem:$src1, u8imm:$src2), 8536 !strconcat(OpcodeStr, 8537 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 8538 [(set VR256:$dst, 8539 (OpVT (X86VPermi (mem_frag addr:$src1), 8540 (i8 imm:$src2))))]>, 8541 Sched<[Sched.Folded, ReadAfterLd]>, VEX, VEX_L; 8542} 8543 8544defm VPERMQ : avx2_perm_imm<0x00, "vpermq", loadv4i64, v4i64, 8545 WriteShuffle256>, VEX_W; 8546let ExeDomain = SSEPackedDouble in 8547defm VPERMPD : avx2_perm_imm<0x01, "vpermpd", loadv4f64, v4f64, 8548 WriteFShuffle256>, VEX_W; 8549 8550//===----------------------------------------------------------------------===// 8551// VPERM2I128 - Permute Floating-Point Values in 128-bit chunks 8552// 8553def VPERM2I128rr : AVX2AIi8<0x46, MRMSrcReg, (outs VR256:$dst), 8554 (ins VR256:$src1, VR256:$src2, u8imm:$src3), 8555 "vperm2i128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", 8556 [(set VR256:$dst, (v4i64 (X86VPerm2x128 VR256:$src1, VR256:$src2, 8557 (i8 imm:$src3))))]>, Sched<[WriteShuffle256]>, 8558 VEX_4V, VEX_L; 8559def VPERM2I128rm : AVX2AIi8<0x46, MRMSrcMem, (outs VR256:$dst), 8560 (ins VR256:$src1, f256mem:$src2, u8imm:$src3), 8561 "vperm2i128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", 8562 [(set VR256:$dst, (X86VPerm2x128 VR256:$src1, (loadv4i64 addr:$src2), 8563 (i8 imm:$src3)))]>, 8564 Sched<[WriteShuffle256Ld, ReadAfterLd]>, VEX_4V, VEX_L; 8565 8566let Predicates = [HasAVX2] in { 8567def : Pat<(v8i32 (X86VPerm2x128 VR256:$src1, VR256:$src2, (i8 imm:$imm))), 8568 (VPERM2I128rr VR256:$src1, VR256:$src2, imm:$imm)>; 8569def : Pat<(v32i8 (X86VPerm2x128 VR256:$src1, VR256:$src2, (i8 imm:$imm))), 8570 (VPERM2I128rr VR256:$src1, VR256:$src2, imm:$imm)>; 8571def : Pat<(v16i16 (X86VPerm2x128 VR256:$src1, VR256:$src2, (i8 imm:$imm))), 8572 (VPERM2I128rr VR256:$src1, VR256:$src2, imm:$imm)>; 8573 8574def : Pat<(v32i8 (X86VPerm2x128 VR256:$src1, (bc_v32i8 (loadv4i64 addr:$src2)), 8575 (i8 imm:$imm))), 8576 (VPERM2I128rm VR256:$src1, addr:$src2, imm:$imm)>; 8577def : Pat<(v16i16 (X86VPerm2x128 VR256:$src1, 8578 (bc_v16i16 (loadv4i64 addr:$src2)), (i8 imm:$imm))), 8579 (VPERM2I128rm VR256:$src1, addr:$src2, imm:$imm)>; 8580def : Pat<(v8i32 (X86VPerm2x128 VR256:$src1, (bc_v8i32 (loadv4i64 addr:$src2)), 8581 (i8 imm:$imm))), 8582 (VPERM2I128rm VR256:$src1, addr:$src2, imm:$imm)>; 8583} 8584 8585 8586//===----------------------------------------------------------------------===// 8587// VINSERTI128 - Insert packed integer values 8588// 8589let hasSideEffects = 0 in { 8590def VINSERTI128rr : AVX2AIi8<0x38, MRMSrcReg, (outs VR256:$dst), 8591 (ins VR256:$src1, VR128:$src2, u8imm:$src3), 8592 "vinserti128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", 8593 []>, Sched<[WriteShuffle256]>, VEX_4V, VEX_L; 8594let mayLoad = 1 in 8595def VINSERTI128rm : AVX2AIi8<0x38, MRMSrcMem, (outs VR256:$dst), 8596 (ins VR256:$src1, i128mem:$src2, u8imm:$src3), 8597 "vinserti128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", 8598 []>, Sched<[WriteShuffle256Ld, ReadAfterLd]>, VEX_4V, VEX_L; 8599} 8600 8601let Predicates = [HasAVX2] in { 8602def : Pat<(vinsert128_insert:$ins (v4i64 VR256:$src1), (v2i64 VR128:$src2), 8603 (iPTR imm)), 8604 (VINSERTI128rr VR256:$src1, VR128:$src2, 8605 (INSERT_get_vinsert128_imm VR256:$ins))>; 8606def : Pat<(vinsert128_insert:$ins (v8i32 VR256:$src1), (v4i32 VR128:$src2), 8607 (iPTR imm)), 8608 (VINSERTI128rr VR256:$src1, VR128:$src2, 8609 (INSERT_get_vinsert128_imm VR256:$ins))>; 8610def : Pat<(vinsert128_insert:$ins (v32i8 VR256:$src1), (v16i8 VR128:$src2), 8611 (iPTR imm)), 8612 (VINSERTI128rr VR256:$src1, VR128:$src2, 8613 (INSERT_get_vinsert128_imm VR256:$ins))>; 8614def : Pat<(vinsert128_insert:$ins (v16i16 VR256:$src1), (v8i16 VR128:$src2), 8615 (iPTR imm)), 8616 (VINSERTI128rr VR256:$src1, VR128:$src2, 8617 (INSERT_get_vinsert128_imm VR256:$ins))>; 8618 8619def : Pat<(vinsert128_insert:$ins (v4i64 VR256:$src1), (loadv2i64 addr:$src2), 8620 (iPTR imm)), 8621 (VINSERTI128rm VR256:$src1, addr:$src2, 8622 (INSERT_get_vinsert128_imm VR256:$ins))>; 8623def : Pat<(vinsert128_insert:$ins (v8i32 VR256:$src1), 8624 (bc_v4i32 (loadv2i64 addr:$src2)), 8625 (iPTR imm)), 8626 (VINSERTI128rm VR256:$src1, addr:$src2, 8627 (INSERT_get_vinsert128_imm VR256:$ins))>; 8628def : Pat<(vinsert128_insert:$ins (v32i8 VR256:$src1), 8629 (bc_v16i8 (loadv2i64 addr:$src2)), 8630 (iPTR imm)), 8631 (VINSERTI128rm VR256:$src1, addr:$src2, 8632 (INSERT_get_vinsert128_imm VR256:$ins))>; 8633def : Pat<(vinsert128_insert:$ins (v16i16 VR256:$src1), 8634 (bc_v8i16 (loadv2i64 addr:$src2)), 8635 (iPTR imm)), 8636 (VINSERTI128rm VR256:$src1, addr:$src2, 8637 (INSERT_get_vinsert128_imm VR256:$ins))>; 8638} 8639 8640//===----------------------------------------------------------------------===// 8641// VEXTRACTI128 - Extract packed integer values 8642// 8643def VEXTRACTI128rr : AVX2AIi8<0x39, MRMDestReg, (outs VR128:$dst), 8644 (ins VR256:$src1, u8imm:$src2), 8645 "vextracti128\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>, 8646 Sched<[WriteShuffle256]>, VEX, VEX_L; 8647let hasSideEffects = 0, mayStore = 1 in 8648def VEXTRACTI128mr : AVX2AIi8<0x39, MRMDestMem, (outs), 8649 (ins i128mem:$dst, VR256:$src1, u8imm:$src2), 8650 "vextracti128\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>, 8651 Sched<[WriteStore]>, VEX, VEX_L; 8652 8653let Predicates = [HasAVX2] in { 8654def : Pat<(vextract128_extract:$ext VR256:$src1, (iPTR imm)), 8655 (v2i64 (VEXTRACTI128rr 8656 (v4i64 VR256:$src1), 8657 (EXTRACT_get_vextract128_imm VR128:$ext)))>; 8658def : Pat<(vextract128_extract:$ext VR256:$src1, (iPTR imm)), 8659 (v4i32 (VEXTRACTI128rr 8660 (v8i32 VR256:$src1), 8661 (EXTRACT_get_vextract128_imm VR128:$ext)))>; 8662def : Pat<(vextract128_extract:$ext VR256:$src1, (iPTR imm)), 8663 (v8i16 (VEXTRACTI128rr 8664 (v16i16 VR256:$src1), 8665 (EXTRACT_get_vextract128_imm VR128:$ext)))>; 8666def : Pat<(vextract128_extract:$ext VR256:$src1, (iPTR imm)), 8667 (v16i8 (VEXTRACTI128rr 8668 (v32i8 VR256:$src1), 8669 (EXTRACT_get_vextract128_imm VR128:$ext)))>; 8670 8671def : Pat<(store (v2i64 (vextract128_extract:$ext (v4i64 VR256:$src1), 8672 (iPTR imm))), addr:$dst), 8673 (VEXTRACTI128mr addr:$dst, VR256:$src1, 8674 (EXTRACT_get_vextract128_imm VR128:$ext))>; 8675def : Pat<(store (v4i32 (vextract128_extract:$ext (v8i32 VR256:$src1), 8676 (iPTR imm))), addr:$dst), 8677 (VEXTRACTI128mr addr:$dst, VR256:$src1, 8678 (EXTRACT_get_vextract128_imm VR128:$ext))>; 8679def : Pat<(store (v8i16 (vextract128_extract:$ext (v16i16 VR256:$src1), 8680 (iPTR imm))), addr:$dst), 8681 (VEXTRACTI128mr addr:$dst, VR256:$src1, 8682 (EXTRACT_get_vextract128_imm VR128:$ext))>; 8683def : Pat<(store (v16i8 (vextract128_extract:$ext (v32i8 VR256:$src1), 8684 (iPTR imm))), addr:$dst), 8685 (VEXTRACTI128mr addr:$dst, VR256:$src1, 8686 (EXTRACT_get_vextract128_imm VR128:$ext))>; 8687} 8688 8689//===----------------------------------------------------------------------===// 8690// VPMASKMOV - Conditional SIMD Integer Packed Loads and Stores 8691// 8692multiclass avx2_pmovmask<string OpcodeStr, 8693 Intrinsic IntLd128, Intrinsic IntLd256, 8694 Intrinsic IntSt128, Intrinsic IntSt256> { 8695 def rm : AVX28I<0x8c, MRMSrcMem, (outs VR128:$dst), 8696 (ins VR128:$src1, i128mem:$src2), 8697 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 8698 [(set VR128:$dst, (IntLd128 addr:$src2, VR128:$src1))]>, VEX_4V; 8699 def Yrm : AVX28I<0x8c, MRMSrcMem, (outs VR256:$dst), 8700 (ins VR256:$src1, i256mem:$src2), 8701 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 8702 [(set VR256:$dst, (IntLd256 addr:$src2, VR256:$src1))]>, 8703 VEX_4V, VEX_L; 8704 def mr : AVX28I<0x8e, MRMDestMem, (outs), 8705 (ins i128mem:$dst, VR128:$src1, VR128:$src2), 8706 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 8707 [(IntSt128 addr:$dst, VR128:$src1, VR128:$src2)]>, VEX_4V; 8708 def Ymr : AVX28I<0x8e, MRMDestMem, (outs), 8709 (ins i256mem:$dst, VR256:$src1, VR256:$src2), 8710 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 8711 [(IntSt256 addr:$dst, VR256:$src1, VR256:$src2)]>, VEX_4V, VEX_L; 8712} 8713 8714defm VPMASKMOVD : avx2_pmovmask<"vpmaskmovd", 8715 int_x86_avx2_maskload_d, 8716 int_x86_avx2_maskload_d_256, 8717 int_x86_avx2_maskstore_d, 8718 int_x86_avx2_maskstore_d_256>; 8719defm VPMASKMOVQ : avx2_pmovmask<"vpmaskmovq", 8720 int_x86_avx2_maskload_q, 8721 int_x86_avx2_maskload_q_256, 8722 int_x86_avx2_maskstore_q, 8723 int_x86_avx2_maskstore_q_256>, VEX_W; 8724 8725def: Pat<(masked_store addr:$ptr, (v8i32 VR256:$mask), (v8f32 VR256:$src)), 8726 (VMASKMOVPSYmr addr:$ptr, VR256:$mask, VR256:$src)>; 8727 8728def: Pat<(masked_store addr:$ptr, (v8i32 VR256:$mask), (v8i32 VR256:$src)), 8729 (VPMASKMOVDYmr addr:$ptr, VR256:$mask, VR256:$src)>; 8730 8731def: Pat<(masked_store addr:$ptr, (v4i32 VR128:$mask), (v4f32 VR128:$src)), 8732 (VMASKMOVPSmr addr:$ptr, VR128:$mask, VR128:$src)>; 8733 8734def: Pat<(masked_store addr:$ptr, (v4i32 VR128:$mask), (v4i32 VR128:$src)), 8735 (VPMASKMOVDmr addr:$ptr, VR128:$mask, VR128:$src)>; 8736 8737def: Pat<(v8f32 (masked_load addr:$ptr, (v8i32 VR256:$mask), undef)), 8738 (VMASKMOVPSYrm VR256:$mask, addr:$ptr)>; 8739 8740def: Pat<(v8f32 (masked_load addr:$ptr, (v8i32 VR256:$mask), 8741 (bc_v8f32 (v8i32 immAllZerosV)))), 8742 (VMASKMOVPSYrm VR256:$mask, addr:$ptr)>; 8743 8744def: Pat<(v8f32 (masked_load addr:$ptr, (v8i32 VR256:$mask), (v8f32 VR256:$src0))), 8745 (VBLENDVPSYrr VR256:$src0, (VMASKMOVPSYrm VR256:$mask, addr:$ptr), 8746 VR256:$mask)>; 8747 8748def: Pat<(v8i32 (masked_load addr:$ptr, (v8i32 VR256:$mask), undef)), 8749 (VPMASKMOVDYrm VR256:$mask, addr:$ptr)>; 8750 8751def: Pat<(v8i32 (masked_load addr:$ptr, (v8i32 VR256:$mask), (v8i32 immAllZerosV))), 8752 (VPMASKMOVDYrm VR256:$mask, addr:$ptr)>; 8753 8754def: Pat<(v8i32 (masked_load addr:$ptr, (v8i32 VR256:$mask), (v8i32 VR256:$src0))), 8755 (VBLENDVPSYrr VR256:$src0, (VPMASKMOVDYrm VR256:$mask, addr:$ptr), 8756 VR256:$mask)>; 8757 8758def: Pat<(v4f32 (masked_load addr:$ptr, (v4i32 VR128:$mask), undef)), 8759 (VMASKMOVPSrm VR128:$mask, addr:$ptr)>; 8760 8761def: Pat<(v4f32 (masked_load addr:$ptr, (v4i32 VR128:$mask), 8762 (bc_v4f32 (v4i32 immAllZerosV)))), 8763 (VMASKMOVPSrm VR128:$mask, addr:$ptr)>; 8764 8765def: Pat<(v4f32 (masked_load addr:$ptr, (v4i32 VR128:$mask), (v4f32 VR128:$src0))), 8766 (VBLENDVPSrr VR128:$src0, (VMASKMOVPSrm VR128:$mask, addr:$ptr), 8767 VR128:$mask)>; 8768 8769def: Pat<(v4i32 (masked_load addr:$ptr, (v4i32 VR128:$mask), undef)), 8770 (VPMASKMOVDrm VR128:$mask, addr:$ptr)>; 8771 8772def: Pat<(v4i32 (masked_load addr:$ptr, (v4i32 VR128:$mask), (v4i32 immAllZerosV))), 8773 (VPMASKMOVDrm VR128:$mask, addr:$ptr)>; 8774 8775def: Pat<(v4i32 (masked_load addr:$ptr, (v4i32 VR128:$mask), (v4i32 VR128:$src0))), 8776 (VBLENDVPSrr VR128:$src0, (VPMASKMOVDrm VR128:$mask, addr:$ptr), 8777 VR128:$mask)>; 8778 8779def: Pat<(masked_store addr:$ptr, (v4i64 VR256:$mask), (v4f64 VR256:$src)), 8780 (VMASKMOVPDYmr addr:$ptr, VR256:$mask, VR256:$src)>; 8781 8782def: Pat<(masked_store addr:$ptr, (v4i64 VR256:$mask), (v4i64 VR256:$src)), 8783 (VPMASKMOVQYmr addr:$ptr, VR256:$mask, VR256:$src)>; 8784 8785def: Pat<(v4f64 (masked_load addr:$ptr, (v4i64 VR256:$mask), undef)), 8786 (VMASKMOVPDYrm VR256:$mask, addr:$ptr)>; 8787 8788def: Pat<(v4f64 (masked_load addr:$ptr, (v4i64 VR256:$mask), 8789 (v4f64 immAllZerosV))), 8790 (VMASKMOVPDYrm VR256:$mask, addr:$ptr)>; 8791 8792def: Pat<(v4f64 (masked_load addr:$ptr, (v4i64 VR256:$mask), (v4f64 VR256:$src0))), 8793 (VBLENDVPDYrr VR256:$src0, (VMASKMOVPDYrm VR256:$mask, addr:$ptr), 8794 VR256:$mask)>; 8795 8796def: Pat<(v4i64 (masked_load addr:$ptr, (v4i64 VR256:$mask), undef)), 8797 (VPMASKMOVQYrm VR256:$mask, addr:$ptr)>; 8798 8799def: Pat<(v4i64 (masked_load addr:$ptr, (v4i64 VR256:$mask), 8800 (bc_v4i64 (v8i32 immAllZerosV)))), 8801 (VPMASKMOVQYrm VR256:$mask, addr:$ptr)>; 8802 8803def: Pat<(v4i64 (masked_load addr:$ptr, (v4i64 VR256:$mask), (v4i64 VR256:$src0))), 8804 (VBLENDVPDYrr VR256:$src0, (VPMASKMOVQYrm VR256:$mask, addr:$ptr), 8805 VR256:$mask)>; 8806 8807def: Pat<(masked_store addr:$ptr, (v2i64 VR128:$mask), (v2f64 VR128:$src)), 8808 (VMASKMOVPDmr addr:$ptr, VR128:$mask, VR128:$src)>; 8809 8810def: Pat<(masked_store addr:$ptr, (v2i64 VR128:$mask), (v2i64 VR128:$src)), 8811 (VPMASKMOVQmr addr:$ptr, VR128:$mask, VR128:$src)>; 8812 8813def: Pat<(v2f64 (masked_load addr:$ptr, (v2i64 VR128:$mask), undef)), 8814 (VMASKMOVPDrm VR128:$mask, addr:$ptr)>; 8815 8816def: Pat<(v2f64 (masked_load addr:$ptr, (v2i64 VR128:$mask), 8817 (v2f64 immAllZerosV))), 8818 (VMASKMOVPDrm VR128:$mask, addr:$ptr)>; 8819 8820def: Pat<(v2f64 (masked_load addr:$ptr, (v2i64 VR128:$mask), (v2f64 VR128:$src0))), 8821 (VBLENDVPDrr VR128:$src0, (VMASKMOVPDrm VR128:$mask, addr:$ptr), 8822 VR128:$mask)>; 8823 8824def: Pat<(v2i64 (masked_load addr:$ptr, (v2i64 VR128:$mask), undef)), 8825 (VPMASKMOVQrm VR128:$mask, addr:$ptr)>; 8826 8827def: Pat<(v2i64 (masked_load addr:$ptr, (v2i64 VR128:$mask), 8828 (bc_v2i64 (v4i32 immAllZerosV)))), 8829 (VPMASKMOVQrm VR128:$mask, addr:$ptr)>; 8830 8831def: Pat<(v2i64 (masked_load addr:$ptr, (v2i64 VR128:$mask), (v2i64 VR128:$src0))), 8832 (VBLENDVPDrr VR128:$src0, (VPMASKMOVQrm VR128:$mask, addr:$ptr), 8833 VR128:$mask)>; 8834 8835//===----------------------------------------------------------------------===// 8836// Variable Bit Shifts 8837// 8838multiclass avx2_var_shift<bits<8> opc, string OpcodeStr, SDNode OpNode, 8839 ValueType vt128, ValueType vt256> { 8840 def rr : AVX28I<opc, MRMSrcReg, (outs VR128:$dst), 8841 (ins VR128:$src1, VR128:$src2), 8842 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 8843 [(set VR128:$dst, 8844 (vt128 (OpNode VR128:$src1, (vt128 VR128:$src2))))]>, 8845 VEX_4V, Sched<[WriteVarVecShift]>; 8846 def rm : AVX28I<opc, MRMSrcMem, (outs VR128:$dst), 8847 (ins VR128:$src1, i128mem:$src2), 8848 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 8849 [(set VR128:$dst, 8850 (vt128 (OpNode VR128:$src1, 8851 (vt128 (bitconvert (loadv2i64 addr:$src2))))))]>, 8852 VEX_4V, Sched<[WriteVarVecShiftLd, ReadAfterLd]>; 8853 def Yrr : AVX28I<opc, MRMSrcReg, (outs VR256:$dst), 8854 (ins VR256:$src1, VR256:$src2), 8855 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 8856 [(set VR256:$dst, 8857 (vt256 (OpNode VR256:$src1, (vt256 VR256:$src2))))]>, 8858 VEX_4V, VEX_L, Sched<[WriteVarVecShift]>; 8859 def Yrm : AVX28I<opc, MRMSrcMem, (outs VR256:$dst), 8860 (ins VR256:$src1, i256mem:$src2), 8861 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 8862 [(set VR256:$dst, 8863 (vt256 (OpNode VR256:$src1, 8864 (vt256 (bitconvert (loadv4i64 addr:$src2))))))]>, 8865 VEX_4V, VEX_L, Sched<[WriteVarVecShiftLd, ReadAfterLd]>; 8866} 8867 8868defm VPSLLVD : avx2_var_shift<0x47, "vpsllvd", shl, v4i32, v8i32>; 8869defm VPSLLVQ : avx2_var_shift<0x47, "vpsllvq", shl, v2i64, v4i64>, VEX_W; 8870defm VPSRLVD : avx2_var_shift<0x45, "vpsrlvd", srl, v4i32, v8i32>; 8871defm VPSRLVQ : avx2_var_shift<0x45, "vpsrlvq", srl, v2i64, v4i64>, VEX_W; 8872defm VPSRAVD : avx2_var_shift<0x46, "vpsravd", sra, v4i32, v8i32>; 8873 8874//===----------------------------------------------------------------------===// 8875// VGATHER - GATHER Operations 8876multiclass avx2_gather<bits<8> opc, string OpcodeStr, RegisterClass RC256, 8877 X86MemOperand memop128, X86MemOperand memop256> { 8878 def rm : AVX28I<opc, MRMSrcMem, (outs VR128:$dst, VR128:$mask_wb), 8879 (ins VR128:$src1, memop128:$src2, VR128:$mask), 8880 !strconcat(OpcodeStr, 8881 "\t{$mask, $src2, $dst|$dst, $src2, $mask}"), 8882 []>, VEX_4VOp3; 8883 def Yrm : AVX28I<opc, MRMSrcMem, (outs RC256:$dst, RC256:$mask_wb), 8884 (ins RC256:$src1, memop256:$src2, RC256:$mask), 8885 !strconcat(OpcodeStr, 8886 "\t{$mask, $src2, $dst|$dst, $src2, $mask}"), 8887 []>, VEX_4VOp3, VEX_L; 8888} 8889 8890let mayLoad = 1, Constraints 8891 = "@earlyclobber $dst,@earlyclobber $mask_wb, $src1 = $dst, $mask = $mask_wb" 8892 in { 8893 defm VPGATHERDQ : avx2_gather<0x90, "vpgatherdq", VR256, vx64mem, vx64mem>, VEX_W; 8894 defm VPGATHERQQ : avx2_gather<0x91, "vpgatherqq", VR256, vx64mem, vy64mem>, VEX_W; 8895 defm VPGATHERDD : avx2_gather<0x90, "vpgatherdd", VR256, vx32mem, vy32mem>; 8896 defm VPGATHERQD : avx2_gather<0x91, "vpgatherqd", VR128, vx32mem, vy32mem>; 8897 8898 let ExeDomain = SSEPackedDouble in { 8899 defm VGATHERDPD : avx2_gather<0x92, "vgatherdpd", VR256, vx64mem, vx64mem>, VEX_W; 8900 defm VGATHERQPD : avx2_gather<0x93, "vgatherqpd", VR256, vx64mem, vy64mem>, VEX_W; 8901 } 8902 8903 let ExeDomain = SSEPackedSingle in { 8904 defm VGATHERDPS : avx2_gather<0x92, "vgatherdps", VR256, vx32mem, vy32mem>; 8905 defm VGATHERQPS : avx2_gather<0x93, "vgatherqps", VR128, vx32mem, vy32mem>; 8906 } 8907} 8908