Lines Matching refs:VT
202 for (MVT VT : MVT::integer_valuetypes()) in X86TargetLowering() local
203 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); in X86TargetLowering()
216 for (auto VT : {MVT::f32, MVT::f64, MVT::f80}) { in X86TargetLowering()
217 setCondCodeAction(ISD::SETOEQ, VT, Expand); in X86TargetLowering()
218 setCondCodeAction(ISD::SETUNE, VT, Expand); in X86TargetLowering()
330 for (MVT VT : { MVT::i8, MVT::i16, MVT::i32 }) { in X86TargetLowering()
331 setOperationAction(ISD::FP_TO_UINT_SAT, VT, Custom); in X86TargetLowering()
332 setOperationAction(ISD::FP_TO_SINT_SAT, VT, Custom); in X86TargetLowering()
366 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) { in X86TargetLowering()
367 setOperationAction(ISD::MULHS, VT, Expand); in X86TargetLowering()
368 setOperationAction(ISD::MULHU, VT, Expand); in X86TargetLowering()
369 setOperationAction(ISD::SDIV, VT, Expand); in X86TargetLowering()
370 setOperationAction(ISD::UDIV, VT, Expand); in X86TargetLowering()
371 setOperationAction(ISD::SREM, VT, Expand); in X86TargetLowering()
372 setOperationAction(ISD::UREM, VT, Expand); in X86TargetLowering()
377 for (auto VT : { MVT::f32, MVT::f64, MVT::f80, MVT::f128, in X86TargetLowering()
379 setOperationAction(ISD::BR_CC, VT, Expand); in X86TargetLowering()
380 setOperationAction(ISD::SELECT_CC, VT, Expand); in X86TargetLowering()
426 for (auto VT : {MVT::i8, MVT::i16, MVT::i32, MVT::i64}) { in X86TargetLowering()
427 if (VT == MVT::i64 && !Subtarget.is64Bit()) in X86TargetLowering()
429 setOperationAction(ISD::CTLZ , VT, Custom); in X86TargetLowering()
430 setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Custom); in X86TargetLowering()
448 for (auto VT : {MVT::f32, MVT::f64, MVT::f80, MVT::f128}) { in X86TargetLowering()
449 setOperationAction(ISD::STRICT_FP_TO_BF16, VT, Expand); in X86TargetLowering()
450 setOperationAction(ISD::STRICT_BF16_TO_FP, VT, Expand); in X86TargetLowering()
453 for (MVT VT : {MVT::f32, MVT::f64, MVT::f80, MVT::f128}) { in X86TargetLowering()
454 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand); in X86TargetLowering()
455 setLoadExtAction(ISD::EXTLOAD, VT, MVT::bf16, Expand); in X86TargetLowering()
456 setTruncStoreAction(VT, MVT::f16, Expand); in X86TargetLowering()
457 setTruncStoreAction(VT, MVT::bf16, Expand); in X86TargetLowering()
459 setOperationAction(ISD::BF16_TO_FP, VT, Expand); in X86TargetLowering()
460 setOperationAction(ISD::FP_TO_BF16, VT, Custom); in X86TargetLowering()
486 for (auto VT : { MVT::f32, MVT::f64, MVT::f80, MVT::f128 }) { in X86TargetLowering()
487 setOperationAction(ISD::SELECT, VT, Custom); in X86TargetLowering()
488 setOperationAction(ISD::SETCC, VT, Custom); in X86TargetLowering()
489 setOperationAction(ISD::STRICT_FSETCC, VT, Custom); in X86TargetLowering()
490 setOperationAction(ISD::STRICT_FSETCCS, VT, Custom); in X86TargetLowering()
492 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) { in X86TargetLowering()
493 if (VT == MVT::i64 && !Subtarget.is64Bit()) in X86TargetLowering()
495 setOperationAction(ISD::SELECT, VT, Custom); in X86TargetLowering()
496 setOperationAction(ISD::SETCC, VT, Custom); in X86TargetLowering()
513 for (auto VT : { MVT::i32, MVT::i64 }) { in X86TargetLowering()
514 if (VT == MVT::i64 && !Subtarget.is64Bit()) in X86TargetLowering()
516 setOperationAction(ISD::ConstantPool , VT, Custom); in X86TargetLowering()
517 setOperationAction(ISD::JumpTable , VT, Custom); in X86TargetLowering()
518 setOperationAction(ISD::GlobalAddress , VT, Custom); in X86TargetLowering()
519 setOperationAction(ISD::GlobalTLSAddress, VT, Custom); in X86TargetLowering()
520 setOperationAction(ISD::ExternalSymbol , VT, Custom); in X86TargetLowering()
521 setOperationAction(ISD::BlockAddress , VT, Custom); in X86TargetLowering()
525 for (auto VT : { MVT::i32, MVT::i64 }) { in X86TargetLowering()
526 if (VT == MVT::i64 && !Subtarget.is64Bit()) in X86TargetLowering()
528 setOperationAction(ISD::SHL_PARTS, VT, Custom); in X86TargetLowering()
529 setOperationAction(ISD::SRA_PARTS, VT, Custom); in X86TargetLowering()
530 setOperationAction(ISD::SRL_PARTS, VT, Custom); in X86TargetLowering()
539 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) { in X86TargetLowering()
540 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT, Custom); in X86TargetLowering()
541 setOperationAction(ISD::ATOMIC_LOAD_SUB, VT, Custom); in X86TargetLowering()
542 setOperationAction(ISD::ATOMIC_LOAD_ADD, VT, Custom); in X86TargetLowering()
543 setOperationAction(ISD::ATOMIC_LOAD_OR, VT, Custom); in X86TargetLowering()
544 setOperationAction(ISD::ATOMIC_LOAD_XOR, VT, Custom); in X86TargetLowering()
545 setOperationAction(ISD::ATOMIC_LOAD_AND, VT, Custom); in X86TargetLowering()
546 setOperationAction(ISD::ATOMIC_STORE, VT, Custom); in X86TargetLowering()
600 auto setF16Action = [&] (MVT VT, LegalizeAction Action) { in X86TargetLowering() argument
601 setOperationAction(ISD::FABS, VT, Action); in X86TargetLowering()
602 setOperationAction(ISD::FNEG, VT, Action); in X86TargetLowering()
603 setOperationAction(ISD::FCOPYSIGN, VT, Expand); in X86TargetLowering()
604 setOperationAction(ISD::FREM, VT, Action); in X86TargetLowering()
605 setOperationAction(ISD::FMA, VT, Action); in X86TargetLowering()
606 setOperationAction(ISD::FMINNUM, VT, Action); in X86TargetLowering()
607 setOperationAction(ISD::FMAXNUM, VT, Action); in X86TargetLowering()
608 setOperationAction(ISD::FMINIMUM, VT, Action); in X86TargetLowering()
609 setOperationAction(ISD::FMAXIMUM, VT, Action); in X86TargetLowering()
610 setOperationAction(ISD::FSIN, VT, Action); in X86TargetLowering()
611 setOperationAction(ISD::FCOS, VT, Action); in X86TargetLowering()
612 setOperationAction(ISD::FSINCOS, VT, Action); in X86TargetLowering()
613 setOperationAction(ISD::FTAN, VT, Action); in X86TargetLowering()
614 setOperationAction(ISD::FSQRT, VT, Action); in X86TargetLowering()
615 setOperationAction(ISD::FPOW, VT, Action); in X86TargetLowering()
616 setOperationAction(ISD::FLOG, VT, Action); in X86TargetLowering()
617 setOperationAction(ISD::FLOG2, VT, Action); in X86TargetLowering()
618 setOperationAction(ISD::FLOG10, VT, Action); in X86TargetLowering()
619 setOperationAction(ISD::FEXP, VT, Action); in X86TargetLowering()
620 setOperationAction(ISD::FEXP2, VT, Action); in X86TargetLowering()
621 setOperationAction(ISD::FEXP10, VT, Action); in X86TargetLowering()
622 setOperationAction(ISD::FCEIL, VT, Action); in X86TargetLowering()
623 setOperationAction(ISD::FFLOOR, VT, Action); in X86TargetLowering()
624 setOperationAction(ISD::FNEARBYINT, VT, Action); in X86TargetLowering()
625 setOperationAction(ISD::FRINT, VT, Action); in X86TargetLowering()
626 setOperationAction(ISD::BR_CC, VT, Action); in X86TargetLowering()
627 setOperationAction(ISD::SETCC, VT, Action); in X86TargetLowering()
628 setOperationAction(ISD::SELECT, VT, Custom); in X86TargetLowering()
629 setOperationAction(ISD::SELECT_CC, VT, Action); in X86TargetLowering()
630 setOperationAction(ISD::FROUND, VT, Action); in X86TargetLowering()
631 setOperationAction(ISD::FROUNDEVEN, VT, Action); in X86TargetLowering()
632 setOperationAction(ISD::FTRUNC, VT, Action); in X86TargetLowering()
633 setOperationAction(ISD::FLDEXP, VT, Action); in X86TargetLowering()
652 for (auto VT : { MVT::f32, MVT::f64 }) { in X86TargetLowering()
654 setOperationAction(ISD::FABS, VT, Custom); in X86TargetLowering()
657 setOperationAction(ISD::FNEG, VT, Custom); in X86TargetLowering()
660 setOperationAction(ISD::FCOPYSIGN, VT, Custom); in X86TargetLowering()
663 setOperationAction(ISD::FADD, VT, Custom); in X86TargetLowering()
664 setOperationAction(ISD::FSUB, VT, Custom); in X86TargetLowering()
667 setOperationAction(ISD::FSIN , VT, Expand); in X86TargetLowering()
668 setOperationAction(ISD::FCOS , VT, Expand); in X86TargetLowering()
669 setOperationAction(ISD::FSINCOS, VT, Expand); in X86TargetLowering()
758 for (auto VT : { MVT::f32, MVT::f64 }) { in X86TargetLowering()
759 setOperationAction(ISD::UNDEF, VT, Expand); in X86TargetLowering()
760 setOperationAction(ISD::FCOPYSIGN, VT, Expand); in X86TargetLowering()
763 setOperationAction(ISD::FSIN , VT, Expand); in X86TargetLowering()
764 setOperationAction(ISD::FCOS , VT, Expand); in X86TargetLowering()
765 setOperationAction(ISD::FSINCOS, VT, Expand); in X86TargetLowering()
953 for (auto VT : { MVT::v8f16, MVT::v16f16, MVT::v32f16, in X86TargetLowering()
957 setOperationAction(ISD::FSIN, VT, Expand); in X86TargetLowering()
958 setOperationAction(ISD::FSINCOS, VT, Expand); in X86TargetLowering()
959 setOperationAction(ISD::FCOS, VT, Expand); in X86TargetLowering()
960 setOperationAction(ISD::FTAN, VT, Expand); in X86TargetLowering()
961 setOperationAction(ISD::FREM, VT, Expand); in X86TargetLowering()
962 setOperationAction(ISD::FCOPYSIGN, VT, Expand); in X86TargetLowering()
963 setOperationAction(ISD::FPOW, VT, Expand); in X86TargetLowering()
964 setOperationAction(ISD::FLOG, VT, Expand); in X86TargetLowering()
965 setOperationAction(ISD::FLOG2, VT, Expand); in X86TargetLowering()
966 setOperationAction(ISD::FLOG10, VT, Expand); in X86TargetLowering()
967 setOperationAction(ISD::FEXP, VT, Expand); in X86TargetLowering()
968 setOperationAction(ISD::FEXP2, VT, Expand); in X86TargetLowering()
969 setOperationAction(ISD::FEXP10, VT, Expand); in X86TargetLowering()
976 for (MVT VT : MVT::fixedlen_vector_valuetypes()) { in X86TargetLowering() local
977 setOperationAction(ISD::SDIV, VT, Expand); in X86TargetLowering()
978 setOperationAction(ISD::UDIV, VT, Expand); in X86TargetLowering()
979 setOperationAction(ISD::SREM, VT, Expand); in X86TargetLowering()
980 setOperationAction(ISD::UREM, VT, Expand); in X86TargetLowering()
981 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT,Expand); in X86TargetLowering()
982 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Expand); in X86TargetLowering()
983 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT,Expand); in X86TargetLowering()
984 setOperationAction(ISD::INSERT_SUBVECTOR, VT,Expand); in X86TargetLowering()
985 setOperationAction(ISD::FMA, VT, Expand); in X86TargetLowering()
986 setOperationAction(ISD::FFLOOR, VT, Expand); in X86TargetLowering()
987 setOperationAction(ISD::FCEIL, VT, Expand); in X86TargetLowering()
988 setOperationAction(ISD::FTRUNC, VT, Expand); in X86TargetLowering()
989 setOperationAction(ISD::FRINT, VT, Expand); in X86TargetLowering()
990 setOperationAction(ISD::FNEARBYINT, VT, Expand); in X86TargetLowering()
991 setOperationAction(ISD::FROUNDEVEN, VT, Expand); in X86TargetLowering()
992 setOperationAction(ISD::SMUL_LOHI, VT, Expand); in X86TargetLowering()
993 setOperationAction(ISD::MULHS, VT, Expand); in X86TargetLowering()
994 setOperationAction(ISD::UMUL_LOHI, VT, Expand); in X86TargetLowering()
995 setOperationAction(ISD::MULHU, VT, Expand); in X86TargetLowering()
996 setOperationAction(ISD::SDIVREM, VT, Expand); in X86TargetLowering()
997 setOperationAction(ISD::UDIVREM, VT, Expand); in X86TargetLowering()
998 setOperationAction(ISD::CTPOP, VT, Expand); in X86TargetLowering()
999 setOperationAction(ISD::CTTZ, VT, Expand); in X86TargetLowering()
1000 setOperationAction(ISD::CTLZ, VT, Expand); in X86TargetLowering()
1001 setOperationAction(ISD::ROTL, VT, Expand); in X86TargetLowering()
1002 setOperationAction(ISD::ROTR, VT, Expand); in X86TargetLowering()
1003 setOperationAction(ISD::BSWAP, VT, Expand); in X86TargetLowering()
1004 setOperationAction(ISD::SETCC, VT, Expand); in X86TargetLowering()
1005 setOperationAction(ISD::FP_TO_UINT, VT, Expand); in X86TargetLowering()
1006 setOperationAction(ISD::FP_TO_SINT, VT, Expand); in X86TargetLowering()
1007 setOperationAction(ISD::UINT_TO_FP, VT, Expand); in X86TargetLowering()
1008 setOperationAction(ISD::SINT_TO_FP, VT, Expand); in X86TargetLowering()
1009 setOperationAction(ISD::SIGN_EXTEND_INREG, VT,Expand); in X86TargetLowering()
1010 setOperationAction(ISD::TRUNCATE, VT, Expand); in X86TargetLowering()
1011 setOperationAction(ISD::SIGN_EXTEND, VT, Expand); in X86TargetLowering()
1012 setOperationAction(ISD::ZERO_EXTEND, VT, Expand); in X86TargetLowering()
1013 setOperationAction(ISD::ANY_EXTEND, VT, Expand); in X86TargetLowering()
1014 setOperationAction(ISD::SELECT_CC, VT, Expand); in X86TargetLowering()
1016 setTruncStoreAction(InnerVT, VT, Expand); in X86TargetLowering()
1018 setLoadExtAction(ISD::SEXTLOAD, InnerVT, VT, Expand); in X86TargetLowering()
1019 setLoadExtAction(ISD::ZEXTLOAD, InnerVT, VT, Expand); in X86TargetLowering()
1025 if (VT.getVectorElementType() == MVT::i1) in X86TargetLowering()
1026 setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand); in X86TargetLowering()
1030 if (VT.getVectorElementType() == MVT::f16 || in X86TargetLowering()
1031 VT.getVectorElementType() == MVT::bf16) in X86TargetLowering()
1032 setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand); in X86TargetLowering()
1086 for (auto VT : { MVT::f64, MVT::v4f32, MVT::v2f64 }) { in X86TargetLowering()
1087 setOperationAction(ISD::FMAXIMUM, VT, Custom); in X86TargetLowering()
1088 setOperationAction(ISD::FMINIMUM, VT, Custom); in X86TargetLowering()
1091 for (auto VT : { MVT::v2i8, MVT::v4i8, MVT::v8i8, in X86TargetLowering()
1093 setOperationAction(ISD::SDIV, VT, Custom); in X86TargetLowering()
1094 setOperationAction(ISD::SREM, VT, Custom); in X86TargetLowering()
1095 setOperationAction(ISD::UDIV, VT, Custom); in X86TargetLowering()
1096 setOperationAction(ISD::UREM, VT, Custom); in X86TargetLowering()
1126 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) { in X86TargetLowering()
1127 setOperationAction(ISD::SMAX, VT, VT == MVT::v8i16 ? Legal : Custom); in X86TargetLowering()
1128 setOperationAction(ISD::SMIN, VT, VT == MVT::v8i16 ? Legal : Custom); in X86TargetLowering()
1129 setOperationAction(ISD::UMAX, VT, VT == MVT::v16i8 ? Legal : Custom); in X86TargetLowering()
1130 setOperationAction(ISD::UMIN, VT, VT == MVT::v16i8 ? Legal : Custom); in X86TargetLowering()
1149 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) { in X86TargetLowering()
1150 setOperationAction(ISD::SETCC, VT, Custom); in X86TargetLowering()
1151 setOperationAction(ISD::CTPOP, VT, Custom); in X86TargetLowering()
1152 setOperationAction(ISD::ABS, VT, Custom); in X86TargetLowering()
1153 setOperationAction(ISD::ABDS, VT, Custom); in X86TargetLowering()
1154 setOperationAction(ISD::ABDU, VT, Custom); in X86TargetLowering()
1158 setCondCodeAction(ISD::SETLT, VT, Custom); in X86TargetLowering()
1159 setCondCodeAction(ISD::SETLE, VT, Custom); in X86TargetLowering()
1169 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32 }) { in X86TargetLowering()
1170 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom); in X86TargetLowering()
1171 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); in X86TargetLowering()
1172 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); in X86TargetLowering()
1173 setOperationAction(ISD::VSELECT, VT, Custom); in X86TargetLowering()
1174 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); in X86TargetLowering()
1177 for (auto VT : { MVT::v8f16, MVT::v2f64, MVT::v2i64 }) { in X86TargetLowering()
1178 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); in X86TargetLowering()
1179 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); in X86TargetLowering()
1180 setOperationAction(ISD::VSELECT, VT, Custom); in X86TargetLowering()
1182 if (VT == MVT::v2i64 && !Subtarget.is64Bit()) in X86TargetLowering()
1185 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); in X86TargetLowering()
1186 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); in X86TargetLowering()
1213 for (auto VT : {MVT::v2i8, MVT::v4i8, MVT::v8i8, MVT::v2i16, MVT::v4i16}) { in X86TargetLowering()
1214 setOperationAction(ISD::FP_TO_SINT, VT, Custom); in X86TargetLowering()
1215 setOperationAction(ISD::FP_TO_UINT, VT, Custom); in X86TargetLowering()
1216 setOperationAction(ISD::STRICT_FP_TO_SINT, VT, Custom); in X86TargetLowering()
1217 setOperationAction(ISD::STRICT_FP_TO_UINT, VT, Custom); in X86TargetLowering()
1287 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) { in X86TargetLowering()
1288 setOperationAction(ISD::SRL, VT, Custom); in X86TargetLowering()
1289 setOperationAction(ISD::SHL, VT, Custom); in X86TargetLowering()
1290 setOperationAction(ISD::SRA, VT, Custom); in X86TargetLowering()
1291 if (VT == MVT::v2i64) continue; in X86TargetLowering()
1292 setOperationAction(ISD::ROTL, VT, Custom); in X86TargetLowering()
1293 setOperationAction(ISD::ROTR, VT, Custom); in X86TargetLowering()
1294 setOperationAction(ISD::FSHL, VT, Custom); in X86TargetLowering()
1295 setOperationAction(ISD::FSHR, VT, Custom); in X86TargetLowering()
1317 for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64}) { in X86TargetLowering()
1318 setOperationAction(ISD::BITREVERSE, VT, Custom); in X86TargetLowering()
1319 setOperationAction(ISD::CTLZ, VT, Custom); in X86TargetLowering()
1370 for (auto VT : { MVT::v8i16, MVT::v4i32, MVT::v2i64 }) { in X86TargetLowering()
1371 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Legal); in X86TargetLowering()
1372 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Legal); in X86TargetLowering()
1402 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, in X86TargetLowering()
1404 setOperationAction(ISD::ROTL, VT, Custom); in X86TargetLowering()
1405 setOperationAction(ISD::ROTR, VT, Custom); in X86TargetLowering()
1409 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) in X86TargetLowering()
1410 setOperationAction(ISD::BITREVERSE, VT, Custom); in X86TargetLowering()
1431 for (auto VT : { MVT::v8f32, MVT::v4f64 }) { in X86TargetLowering()
1432 setOperationAction(ISD::FFLOOR, VT, Legal); in X86TargetLowering()
1433 setOperationAction(ISD::STRICT_FFLOOR, VT, Legal); in X86TargetLowering()
1434 setOperationAction(ISD::FCEIL, VT, Legal); in X86TargetLowering()
1435 setOperationAction(ISD::STRICT_FCEIL, VT, Legal); in X86TargetLowering()
1436 setOperationAction(ISD::FTRUNC, VT, Legal); in X86TargetLowering()
1437 setOperationAction(ISD::STRICT_FTRUNC, VT, Legal); in X86TargetLowering()
1438 setOperationAction(ISD::FRINT, VT, Legal); in X86TargetLowering()
1439 setOperationAction(ISD::STRICT_FRINT, VT, Legal); in X86TargetLowering()
1440 setOperationAction(ISD::FNEARBYINT, VT, Legal); in X86TargetLowering()
1441 setOperationAction(ISD::STRICT_FNEARBYINT, VT, Legal); in X86TargetLowering()
1442 setOperationAction(ISD::FROUNDEVEN, VT, Legal); in X86TargetLowering()
1443 setOperationAction(ISD::STRICT_FROUNDEVEN, VT, Legal); in X86TargetLowering()
1445 setOperationAction(ISD::FROUND, VT, Custom); in X86TargetLowering()
1447 setOperationAction(ISD::FNEG, VT, Custom); in X86TargetLowering()
1448 setOperationAction(ISD::FABS, VT, Custom); in X86TargetLowering()
1449 setOperationAction(ISD::FCOPYSIGN, VT, Custom); in X86TargetLowering()
1451 setOperationAction(ISD::FMAXIMUM, VT, Custom); in X86TargetLowering()
1452 setOperationAction(ISD::FMINIMUM, VT, Custom); in X86TargetLowering()
1492 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) { in X86TargetLowering()
1493 setOperationAction(ISD::SRL, VT, Custom); in X86TargetLowering()
1494 setOperationAction(ISD::SHL, VT, Custom); in X86TargetLowering()
1495 setOperationAction(ISD::SRA, VT, Custom); in X86TargetLowering()
1496 setOperationAction(ISD::ABDS, VT, Custom); in X86TargetLowering()
1497 setOperationAction(ISD::ABDU, VT, Custom); in X86TargetLowering()
1498 if (VT == MVT::v4i64) continue; in X86TargetLowering()
1499 setOperationAction(ISD::ROTL, VT, Custom); in X86TargetLowering()
1500 setOperationAction(ISD::ROTR, VT, Custom); in X86TargetLowering()
1501 setOperationAction(ISD::FSHL, VT, Custom); in X86TargetLowering()
1502 setOperationAction(ISD::FSHR, VT, Custom); in X86TargetLowering()
1519 for (auto VT : { MVT::v16i16, MVT::v8i32, MVT::v4i64 }) { in X86TargetLowering()
1520 setOperationAction(ISD::SIGN_EXTEND, VT, Custom); in X86TargetLowering()
1521 setOperationAction(ISD::ZERO_EXTEND, VT, Custom); in X86TargetLowering()
1522 setOperationAction(ISD::ANY_EXTEND, VT, Custom); in X86TargetLowering()
1530 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) { in X86TargetLowering()
1531 setOperationAction(ISD::SETCC, VT, Custom); in X86TargetLowering()
1532 setOperationAction(ISD::CTPOP, VT, Custom); in X86TargetLowering()
1533 setOperationAction(ISD::CTLZ, VT, Custom); in X86TargetLowering()
1534 setOperationAction(ISD::BITREVERSE, VT, Custom); in X86TargetLowering()
1538 setCondCodeAction(ISD::SETLT, VT, Custom); in X86TargetLowering()
1539 setCondCodeAction(ISD::SETLE, VT, Custom); in X86TargetLowering()
1550 for (auto VT : { MVT::f32, MVT::f64, MVT::v4f32, MVT::v8f32, in X86TargetLowering()
1552 setOperationAction(ISD::FMA, VT, Legal); in X86TargetLowering()
1553 setOperationAction(ISD::STRICT_FMA, VT, Legal); in X86TargetLowering()
1557 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) { in X86TargetLowering()
1558 setOperationAction(ISD::ADD, VT, HasInt256 ? Legal : Custom); in X86TargetLowering()
1559 setOperationAction(ISD::SUB, VT, HasInt256 ? Legal : Custom); in X86TargetLowering()
1598 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32 }) { in X86TargetLowering()
1599 setOperationAction(ISD::ABS, VT, HasInt256 ? Legal : Custom); in X86TargetLowering()
1600 setOperationAction(ISD::SMAX, VT, HasInt256 ? Legal : Custom); in X86TargetLowering()
1601 setOperationAction(ISD::UMAX, VT, HasInt256 ? Legal : Custom); in X86TargetLowering()
1602 setOperationAction(ISD::SMIN, VT, HasInt256 ? Legal : Custom); in X86TargetLowering()
1603 setOperationAction(ISD::UMIN, VT, HasInt256 ? Legal : Custom); in X86TargetLowering()
1606 for (auto VT : {MVT::v16i16, MVT::v8i32, MVT::v4i64}) { in X86TargetLowering()
1607 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Custom); in X86TargetLowering()
1608 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Custom); in X86TargetLowering()
1628 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64, in X86TargetLowering()
1630 setOperationAction(ISD::MLOAD, VT, Subtarget.hasVLX() ? Legal : Custom); in X86TargetLowering()
1631 setOperationAction(ISD::MSTORE, VT, Legal); in X86TargetLowering()
1636 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, in X86TargetLowering()
1638 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal); in X86TargetLowering()
1642 for (MVT VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64, in X86TargetLowering()
1644 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); in X86TargetLowering()
1645 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); in X86TargetLowering()
1646 setOperationAction(ISD::VSELECT, VT, Custom); in X86TargetLowering()
1647 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); in X86TargetLowering()
1648 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); in X86TargetLowering()
1649 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom); in X86TargetLowering()
1650 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Legal); in X86TargetLowering()
1651 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom); in X86TargetLowering()
1652 setOperationAction(ISD::STORE, VT, Custom); in X86TargetLowering()
1670 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64, in X86TargetLowering()
1672 setOperationAction(ISD::MGATHER, VT, Custom); in X86TargetLowering()
1678 for (MVT VT : { MVT::f16, MVT::v2f16, MVT::v4f16, MVT::v8f16 }) { in X86TargetLowering()
1679 setOperationAction(ISD::FP_ROUND, VT, Custom); in X86TargetLowering()
1680 setOperationAction(ISD::STRICT_FP_ROUND, VT, Custom); in X86TargetLowering()
1682 for (MVT VT : { MVT::f32, MVT::v2f32, MVT::v4f32, MVT::v8f32 }) { in X86TargetLowering()
1683 setOperationAction(ISD::FP_EXTEND, VT, Custom); in X86TargetLowering()
1684 setOperationAction(ISD::STRICT_FP_EXTEND, VT, Custom); in X86TargetLowering()
1733 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) { in X86TargetLowering()
1734 setOperationAction(ISD::SIGN_EXTEND, VT, Custom); in X86TargetLowering()
1735 setOperationAction(ISD::ZERO_EXTEND, VT, Custom); in X86TargetLowering()
1736 setOperationAction(ISD::ANY_EXTEND, VT, Custom); in X86TargetLowering()
1739 for (auto VT : { MVT::v1i1, MVT::v2i1, MVT::v4i1, MVT::v8i1, MVT::v16i1 }) in X86TargetLowering()
1740 setOperationAction(ISD::VSELECT, VT, Expand); in X86TargetLowering()
1742 for (auto VT : { MVT::v2i1, MVT::v4i1, MVT::v8i1, MVT::v16i1 }) { in X86TargetLowering()
1743 setOperationAction(ISD::SETCC, VT, Custom); in X86TargetLowering()
1744 setOperationAction(ISD::SELECT, VT, Custom); in X86TargetLowering()
1745 setOperationAction(ISD::TRUNCATE, VT, Custom); in X86TargetLowering()
1747 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); in X86TargetLowering()
1748 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom); in X86TargetLowering()
1749 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); in X86TargetLowering()
1750 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom); in X86TargetLowering()
1751 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); in X86TargetLowering()
1752 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); in X86TargetLowering()
1755 for (auto VT : { MVT::v1i1, MVT::v2i1, MVT::v4i1, MVT::v8i1 }) in X86TargetLowering()
1756 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom); in X86TargetLowering()
1759 for (MVT VT : {MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64}) { in X86TargetLowering()
1760 setOperationAction(ISD::LRINT, VT, Legal); in X86TargetLowering()
1761 setOperationAction(ISD::LLRINT, VT, Legal); in X86TargetLowering()
1789 for (MVT VT : { MVT::v16f32, MVT::v8f64 }) { in X86TargetLowering()
1790 setOperationAction(ISD::FMAXIMUM, VT, Custom); in X86TargetLowering()
1791 setOperationAction(ISD::FMINIMUM, VT, Custom); in X86TargetLowering()
1792 setOperationAction(ISD::FNEG, VT, Custom); in X86TargetLowering()
1793 setOperationAction(ISD::FABS, VT, Custom); in X86TargetLowering()
1794 setOperationAction(ISD::FMA, VT, Legal); in X86TargetLowering()
1795 setOperationAction(ISD::STRICT_FMA, VT, Legal); in X86TargetLowering()
1796 setOperationAction(ISD::FCOPYSIGN, VT, Custom); in X86TargetLowering()
1805 for (MVT VT : { MVT::v16i1, MVT::v16i8 }) { in X86TargetLowering()
1806 setOperationPromotedToType(ISD::FP_TO_SINT , VT, MVT::v16i32); in X86TargetLowering()
1807 setOperationPromotedToType(ISD::FP_TO_UINT , VT, MVT::v16i32); in X86TargetLowering()
1808 setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, VT, MVT::v16i32); in X86TargetLowering()
1809 setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, VT, MVT::v16i32); in X86TargetLowering()
1812 for (MVT VT : { MVT::v16i16, MVT::v16i32 }) { in X86TargetLowering()
1813 setOperationAction(ISD::FP_TO_SINT, VT, Custom); in X86TargetLowering()
1814 setOperationAction(ISD::FP_TO_UINT, VT, Custom); in X86TargetLowering()
1815 setOperationAction(ISD::STRICT_FP_TO_SINT, VT, Custom); in X86TargetLowering()
1816 setOperationAction(ISD::STRICT_FP_TO_UINT, VT, Custom); in X86TargetLowering()
1850 for (auto VT : {MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64, in X86TargetLowering()
1852 setOperationAction(ISD::MLOAD, VT, Custom); in X86TargetLowering()
1853 setOperationAction(ISD::MSTORE, VT, Custom); in X86TargetLowering()
1877 for (auto VT : { MVT::v16f32, MVT::v8f64 }) { in X86TargetLowering()
1878 setOperationAction(ISD::FFLOOR, VT, Legal); in X86TargetLowering()
1879 setOperationAction(ISD::STRICT_FFLOOR, VT, Legal); in X86TargetLowering()
1880 setOperationAction(ISD::FCEIL, VT, Legal); in X86TargetLowering()
1881 setOperationAction(ISD::STRICT_FCEIL, VT, Legal); in X86TargetLowering()
1882 setOperationAction(ISD::FTRUNC, VT, Legal); in X86TargetLowering()
1883 setOperationAction(ISD::STRICT_FTRUNC, VT, Legal); in X86TargetLowering()
1884 setOperationAction(ISD::FRINT, VT, Legal); in X86TargetLowering()
1885 setOperationAction(ISD::STRICT_FRINT, VT, Legal); in X86TargetLowering()
1886 setOperationAction(ISD::FNEARBYINT, VT, Legal); in X86TargetLowering()
1887 setOperationAction(ISD::STRICT_FNEARBYINT, VT, Legal); in X86TargetLowering()
1888 setOperationAction(ISD::FROUNDEVEN, VT, Legal); in X86TargetLowering()
1889 setOperationAction(ISD::STRICT_FROUNDEVEN, VT, Legal); in X86TargetLowering()
1891 setOperationAction(ISD::FROUND, VT, Custom); in X86TargetLowering()
1894 for (auto VT : {MVT::v32i16, MVT::v16i32, MVT::v8i64}) { in X86TargetLowering()
1895 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Custom); in X86TargetLowering()
1896 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Custom); in X86TargetLowering()
1921 for (auto VT : { MVT::v64i8, MVT::v32i16, MVT::v16i32, MVT::v8i64 }) { in X86TargetLowering()
1922 setOperationAction(ISD::SRL, VT, Custom); in X86TargetLowering()
1923 setOperationAction(ISD::SHL, VT, Custom); in X86TargetLowering()
1924 setOperationAction(ISD::SRA, VT, Custom); in X86TargetLowering()
1925 setOperationAction(ISD::ROTL, VT, Custom); in X86TargetLowering()
1926 setOperationAction(ISD::ROTR, VT, Custom); in X86TargetLowering()
1927 setOperationAction(ISD::SETCC, VT, Custom); in X86TargetLowering()
1928 setOperationAction(ISD::ABDS, VT, Custom); in X86TargetLowering()
1929 setOperationAction(ISD::ABDU, VT, Custom); in X86TargetLowering()
1930 setOperationAction(ISD::BITREVERSE, VT, Custom); in X86TargetLowering()
1934 setCondCodeAction(ISD::SETLT, VT, Custom); in X86TargetLowering()
1935 setCondCodeAction(ISD::SETLE, VT, Custom); in X86TargetLowering()
1945 for (auto VT : { MVT::v16i32, MVT::v8i64 }) { in X86TargetLowering()
1946 setOperationAction(ISD::SMAX, VT, Legal); in X86TargetLowering()
1947 setOperationAction(ISD::UMAX, VT, Legal); in X86TargetLowering()
1948 setOperationAction(ISD::SMIN, VT, Legal); in X86TargetLowering()
1949 setOperationAction(ISD::UMIN, VT, Legal); in X86TargetLowering()
1950 setOperationAction(ISD::ABS, VT, Legal); in X86TargetLowering()
1951 setOperationAction(ISD::CTPOP, VT, Custom); in X86TargetLowering()
1954 for (auto VT : { MVT::v64i8, MVT::v32i16 }) { in X86TargetLowering()
1955 setOperationAction(ISD::ABS, VT, HasBWI ? Legal : Custom); in X86TargetLowering()
1956 setOperationAction(ISD::CTPOP, VT, Subtarget.hasBITALG() ? Legal : Custom); in X86TargetLowering()
1957 setOperationAction(ISD::CTLZ, VT, Custom); in X86TargetLowering()
1958 setOperationAction(ISD::SMAX, VT, HasBWI ? Legal : Custom); in X86TargetLowering()
1959 setOperationAction(ISD::UMAX, VT, HasBWI ? Legal : Custom); in X86TargetLowering()
1960 setOperationAction(ISD::SMIN, VT, HasBWI ? Legal : Custom); in X86TargetLowering()
1961 setOperationAction(ISD::UMIN, VT, HasBWI ? Legal : Custom); in X86TargetLowering()
1962 setOperationAction(ISD::UADDSAT, VT, HasBWI ? Legal : Custom); in X86TargetLowering()
1963 setOperationAction(ISD::SADDSAT, VT, HasBWI ? Legal : Custom); in X86TargetLowering()
1964 setOperationAction(ISD::USUBSAT, VT, HasBWI ? Legal : Custom); in X86TargetLowering()
1965 setOperationAction(ISD::SSUBSAT, VT, HasBWI ? Legal : Custom); in X86TargetLowering()
1985 for (auto VT : { MVT::v16i32, MVT::v8i64} ) { in X86TargetLowering()
1986 setOperationAction(ISD::CTLZ, VT, Legal); in X86TargetLowering()
1991 for (auto VT : { MVT::v16i32, MVT::v8i64 }) in X86TargetLowering()
1992 setOperationAction(ISD::CTPOP, VT, Legal); in X86TargetLowering()
1998 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64, in X86TargetLowering()
2000 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal); in X86TargetLowering()
2002 for (auto VT : { MVT::v64i8, MVT::v32i16, MVT::v16i32, MVT::v8i64, in X86TargetLowering()
2004 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom); in X86TargetLowering()
2005 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Legal); in X86TargetLowering()
2006 setOperationAction(ISD::SELECT, VT, Custom); in X86TargetLowering()
2007 setOperationAction(ISD::VSELECT, VT, Custom); in X86TargetLowering()
2008 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); in X86TargetLowering()
2009 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); in X86TargetLowering()
2010 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); in X86TargetLowering()
2011 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom); in X86TargetLowering()
2012 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); in X86TargetLowering()
2022 for (auto VT : { MVT::v16i32, MVT::v8i64, MVT::v16f32, MVT::v8f64 }) { in X86TargetLowering()
2023 setOperationAction(ISD::MLOAD, VT, Legal); in X86TargetLowering()
2024 setOperationAction(ISD::MSTORE, VT, Legal); in X86TargetLowering()
2025 setOperationAction(ISD::MGATHER, VT, Custom); in X86TargetLowering()
2026 setOperationAction(ISD::MSCATTER, VT, Custom); in X86TargetLowering()
2029 for (auto VT : { MVT::v64i8, MVT::v32i16 }) { in X86TargetLowering()
2030 setOperationAction(ISD::MLOAD, VT, Legal); in X86TargetLowering()
2031 setOperationAction(ISD::MSTORE, VT, Legal); in X86TargetLowering()
2039 for (auto VT : {MVT::v32i16, MVT::v16i32, MVT::v8i64}) { in X86TargetLowering()
2040 setOperationAction(ISD::FSHL, VT, Custom); in X86TargetLowering()
2041 setOperationAction(ISD::FSHR, VT, Custom); in X86TargetLowering()
2054 for (auto VT : {MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v16i16, MVT::v8i32, in X86TargetLowering()
2056 setOperationAction(ISD::FSHL, VT, Custom); in X86TargetLowering()
2057 setOperationAction(ISD::FSHR, VT, Custom); in X86TargetLowering()
2085 for (auto VT : { MVT::v2i64, MVT::v4i64 }) { in X86TargetLowering()
2086 setOperationAction(ISD::SMAX, VT, Legal); in X86TargetLowering()
2087 setOperationAction(ISD::UMAX, VT, Legal); in X86TargetLowering()
2088 setOperationAction(ISD::SMIN, VT, Legal); in X86TargetLowering()
2089 setOperationAction(ISD::UMIN, VT, Legal); in X86TargetLowering()
2090 setOperationAction(ISD::ABS, VT, Legal); in X86TargetLowering()
2093 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 }) { in X86TargetLowering()
2094 setOperationAction(ISD::ROTL, VT, Custom); in X86TargetLowering()
2095 setOperationAction(ISD::ROTR, VT, Custom); in X86TargetLowering()
2102 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64, in X86TargetLowering()
2104 setOperationAction(ISD::MSCATTER, VT, Custom); in X86TargetLowering()
2118 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 }) { in X86TargetLowering()
2119 setOperationAction(ISD::CTLZ, VT, Legal); in X86TargetLowering()
2124 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 }) in X86TargetLowering()
2125 setOperationAction(ISD::CTPOP, VT, Legal); in X86TargetLowering()
2135 for (auto VT : { MVT::v32i1, MVT::v64i1 }) { in X86TargetLowering()
2136 setOperationAction(ISD::VSELECT, VT, Expand); in X86TargetLowering()
2137 setOperationAction(ISD::TRUNCATE, VT, Custom); in X86TargetLowering()
2138 setOperationAction(ISD::SETCC, VT, Custom); in X86TargetLowering()
2139 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); in X86TargetLowering()
2140 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); in X86TargetLowering()
2141 setOperationAction(ISD::SELECT, VT, Custom); in X86TargetLowering()
2142 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); in X86TargetLowering()
2143 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); in X86TargetLowering()
2144 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom); in X86TargetLowering()
2145 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom); in X86TargetLowering()
2148 for (auto VT : { MVT::v16i1, MVT::v32i1 }) in X86TargetLowering()
2149 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom); in X86TargetLowering()
2156 for (auto VT : { MVT::v32i8, MVT::v16i8, MVT::v16i16, MVT::v8i16 }) { in X86TargetLowering()
2157 setOperationAction(ISD::MLOAD, VT, Subtarget.hasVLX() ? Legal : Custom); in X86TargetLowering()
2158 setOperationAction(ISD::MSTORE, VT, Subtarget.hasVLX() ? Legal : Custom); in X86TargetLowering()
2166 for (auto VT : { MVT::v16i8, MVT::v32i8, MVT::v8i16, MVT::v16i16 }) in X86TargetLowering()
2167 setOperationAction(ISD::CTPOP, VT, Legal); in X86TargetLowering()
2172 auto setGroup = [&] (MVT VT) { in X86TargetLowering() argument
2173 setOperationAction(ISD::FADD, VT, Legal); in X86TargetLowering()
2174 setOperationAction(ISD::STRICT_FADD, VT, Legal); in X86TargetLowering()
2175 setOperationAction(ISD::FSUB, VT, Legal); in X86TargetLowering()
2176 setOperationAction(ISD::STRICT_FSUB, VT, Legal); in X86TargetLowering()
2177 setOperationAction(ISD::FMUL, VT, Legal); in X86TargetLowering()
2178 setOperationAction(ISD::STRICT_FMUL, VT, Legal); in X86TargetLowering()
2179 setOperationAction(ISD::FDIV, VT, Legal); in X86TargetLowering()
2180 setOperationAction(ISD::STRICT_FDIV, VT, Legal); in X86TargetLowering()
2181 setOperationAction(ISD::FSQRT, VT, Legal); in X86TargetLowering()
2182 setOperationAction(ISD::STRICT_FSQRT, VT, Legal); in X86TargetLowering()
2184 setOperationAction(ISD::FFLOOR, VT, Legal); in X86TargetLowering()
2185 setOperationAction(ISD::STRICT_FFLOOR, VT, Legal); in X86TargetLowering()
2186 setOperationAction(ISD::FCEIL, VT, Legal); in X86TargetLowering()
2187 setOperationAction(ISD::STRICT_FCEIL, VT, Legal); in X86TargetLowering()
2188 setOperationAction(ISD::FTRUNC, VT, Legal); in X86TargetLowering()
2189 setOperationAction(ISD::STRICT_FTRUNC, VT, Legal); in X86TargetLowering()
2190 setOperationAction(ISD::FRINT, VT, Legal); in X86TargetLowering()
2191 setOperationAction(ISD::STRICT_FRINT, VT, Legal); in X86TargetLowering()
2192 setOperationAction(ISD::FNEARBYINT, VT, Legal); in X86TargetLowering()
2193 setOperationAction(ISD::STRICT_FNEARBYINT, VT, Legal); in X86TargetLowering()
2194 setOperationAction(ISD::FROUNDEVEN, VT, Legal); in X86TargetLowering()
2195 setOperationAction(ISD::STRICT_FROUNDEVEN, VT, Legal); in X86TargetLowering()
2197 setOperationAction(ISD::FROUND, VT, Custom); in X86TargetLowering()
2199 setOperationAction(ISD::LOAD, VT, Legal); in X86TargetLowering()
2200 setOperationAction(ISD::STORE, VT, Legal); in X86TargetLowering()
2202 setOperationAction(ISD::FMA, VT, Legal); in X86TargetLowering()
2203 setOperationAction(ISD::STRICT_FMA, VT, Legal); in X86TargetLowering()
2204 setOperationAction(ISD::VSELECT, VT, Legal); in X86TargetLowering()
2205 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); in X86TargetLowering()
2206 setOperationAction(ISD::SELECT, VT, Custom); in X86TargetLowering()
2208 setOperationAction(ISD::FNEG, VT, Custom); in X86TargetLowering()
2209 setOperationAction(ISD::FABS, VT, Custom); in X86TargetLowering()
2210 setOperationAction(ISD::FCOPYSIGN, VT, Custom); in X86TargetLowering()
2211 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); in X86TargetLowering()
2212 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); in X86TargetLowering()
2214 setOperationAction(ISD::SETCC, VT, Custom); in X86TargetLowering()
2215 setOperationAction(ISD::STRICT_FSETCC, VT, Custom); in X86TargetLowering()
2216 setOperationAction(ISD::STRICT_FSETCCS, VT, Custom); in X86TargetLowering()
2334 for (auto VT : {MVT::v8bf16, MVT::v16bf16}) { in X86TargetLowering()
2335 setF16Action(VT, Expand); in X86TargetLowering()
2336 setOperationAction(ISD::BUILD_VECTOR, VT, Custom); in X86TargetLowering()
2337 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); in X86TargetLowering()
2338 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Legal); in X86TargetLowering()
2339 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom); in X86TargetLowering()
2429 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) { in X86TargetLowering()
2430 if (VT == MVT::i64 && !Subtarget.is64Bit()) in X86TargetLowering()
2433 setOperationAction(ISD::SADDO, VT, Custom); in X86TargetLowering()
2434 setOperationAction(ISD::UADDO, VT, Custom); in X86TargetLowering()
2435 setOperationAction(ISD::SSUBO, VT, Custom); in X86TargetLowering()
2436 setOperationAction(ISD::USUBO, VT, Custom); in X86TargetLowering()
2437 setOperationAction(ISD::SMULO, VT, Custom); in X86TargetLowering()
2438 setOperationAction(ISD::UMULO, VT, Custom); in X86TargetLowering()
2441 setOperationAction(ISD::UADDO_CARRY, VT, Custom); in X86TargetLowering()
2442 setOperationAction(ISD::USUBO_CARRY, VT, Custom); in X86TargetLowering()
2443 setOperationAction(ISD::SETCCCARRY, VT, Custom); in X86TargetLowering()
2444 setOperationAction(ISD::SADDO_CARRY, VT, Custom); in X86TargetLowering()
2445 setOperationAction(ISD::SSUBO_CARRY, VT, Custom); in X86TargetLowering()
2614 X86TargetLowering::getPreferredVectorAction(MVT VT) const { in getPreferredVectorAction()
2615 if ((VT == MVT::v32i1 || VT == MVT::v64i1) && Subtarget.hasAVX512() && in getPreferredVectorAction()
2619 if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 && in getPreferredVectorAction()
2620 !Subtarget.hasF16C() && VT.getVectorElementType() == MVT::f16) in getPreferredVectorAction()
2623 if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 && in getPreferredVectorAction()
2624 VT.getVectorElementType() != MVT::i1) in getPreferredVectorAction()
2627 return TargetLoweringBase::getPreferredVectorAction(VT); in getPreferredVectorAction()
2937 static bool useVPTERNLOG(const X86Subtarget &Subtarget, MVT VT) { in useVPTERNLOG() argument
2939 VT.is512BitVector(); in useVPTERNLOG()
3042 MVT VT = MVT::getVT(I.getArgOperand(1)->getType()); in getTgtMemIntrinsic() local
3051 Info.memVT = MVT::getVectorVT(ScalarVT, VT.getVectorNumElements()); in getTgtMemIntrinsic()
3091 bool X86TargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT, in isFPImmLegal() argument
3114 EVT VT = Load->getValueType(0); in shouldReduceLoadWidth() local
3115 if ((VT.is256BitVector() || VT.is512BitVector()) && !Load->hasOneUse()) { in shouldReduceLoadWidth()
3154 bool X86TargetLowering::convertSelectOfConstantsToMath(EVT VT) const { in convertSelectOfConstantsToMath()
3157 if (VT.isVector() && Subtarget.hasAVX512()) in convertSelectOfConstantsToMath()
3163 bool X86TargetLowering::decomposeMulByConstant(LLVMContext &Context, EVT VT, in decomposeMulByConstant() argument
3177 while (getTypeAction(Context, VT) != TypeLegal) in decomposeMulByConstant()
3178 VT = getTypeToTransformTo(Context, VT); in decomposeMulByConstant()
3185 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in decomposeMulByConstant()
3186 if (isOperationLegal(ISD::MUL, VT) && EltSizeInBits <= 32 && in decomposeMulByConstant()
3228 bool X86TargetLowering::shouldFormOverflowOp(unsigned Opcode, EVT VT, in shouldFormOverflowOp() argument
3231 if (VT.isVector()) in shouldFormOverflowOp()
3233 return VT.isSimple() || !isOperationExpand(Opcode, VT); in shouldFormOverflowOp()
3247 bool X86TargetLowering::ShouldShrinkFPConstant(EVT VT) const { in ShouldShrinkFPConstant()
3251 return !Subtarget.hasSSE2() || VT == MVT::f80; in ShouldShrinkFPConstant()
3254 bool X86TargetLowering::isScalarFPTypeInSSEReg(EVT VT) const { in isScalarFPTypeInSSEReg()
3255 return (VT == MVT::f64 && Subtarget.hasSSE2()) || in isScalarFPTypeInSSEReg()
3256 (VT == MVT::f32 && Subtarget.hasSSE1()) || VT == MVT::f16; in isScalarFPTypeInSSEReg()
3305 EVT VT = Y.getValueType(); in hasAndNotCompare() local
3307 if (VT.isVector()) in hasAndNotCompare()
3314 if (VT != MVT::i32 && VT != MVT::i64) in hasAndNotCompare()
3321 EVT VT = Y.getValueType(); in hasAndNot() local
3323 if (!VT.isVector()) in hasAndNot()
3328 if (!Subtarget.hasSSE1() || VT.getSizeInBits() < 128) in hasAndNot()
3331 if (VT == MVT::v4i32) in hasAndNot()
3365 EVT VT, unsigned ShiftOpc, bool MayTransformRotate, in preferedOpcodeForCmpEqPiecesOfOperand() argument
3367 if (!VT.isInteger()) in preferedOpcodeForCmpEqPiecesOfOperand()
3371 if (VT.isVector()) { in preferedOpcodeForCmpEqPiecesOfOperand()
3374 PreferRotate = Subtarget.hasAVX512() && (VT.getScalarType() == MVT::i32 || in preferedOpcodeForCmpEqPiecesOfOperand()
3375 VT.getScalarType() == MVT::i64); in preferedOpcodeForCmpEqPiecesOfOperand()
3382 VT.getScalarSizeInBits() - ShiftOrRotateAmt.getZExtValue(); in preferedOpcodeForCmpEqPiecesOfOperand()
3396 if (VT.isVector()) in preferedOpcodeForCmpEqPiecesOfOperand()
3403 if (VT == MVT::i64) in preferedOpcodeForCmpEqPiecesOfOperand()
3413 if (VT == MVT::i64) in preferedOpcodeForCmpEqPiecesOfOperand()
3424 if (PreferRotate || !MayTransformRotate || VT.isVector()) in preferedOpcodeForCmpEqPiecesOfOperand()
3464 EVT VT = N->getValueType(0); in shouldFoldConstantShiftPairToMask() local
3465 if ((Subtarget.hasFastVectorShiftMasks() && VT.isVector()) || in shouldFoldConstantShiftPairToMask()
3466 (Subtarget.hasFastScalarShiftMasks() && !VT.isVector())) { in shouldFoldConstantShiftPairToMask()
3476 EVT VT = Y.getValueType(); in shouldFoldMaskToVariableShiftPair() local
3479 if (VT.isVector()) in shouldFoldMaskToVariableShiftPair()
3483 if (VT == MVT::i64 && !Subtarget.is64Bit()) in shouldFoldMaskToVariableShiftPair()
3499 bool X86TargetLowering::shouldSplatInsEltVarIndex(EVT VT) const { in shouldSplatInsEltVarIndex()
3502 return isTypeLegal(VT); in shouldSplatInsEltVarIndex()
3506 MVT VT = MVT::getIntegerVT(NumBits); in hasFastEqualityCompare() local
3507 if (isTypeLegal(VT)) in hasFastEqualityCompare()
3508 return VT; in hasFastEqualityCompare()
3786 static SDValue getConstVector(ArrayRef<int> Values, MVT VT, SelectionDAG &DAG, in getConstVector() argument
3792 MVT ConstVecVT = VT; in getConstVector()
3793 unsigned NumElts = VT.getVectorNumElements(); in getConstVector()
3795 if (!In64BitMode && VT.getVectorElementType() == MVT::i64) { in getConstVector()
3812 ConstsNode = DAG.getBitcast(VT, ConstsNode); in getConstVector()
3817 MVT VT, SelectionDAG &DAG, const SDLoc &dl) { in getConstVector() argument
3823 MVT ConstVecVT = VT; in getConstVector()
3824 unsigned NumElts = VT.getVectorNumElements(); in getConstVector()
3826 if (!In64BitMode && VT.getVectorElementType() == MVT::i64) { in getConstVector()
3838 assert(V.getBitWidth() == VT.getScalarSizeInBits() && "Unexpected sizes"); in getConstVector()
3854 return DAG.getBitcast(VT, ConstsNode); in getConstVector()
3857 static SDValue getConstVector(ArrayRef<APInt> Bits, MVT VT, in getConstVector() argument
3860 return getConstVector(Bits, Undefs, VT, DAG, dl); in getConstVector()
3864 static SDValue getZeroVector(MVT VT, const X86Subtarget &Subtarget, in getZeroVector() argument
3866 assert((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector() || in getZeroVector()
3867 VT.getVectorElementType() == MVT::i1) && in getZeroVector()
3875 if (!Subtarget.hasSSE2() && VT.is128BitVector()) { in getZeroVector()
3877 } else if (VT.isFloatingPoint() && in getZeroVector()
3878 TLI.isTypeLegal(VT.getVectorElementType())) { in getZeroVector()
3879 Vec = DAG.getConstantFP(+0.0, dl, VT); in getZeroVector()
3880 } else if (VT.getVectorElementType() == MVT::i1) { in getZeroVector()
3881 assert((Subtarget.hasBWI() || VT.getVectorNumElements() <= 16) && in getZeroVector()
3883 Vec = DAG.getConstant(0, dl, VT); in getZeroVector()
3885 unsigned Num32BitElts = VT.getSizeInBits() / 32; in getZeroVector()
3888 return DAG.getBitcast(VT, Vec); in getZeroVector()
3916 EVT VT = Vec.getValueType(); in extractSubVector() local
3917 EVT ElVT = VT.getVectorElementType(); in extractSubVector()
3918 unsigned Factor = VT.getSizeInBits() / vectorWidth; in extractSubVector()
3920 VT.getVectorNumElements() / Factor); in extractSubVector()
3973 EVT VT = Vec.getValueType(); in insertSubVector() local
3974 EVT ElVT = VT.getVectorElementType(); in insertSubVector()
4003 static SDValue widenSubVector(MVT VT, SDValue Vec, bool ZeroNewElements, in widenSubVector() argument
4006 assert(Vec.getValueSizeInBits().getFixedValue() <= VT.getFixedSizeInBits() && in widenSubVector()
4007 Vec.getValueType().getScalarType() == VT.getScalarType() && in widenSubVector()
4009 SDValue Res = ZeroNewElements ? getZeroVector(VT, Subtarget, DAG, dl) in widenSubVector()
4010 : DAG.getUNDEF(VT); in widenSubVector()
4011 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, VT, Res, Vec, in widenSubVector()
4025 MVT VT = MVT::getVectorVT(SVT, WideNumElts); in widenSubVector() local
4026 return widenSubVector(VT, Vec, ZeroNewElements, Subtarget, DAG, dl); in widenSubVector()
4031 static MVT widenMaskVectorType(MVT VT, const X86Subtarget &Subtarget) { in widenMaskVectorType() argument
4032 assert(VT.getVectorElementType() == MVT::i1 && "Expected bool vector"); in widenMaskVectorType()
4033 unsigned NumElts = VT.getVectorNumElements(); in widenMaskVectorType()
4036 return VT; in widenMaskVectorType()
4044 MVT VT = widenMaskVectorType(Vec.getSimpleValueType(), Subtarget); in widenMaskVector() local
4045 return widenSubVector(VT, Vec, ZeroNewElements, Subtarget, DAG, dl); in widenMaskVector()
4064 EVT VT = Src.getValueType(); in collectConcatOps() local
4067 if (VT.getSizeInBits() == (SubVT.getSizeInBits() * 2)) { in collectConcatOps()
4074 if (Idx == (VT.getVectorNumElements() / 2)) { in collectConcatOps()
4143 EVT VT = Op.getValueType(); in splitVector() local
4144 unsigned NumElems = VT.getVectorNumElements(); in splitVector()
4145 unsigned SizeInBits = VT.getSizeInBits(); in splitVector()
4162 EVT VT = Op.getValueType(); in splitVectorOp() local
4177 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT); in splitVectorOp()
4178 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, in splitVectorOp()
4189 [[maybe_unused]] EVT VT = Op.getValueType(); in splitVectorIntUnary() local
4192 (VT.is256BitVector() || VT.is512BitVector()) && "Unsupported VT!"); in splitVectorIntUnary()
4194 VT.getVectorNumElements() && in splitVectorIntUnary()
4204 [[maybe_unused]] EVT VT = Op.getValueType(); in splitVectorIntBinary() local
4205 assert(Op.getOperand(0).getValueType() == VT && in splitVectorIntBinary()
4206 Op.getOperand(1).getValueType() == VT && "Unexpected VTs!"); in splitVectorIntBinary()
4207 assert((VT.is256BitVector() || VT.is512BitVector()) && "Unsupported VT!"); in splitVectorIntBinary()
4220 const SDLoc &DL, EVT VT, ArrayRef<SDValue> Ops, in SplitOpsAndApply() argument
4226 if (VT.getSizeInBits() > 512) { in SplitOpsAndApply()
4227 NumSubs = VT.getSizeInBits() / 512; in SplitOpsAndApply()
4228 assert((VT.getSizeInBits() % 512) == 0 && "Illegal vector size"); in SplitOpsAndApply()
4231 if (VT.getSizeInBits() > 256) { in SplitOpsAndApply()
4232 NumSubs = VT.getSizeInBits() / 256; in SplitOpsAndApply()
4233 assert((VT.getSizeInBits() % 256) == 0 && "Illegal vector size"); in SplitOpsAndApply()
4236 if (VT.getSizeInBits() > 128) { in SplitOpsAndApply()
4237 NumSubs = VT.getSizeInBits() / 128; in SplitOpsAndApply()
4238 assert((VT.getSizeInBits() % 128) == 0 && "Illegal vector size"); in SplitOpsAndApply()
4256 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Subs); in SplitOpsAndApply()
4261 static SDValue getAVX512Node(unsigned Opcode, const SDLoc &DL, MVT VT, in getAVX512Node() argument
4265 MVT SVT = VT.getScalarType(); in getAVX512Node()
4291 bool Widen = !(Subtarget.hasVLX() || VT.is512BitVector()); in getAVX512Node()
4293 MVT DstVT = VT; in getAVX512Node()
4304 assert(OpVT == VT && "Vector type mismatch"); in getAVX512Node()
4320 Res = extractSubVector(Res, 0, DAG, DL, VT.getSizeInBits()); in getAVX512Node()
4497 EVT VT = EVT::getVectorVT(*DAG.getContext(), SubSVT, 2 * SubNumElts); in concatSubVectors() local
4498 SDValue V = insertSubVector(DAG.getUNDEF(VT), V1, 0, DAG, dl, SubVectorWidth); in concatSubVectors()
4505 static SDValue getOnesVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl) { in getOnesVector() argument
4506 assert((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()) && in getOnesVector()
4508 unsigned NumElts = VT.getSizeInBits() / 32; in getOnesVector()
4510 return DAG.getBitcast(VT, Vec); in getOnesVector()
4513 static SDValue getEXTEND_VECTOR_INREG(unsigned Opcode, const SDLoc &DL, EVT VT, in getEXTEND_VECTOR_INREG() argument
4516 assert(VT.isVector() && InVT.isVector() && "Expected vector VTs."); in getEXTEND_VECTOR_INREG()
4524 assert(VT.getSizeInBits() == InVT.getSizeInBits() && in getEXTEND_VECTOR_INREG()
4526 unsigned Scale = VT.getScalarSizeInBits() / InVT.getScalarSizeInBits(); in getEXTEND_VECTOR_INREG()
4528 std::max(128U, (unsigned)VT.getSizeInBits() / Scale)); in getEXTEND_VECTOR_INREG()
4532 if (VT.getVectorNumElements() != InVT.getVectorNumElements()) in getEXTEND_VECTOR_INREG()
4535 return DAG.getNode(Opcode, DL, VT, In); in getEXTEND_VECTOR_INREG()
4539 static SDValue getBitSelect(const SDLoc &DL, MVT VT, SDValue LHS, SDValue RHS, in getBitSelect() argument
4541 LHS = DAG.getNode(ISD::AND, DL, VT, LHS, Mask); in getBitSelect()
4542 RHS = DAG.getNode(X86ISD::ANDNP, DL, VT, Mask, RHS); in getBitSelect()
4543 return DAG.getNode(ISD::OR, DL, VT, LHS, RHS); in getBitSelect()
4546 void llvm::createUnpackShuffleMask(EVT VT, SmallVectorImpl<int> &Mask, in createUnpackShuffleMask() argument
4548 assert(VT.getScalarType().isSimple() && (VT.getSizeInBits() % 128) == 0 && in createUnpackShuffleMask()
4551 int NumElts = VT.getVectorNumElements(); in createUnpackShuffleMask()
4552 int NumEltsInLane = 128 / VT.getScalarSizeInBits(); in createUnpackShuffleMask()
4566 void llvm::createSplat2ShuffleMask(MVT VT, SmallVectorImpl<int> &Mask, in createSplat2ShuffleMask() argument
4569 int NumElts = VT.getVectorNumElements(); in createSplat2ShuffleMask()
4578 static SDValue getVectorShuffle(SelectionDAG &DAG, EVT VT, const SDLoc &dl, in getVectorShuffle() argument
4582 SmallVector<SDValue> Ops(Mask.size(), DAG.getUNDEF(VT.getScalarType())); in getVectorShuffle()
4592 return DAG.getBuildVector(VT, dl, Ops); in getVectorShuffle()
4595 return DAG.getVectorShuffle(VT, dl, V1, V2, Mask); in getVectorShuffle()
4599 static SDValue getUnpackl(SelectionDAG &DAG, const SDLoc &dl, EVT VT, in getUnpackl() argument
4602 createUnpackShuffleMask(VT, Mask, /* Lo = */ true, /* Unary = */ false); in getUnpackl()
4603 return getVectorShuffle(DAG, VT, dl, V1, V2, Mask); in getUnpackl()
4607 static SDValue getUnpackh(SelectionDAG &DAG, const SDLoc &dl, EVT VT, in getUnpackh() argument
4610 createUnpackShuffleMask(VT, Mask, /* Lo = */ false, /* Unary = */ false); in getUnpackh()
4611 return getVectorShuffle(DAG, VT, dl, V1, V2, Mask); in getUnpackh()
4618 const SDLoc &dl, MVT VT, SDValue LHS, SDValue RHS, in getPack() argument
4621 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in getPack()
4624 VT.getSizeInBits() == OpVT.getSizeInBits() && in getPack()
4634 int NumElts = VT.getVectorNumElements(); in getPack()
4641 return DAG.getVectorShuffle(VT, dl, DAG.getBitcast(VT, LHS), in getPack()
4642 DAG.getBitcast(VT, RHS), PackMask); in getPack()
4650 return DAG.getNode(X86ISD::PACKUS, dl, VT, LHS, RHS); in getPack()
4654 return DAG.getNode(X86ISD::PACKSS, dl, VT, LHS, RHS); in getPack()
4668 return DAG.getNode(X86ISD::PACKUS, dl, VT, LHS, RHS); in getPack()
4677 return DAG.getNode(X86ISD::PACKSS, dl, VT, LHS, RHS); in getPack()
4688 MVT VT = V2.getSimpleValueType(); in getShuffleVectorZeroOrUndef() local
4690 ? getZeroVector(VT, Subtarget, DAG, SDLoc(V2)) : DAG.getUNDEF(VT); in getShuffleVectorZeroOrUndef()
4691 int NumElems = VT.getVectorNumElements(); in getShuffleVectorZeroOrUndef()
4696 return DAG.getVectorShuffle(VT, SDLoc(V2), V1, V2, MaskVec); in getShuffleVectorZeroOrUndef()
4741 EVT VT = Op.getValueType(); in getTargetConstantBitsFromNode() local
4742 unsigned SizeInBits = VT.getSizeInBits(); in getTargetConstantBitsFromNode()
4864 unsigned SrcEltSizeInBits = VT.getScalarSizeInBits(); in getTargetConstantBitsFromNode()
4898 EltSizeInBits <= VT.getScalarSizeInBits()) { in getTargetConstantBitsFromNode()
4900 if (MemIntr->getMemoryVT().getStoreSizeInBits() != VT.getScalarSizeInBits()) in getTargetConstantBitsFromNode()
4905 unsigned SrcEltSizeInBits = VT.getScalarSizeInBits(); in getTargetConstantBitsFromNode()
4957 unsigned SrcEltSizeInBits = VT.getScalarSizeInBits(); in getTargetConstantBitsFromNode()
4972 unsigned SrcEltSizeInBits = VT.getScalarSizeInBits(); in getTargetConstantBitsFromNode()
4996 if (EltSizeInBits != VT.getScalarSizeInBits()) in getTargetConstantBitsFromNode()
5004 unsigned NumSubElts = VT.getVectorNumElements(); in getTargetConstantBitsFromNode()
5018 if (EltSizeInBits != VT.getScalarSizeInBits()) in getTargetConstantBitsFromNode()
5143 MVT VT = V.getSimpleValueType(); in IsNOT() local
5144 return DAG.getNode(X86ISD::PCMPGT, DL, VT, V.getOperand(1), in IsNOT()
5145 getConstVector(EltBits, UndefElts, VT, DAG, DL)); in IsNOT()
5164 static void createPackShuffleMask(MVT VT, SmallVectorImpl<int> &Mask, in createPackShuffleMask() argument
5167 unsigned NumElts = VT.getVectorNumElements(); in createPackShuffleMask()
5168 unsigned NumLanes = VT.getSizeInBits() / 128; in createPackShuffleMask()
5169 unsigned NumEltsPerLane = 128 / VT.getScalarSizeInBits(); in createPackShuffleMask()
5186 static void getPackDemandedElts(EVT VT, const APInt &DemandedElts, in getPackDemandedElts() argument
5188 int NumLanes = VT.getSizeInBits() / 128; in getPackDemandedElts()
5211 static void getHorizDemandedElts(EVT VT, const APInt &DemandedElts, in getHorizDemandedElts() argument
5213 getHorizDemandedEltsForFirstOperand(VT.getSizeInBits(), DemandedElts, in getHorizDemandedElts()
5232 MVT VT = N.getSimpleValueType(); in getTargetShuffleMask() local
5233 unsigned NumElems = VT.getVectorNumElements(); in getTargetShuffleMask()
5234 unsigned MaskEltSize = VT.getScalarSizeInBits(); in getTargetShuffleMask()
5246 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5247 assert(N.getOperand(1).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5253 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5254 assert(N.getOperand(1).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5260 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5261 assert(N.getOperand(1).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5267 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5277 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5278 assert(N.getOperand(1).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5288 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5289 assert(N.getOperand(1).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5294 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5295 assert(N.getOperand(1).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5300 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5301 assert(N.getOperand(1).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5306 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5307 assert(N.getOperand(1).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5312 assert((VT.getScalarType() == MVT::i32 || VT.getScalarType() == MVT::i64) && in getTargetShuffleMask()
5314 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5315 assert(N.getOperand(1).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5323 assert(VT.getScalarType() == MVT::i8 && "Byte vector expected"); in getTargetShuffleMask()
5324 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5325 assert(N.getOperand(1).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5333 assert(VT.getScalarType() == MVT::i8 && "Byte vector expected"); in getTargetShuffleMask()
5334 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5340 assert(VT.getScalarType() == MVT::i8 && "Byte vector expected"); in getTargetShuffleMask()
5341 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5348 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5354 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5360 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5366 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5374 if (N.getOperand(0).getValueType() == VT) { in getTargetShuffleMask()
5381 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5392 assert(VT.getScalarType() == MVT::i8 && "Byte vector expected"); in getTargetShuffleMask()
5393 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5394 assert(N.getOperand(1).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5404 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5412 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5413 assert(N.getOperand(1).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5417 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5418 assert(N.getOperand(1).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5424 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5425 assert(N.getOperand(1).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5431 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5436 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5441 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5446 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5447 assert(N.getOperand(1).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5463 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5464 assert(N.getOperand(1).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5474 assert(N.getOperand(1).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5487 assert(N.getOperand(0).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5488 assert(N.getOperand(2).getValueType() == VT && "Unexpected value type"); in getTargetShuffleMask()
5638 MVT VT = N.getSimpleValueType(); in getTargetShuffleAndZeroables() local
5650 assert((VT.getSizeInBits() % Size) == 0 && in getTargetShuffleAndZeroables()
5652 unsigned EltSizeInBits = VT.getSizeInBits() / Size; in getTargetShuffleAndZeroables()
5697 if (Idx != 0 && !VT.isFloatingPoint()) in getTargetShuffleAndZeroables()
5727 assert(VT.getVectorNumElements() == (unsigned)Size && in getTargetShuffleAndZeroables()
5813 MVT VT = N.getSimpleValueType(); in getFauxShuffleMask() local
5814 unsigned NumElts = VT.getVectorNumElements(); in getFauxShuffleMask()
5815 unsigned NumSizeInBits = VT.getSizeInBits(); in getFauxShuffleMask()
5816 unsigned NumBitsPerElt = VT.getScalarSizeInBits(); in getFauxShuffleMask()
6092 getPackDemandedElts(VT, DemandedElts, EltsLHS, EltsRHS); in getFauxShuffleMask()
6131 createPackShuffleMask(VT, Mask, IsUnary); in getFauxShuffleMask()
6221 VT.getScalarType()) in getFauxShuffleMask()
6330 EVT VT = Op.getValueType(); in getTargetShuffleInputs() local
6331 if (!VT.isSimple() || !VT.isVector()) in getTargetShuffleInputs()
6361 EVT VT = Op.getValueType(); in getTargetShuffleInputs() local
6362 if (!VT.isSimple() || !VT.isVector()) in getTargetShuffleInputs()
6372 static SDValue getBROADCAST_LOAD(unsigned Opcode, const SDLoc &DL, EVT VT, in getBROADCAST_LOAD() argument
6385 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in getBROADCAST_LOAD()
6402 EVT VT = Op.getValueType(); in getShuffleScalarElt() local
6404 unsigned NumElems = VT.getVectorNumElements(); in getShuffleScalarElt()
6411 return DAG.getUNDEF(VT.getVectorElementType()); in getShuffleScalarElt()
6419 MVT ShufVT = VT.getSimpleVT(); in getShuffleScalarElt()
6489 : DAG.getUNDEF(VT.getVectorElementType()); in getShuffleScalarElt()
6503 MVT VT = Op.getSimpleValueType(); in LowerBuildVectorAsInsert() local
6504 unsigned NumElts = VT.getVectorNumElements(); in LowerBuildVectorAsInsert()
6505 assert(((VT == MVT::v8i16 && Subtarget.hasSSE2()) || in LowerBuildVectorAsInsert()
6506 ((VT == MVT::v16i8 || VT == MVT::v4i32) && Subtarget.hasSSE41())) && in LowerBuildVectorAsInsert()
6523 V = getZeroVector(VT, Subtarget, DAG, DL); in LowerBuildVectorAsInsert()
6528 V = DAG.getBitcast(VT, V); in LowerBuildVectorAsInsert()
6532 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, V, Op.getOperand(i), in LowerBuildVectorAsInsert()
6647 MVT VT = Op.getSimpleValueType(); in LowerBuildVectorv4x32() local
6648 MVT EltVT = VT.getVectorElementType(); in LowerBuildVectorv4x32()
6653 SDValue NewBV = DAG.getBitcast(MVT::v2f64, DAG.getBuildVector(VT, DL, Ops)); in LowerBuildVectorv4x32()
6655 return DAG.getBitcast(VT, Dup); in LowerBuildVectorv4x32()
6680 MVT VT = Elt.getOperand(0).getSimpleValueType(); in LowerBuildVectorv4x32() local
6681 if (!VT.is128BitVector()) in LowerBuildVectorv4x32()
6691 MVT VT = V1.getSimpleValueType(); in LowerBuildVectorv4x32() local
6715 ? DAG.getUNDEF(VT) in LowerBuildVectorv4x32()
6716 : getZeroVector(VT, Subtarget, DAG, DL); in LowerBuildVectorv4x32()
6717 if (V1.getSimpleValueType() != VT) in LowerBuildVectorv4x32()
6718 V1 = DAG.getBitcast(VT, V1); in LowerBuildVectorv4x32()
6719 return DAG.getVectorShuffle(VT, SDLoc(V1), V1, VZeroOrUndef, Mask); in LowerBuildVectorv4x32()
6758 return DAG.getBitcast(VT, Result); in LowerBuildVectorv4x32()
6762 static SDValue getVShift(bool isLeft, EVT VT, SDValue SrcOp, unsigned NumBits, in getVShift() argument
6765 assert(VT.is128BitVector() && "Unknown type for VShift"); in getVShift()
6771 return DAG.getBitcast(VT, DAG.getNode(Opc, dl, ShVT, SrcOp, ShiftVal)); in getVShift()
6774 static SDValue LowerAsSplatVectorLoad(SDValue SrcOp, MVT VT, const SDLoc &dl, in LowerAsSplatVectorLoad() argument
6804 Align RequiredAlign(VT.getSizeInBits() / 8); in LowerAsSplatVectorLoad()
6834 unsigned NumElems = VT.getVectorNumElements(); in LowerAsSplatVectorLoad()
6896 static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts, in EltsFromConsecutiveLoads() argument
6900 if ((VT.getScalarSizeInBits() % 8) != 0) in EltsFromConsecutiveLoads()
6931 if ((NumElems * EltSizeInBits) != VT.getSizeInBits()) in EltsFromConsecutiveLoads()
6949 return DAG.getUNDEF(VT); in EltsFromConsecutiveLoads()
6951 return VT.isInteger() ? DAG.getConstant(0, DL, VT) in EltsFromConsecutiveLoads()
6952 : DAG.getConstantFP(0.0, DL, VT); in EltsFromConsecutiveLoads()
7003 auto CreateLoad = [&DAG, &DL, &Loads](EVT VT, LoadSDNode *LDBase) { in EltsFromConsecutiveLoads() argument
7008 DAG.getLoad(VT, DL, LDBase->getChain(), LDBase->getBasePtr(), in EltsFromConsecutiveLoads()
7019 VT.getSizeInBits() / 8, *DAG.getContext(), DAG.getDataLayout()); in EltsFromConsecutiveLoads()
7029 if (IsAfterLegalize && !TLI.isOperationLegal(ISD::LOAD, VT)) in EltsFromConsecutiveLoads()
7035 VT.is256BitVector() && !Subtarget.hasInt256()) in EltsFromConsecutiveLoads()
7039 return DAG.getBitcast(VT, Elts[FirstLoadedElt]); in EltsFromConsecutiveLoads()
7042 return CreateLoad(VT, LDBase); in EltsFromConsecutiveLoads()
7046 if (!IsAfterLegalize && VT.isVector()) { in EltsFromConsecutiveLoads()
7047 unsigned NumMaskElts = VT.getVectorNumElements(); in EltsFromConsecutiveLoads()
7058 SDValue V = CreateLoad(VT, LDBase); in EltsFromConsecutiveLoads()
7059 SDValue Z = VT.isInteger() ? DAG.getConstant(0, DL, VT) in EltsFromConsecutiveLoads()
7060 : DAG.getConstantFP(0.0, DL, VT); in EltsFromConsecutiveLoads()
7061 return DAG.getVectorShuffle(VT, DL, V, Z, ClearMask); in EltsFromConsecutiveLoads()
7067 if (VT.is256BitVector() || VT.is512BitVector()) { in EltsFromConsecutiveLoads()
7071 EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(), HalfNumElems); in EltsFromConsecutiveLoads()
7076 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), in EltsFromConsecutiveLoads()
7085 ((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()))) { in EltsFromConsecutiveLoads()
7086 MVT VecSVT = VT.isFloatingPoint() ? MVT::getFloatingPointVT(LoadSizeInBits) in EltsFromConsecutiveLoads()
7088 MVT VecVT = MVT::getVectorVT(VecSVT, VT.getSizeInBits() / LoadSizeInBits); in EltsFromConsecutiveLoads()
7091 if (!Subtarget.hasSSE2() && VT == MVT::v4f32) in EltsFromConsecutiveLoads()
7102 return DAG.getBitcast(VT, ResNode); in EltsFromConsecutiveLoads()
7109 (VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector())) { in EltsFromConsecutiveLoads()
7140 VT.isInteger() && (RepeatSize != 64 || TLI.isTypeLegal(MVT::i64)) in EltsFromConsecutiveLoads()
7148 VT.getSizeInBits() / ScalarSize); in EltsFromConsecutiveLoads()
7154 while (Broadcast.getValueSizeInBits() < VT.getSizeInBits()) in EltsFromConsecutiveLoads()
7166 return DAG.getBitcast(VT, Broadcast); in EltsFromConsecutiveLoads()
7178 static SDValue combineToConsecutiveLoads(EVT VT, SDValue Op, const SDLoc &DL, in combineToConsecutiveLoads() argument
7183 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) { in combineToConsecutiveLoads()
7190 assert(Elts.size() == VT.getVectorNumElements()); in combineToConsecutiveLoads()
7191 return EltsFromConsecutiveLoads(VT, Elts, DL, DAG, Subtarget, in combineToConsecutiveLoads()
7195 static Constant *getConstantVector(MVT VT, ArrayRef<APInt> Bits, in getConstantVector() argument
7197 unsigned ScalarSize = VT.getScalarSizeInBits(); in getConstantVector()
7198 Type *Ty = EVT(VT.getScalarType()).getTypeForEVT(C); in getConstantVector()
7201 if (VT.isFloatingPoint()) { in getConstantVector()
7220 static Constant *getConstantVector(MVT VT, const APInt &SplatValue, in getConstantVector() argument
7222 unsigned ScalarSize = VT.getScalarSizeInBits(); in getConstantVector()
7225 if (VT.isFloatingPoint()) { in getConstantVector()
7288 MVT VT = BVOp->getSimpleValueType(0); in lowerBuildVectorAsBroadcast() local
7289 unsigned NumElts = VT.getVectorNumElements(); in lowerBuildVectorAsBroadcast()
7291 assert((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()) && in lowerBuildVectorAsBroadcast()
7325 MVT EltType = MVT::getIntegerVT(VT.getScalarSizeInBits() * SeqLen); in lowerBuildVectorAsBroadcast()
7329 if (!VT.is512BitVector() && !Subtarget.hasVLX()) { in lowerBuildVectorAsBroadcast()
7330 unsigned Scale = 512 / VT.getSizeInBits(); in lowerBuildVectorAsBroadcast()
7334 if (BcstVT.getSizeInBits() != VT.getSizeInBits()) in lowerBuildVectorAsBroadcast()
7335 Bcst = extractSubVector(Bcst, 0, DAG, dl, VT.getSizeInBits()); in lowerBuildVectorAsBroadcast()
7336 return DAG.getBitcast(VT, Bcst); in lowerBuildVectorAsBroadcast()
7348 SplatBitSize > VT.getScalarSizeInBits() && in lowerBuildVectorAsBroadcast()
7349 SplatBitSize < VT.getSizeInBits()) { in lowerBuildVectorAsBroadcast()
7361 Constant *C = getConstantVector(VT, SplatValue, SplatBitSize, *Ctx); in lowerBuildVectorAsBroadcast()
7363 unsigned Repeat = VT.getSizeInBits() / SplatBitSize; in lowerBuildVectorAsBroadcast()
7373 return DAG.getBitcast(VT, Brdcst); in lowerBuildVectorAsBroadcast()
7377 Constant *VecC = getConstantVector(VT, SplatValue, SplatBitSize, *Ctx); in lowerBuildVectorAsBroadcast()
7379 unsigned NumElm = SplatBitSize / VT.getScalarSizeInBits(); in lowerBuildVectorAsBroadcast()
7380 MVT VVT = MVT::getVectorVT(VT.getScalarType(), NumElm); in lowerBuildVectorAsBroadcast()
7382 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in lowerBuildVectorAsBroadcast()
7417 bool IsGE256 = (VT.getSizeInBits() >= 256); in lowerBuildVectorAsBroadcast()
7456 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in lowerBuildVectorAsBroadcast()
7468 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld); in lowerBuildVectorAsBroadcast()
7481 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in lowerBuildVectorAsBroadcast()
7495 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in lowerBuildVectorAsBroadcast()
7505 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld); in lowerBuildVectorAsBroadcast()
7548 MVT VT = Op.getSimpleValueType(); in buildFromShuffleMostly() local
7552 if (!TLI.isOperationLegalOrCustom(ISD::INSERT_VECTOR_ELT, VT)) in buildFromShuffleMostly()
7585 if (ExtractedFromVec.getValueType() != VT) in buildFromShuffleMostly()
7607 VecIn2 = VecIn2.getNode() ? VecIn2 : DAG.getUNDEF(VT); in buildFromShuffleMostly()
7608 SDValue NV = DAG.getVectorShuffle(VT, DL, VecIn1, VecIn2, Mask); in buildFromShuffleMostly()
7611 NV = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, NV, Op.getOperand(Idx), in buildFromShuffleMostly()
7620 MVT VT = Op.getSimpleValueType(); in LowerBUILD_VECTORvXbf16() local
7622 VT.changeVectorElementType(Subtarget.hasFP16() ? MVT::f16 : MVT::i16); in LowerBUILD_VECTORvXbf16()
7628 return DAG.getBitcast(VT, Res); in LowerBUILD_VECTORvXbf16()
7636 MVT VT = Op.getSimpleValueType(); in LowerBUILD_VECTORvXi1() local
7637 assert((VT.getVectorElementType() == MVT::i1) && in LowerBUILD_VECTORvXi1()
7677 if (VT == MVT::v64i1 && !Subtarget.is64Bit()) { in LowerBUILD_VECTORvXi1()
7684 MVT ImmVT = MVT::getIntegerVT(std::max((unsigned)VT.getSizeInBits(), 8U)); in LowerBUILD_VECTORvXi1()
7688 MVT VecVT = VT.getSizeInBits() >= 8 ? VT : MVT::v8i1; in LowerBUILD_VECTORvXi1()
7690 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Select, in LowerBUILD_VECTORvXi1()
7698 if (VT == MVT::v64i1 && !Subtarget.is64Bit()) { in LowerBUILD_VECTORvXi1()
7705 MVT ImmVT = MVT::getIntegerVT(std::max((unsigned)VT.getSizeInBits(), 8U)); in LowerBUILD_VECTORvXi1()
7707 MVT VecVT = VT.getSizeInBits() >= 8 ? VT : MVT::v8i1; in LowerBUILD_VECTORvXi1()
7709 DstVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, DstVec, in LowerBUILD_VECTORvXi1()
7713 DstVec = DAG.getUNDEF(VT); in LowerBUILD_VECTORvXi1()
7716 DstVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, DstVec, in LowerBUILD_VECTORvXi1()
7760 EVT VT = N->getValueType(0); in isHorizontalBinOpPart() local
7761 assert(VT.is256BitVector() && "Only use for matching partial 256-bit h-ops"); in isHorizontalBinOpPart()
7763 assert(VT.isVector() && VT.getVectorNumElements() >= LastIdx && in isHorizontalBinOpPart()
7770 V0 = DAG.getUNDEF(VT); in isHorizontalBinOpPart()
7771 V1 = DAG.getUNDEF(VT); in isHorizontalBinOpPart()
7810 if (V0.getValueType() != VT) in isHorizontalBinOpPart()
7816 if (V1.getValueType() != VT) in isHorizontalBinOpPart()
7874 MVT VT = V0.getSimpleValueType(); in ExpandHorizontalBinOp() local
7875 assert(VT.is256BitVector() && VT == V1.getSimpleValueType() && in ExpandHorizontalBinOp()
7878 unsigned NumElts = VT.getVectorNumElements(); in ExpandHorizontalBinOp()
7903 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LO, HI); in ExpandHorizontalBinOp()
7917 MVT VT = BV->getSimpleValueType(0); in isAddSubOrSubAdd() local
7918 if (!Subtarget.hasSSE3() || !VT.isFloatingPoint()) in isAddSubOrSubAdd()
7921 unsigned NumElts = VT.getVectorNumElements(); in isAddSubOrSubAdd()
7922 SDValue InVec0 = DAG.getUNDEF(VT); in isAddSubOrSubAdd()
7923 SDValue InVec1 = DAG.getUNDEF(VT); in isAddSubOrSubAdd()
7969 if (InVec0.getSimpleValueType() != VT) in isAddSubOrSubAdd()
7974 if (InVec1.getSimpleValueType() != VT) in isAddSubOrSubAdd()
8077 MVT VT = BV->getSimpleValueType(0); in lowerToAddSubOrFMAddSub() local
8083 return DAG.getNode(Opc, DL, VT, Opnd0, Opnd1, Opnd2); in lowerToAddSubOrFMAddSub()
8092 if (VT.is512BitVector()) { in lowerToAddSubOrFMAddSub()
8094 for (int I = 0, E = VT.getVectorNumElements(); I != E; I += 2) { in lowerToAddSubOrFMAddSub()
8098 SDValue Sub = DAG.getNode(ISD::FSUB, DL, VT, Opnd0, Opnd1); in lowerToAddSubOrFMAddSub()
8099 SDValue Add = DAG.getNode(ISD::FADD, DL, VT, Opnd0, Opnd1); in lowerToAddSubOrFMAddSub()
8100 return DAG.getVectorShuffle(VT, DL, Sub, Add, Mask); in lowerToAddSubOrFMAddSub()
8103 return DAG.getNode(X86ISD::ADDSUB, DL, VT, Opnd0, Opnd1); in lowerToAddSubOrFMAddSub()
8109 MVT VT = BV->getSimpleValueType(0); in isHopBuildVector() local
8111 V0 = DAG.getUNDEF(VT); in isHopBuildVector()
8112 V1 = DAG.getUNDEF(VT); in isHopBuildVector()
8117 unsigned NumElts = VT.getVectorNumElements(); in isHopBuildVector()
8119 unsigned Num128BitChunks = VT.is256BitVector() ? 2 : 1; in isHopBuildVector()
8201 MVT VT = BV->getSimpleValueType(0); in getHopForBuildVector() local
8202 unsigned Width = VT.getSizeInBits(); in getHopForBuildVector()
8206 V0 = insertSubVector(DAG.getUNDEF(VT), V0, 0, DAG, DL, Width); in getHopForBuildVector()
8211 V1 = insertSubVector(DAG.getUNDEF(VT), V1, 0, DAG, DL, Width); in getHopForBuildVector()
8213 unsigned NumElts = VT.getVectorNumElements(); in getHopForBuildVector()
8221 if (VT.is256BitVector() && DemandedElts.lshr(HalfNumElts) == 0) { in getHopForBuildVector()
8222 MVT HalfVT = VT.getHalfNumVectorElementsVT(); in getHopForBuildVector()
8226 return insertSubVector(DAG.getUNDEF(VT), Half, 0, DAG, DL, 256); in getHopForBuildVector()
8229 return DAG.getNode(HOpcode, DL, VT, V0, V1); in getHopForBuildVector()
8245 MVT VT = BV->getSimpleValueType(0); in LowerToHorizontalOp() local
8246 if (((VT == MVT::v4f32 || VT == MVT::v2f64) && Subtarget.hasSSE3()) || in LowerToHorizontalOp()
8247 ((VT == MVT::v8i16 || VT == MVT::v4i32) && Subtarget.hasSSSE3()) || in LowerToHorizontalOp()
8248 ((VT == MVT::v8f32 || VT == MVT::v4f64) && Subtarget.hasAVX()) || in LowerToHorizontalOp()
8249 ((VT == MVT::v16i16 || VT == MVT::v8i32) && Subtarget.hasAVX2())) { in LowerToHorizontalOp()
8257 if (!Subtarget.hasAVX() || !VT.is256BitVector()) in LowerToHorizontalOp()
8261 unsigned NumElts = VT.getVectorNumElements(); in LowerToHorizontalOp()
8274 if (VT == MVT::v8i32 || VT == MVT::v16i16) { in LowerToHorizontalOp()
8314 if (VT == MVT::v8f32 || VT == MVT::v4f64 || VT == MVT::v8i32 || in LowerToHorizontalOp()
8315 VT == MVT::v16i16) { in LowerToHorizontalOp()
8360 MVT VT = Op->getSimpleValueType(0); in lowerBuildVectorToBitOp() local
8361 unsigned NumElems = VT.getVectorNumElements(); in lowerBuildVectorToBitOp()
8388 if (!TLI.isOperationLegalOrPromote(Opcode, VT)) in lowerBuildVectorToBitOp()
8403 if (RHS.getValueSizeInBits() != VT.getScalarSizeInBits()) { in lowerBuildVectorToBitOp()
8406 RHS = DAG.getZExtOrTrunc(RHS, DL, VT.getScalarType()); in lowerBuildVectorToBitOp()
8419 SDValue LHS = DAG.getBuildVector(VT, DL, LHSElts); in lowerBuildVectorToBitOp()
8420 SDValue RHS = DAG.getBuildVector(VT, DL, RHSElts); in lowerBuildVectorToBitOp()
8421 SDValue Res = DAG.getNode(Opcode, DL, VT, LHS, RHS); in lowerBuildVectorToBitOp()
8437 MVT VT = Op.getSimpleValueType(); in materializeVectorConstant() local
8447 if (VT == MVT::v4i32 || VT == MVT::v8i32 || VT == MVT::v16i32) in materializeVectorConstant()
8450 return getOnesVector(VT, DAG, DL); in materializeVectorConstant()
8459 static SDValue createVariablePermute(MVT VT, SDValue SrcVec, SDValue IndicesVec, in createVariablePermute() argument
8462 MVT ShuffleVT = VT; in createVariablePermute()
8463 EVT IndicesVT = EVT(VT).changeVectorElementTypeToInteger(); in createVariablePermute()
8464 unsigned NumElts = VT.getVectorNumElements(); in createVariablePermute()
8465 unsigned SizeInBits = VT.getSizeInBits(); in createVariablePermute()
8474 NumElts * VT.getScalarSizeInBits()); in createVariablePermute()
8490 VT = MVT::getVectorVT(VT.getScalarType(), Scale * NumElts); in createVariablePermute()
8491 IndicesVT = EVT(VT).changeVectorElementTypeToInteger(); in createVariablePermute()
8495 createVariablePermute(VT, SrcVec, IndicesVec, DL, DAG, Subtarget); in createVariablePermute()
8501 SrcVec = widenSubVector(VT, SrcVec, false, Subtarget, DAG, SDLoc(SrcVec)); in createVariablePermute()
8531 switch (VT.SimpleTy) { in createVariablePermute()
8568 DAG.getVectorShuffle(VT, DL, SrcVec, SrcVec, {0, 0}), in createVariablePermute()
8569 DAG.getVectorShuffle(VT, DL, SrcVec, SrcVec, {1, 1}), in createVariablePermute()
8582 ISD::CONCAT_VECTORS, DL, VT, in createVariablePermute()
8588 SDValue LoLo = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Lo); in createVariablePermute()
8589 SDValue HiHi = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Hi, Hi); in createVariablePermute()
8596 EVT VT = Idx.getValueType(); in createVariablePermute() local
8597 return DAG.getSelectCC(DL, Idx, DAG.getConstant(15, DL, VT), in createVariablePermute()
8598 DAG.getNode(X86ISD::PSHUFB, DL, VT, Ops[1], Idx), in createVariablePermute()
8599 DAG.getNode(X86ISD::PSHUFB, DL, VT, Ops[0], Idx), in createVariablePermute()
8614 VT, createVariablePermute( in createVariablePermute()
8631 VT, DAG.getNode(X86ISD::VPERMIL2, DL, MVT::v8f32, LoLo, HiHi, in createVariablePermute()
8640 return DAG.getBitcast(VT, Res); in createVariablePermute()
8647 MVT WidenSrcVT = MVT::getVectorVT(VT.getScalarType(), 8); in createVariablePermute()
8667 VT, DAG.getNode(X86ISD::VPERMIL2, DL, MVT::v4f64, LoLo, HiHi, in createVariablePermute()
8676 return DAG.getBitcast(VT, Res); in createVariablePermute()
8698 assert((VT.getSizeInBits() == ShuffleVT.getSizeInBits()) && in createVariablePermute()
8699 (VT.getScalarSizeInBits() % ShuffleVT.getScalarSizeInBits()) == 0 && in createVariablePermute()
8702 uint64_t Scale = VT.getScalarSizeInBits() / ShuffleVT.getScalarSizeInBits(); in createVariablePermute()
8713 return DAG.getBitcast(VT, Res); in createVariablePermute()
8768 MVT VT = V.getSimpleValueType(); in LowerBUILD_VECTORAsVariablePermute() local
8769 return createVariablePermute(VT, SrcVec, IndicesVec, DL, DAG, Subtarget); in LowerBUILD_VECTORAsVariablePermute()
8776 MVT VT = Op.getSimpleValueType(); in LowerBUILD_VECTOR() local
8777 MVT EltVT = VT.getVectorElementType(); in LowerBUILD_VECTOR()
8782 if (VT.getVectorElementType() == MVT::i1 && Subtarget.hasAVX512()) in LowerBUILD_VECTOR()
8785 if (VT.getVectorElementType() == MVT::bf16 && in LowerBUILD_VECTOR()
8826 return DAG.getUNDEF(VT); in LowerBUILD_VECTOR()
8830 return DAG.getFreeze(DAG.getUNDEF(VT)); in LowerBUILD_VECTOR()
8834 return getZeroVector(VT, Subtarget, DAG, dl); in LowerBUILD_VECTOR()
8856 SDValue EltsBV = DAG.getBuildVector(VT, dl, Elts); in LowerBUILD_VECTOR()
8858 SDValue FrozenUndefBV = DAG.getSplatBuildVector(VT, dl, FrozenUndefElt); in LowerBUILD_VECTOR()
8859 return DAG.getVectorShuffle(VT, dl, EltsBV, FrozenUndefBV, BlendMask); in LowerBUILD_VECTOR()
8867 if ((VT.is256BitVector() || VT.is512BitVector()) && in LowerBUILD_VECTOR()
8873 if (VT.is512BitVector() && in LowerBUILD_VECTOR()
8881 return widenSubVector(VT, NewBV, !UndefUpper, Subtarget, DAG, dl); in LowerBUILD_VECTOR()
8905 (isOperationLegalOrCustom(ISD::INSERT_VECTOR_ELT, VT) || in LowerBUILD_VECTOR()
8906 isOperationLegalOrCustom(ISD::VECTOR_SHUFFLE, VT))) { in LowerBUILD_VECTOR()
8929 SDValue DAGConstVec = DAG.getConstantPool(CV, VT); in LowerBUILD_VECTOR()
8940 SDValue Ld = DAG.getLoad(VT, dl, DAG.getEntryNode(), LegalDAGConstVec, MPI); in LowerBUILD_VECTOR()
8942 unsigned NumEltsInLow128Bits = 128 / VT.getScalarSizeInBits(); in LowerBUILD_VECTOR()
8944 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Ld, VarElt, InsIndex); in LowerBUILD_VECTOR()
8948 assert(VT.getSizeInBits() > 128 && "Invalid insertion index?"); in LowerBUILD_VECTOR()
8951 unsigned NumElts = VT.getVectorNumElements(); in LowerBUILD_VECTOR()
8954 SDValue S2V = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, VarElt); in LowerBUILD_VECTOR()
8955 return DAG.getVectorShuffle(VT, dl, Ld, S2V, ShuffleMask); in LowerBUILD_VECTOR()
8969 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item); in LowerBUILD_VECTOR()
8974 assert((VT.is128BitVector() || VT.is256BitVector() || in LowerBUILD_VECTOR()
8975 VT.is512BitVector()) && in LowerBUILD_VECTOR()
8977 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item); in LowerBUILD_VECTOR()
8987 MVT ShufVT = MVT::getVectorVT(MVT::i32, VT.getSizeInBits() / 32); in LowerBUILD_VECTOR()
8990 return DAG.getBitcast(VT, Item); in LowerBUILD_VECTOR()
8998 unsigned NumBits = VT.getSizeInBits(); in LowerBUILD_VECTOR()
8999 return getVShift(true, VT, in LowerBUILD_VECTOR()
9001 VT, Op.getOperand(1)), in LowerBUILD_VECTOR()
9014 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item); in LowerBUILD_VECTOR()
9029 return LowerAsSplatVectorLoad(Item, VT, dl, DAG); in LowerBUILD_VECTOR()
9046 EltsFromConsecutiveLoads(VT, Ops, dl, DAG, Subtarget, false)) in LowerBUILD_VECTOR()
9064 MVT WideEltVT = VT.isFloatingPoint() ? MVT::f64 : MVT::i64; in LowerBUILD_VECTOR()
9071 return DAG.getBitcast(VT, DAG.getNode(X86ISD::VBROADCAST, dl, BcastVT, in LowerBUILD_VECTOR()
9078 if (VT.getSizeInBits() > 128) { in LowerBUILD_VECTOR()
9096 SDValue V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, in LowerBUILD_VECTOR()
9125 Ops[i] = getZeroVector(VT, Subtarget, DAG, dl); in LowerBUILD_VECTOR()
9127 Ops[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i)); in LowerBUILD_VECTOR()
9137 Ops[i] = getMOVL(DAG, dl, VT, Ops[i*2+1], Ops[i*2]); in LowerBUILD_VECTOR()
9140 Ops[i] = getMOVL(DAG, dl, VT, Ops[i*2], Ops[i*2+1]); in LowerBUILD_VECTOR()
9143 Ops[i] = getUnpackl(DAG, dl, VT, Ops[i*2], Ops[i*2+1]); in LowerBUILD_VECTOR()
9156 return DAG.getVectorShuffle(VT, dl, Ops[0], Ops[1], MaskVec); in LowerBUILD_VECTOR()
9169 Result = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(0)); in LowerBUILD_VECTOR()
9171 Result = DAG.getUNDEF(VT); in LowerBUILD_VECTOR()
9175 Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Result, in LowerBUILD_VECTOR()
9187 Ops[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i)); in LowerBUILD_VECTOR()
9189 Ops[i] = DAG.getUNDEF(VT); in LowerBUILD_VECTOR()
9206 Ops[i] = DAG.getVectorShuffle(VT, dl, Ops[2*i], Ops[(2*i)+1], Mask); in LowerBUILD_VECTOR()
9359 MVT VT = Op.getSimpleValueType(); in LowerCONCAT_VECTORS() local
9360 if (VT.getVectorElementType() == MVT::i1) in LowerCONCAT_VECTORS()
9363 assert((VT.is256BitVector() && Op.getNumOperands() == 2) || in LowerCONCAT_VECTORS()
9364 (VT.is512BitVector() && (Op.getNumOperands() == 2 || in LowerCONCAT_VECTORS()
9422 static bool is128BitLaneCrossingShuffleMask(MVT VT, ArrayRef<int> Mask) { in is128BitLaneCrossingShuffleMask() argument
9423 return isLaneCrossingShuffleMask(128, VT.getScalarSizeInBits(), Mask); in is128BitLaneCrossingShuffleMask()
9466 static bool isRepeatedShuffleMask(unsigned LaneSizeInBits, MVT VT, in isRepeatedShuffleMask() argument
9469 auto LaneSize = LaneSizeInBits / VT.getScalarSizeInBits(); in isRepeatedShuffleMask()
9496 is128BitLaneRepeatedShuffleMask(MVT VT, ArrayRef<int> Mask, in is128BitLaneRepeatedShuffleMask() argument
9498 return isRepeatedShuffleMask(128, VT, Mask, RepeatedMask); in is128BitLaneRepeatedShuffleMask()
9502 is128BitLaneRepeatedShuffleMask(MVT VT, ArrayRef<int> Mask) { in is128BitLaneRepeatedShuffleMask() argument
9504 return isRepeatedShuffleMask(128, VT, Mask, RepeatedMask); in is128BitLaneRepeatedShuffleMask()
9509 is256BitLaneRepeatedShuffleMask(MVT VT, ArrayRef<int> Mask, in is256BitLaneRepeatedShuffleMask() argument
9511 return isRepeatedShuffleMask(256, VT, Mask, RepeatedMask); in is256BitLaneRepeatedShuffleMask()
9553 static bool isRepeatedTargetShuffleMask(unsigned LaneSizeInBits, MVT VT, in isRepeatedTargetShuffleMask() argument
9556 return isRepeatedTargetShuffleMask(LaneSizeInBits, VT.getScalarSizeInBits(), in isRepeatedTargetShuffleMask()
9593 MVT VT = Op.getSimpleValueType(); in IsElementEquivalent() local
9594 int NumElts = VT.getVectorNumElements(); in IsElementEquivalent()
9596 int NumLanes = VT.getSizeInBits() / 128; in IsElementEquivalent()
9654 static bool isTargetShuffleEquivalent(MVT VT, ArrayRef<int> Mask, in isTargetShuffleEquivalent() argument
9671 if (V1 && (V1.getValueSizeInBits() != VT.getSizeInBits() || in isTargetShuffleEquivalent()
9674 if (V2 && (V2.getValueSizeInBits() != VT.getSizeInBits() || in isTargetShuffleEquivalent()
9715 static bool isUnpackWdShuffleMask(ArrayRef<int> Mask, MVT VT, in isUnpackWdShuffleMask() argument
9717 if (VT != MVT::v8i32 && VT != MVT::v8f32) in isUnpackWdShuffleMask()
9726 bool IsUnpackwdMask = (isTargetShuffleEquivalent(VT, Mask, Unpcklwd, DAG) || in isUnpackWdShuffleMask()
9727 isTargetShuffleEquivalent(VT, Mask, Unpckhwd, DAG)); in isUnpackWdShuffleMask()
9735 MVT VT = MVT::getVectorVT(EltVT, Mask.size()); in is128BitUnpackShuffleMask() local
9744 createUnpackShuffleMask(VT, UnpackMask, (i >> 1) % 2, i % 2); in is128BitUnpackShuffleMask()
9745 if (isTargetShuffleEquivalent(VT, Mask, UnpackMask, DAG) || in is128BitUnpackShuffleMask()
9746 isTargetShuffleEquivalent(VT, CommutedMask, UnpackMask, DAG)) in is128BitUnpackShuffleMask()
9836 static SDValue lowerShuffleWithPSHUFB(const SDLoc &DL, MVT VT, in lowerShuffleWithPSHUFB() argument
9842 int LaneSize = 128 / VT.getScalarSizeInBits(); in lowerShuffleWithPSHUFB()
9843 const int NumBytes = VT.getSizeInBits() / 8; in lowerShuffleWithPSHUFB()
9844 const int NumEltBytes = VT.getScalarSizeInBits() / 8; in lowerShuffleWithPSHUFB()
9846 assert((Subtarget.hasSSSE3() && VT.is128BitVector()) || in lowerShuffleWithPSHUFB()
9847 (Subtarget.hasAVX2() && VT.is256BitVector()) || in lowerShuffleWithPSHUFB()
9848 (Subtarget.hasBWI() && VT.is512BitVector())); in lowerShuffleWithPSHUFB()
9885 VT, DAG.getNode(X86ISD::PSHUFB, DL, I8VT, DAG.getBitcast(I8VT, V), in lowerShuffleWithPSHUFB()
9894 static SDValue lowerShuffleToEXPAND(const SDLoc &DL, MVT VT, in lowerShuffleToEXPAND() argument
9905 MVT::getIntegerVT(std::max((int)VT.getVectorNumElements(), 8)); in lowerShuffleToEXPAND()
9907 unsigned NumElts = VT.getVectorNumElements(); in lowerShuffleToEXPAND()
9912 SDValue ZeroVector = getZeroVector(VT, Subtarget, DAG, DL); in lowerShuffleToEXPAND()
9914 return DAG.getNode(X86ISD::EXPAND, DL, VT, ExpandedVector, ZeroVector, VMask); in lowerShuffleToEXPAND()
9917 static bool matchShuffleWithUNPCK(MVT VT, SDValue &V1, SDValue &V2, in matchShuffleWithUNPCK() argument
9922 int NumElts = VT.getVectorNumElements(); in matchShuffleWithUNPCK()
9938 createUnpackShuffleMask(VT, Unpckl, /* Lo = */ true, IsUnary); in matchShuffleWithUNPCK()
9939 if (isTargetShuffleEquivalent(VT, TargetMask, Unpckl, DAG, V1, in matchShuffleWithUNPCK()
9942 V2 = (Undef2 ? DAG.getUNDEF(VT) : (IsUnary ? V1 : V2)); in matchShuffleWithUNPCK()
9943 V1 = (Undef1 ? DAG.getUNDEF(VT) : V1); in matchShuffleWithUNPCK()
9947 createUnpackShuffleMask(VT, Unpckh, /* Lo = */ false, IsUnary); in matchShuffleWithUNPCK()
9948 if (isTargetShuffleEquivalent(VT, TargetMask, Unpckh, DAG, V1, in matchShuffleWithUNPCK()
9951 V2 = (Undef2 ? DAG.getUNDEF(VT) : (IsUnary ? V1 : V2)); in matchShuffleWithUNPCK()
9952 V1 = (Undef1 ? DAG.getUNDEF(VT) : V1); in matchShuffleWithUNPCK()
9959 if ((Subtarget.hasSSE41() || VT == MVT::v2i64 || VT == MVT::v2f64) && in matchShuffleWithUNPCK()
9978 V2 = Zero2 ? getZeroVector(VT, Subtarget, DAG, DL) : V1; in matchShuffleWithUNPCK()
9979 V1 = Zero1 ? getZeroVector(VT, Subtarget, DAG, DL) : V1; in matchShuffleWithUNPCK()
9987 if (isTargetShuffleEquivalent(VT, TargetMask, Unpckl, DAG)) { in matchShuffleWithUNPCK()
9994 if (isTargetShuffleEquivalent(VT, TargetMask, Unpckh, DAG)) { in matchShuffleWithUNPCK()
10006 static SDValue lowerShuffleWithUNPCK(const SDLoc &DL, MVT VT, in lowerShuffleWithUNPCK() argument
10010 createUnpackShuffleMask(VT, Unpckl, /* Lo = */ true, /* Unary = */ false); in lowerShuffleWithUNPCK()
10012 return DAG.getNode(X86ISD::UNPCKL, DL, VT, V1, V2); in lowerShuffleWithUNPCK()
10015 createUnpackShuffleMask(VT, Unpckh, /* Lo = */ false, /* Unary = */ false); in lowerShuffleWithUNPCK()
10017 return DAG.getNode(X86ISD::UNPCKH, DL, VT, V1, V2); in lowerShuffleWithUNPCK()
10022 return DAG.getNode(X86ISD::UNPCKL, DL, VT, V2, V1); in lowerShuffleWithUNPCK()
10026 return DAG.getNode(X86ISD::UNPCKH, DL, VT, V2, V1); in lowerShuffleWithUNPCK()
10033 static SDValue lowerShuffleWithUNPCK256(const SDLoc &DL, MVT VT, in lowerShuffleWithUNPCK256() argument
10037 createSplat2ShuffleMask(VT, Unpckl, /* Lo */ true); in lowerShuffleWithUNPCK256()
10038 createSplat2ShuffleMask(VT, Unpckh, /* Lo */ false); in lowerShuffleWithUNPCK256()
10053 V1 = DAG.getBitcast(VT, V1); in lowerShuffleWithUNPCK256()
10054 return DAG.getNode(UnpackOpcode, DL, VT, V1, V1); in lowerShuffleWithUNPCK256()
10059 static bool matchShuffleAsVTRUNC(MVT &SrcVT, MVT &DstVT, MVT VT, in matchShuffleAsVTRUNC() argument
10062 if (!VT.is512BitVector() && !Subtarget.hasVLX()) in matchShuffleAsVTRUNC()
10066 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in matchShuffleAsVTRUNC()
10158 static SDValue lowerShuffleWithVPMOV(const SDLoc &DL, MVT VT, SDValue V1, in lowerShuffleWithVPMOV() argument
10163 assert((VT == MVT::v16i8 || VT == MVT::v8i16) && "Unexpected VTRUNC type"); in lowerShuffleWithVPMOV()
10167 unsigned NumElts = VT.getVectorNumElements(); in lowerShuffleWithVPMOV()
10168 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in lowerShuffleWithVPMOV()
10201 return getAVX512TruncNode(DL, VT, Src, Subtarget, DAG, !UndefUppers); in lowerShuffleWithVPMOV()
10208 static SDValue lowerShuffleAsVTRUNC(const SDLoc &DL, MVT VT, SDValue V1, in lowerShuffleAsVTRUNC() argument
10213 assert((VT.is128BitVector() || VT.is256BitVector()) && in lowerShuffleAsVTRUNC()
10218 unsigned NumElts = VT.getVectorNumElements(); in lowerShuffleAsVTRUNC()
10219 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in lowerShuffleAsVTRUNC()
10265 MVT ConcatVT = MVT::getVectorVT(VT.getScalarType(), NumElts * 2); in lowerShuffleAsVTRUNC()
10279 return getAVX512TruncNode(DL, VT, Src, Subtarget, DAG, !UndefUppers); in lowerShuffleAsVTRUNC()
10362 static bool matchShuffleWithPACK(MVT VT, MVT &SrcVT, SDValue &V1, SDValue &V2, in matchShuffleWithPACK() argument
10367 unsigned NumElts = VT.getVectorNumElements(); in matchShuffleWithPACK()
10368 unsigned BitSize = VT.getScalarSizeInBits(); in matchShuffleWithPACK()
10417 createPackShuffleMask(VT, BinaryMask, false, NumStages); in matchShuffleWithPACK()
10418 if (isTargetShuffleEquivalent(VT, TargetMask, BinaryMask, DAG, V1, V2)) in matchShuffleWithPACK()
10424 createPackShuffleMask(VT, UnaryMask, true, NumStages); in matchShuffleWithPACK()
10425 if (isTargetShuffleEquivalent(VT, TargetMask, UnaryMask, DAG, V1)) in matchShuffleWithPACK()
10433 static SDValue lowerShuffleWithPACK(const SDLoc &DL, MVT VT, ArrayRef<int> Mask, in lowerShuffleWithPACK() argument
10438 unsigned SizeBits = VT.getSizeInBits(); in lowerShuffleWithPACK()
10439 unsigned EltBits = VT.getScalarSizeInBits(); in lowerShuffleWithPACK()
10441 if (!matchShuffleWithPACK(VT, PackVT, V1, V2, PackOpcode, Mask, DAG, in lowerShuffleWithPACK()
10473 assert(Res && Res.getValueType() == VT && in lowerShuffleWithPACK()
10482 static SDValue lowerShuffleAsBitMask(const SDLoc &DL, MVT VT, SDValue V1, in lowerShuffleAsBitMask() argument
10487 MVT MaskVT = VT; in lowerShuffleAsBitMask()
10488 MVT EltVT = VT.getVectorElementType(); in lowerShuffleAsBitMask()
10496 MVT LogicVT = VT; in lowerShuffleAsBitMask()
10530 return DAG.getBitcast(VT, And); in lowerShuffleAsBitMask()
10538 static SDValue lowerShuffleAsBitBlend(const SDLoc &DL, MVT VT, SDValue V1, in lowerShuffleAsBitBlend() argument
10541 assert(VT.isInteger() && "Only supports integer vector types!"); in lowerShuffleAsBitBlend()
10542 MVT EltVT = VT.getVectorElementType(); in lowerShuffleAsBitBlend()
10552 SDValue V1Mask = DAG.getBuildVector(VT, DL, MaskOps); in lowerShuffleAsBitBlend()
10553 return getBitSelect(DL, VT, V1, V2, V1Mask, DAG); in lowerShuffleAsBitBlend()
10561 static bool matchShuffleAsBlend(MVT VT, SDValue V1, SDValue V2, in matchShuffleAsBlend() argument
10575 int NumLanes = VT.getSizeInBits() / 128; in matchShuffleAsBlend()
10582 VT.is256BitVector() && VT.getScalarSizeInBits() >= 32; in matchShuffleAsBlend()
10645 static SDValue lowerShuffleAsBlend(const SDLoc &DL, MVT VT, SDValue V1, in lowerShuffleAsBlend() argument
10653 if (!matchShuffleAsBlend(VT, V1, V2, Mask, Zeroable, ForceV1Zero, ForceV2Zero, in lowerShuffleAsBlend()
10659 V1 = getZeroVector(VT, Subtarget, DAG, DL); in lowerShuffleAsBlend()
10661 V2 = getZeroVector(VT, Subtarget, DAG, DL); in lowerShuffleAsBlend()
10663 unsigned NumElts = VT.getVectorNumElements(); in lowerShuffleAsBlend()
10665 switch (VT.SimpleTy) { in lowerShuffleAsBlend()
10680 return DAG.getNode(X86ISD::BLENDI, DL, VT, V1, V2, in lowerShuffleAsBlend()
10718 if (SDValue Masked = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable, in lowerShuffleAsBlend()
10731 lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG)) in lowerShuffleAsBlend()
10735 int Scale = VT.getScalarSizeInBits() / 8; in lowerShuffleAsBlend()
10739 MVT BlendVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8); in lowerShuffleAsBlend()
10772 VT, in lowerShuffleAsBlend()
10785 if (SDValue Masked = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable, in lowerShuffleAsBlend()
10806 static SDValue lowerShuffleAsBlendAndPermute(const SDLoc &DL, MVT VT, in lowerShuffleAsBlendAndPermute() argument
10832 unsigned EltSize = VT.getScalarSizeInBits(); in lowerShuffleAsBlendAndPermute()
10836 SDValue V = DAG.getVectorShuffle(VT, DL, V1, V2, BlendMask); in lowerShuffleAsBlendAndPermute()
10837 return DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), PermuteMask); in lowerShuffleAsBlendAndPermute()
10845 static SDValue lowerShuffleAsUNPCKAndPermute(const SDLoc &DL, MVT VT, in lowerShuffleAsUNPCKAndPermute() argument
10850 int NumLanes = VT.getSizeInBits() / 128; in lowerShuffleAsUNPCKAndPermute()
10855 SDValue Ops[2] = {DAG.getUNDEF(VT), DAG.getUNDEF(VT)}; in lowerShuffleAsUNPCKAndPermute()
10915 SDValue Unpck = DAG.getNode(UnpckOp, DL, VT, Ops); in lowerShuffleAsUNPCKAndPermute()
10916 return DAG.getVectorShuffle(VT, DL, Unpck, DAG.getUNDEF(VT), PermuteMask); in lowerShuffleAsUNPCKAndPermute()
10928 static SDValue lowerShuffleAsPermuteAndUnpack(const SDLoc &DL, MVT VT, in lowerShuffleAsPermuteAndUnpack() argument
10937 if (VT.isFloatingPoint() || !VT.is128BitVector() || V2.isUndef()) in lowerShuffleAsPermuteAndUnpack()
10977 V1 = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT), V1Mask); in lowerShuffleAsPermuteAndUnpack()
10978 V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Mask); in lowerShuffleAsPermuteAndUnpack()
10988 VT, DAG.getNode(UnpackLo ? X86ISD::UNPCKL : X86ISD::UNPCKH, DL, in lowerShuffleAsPermuteAndUnpack()
10994 int OrigScalarSize = VT.getScalarSizeInBits(); in lowerShuffleAsPermuteAndUnpack()
11028 VT, DL, in lowerShuffleAsPermuteAndUnpack()
11029 DAG.getNode(NumLoInputs == 0 ? X86ISD::UNPCKH : X86ISD::UNPCKL, DL, VT, in lowerShuffleAsPermuteAndUnpack()
11031 DAG.getUNDEF(VT), PermMask); in lowerShuffleAsPermuteAndUnpack()
11040 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask, in lowerShuffleAsByteRotateAndPermute() argument
11042 if ((VT.is128BitVector() && !Subtarget.hasSSSE3()) || in lowerShuffleAsByteRotateAndPermute()
11043 (VT.is256BitVector() && !Subtarget.hasAVX2()) || in lowerShuffleAsByteRotateAndPermute()
11044 (VT.is512BitVector() && !Subtarget.hasBWI())) in lowerShuffleAsByteRotateAndPermute()
11048 if (is128BitLaneCrossingShuffleMask(VT, Mask)) in lowerShuffleAsByteRotateAndPermute()
11051 int Scale = VT.getScalarSizeInBits() / 8; in lowerShuffleAsByteRotateAndPermute()
11052 int NumLanes = VT.getSizeInBits() / 128; in lowerShuffleAsByteRotateAndPermute()
11053 int NumElts = VT.getVectorNumElements(); in lowerShuffleAsByteRotateAndPermute()
11090 if (VT.getSizeInBits() > 128 && (Blend1 || Blend2)) in lowerShuffleAsByteRotateAndPermute()
11095 MVT ByteVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8); in lowerShuffleAsByteRotateAndPermute()
11097 VT, DAG.getNode(X86ISD::PALIGNR, DL, ByteVT, DAG.getBitcast(ByteVT, Hi), in lowerShuffleAsByteRotateAndPermute()
11112 return DAG.getVectorShuffle(VT, DL, Rotate, DAG.getUNDEF(VT), PermMask); in lowerShuffleAsByteRotateAndPermute()
11157 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask, in lowerShuffleAsDecomposedShuffleMerge() argument
11160 int NumLanes = VT.getSizeInBits() / 128; in lowerShuffleAsDecomposedShuffleMerge()
11185 auto canonicalizeBroadcastableInput = [DL, VT, &Subtarget, in lowerShuffleAsDecomposedShuffleMerge()
11196 Input = DAG.getNode(X86ISD::VBROADCAST, DL, VT, Input); in lowerShuffleAsDecomposedShuffleMerge()
11221 if (SDValue BlendPerm = lowerShuffleAsBlendAndPermute(DL, VT, V1, V2, Mask, in lowerShuffleAsDecomposedShuffleMerge()
11233 lowerShuffleAsUNPCKAndPermute(DL, VT, V1, V2, Mask, DAG)) in lowerShuffleAsDecomposedShuffleMerge()
11236 DL, VT, V1, V2, Mask, Subtarget, DAG)) in lowerShuffleAsDecomposedShuffleMerge()
11239 if (SDValue BlendPerm = lowerShuffleAsBlendAndPermute(DL, VT, V1, V2, Mask, in lowerShuffleAsDecomposedShuffleMerge()
11242 if (VT.getScalarSizeInBits() >= 32) in lowerShuffleAsDecomposedShuffleMerge()
11244 DL, VT, V1, V2, Mask, Subtarget, DAG)) in lowerShuffleAsDecomposedShuffleMerge()
11252 if (IsAlternating && VT.getScalarSizeInBits() < 32) { in lowerShuffleAsDecomposedShuffleMerge()
11269 V1 = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT), V1Mask); in lowerShuffleAsDecomposedShuffleMerge()
11270 V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Mask); in lowerShuffleAsDecomposedShuffleMerge()
11271 return DAG.getVectorShuffle(VT, DL, V1, V2, FinalMask); in lowerShuffleAsDecomposedShuffleMerge()
11294 static SDValue lowerShuffleAsBitRotate(const SDLoc &DL, MVT VT, SDValue V1, in lowerShuffleAsBitRotate() argument
11301 (VT.is128BitVector() && Subtarget.hasXOP()) || Subtarget.hasAVX512(); in lowerShuffleAsBitRotate()
11306 int RotateAmt = matchShuffleAsBitRotate(RotateVT, VT.getScalarSizeInBits(), in lowerShuffleAsBitRotate()
11326 return DAG.getBitcast(VT, Rot); in lowerShuffleAsBitRotate()
11332 return DAG.getBitcast(VT, Rot); in lowerShuffleAsBitRotate()
11423 static int matchShuffleAsByteRotate(MVT VT, SDValue &V1, SDValue &V2, in matchShuffleAsByteRotate() argument
11431 if (!is128BitLaneRepeatedShuffleMask(VT, Mask, RepeatedMask)) in matchShuffleAsByteRotate()
11445 static SDValue lowerShuffleAsByteRotate(const SDLoc &DL, MVT VT, SDValue V1, in lowerShuffleAsByteRotate() argument
11452 int ByteRotation = matchShuffleAsByteRotate(VT, Lo, Hi, Mask); in lowerShuffleAsByteRotate()
11458 MVT ByteVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8); in lowerShuffleAsByteRotate()
11464 assert((!VT.is512BitVector() || Subtarget.hasBWI()) && in lowerShuffleAsByteRotate()
11467 VT, DAG.getNode(X86ISD::PALIGNR, DL, ByteVT, Lo, Hi, in lowerShuffleAsByteRotate()
11471 assert(VT.is128BitVector() && in lowerShuffleAsByteRotate()
11488 return DAG.getBitcast(VT, in lowerShuffleAsByteRotate()
11502 static SDValue lowerShuffleAsVALIGN(const SDLoc &DL, MVT VT, SDValue V1, in lowerShuffleAsVALIGN() argument
11507 assert((VT.getScalarType() == MVT::i32 || VT.getScalarType() == MVT::i64) && in lowerShuffleAsVALIGN()
11511 assert((Subtarget.hasVLX() || (!VT.is128BitVector() && !VT.is256BitVector())) in lowerShuffleAsVALIGN()
11517 return DAG.getNode(X86ISD::VALIGN, DL, VT, Lo, Hi, in lowerShuffleAsVALIGN()
11535 return DAG.getNode(X86ISD::VALIGN, DL, VT, Src, in lowerShuffleAsVALIGN()
11536 getZeroVector(VT, Subtarget, DAG, DL), in lowerShuffleAsVALIGN()
11544 return DAG.getNode(X86ISD::VALIGN, DL, VT, in lowerShuffleAsVALIGN()
11545 getZeroVector(VT, Subtarget, DAG, DL), Src, in lowerShuffleAsVALIGN()
11553 static SDValue lowerShuffleAsByteShiftMask(const SDLoc &DL, MVT VT, SDValue V1, in lowerShuffleAsByteShiftMask() argument
11559 assert(VT.is128BitVector() && "Only 128-bit vectors supported"); in lowerShuffleAsByteShiftMask()
11573 unsigned Scale = VT.getScalarSizeInBits() / 8; in lowerShuffleAsByteShiftMask()
11614 return DAG.getBitcast(VT, Res); in lowerShuffleAsByteShiftMask()
11702 static SDValue lowerShuffleAsShift(const SDLoc &DL, MVT VT, SDValue V1, in lowerShuffleAsShift() argument
11708 assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size"); in lowerShuffleAsShift()
11715 int ShiftAmt = matchShuffleAsShift(ShiftVT, Opcode, VT.getScalarSizeInBits(), in lowerShuffleAsShift()
11720 ShiftAmt = matchShuffleAsShift(ShiftVT, Opcode, VT.getScalarSizeInBits(), in lowerShuffleAsShift()
11736 return DAG.getBitcast(VT, V); in lowerShuffleAsShift()
11741 static bool matchShuffleAsEXTRQ(MVT VT, SDValue &V1, SDValue &V2, in matchShuffleAsEXTRQ() argument
11746 assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size"); in matchShuffleAsEXTRQ()
11788 BitLen = (Len * VT.getScalarSizeInBits()) & 0x3f; in matchShuffleAsEXTRQ()
11789 BitIdx = (Idx * VT.getScalarSizeInBits()) & 0x3f; in matchShuffleAsEXTRQ()
11797 static bool matchShuffleAsINSERTQ(MVT VT, SDValue &V1, SDValue &V2, in matchShuffleAsINSERTQ() argument
11802 assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size"); in matchShuffleAsINSERTQ()
11851 BitLen = (Len * VT.getScalarSizeInBits()) & 0x3f; in matchShuffleAsINSERTQ()
11852 BitIdx = (Idx * VT.getScalarSizeInBits()) & 0x3f; in matchShuffleAsINSERTQ()
11863 static SDValue lowerShuffleWithSSE4A(const SDLoc &DL, MVT VT, SDValue V1, in lowerShuffleWithSSE4A() argument
11867 if (matchShuffleAsEXTRQ(VT, V1, V2, Mask, BitLen, BitIdx, Zeroable)) in lowerShuffleWithSSE4A()
11868 return DAG.getNode(X86ISD::EXTRQI, DL, VT, V1, in lowerShuffleWithSSE4A()
11872 if (matchShuffleAsINSERTQ(VT, V1, V2, Mask, BitLen, BitIdx)) in lowerShuffleWithSSE4A()
11873 return DAG.getNode(X86ISD::INSERTQI, DL, VT, V1 ? V1 : DAG.getUNDEF(VT), in lowerShuffleWithSSE4A()
11874 V2 ? V2 : DAG.getUNDEF(VT), in lowerShuffleWithSSE4A()
11891 const SDLoc &DL, MVT VT, int Scale, int Offset, bool AnyExt, SDValue InputV, in lowerShuffleAsSpecificZeroOrAnyExtend() argument
11894 int EltBits = VT.getScalarSizeInBits(); in lowerShuffleAsSpecificZeroOrAnyExtend()
11895 int NumElements = VT.getVectorNumElements(); in lowerShuffleAsSpecificZeroOrAnyExtend()
11920 return DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), ShMask); in lowerShuffleAsSpecificZeroOrAnyExtend()
11928 if (Offset && Scale == 2 && VT.is128BitVector()) in lowerShuffleAsSpecificZeroOrAnyExtend()
11932 InputV = DAG.getBitcast(VT, InputV); in lowerShuffleAsSpecificZeroOrAnyExtend()
11936 return DAG.getBitcast(VT, InputV); in lowerShuffleAsSpecificZeroOrAnyExtend()
11939 assert(VT.is128BitVector() && "Only 128-bit vectors can be extended."); in lowerShuffleAsSpecificZeroOrAnyExtend()
11940 InputV = DAG.getBitcast(VT, InputV); in lowerShuffleAsSpecificZeroOrAnyExtend()
11948 VT, DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, in lowerShuffleAsSpecificZeroOrAnyExtend()
11961 VT, DAG.getNode(OddEvenOp, DL, MVT::v8i16, in lowerShuffleAsSpecificZeroOrAnyExtend()
11970 assert(VT.is128BitVector() && "Unexpected vector width!"); in lowerShuffleAsSpecificZeroOrAnyExtend()
11974 MVT::v2i64, DAG.getNode(X86ISD::EXTRQI, DL, VT, InputV, in lowerShuffleAsSpecificZeroOrAnyExtend()
11979 return DAG.getBitcast(VT, Lo); in lowerShuffleAsSpecificZeroOrAnyExtend()
11983 MVT::v2i64, DAG.getNode(X86ISD::EXTRQI, DL, VT, InputV, in lowerShuffleAsSpecificZeroOrAnyExtend()
11986 return DAG.getBitcast(VT, in lowerShuffleAsSpecificZeroOrAnyExtend()
12007 VT, DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8, InputV, in lowerShuffleAsSpecificZeroOrAnyExtend()
12018 InputV = DAG.getVectorShuffle(VT, DL, InputV, DAG.getUNDEF(VT), ShMask); in lowerShuffleAsSpecificZeroOrAnyExtend()
12039 return DAG.getBitcast(VT, InputV); in lowerShuffleAsSpecificZeroOrAnyExtend()
12055 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask, in lowerShuffleAsZeroOrAnyExtend() argument
12058 int Bits = VT.getSizeInBits(); in lowerShuffleAsZeroOrAnyExtend()
12060 int NumElements = VT.getVectorNumElements(); in lowerShuffleAsZeroOrAnyExtend()
12062 assert(VT.getScalarSizeInBits() <= 32 && in lowerShuffleAsZeroOrAnyExtend()
12125 return lowerShuffleAsSpecificZeroOrAnyExtend(DL, VT, Scale, Offset, AnyExt, in lowerShuffleAsZeroOrAnyExtend()
12163 return DAG.getBitcast(VT, V); in lowerShuffleAsZeroOrAnyExtend()
12175 MVT VT = V.getSimpleValueType(); in getScalarValueForVectorElement() local
12176 MVT EltVT = VT.getVectorElementType(); in getScalarValueForVectorElement()
12182 if (!NewVT.isVector() || NewVT.getScalarSizeInBits() != VT.getScalarSizeInBits()) in getScalarValueForVectorElement()
12207 static bool isSoftF16(T VT, const X86Subtarget &Subtarget) { in isSoftF16() argument
12208 T EltVT = VT.getScalarType(); in isSoftF16()
12217 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask, in lowerShuffleAsElementInsertion() argument
12220 MVT ExtVT = VT; in lowerShuffleAsElementInsertion()
12221 MVT EltVT = VT.getVectorElementType(); in lowerShuffleAsElementInsertion()
12222 unsigned NumElts = VT.getVectorNumElements(); in lowerShuffleAsElementInsertion()
12223 unsigned EltBits = VT.getScalarSizeInBits(); in lowerShuffleAsElementInsertion()
12273 SDValue BitMask = getConstVector(Bits, VT, DAG, DL); in lowerShuffleAsElementInsertion()
12274 V1 = DAG.getNode(ISD::AND, DL, VT, V1, BitMask); in lowerShuffleAsElementInsertion()
12276 V2 = DAG.getBitcast(VT, DAG.getNode(X86ISD::VZEXT_MOVL, DL, ExtVT, V2)); in lowerShuffleAsElementInsertion()
12277 return DAG.getNode(ISD::OR, DL, VT, V1, V2); in lowerShuffleAsElementInsertion()
12291 assert(VT == ExtVT && "Cannot change extended type when non-zeroable!"); in lowerShuffleAsElementInsertion()
12292 if (!VT.isFloatingPoint() || V2Index != 0) in lowerShuffleAsElementInsertion()
12294 if (!VT.is128BitVector()) in lowerShuffleAsElementInsertion()
12311 if (VT.isFloatingPoint() && V2Index != 0) in lowerShuffleAsElementInsertion()
12315 if (ExtVT != VT) in lowerShuffleAsElementInsertion()
12316 V2 = DAG.getBitcast(VT, V2); in lowerShuffleAsElementInsertion()
12323 if (VT.isFloatingPoint() || NumElts <= 4) { in lowerShuffleAsElementInsertion()
12326 V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Shuffle); in lowerShuffleAsElementInsertion()
12332 V2 = DAG.getBitcast(VT, V2); in lowerShuffleAsElementInsertion()
12342 static SDValue lowerShuffleAsTruncBroadcast(const SDLoc &DL, MVT VT, SDValue V0, in lowerShuffleAsTruncBroadcast() argument
12349 MVT EltVT = VT.getVectorElementType(); in lowerShuffleAsTruncBroadcast()
12352 assert(VT.isInteger() && "Unexpected non-integer trunc broadcast!"); in lowerShuffleAsTruncBroadcast()
12387 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, in lowerShuffleAsTruncBroadcast()
12434 MVT VT = N0.getSimpleValueType(); in lowerShuffleOfExtractsAsVperm() local
12435 assert((VT.is128BitVector() && in lowerShuffleOfExtractsAsVperm()
12436 (VT.getScalarSizeInBits() == 32 || VT.getScalarSizeInBits() == 64)) && in lowerShuffleOfExtractsAsVperm()
12453 unsigned NumElts = VT.getVectorNumElements(); in lowerShuffleOfExtractsAsVperm()
12476 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Shuf, in lowerShuffleOfExtractsAsVperm()
12485 static SDValue lowerShuffleAsBroadcast(const SDLoc &DL, MVT VT, SDValue V1, in lowerShuffleAsBroadcast() argument
12489 MVT EltVT = VT.getVectorElementType(); in lowerShuffleAsBroadcast()
12490 if (!((Subtarget.hasSSE3() && VT == MVT::v2f64) || in lowerShuffleAsBroadcast()
12492 (Subtarget.hasAVX2() && (VT.isInteger() || EltVT == MVT::f16)))) in lowerShuffleAsBroadcast()
12497 unsigned NumEltBits = VT.getScalarSizeInBits(); in lowerShuffleAsBroadcast()
12498 unsigned Opcode = (VT == MVT::v2f64 && !Subtarget.hasAVX2()) in lowerShuffleAsBroadcast()
12568 if (BitCastSrc && VT.isInteger()) in lowerShuffleAsBroadcast()
12570 DL, VT, V, BroadcastIdx, Subtarget, DAG)) in lowerShuffleAsBroadcast()
12591 MVT SVT = VT.getScalarType(); in lowerShuffleAsBroadcast()
12601 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in lowerShuffleAsBroadcast()
12608 return DAG.getBitcast(VT, V); in lowerShuffleAsBroadcast()
12622 if (!VT.is256BitVector() && !VT.is512BitVector()) in lowerShuffleAsBroadcast()
12626 if (VT == MVT::v4f64 || VT == MVT::v4i64) in lowerShuffleAsBroadcast()
12646 return DAG.getBitcast(VT, V); in lowerShuffleAsBroadcast()
12656 VT.getVectorNumElements()); in lowerShuffleAsBroadcast()
12657 return DAG.getBitcast(VT, DAG.getNode(Opcode, DL, BroadcastVT, V)); in lowerShuffleAsBroadcast()
12669 MVT CastVT = MVT::getVectorVT(VT.getVectorElementType(), NumSrcElts); in lowerShuffleAsBroadcast()
12670 return DAG.getNode(Opcode, DL, VT, DAG.getBitcast(CastVT, V)); in lowerShuffleAsBroadcast()
12974 static SDValue lowerShuffleWithSHUFPS(const SDLoc &DL, MVT VT, in lowerShuffleWithSHUFPS() argument
13000 V2 = DAG.getNode(X86ISD::SHUFP, DL, VT, V2, V1, in lowerShuffleWithSHUFPS()
13039 V1 = DAG.getNode(X86ISD::SHUFP, DL, VT, V1, V2, in lowerShuffleWithSHUFPS()
13055 return lowerShuffleWithSHUFPS(DL, VT, NewMask, V2, V1, DAG); in lowerShuffleWithSHUFPS()
13057 return DAG.getNode(X86ISD::SHUFP, DL, VT, LowV, HighV, in lowerShuffleWithSHUFPS()
13312 const SDLoc &DL, MVT VT, SDValue V, MutableArrayRef<int> Mask, in lowerV8I16GeneralSingleInputShuffle() argument
13314 assert(VT.getVectorElementType() == MVT::i16 && "Bad input type!"); in lowerV8I16GeneralSingleInputShuffle()
13315 MVT PSHUFDVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() / 2); in lowerV8I16GeneralSingleInputShuffle()
13324 return DAG.getNode(X86ISD::PSHUFLW, DL, VT, V, in lowerV8I16GeneralSingleInputShuffle()
13331 return DAG.getNode(X86ISD::PSHUFHW, DL, VT, V, in lowerV8I16GeneralSingleInputShuffle()
13357 V = DAG.getNode(ShufWOp, DL, VT, V, in lowerV8I16GeneralSingleInputShuffle()
13362 return DAG.getBitcast(VT, V); in lowerV8I16GeneralSingleInputShuffle()
13534 VT, in lowerV8I16GeneralSingleInputShuffle()
13547 return lowerV8I16GeneralSingleInputShuffle(DL, VT, V, Mask, Subtarget, DAG); in lowerV8I16GeneralSingleInputShuffle()
13769 V = DAG.getNode(X86ISD::PSHUFLW, DL, VT, V, in lowerV8I16GeneralSingleInputShuffle()
13772 V = DAG.getNode(X86ISD::PSHUFHW, DL, VT, V, in lowerV8I16GeneralSingleInputShuffle()
13776 VT, in lowerV8I16GeneralSingleInputShuffle()
13789 V = DAG.getNode(X86ISD::PSHUFLW, DL, VT, V, in lowerV8I16GeneralSingleInputShuffle()
13797 V = DAG.getNode(X86ISD::PSHUFHW, DL, VT, V, in lowerV8I16GeneralSingleInputShuffle()
13806 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask, in lowerShuffleAsBlendOfPSHUFBs() argument
13808 assert(!is128BitLaneCrossingShuffleMask(VT, Mask) && in lowerShuffleAsBlendOfPSHUFBs()
13811 int NumBytes = VT.getSizeInBits() / 8; in lowerShuffleAsBlendOfPSHUFBs()
13853 return DAG.getBitcast(VT, V); in lowerShuffleAsBlendOfPSHUFBs()
14103 static SDValue lowerShuffleWithPERMV(const SDLoc &DL, MVT VT, in lowerShuffleWithPERMV() argument
14107 MVT MaskVT = VT.changeTypeToInteger(); in lowerShuffleWithPERMV()
14109 MVT ShuffleVT = VT; in lowerShuffleWithPERMV()
14110 if (!VT.is512BitVector() && !Subtarget.hasVLX()) { in lowerShuffleWithPERMV()
14116 int NumElts = VT.getVectorNumElements(); in lowerShuffleWithPERMV()
14117 unsigned Scale = 512 / VT.getSizeInBits(); in lowerShuffleWithPERMV()
14134 if (VT != ShuffleVT) in lowerShuffleWithPERMV()
14135 Result = extractSubVector(Result, 0, DAG, DL, VT.getSizeInBits()); in lowerShuffleWithPERMV()
14502 MVT VT, SDValue V1, SDValue V2, in lower128BitShuffle() argument
14506 if (VT == MVT::v8bf16) { in lower128BitShuffle()
14509 return DAG.getBitcast(VT, in lower128BitShuffle()
14513 switch (VT.SimpleTy) { in lower128BitShuffle()
14539 static SDValue splitAndLowerShuffle(const SDLoc &DL, MVT VT, SDValue V1, in splitAndLowerShuffle() argument
14542 assert(VT.getSizeInBits() >= 256 && in splitAndLowerShuffle()
14544 assert(V1.getSimpleValueType() == VT && "Bad operand type!"); in splitAndLowerShuffle()
14545 assert(V2.getSimpleValueType() == VT && "Bad operand type!"); in splitAndLowerShuffle()
14550 int NumElements = VT.getVectorNumElements(); in splitAndLowerShuffle()
14552 MVT ScalarVT = VT.getVectorElementType(); in splitAndLowerShuffle()
14657 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi); in splitAndLowerShuffle()
14668 static SDValue lowerShuffleAsSplitOrBlend(const SDLoc &DL, MVT VT, SDValue V1, in lowerShuffleAsSplitOrBlend() argument
14696 return lowerShuffleAsDecomposedShuffleMerge(DL, VT, V1, V2, Mask, Subtarget, in lowerShuffleAsSplitOrBlend()
14702 int LaneCount = VT.getSizeInBits() / 128; in lowerShuffleAsSplitOrBlend()
14711 return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG, in lowerShuffleAsSplitOrBlend()
14716 return lowerShuffleAsDecomposedShuffleMerge(DL, VT, V1, V2, Mask, Subtarget, in lowerShuffleAsSplitOrBlend()
14722 static SDValue lowerShuffleAsLanePermuteAndSHUFP(const SDLoc &DL, MVT VT, in lowerShuffleAsLanePermuteAndSHUFP() argument
14726 assert(VT == MVT::v4f64 && "Only for v4f64 shuffles"); in lowerShuffleAsLanePermuteAndSHUFP()
14744 SDValue LHS = DAG.getVectorShuffle(VT, DL, V1, V2, LHSMask); in lowerShuffleAsLanePermuteAndSHUFP()
14745 SDValue RHS = DAG.getVectorShuffle(VT, DL, V1, V2, RHSMask); in lowerShuffleAsLanePermuteAndSHUFP()
14746 return DAG.getNode(X86ISD::SHUFP, DL, VT, LHS, RHS, in lowerShuffleAsLanePermuteAndSHUFP()
14759 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask, in lowerShuffleAsLanePermuteAndPermute() argument
14761 int NumElts = VT.getVectorNumElements(); in lowerShuffleAsLanePermuteAndPermute()
14762 int NumLanes = VT.getSizeInBits() / 128; in lowerShuffleAsLanePermuteAndPermute()
14835 SDValue CrossLane = DAG.getVectorShuffle(VT, DL, V1, V2, CrossLaneMask); in lowerShuffleAsLanePermuteAndPermute()
14836 return DAG.getVectorShuffle(VT, DL, CrossLane, DAG.getUNDEF(VT), in lowerShuffleAsLanePermuteAndPermute()
14882 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask, in lowerShuffleAsLanePermuteAndShuffle() argument
14885 assert(VT.is256BitVector() && "Only for 256-bit vector shuffles!"); in lowerShuffleAsLanePermuteAndShuffle()
14892 if (VT == MVT::v4f64 && in lowerShuffleAsLanePermuteAndShuffle()
14894 return lowerShuffleAsLanePermuteAndSHUFP(DL, VT, V1, V2, Mask, DAG); in lowerShuffleAsLanePermuteAndShuffle()
14921 assert(!is128BitLaneCrossingShuffleMask(VT, InLaneMask) && in lowerShuffleAsLanePermuteAndShuffle()
14926 if (!AllLanes && !is128BitLaneRepeatedShuffleMask(VT, InLaneMask)) in lowerShuffleAsLanePermuteAndShuffle()
14927 return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG, in lowerShuffleAsLanePermuteAndShuffle()
14931 MVT PVT = VT.isFloatingPoint() ? MVT::v4f64 : MVT::v4i64; in lowerShuffleAsLanePermuteAndShuffle()
14935 Flipped = DAG.getBitcast(VT, Flipped); in lowerShuffleAsLanePermuteAndShuffle()
14936 return DAG.getVectorShuffle(VT, DL, V1, Flipped, InLaneMask); in lowerShuffleAsLanePermuteAndShuffle()
14940 static SDValue lowerV2X128Shuffle(const SDLoc &DL, MVT VT, SDValue V1, in lowerV2X128Shuffle() argument
14951 MVT MemVT = VT.getHalfNumVectorElementsVT(); in lowerV2X128Shuffle()
14955 VT, MemVT, Ld, Ofs, DAG)) in lowerV2X128Shuffle()
14975 MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), 2); in lowerV2X128Shuffle()
14978 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, in lowerV2X128Shuffle()
14979 getZeroVector(VT, Subtarget, DAG, DL), LoV, in lowerV2X128Shuffle()
14988 if (SDValue Blend = lowerShuffleAsBlend(DL, VT, V1, V2, Mask, Zeroable, in lowerV2X128Shuffle()
15003 MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), 2); in lowerV2X128Shuffle()
15007 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, V1, SubVec, in lowerV2X128Shuffle()
15017 return DAG.getNode(X86ISD::SHUF128, DL, VT, V1, V2, in lowerV2X128Shuffle()
15045 V1 = DAG.getUNDEF(VT); in lowerV2X128Shuffle()
15047 V2 = DAG.getUNDEF(VT); in lowerV2X128Shuffle()
15049 return DAG.getNode(X86ISD::VPERM2X128, DL, VT, V1, V2, in lowerV2X128Shuffle()
15061 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask, in lowerShuffleAsLanePermuteAndRepeatedMask() argument
15065 if (is128BitLaneRepeatedShuffleMask(VT, Mask)) in lowerShuffleAsLanePermuteAndRepeatedMask()
15069 int NumLanes = VT.getSizeInBits() / 128; in lowerShuffleAsLanePermuteAndRepeatedMask()
15070 int NumLaneElts = 128 / VT.getScalarSizeInBits(); in lowerShuffleAsLanePermuteAndRepeatedMask()
15187 SDValue NewV1 = DAG.getVectorShuffle(VT, DL, V1, V2, NewMask); in lowerShuffleAsLanePermuteAndRepeatedMask()
15204 SDValue NewV2 = DAG.getVectorShuffle(VT, DL, V1, V2, NewMask); in lowerShuffleAsLanePermuteAndRepeatedMask()
15223 return DAG.getVectorShuffle(VT, DL, NewV1, NewV2, NewMask); in lowerShuffleAsLanePermuteAndRepeatedMask()
15291 MVT VT = V1.getSimpleValueType(); in getShuffleHalfVectors() local
15292 MVT HalfVT = VT.getHalfNumVectorElementsVT(); in getShuffleHalfVectors()
15313 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Op0, Op1); in getShuffleHalfVectors()
15317 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), V, in getShuffleHalfVectors()
15324 static SDValue lowerShuffleWithUndefHalf(const SDLoc &DL, MVT VT, SDValue V1, in lowerShuffleWithUndefHalf() argument
15328 assert((VT.is256BitVector() || VT.is512BitVector()) && in lowerShuffleWithUndefHalf()
15340 MVT HalfVT = VT.getHalfNumVectorElementsVT(); in lowerShuffleWithUndefHalf()
15346 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), Hi, in lowerShuffleWithUndefHalf()
15356 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), Hi, in lowerShuffleWithUndefHalf()
15376 unsigned EltWidth = VT.getVectorElementType().getSizeInBits(); in lowerShuffleWithUndefHalf()
15401 if (Subtarget.hasAVX512() && VT.is512BitVector()) in lowerShuffleWithUndefHalf()
15420 if (Subtarget.hasAVX512() && VT.is512BitVector()) in lowerShuffleWithUndefHalf()
15436 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask, in lowerShuffleAsRepeatedMaskAndLanePermute() argument
15438 int NumElts = VT.getVectorNumElements(); in lowerShuffleAsRepeatedMaskAndLanePermute()
15439 int NumLanes = VT.getSizeInBits() / 128; in lowerShuffleAsRepeatedMaskAndLanePermute()
15446 if (BroadcastSize <= VT.getScalarSizeInBits()) in lowerShuffleAsRepeatedMaskAndLanePermute()
15448 int NumBroadcastElts = BroadcastSize / VT.getScalarSizeInBits(); in lowerShuffleAsRepeatedMaskAndLanePermute()
15474 SDValue RepeatShuf = DAG.getVectorShuffle(VT, DL, V1, V2, RepeatMask); in lowerShuffleAsRepeatedMaskAndLanePermute()
15487 return DAG.getVectorShuffle(VT, DL, RepeatShuf, DAG.getUNDEF(VT), in lowerShuffleAsRepeatedMaskAndLanePermute()
15493 if (!is128BitLaneCrossingShuffleMask(VT, Mask)) in lowerShuffleAsRepeatedMaskAndLanePermute()
15497 if (is128BitLaneRepeatedShuffleMask(VT, Mask)) in lowerShuffleAsRepeatedMaskAndLanePermute()
15607 DAG.getVectorShuffle(VT, DL, V1, V2, RepeatedMask); in lowerShuffleAsRepeatedMaskAndLanePermute()
15609 return DAG.getVectorShuffle(VT, DL, RepeatedShuffle, DAG.getUNDEF(VT), in lowerShuffleAsRepeatedMaskAndLanePermute()
15618 if (Subtarget.hasAVX2() && VT.is256BitVector()) { in lowerShuffleAsRepeatedMaskAndLanePermute()
15622 (!OnlyLowestElts && V2.isUndef() && VT == MVT::v32i8) ? 4 : 2; in lowerShuffleAsRepeatedMaskAndLanePermute()
15624 if (Subtarget.hasBWI() && VT == MVT::v64i8) in lowerShuffleAsRepeatedMaskAndLanePermute()
15634 static bool matchShuffleWithSHUFPD(MVT VT, SDValue &V1, SDValue &V2, in matchShuffleWithSHUFPD() argument
15638 int NumElts = VT.getVectorNumElements(); in matchShuffleWithSHUFPD()
15639 assert(VT.getScalarSizeInBits() == 64 && in matchShuffleWithSHUFPD()
15679 static SDValue lowerShuffleWithSHUFPD(const SDLoc &DL, MVT VT, SDValue V1, in lowerShuffleWithSHUFPD() argument
15684 assert((VT == MVT::v2f64 || VT == MVT::v4f64 || VT == MVT::v8f64) && in lowerShuffleWithSHUFPD()
15689 if (!matchShuffleWithSHUFPD(VT, V1, V2, ForceV1Zero, ForceV2Zero, Immediate, in lowerShuffleWithSHUFPD()
15695 V1 = getZeroVector(VT, Subtarget, DAG, DL); in lowerShuffleWithSHUFPD()
15697 V2 = getZeroVector(VT, Subtarget, DAG, DL); in lowerShuffleWithSHUFPD()
15699 return DAG.getNode(X86ISD::SHUFP, DL, VT, V1, V2, in lowerShuffleWithSHUFPD()
15706 static SDValue lowerShuffleAsVTRUNCAndUnpack(const SDLoc &DL, MVT VT, in lowerShuffleAsVTRUNCAndUnpack() argument
15711 assert(VT == MVT::v32i8 && "Unexpected type!"); in lowerShuffleAsVTRUNCAndUnpack()
15754 static SDValue lowerShufflePairAsUNPCKAndPermute(const SDLoc &DL, MVT VT, in lowerShufflePairAsUNPCKAndPermute() argument
15758 if (VT != MVT::v8f32 && VT != MVT::v8i32 && VT != MVT::v16i16 && in lowerShufflePairAsUNPCKAndPermute()
15759 VT != MVT::v32i8) in lowerShufflePairAsUNPCKAndPermute()
15774 int NumElts = VT.getVectorNumElements(); in lowerShufflePairAsUNPCKAndPermute()
15809 SDValue Unpckl = DAG.getNode(X86ISD::UNPCKL, DL, VT, V1, V2); in lowerShufflePairAsUNPCKAndPermute()
15810 SDValue Unpckh = DAG.getNode(X86ISD::UNPCKH, DL, VT, V1, V2); in lowerShufflePairAsUNPCKAndPermute()
15811 SDValue Perm1 = DAG.getNode(X86ISD::VPERM2X128, DL, VT, Unpckl, Unpckh, in lowerShufflePairAsUNPCKAndPermute()
15813 SDValue Perm2 = DAG.getNode(X86ISD::VPERM2X128, DL, VT, Unpckl, Unpckh, in lowerShufflePairAsUNPCKAndPermute()
16569 static SDValue lower256BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT, in lower256BitShuffle() argument
16575 int NumElts = VT.getVectorNumElements(); in lower256BitShuffle()
16580 DL, VT, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lower256BitShuffle()
16585 lowerShuffleWithUndefHalf(DL, VT, V1, V2, Mask, Subtarget, DAG)) in lower256BitShuffle()
16594 if (VT.isInteger() && !Subtarget.hasAVX2()) { in lower256BitShuffle()
16595 int ElementBits = VT.getScalarSizeInBits(); in lower256BitShuffle()
16599 if (SDValue V = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable, in lower256BitShuffle()
16602 if (SDValue V = lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG)) in lower256BitShuffle()
16604 return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG, /*SimpleOnly*/ false); in lower256BitShuffle()
16608 VT.getVectorNumElements()); in lower256BitShuffle()
16611 return DAG.getBitcast(VT, DAG.getVectorShuffle(FpVT, DL, V1, V2, Mask)); in lower256BitShuffle()
16614 if (VT == MVT::v16f16 || VT == MVT::v16bf16) { in lower256BitShuffle()
16617 return DAG.getBitcast(VT, in lower256BitShuffle()
16621 switch (VT.SimpleTy) { in lower256BitShuffle()
16641 static SDValue lowerV4X128Shuffle(const SDLoc &DL, MVT VT, ArrayRef<int> Mask, in lowerV4X128Shuffle() argument
16645 assert(VT.getScalarSizeInBits() == 64 && in lowerV4X128Shuffle()
16650 assert(VT.is512BitVector() && "Unexpected vector size for 512bit shuffle."); in lowerV4X128Shuffle()
16662 MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), NumElts); in lowerV4X128Shuffle()
16665 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, in lowerV4X128Shuffle()
16666 getZeroVector(VT, Subtarget, DAG, DL), LoV, in lowerV4X128Shuffle()
16675 MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), 4); in lowerV4X128Shuffle()
16679 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, V1, SubVec, in lowerV4X128Shuffle()
16707 MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), 2); in lowerV4X128Shuffle()
16724 SDValue Ops[2] = {DAG.getUNDEF(VT), DAG.getUNDEF(VT)}; in lowerV4X128Shuffle()
16742 return DAG.getNode(X86ISD::SHUF128, DL, VT, Ops[0], Ops[1], in lowerV4X128Shuffle()
17201 MVT VT, SDValue V1, SDValue V2, in lower512BitShuffle() argument
17215 DL, VT, V1, V2, Mask, Zeroable, Subtarget, DAG)) in lower512BitShuffle()
17220 lowerShuffleWithUndefHalf(DL, VT, V1, V2, Mask, Subtarget, DAG)) in lower512BitShuffle()
17224 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, VT, V1, V2, Mask, in lower512BitShuffle()
17228 if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI()) { in lower512BitShuffle()
17231 if (SDValue V = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable, in lower512BitShuffle()
17234 if (SDValue V = lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG)) in lower512BitShuffle()
17237 return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG, /*SimpleOnly*/ false); in lower512BitShuffle()
17240 if (VT == MVT::v32f16 || VT == MVT::v32bf16) { in lower512BitShuffle()
17242 return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG, in lower512BitShuffle()
17247 return DAG.getBitcast(VT, in lower512BitShuffle()
17255 switch (VT.SimpleTy) { in lower512BitShuffle()
17275 MVT VT, SDValue V1, SDValue V2, in lower1BitShuffleAsKSHIFTR() argument
17308 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res, in lower1BitShuffleAsKSHIFTR()
17350 MVT VT, SDValue V1, SDValue V2, in lower1BitShuffle() argument
17388 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, in lower1BitShuffle()
17389 DAG.getConstant(0, DL, VT), in lower1BitShuffle()
17394 if (SDValue Shift = lower1BitShuffleAsKSHIFTR(DL, Mask, VT, V1, V2, Subtarget, in lower1BitShuffle()
17407 if (Opcode == X86ISD::KSHIFTR && WideVT != VT) { in lower1BitShuffle()
17418 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res, in lower1BitShuffle()
17434 DL, VT, DAG.getVectorShuffle(OpVT, DL, Op0, DAG.getUNDEF(OpVT), Mask), in lower1BitShuffle()
17439 switch (VT.SimpleTy) { in lower1BitShuffle()
17478 int NumElems = VT.getVectorNumElements(); in lower1BitShuffle()
17481 return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, ExtVT), in lower1BitShuffle()
17484 return DAG.getNode(ISD::TRUNCATE, DL, VT, Shuffle); in lower1BitShuffle()
17559 MVT VT = V.getSimpleValueType().getScalarType(); in canCombineAsMaskOperation() local
17560 if ((VT == MVT::i16 || VT == MVT::i8) && !Subtarget.hasBWI()) in canCombineAsMaskOperation()
17565 if ((VT == MVT::i16 || VT == MVT::i8) && in canCombineAsMaskOperation()
17622 MVT VT = Op.getSimpleValueType(); in lowerVECTOR_SHUFFLE() local
17623 int NumElements = VT.getVectorNumElements(); in lowerVECTOR_SHUFFLE()
17625 bool Is1BitVector = (VT.getVectorElementType() == MVT::i1); in lowerVECTOR_SHUFFLE()
17627 assert((VT.getSizeInBits() != 64 || Is1BitVector) && in lowerVECTOR_SHUFFLE()
17633 return DAG.getUNDEF(VT); in lowerVECTOR_SHUFFLE()
17650 return DAG.getVectorShuffle(VT, DL, V1, V2, NewMask); in lowerVECTOR_SHUFFLE()
17668 return getZeroVector(VT, Subtarget, DAG, DL); in lowerVECTOR_SHUFFLE()
17677 if (VT.getScalarSizeInBits() < 64 && !Is1BitVector && in lowerVECTOR_SHUFFLE()
17685 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, VT, V1, V2, OrigMask, in lowerVECTOR_SHUFFLE()
17689 MVT NewEltVT = VT.isFloatingPoint() in lowerVECTOR_SHUFFLE()
17690 ? MVT::getFloatingPointVT(VT.getScalarSizeInBits() * 2) in lowerVECTOR_SHUFFLE()
17691 : MVT::getIntegerVT(VT.getScalarSizeInBits() * 2); in lowerVECTOR_SHUFFLE()
17716 VT, DAG.getVectorShuffle(NewVT, DL, V1, V2, WidenedMask)); in lowerVECTOR_SHUFFLE()
17726 Ops, Mask, VT.getSizeInBits(), DL, DAG, Subtarget)) in lowerVECTOR_SHUFFLE()
17727 return DAG.getBitcast(VT, HOp); in lowerVECTOR_SHUFFLE()
17729 V1 = DAG.getBitcast(VT, Ops[0]); in lowerVECTOR_SHUFFLE()
17730 V2 = DAG.getBitcast(VT, Ops[1]); in lowerVECTOR_SHUFFLE()
17742 if (VT.is128BitVector()) in lowerVECTOR_SHUFFLE()
17743 return lower128BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG); in lowerVECTOR_SHUFFLE()
17745 if (VT.is256BitVector()) in lowerVECTOR_SHUFFLE()
17746 return lower256BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG); in lowerVECTOR_SHUFFLE()
17748 if (VT.is512BitVector()) in lowerVECTOR_SHUFFLE()
17749 return lower512BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG); in lowerVECTOR_SHUFFLE()
17752 return lower1BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG); in lowerVECTOR_SHUFFLE()
17764 MVT VT = Op.getSimpleValueType(); in lowerVSELECTtoVectorShuffle() local
17771 return DAG.getVectorShuffle(VT, SDLoc(Op), LHS, RHS, Mask); in lowerVSELECTtoVectorShuffle()
17783 MVT VT = Op.getSimpleValueType(); in LowerVSELECT() local
17784 if (isSoftF16(VT, Subtarget)) { in LowerVSELECT()
17785 MVT NVT = VT.changeVectorElementTypeToInteger(); in LowerVSELECT()
17786 return DAG.getBitcast(VT, DAG.getNode(ISD::VSELECT, dl, NVT, Cond, in LowerVSELECT()
17814 unsigned EltSize = VT.getScalarSizeInBits(); in LowerVSELECT()
17815 unsigned NumElts = VT.getVectorNumElements(); in LowerVSELECT()
17818 if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI()) in LowerVSELECT()
17824 if (VT.getSizeInBits() == 512) { in LowerVSELECT()
17831 return DAG.getSelect(dl, VT, Mask, LHS, RHS); in LowerVSELECT()
17843 return DAG.getNode(ISD::VSELECT, dl, VT, Cond, LHS, RHS); in LowerVSELECT()
17851 if (EltSize < 32 && VT.is256BitVector() && !Subtarget.hasAVX2() && in LowerVSELECT()
17865 switch (VT.SimpleTy) { in LowerVSELECT()
17885 return DAG.getBitcast(VT, Select); in LowerVSELECT()
17891 MVT VT = Op.getSimpleValueType(); in LowerEXTRACT_VECTOR_ELT_SSE4() local
17900 if (VT.getSizeInBits() == 8) { in LowerEXTRACT_VECTOR_ELT_SSE4()
17912 return DAG.getNode(ISD::TRUNCATE, dl, VT, Extract); in LowerEXTRACT_VECTOR_ELT_SSE4()
17915 if (VT == MVT::f32) { in LowerEXTRACT_VECTOR_ELT_SSE4()
17933 if (VT == MVT::i32 || VT == MVT::i64) in LowerEXTRACT_VECTOR_ELT_SSE4()
17988 MVT VT = N->getSimpleValueType(0); in getExtractedDemandedElts() local
17989 unsigned NumElts = VT.getVectorNumElements(); in getExtractedDemandedElts()
18087 MVT VT = Op.getSimpleValueType(); in LowerEXTRACT_VECTOR_ELT() local
18089 if (VT == MVT::i16) { in LowerEXTRACT_VECTOR_ELT()
18104 return DAG.getNode(ISD::TRUNCATE, dl, VT, Extract); in LowerEXTRACT_VECTOR_ELT()
18114 if (VT == MVT::i8) { in LowerEXTRACT_VECTOR_ELT()
18128 return DAG.getNode(ISD::TRUNCATE, dl, VT, Res); in LowerEXTRACT_VECTOR_ELT()
18140 return DAG.getNode(ISD::TRUNCATE, dl, VT, Res); in LowerEXTRACT_VECTOR_ELT()
18144 if (VT == MVT::f16 || VT.getSizeInBits() == 32) { in LowerEXTRACT_VECTOR_ELT()
18152 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec, in LowerEXTRACT_VECTOR_ELT()
18156 if (VT.getSizeInBits() == 64) { in LowerEXTRACT_VECTOR_ELT()
18168 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec, in LowerEXTRACT_VECTOR_ELT()
18204 MVT VT = Op.getSimpleValueType(); in LowerINSERT_VECTOR_ELT() local
18205 MVT EltVT = VT.getVectorElementType(); in LowerINSERT_VECTOR_ELT()
18206 unsigned NumElts = VT.getVectorNumElements(); in LowerINSERT_VECTOR_ELT()
18219 MVT IVT = VT.changeVectorElementTypeToInteger(); in LowerINSERT_VECTOR_ELT()
18223 return DAG.getBitcast(VT, Res); in LowerINSERT_VECTOR_ELT()
18242 SDValue EltSplat = DAG.getSplatBuildVector(VT, dl, N1); in LowerINSERT_VECTOR_ELT()
18259 bool IsAllOnesElt = VT.isInteger() && llvm::isAllOnesConstant(N1); in LowerINSERT_VECTOR_ELT()
18265 ((VT == MVT::v16i8 && !Subtarget.hasSSE41()) || in LowerINSERT_VECTOR_ELT()
18266 ((VT == MVT::v32i8 || VT == MVT::v16i16) && !Subtarget.hasInt256()))) { in LowerINSERT_VECTOR_ELT()
18267 SDValue ZeroCst = DAG.getConstant(0, dl, VT.getScalarType()); in LowerINSERT_VECTOR_ELT()
18268 SDValue OnesCst = DAG.getAllOnesConstant(dl, VT.getScalarType()); in LowerINSERT_VECTOR_ELT()
18271 SDValue CstVector = DAG.getBuildVector(VT, dl, CstVectorElts); in LowerINSERT_VECTOR_ELT()
18272 return DAG.getNode(ISD::OR, dl, VT, N0, CstVector); in LowerINSERT_VECTOR_ELT()
18277 (EltSizeInBits >= 16 || (IsZeroElt && !VT.is128BitVector()))) { in LowerINSERT_VECTOR_ELT()
18281 SDValue CstVector = IsZeroElt ? getZeroVector(VT, Subtarget, DAG, dl) in LowerINSERT_VECTOR_ELT()
18282 : getOnesVector(VT, DAG, dl); in LowerINSERT_VECTOR_ELT()
18283 return DAG.getVectorShuffle(VT, dl, N0, CstVector, BlendMask); in LowerINSERT_VECTOR_ELT()
18289 if (VT.is256BitVector() || VT.is512BitVector()) { in LowerINSERT_VECTOR_ELT()
18292 if (VT.is256BitVector() && IdxVal == 0) { in LowerINSERT_VECTOR_ELT()
18298 SDValue N1Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, N1); in LowerINSERT_VECTOR_ELT()
18299 return DAG.getNode(X86ISD::BLENDI, dl, VT, N0, N1Vec, in LowerINSERT_VECTOR_ELT()
18315 SDValue N1SplatVec = DAG.getSplatBuildVector(VT, dl, N1); in LowerINSERT_VECTOR_ELT()
18319 return DAG.getVectorShuffle(VT, dl, N0, N1SplatVec, BlendMask); in LowerINSERT_VECTOR_ELT()
18335 assert(VT.is128BitVector() && "Only 128-bit vector types should be left!"); in LowerINSERT_VECTOR_ELT()
18341 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, N1); in LowerINSERT_VECTOR_ELT()
18349 MVT ShufVT = MVT::getVectorVT(MVT::i32, VT.getSizeInBits() / 32); in LowerINSERT_VECTOR_ELT()
18352 return DAG.getBitcast(VT, N1); in LowerINSERT_VECTOR_ELT()
18358 if (VT == MVT::v8i16 || (VT == MVT::v16i8 && Subtarget.hasSSE41())) { in LowerINSERT_VECTOR_ELT()
18360 if (VT == MVT::v8i16) { in LowerINSERT_VECTOR_ELT()
18364 assert(VT == MVT::v16i8 && "PINSRB requires v16i8 vector"); in LowerINSERT_VECTOR_ELT()
18372 return DAG.getNode(Opc, dl, VT, N0, N1, N2); in LowerINSERT_VECTOR_ELT()
18396 return DAG.getNode(X86ISD::BLENDI, dl, VT, N0, N1, in LowerINSERT_VECTOR_ELT()
18401 return DAG.getNode(X86ISD::INSERTPS, dl, VT, N0, N1, in LowerINSERT_VECTOR_ELT()
19031 MVT VT = Op.getSimpleValueType(); in LowerI64IntToFP_AVX512DQ() local
19034 (VT != MVT::f32 && VT != MVT::f64)) in LowerI64IntToFP_AVX512DQ()
19042 MVT VecVT = MVT::getVectorVT(VT, NumElts); in LowerI64IntToFP_AVX512DQ()
19049 SDValue Value = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, CvtVec, in LowerI64IntToFP_AVX512DQ()
19056 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, CvtVec, in LowerI64IntToFP_AVX512DQ()
19071 MVT VT = Op.getSimpleValueType(); in LowerI64IntToFP16() local
19073 if (SrcVT != MVT::i64 || Subtarget.is64Bit() || VT != MVT::f16) in LowerI64IntToFP16()
19085 SDValue Value = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, CvtVec, in LowerI64IntToFP16()
19092 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, CvtVec, in LowerI64IntToFP16()
19168 MVT VT = CastToFP.getSimpleValueType(); in lowerFPToIntToFP() local
19169 if (CastToInt.getOpcode() != ISD::FP_TO_SINT || VT.isVector()) in lowerFPToIntToFP()
19180 if (!Subtarget.hasSSE2() || (VT != MVT::f32 && VT != MVT::f64) || in lowerFPToIntToFP()
19186 unsigned VTSize = VT.getSizeInBits(); in lowerFPToIntToFP()
19189 MVT VecVT = MVT::getVectorVT(VT, 128 / VTSize); in lowerFPToIntToFP()
19207 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, VCastToFP, ZeroIdx); in lowerFPToIntToFP()
19214 MVT VT = Op->getSimpleValueType(0); in lowerINT_TO_FP_vXi64() local
19225 assert((VT == MVT::v4f32 || VT == MVT::v2f64 || VT == MVT::v4f64) && in lowerINT_TO_FP_vXi64()
19227 MVT WideVT = VT == MVT::v4f32 ? MVT::v8f32 : MVT::v8f64; in lowerINT_TO_FP_vXi64()
19244 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res, in lowerINT_TO_FP_vXi64()
19254 if (VT != MVT::v4f32 || IsSigned) in lowerINT_TO_FP_vXi64()
19278 SDValue SignCvt = DAG.getBuildVector(VT, DL, SignCvts); in lowerINT_TO_FP_vXi64()
19304 MVT VT = Op.getSimpleValueType(); in promoteXINT_TO_FP() local
19305 MVT NVT = VT.isVector() ? VT.changeVectorElementType(MVT::f32) : MVT::f32; in promoteXINT_TO_FP()
19310 ISD::STRICT_FP_ROUND, dl, {VT, MVT::Other}, in promoteXINT_TO_FP()
19314 return DAG.getNode(ISD::FP_ROUND, dl, VT, in promoteXINT_TO_FP()
19318 static bool isLegalConversion(MVT VT, bool IsSigned, in isLegalConversion() argument
19320 if (VT == MVT::v4i32 && Subtarget.hasSSE2() && IsSigned) in isLegalConversion()
19322 if (VT == MVT::v8i32 && Subtarget.hasAVX() && IsSigned) in isLegalConversion()
19324 if (Subtarget.hasVLX() && (VT == MVT::v4i32 || VT == MVT::v8i32)) in isLegalConversion()
19327 if (VT == MVT::v16i32) in isLegalConversion()
19329 if (VT == MVT::v8i64 && Subtarget.hasDQI()) in isLegalConversion()
19333 (VT == MVT::v2i64 || VT == MVT::v4i64)) in isLegalConversion()
19345 MVT VT = Op.getSimpleValueType(); in LowerSINT_TO_FP() local
19348 if (isSoftF16(VT, Subtarget)) in LowerSINT_TO_FP()
19363 if (SrcVT == MVT::v2i32 && VT == MVT::v2f64) { in LowerSINT_TO_FP()
19368 X86ISD::STRICT_CVTSI2P, dl, {VT, MVT::Other}, in LowerSINT_TO_FP()
19371 return DAG.getNode(X86ISD::CVTSI2P, dl, VT, in LowerSINT_TO_FP()
19384 bool UseSSEReg = isScalarFPTypeInSSEReg(VT); in LowerSINT_TO_FP()
19399 if (SrcVT == MVT::i16 && (UseSSEReg || VT == MVT::f128)) { in LowerSINT_TO_FP()
19402 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other}, in LowerSINT_TO_FP()
19405 return DAG.getNode(ISD::SINT_TO_FP, dl, VT, Ext); in LowerSINT_TO_FP()
19408 if (VT == MVT::f128 || !Subtarget.hasX87()) in LowerSINT_TO_FP()
19428 BuildFILD(VT, SrcVT, dl, Chain, StackSlot, MPI, Alignment, DAG); in LowerSINT_TO_FP()
19671 MVT VT = Op->getSimpleValueType(0); in lowerUINT_TO_FP_vXi32() local
19674 if (VT == MVT::v8f64) in lowerUINT_TO_FP_vXi32()
19677 assert((VT == MVT::v4f32 || VT == MVT::v8f32 || VT == MVT::v4f64) && in lowerUINT_TO_FP_vXi32()
19679 MVT WideVT = VT == MVT::v4f64 ? MVT::v8f64 : MVT::v16f32; in lowerUINT_TO_FP_vXi32()
19680 MVT WideIntVT = VT == MVT::v4f64 ? MVT::v8i32 : MVT::v16i32; in lowerUINT_TO_FP_vXi32()
19696 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res, in lowerUINT_TO_FP_vXi32()
20152 MVT VT = Op.getSimpleValueType(); in LowerAVXExtend() local
20157 assert(VT.isVector() && InVT.isVector() && "Expected vector type"); in LowerAVXExtend()
20160 assert(VT.getVectorNumElements() == InVT.getVectorNumElements() && in LowerAVXExtend()
20162 assert((VT.getVectorElementType() == MVT::i16 || in LowerAVXExtend()
20163 VT.getVectorElementType() == MVT::i32 || in LowerAVXExtend()
20164 VT.getVectorElementType() == MVT::i64) && in LowerAVXExtend()
20173 if (VT == MVT::v32i16 && !Subtarget.hasBWI()) { in LowerAVXExtend()
20193 MVT HalfVT = VT.getHalfNumVectorElementsVT(); in LowerAVXExtend()
20200 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpLo); in LowerAVXExtend()
20208 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpHi); in LowerAVXExtend()
20212 static SDValue SplitAndExtendv16i1(unsigned ExtOpc, MVT VT, SDValue In, in SplitAndExtendv16i1() argument
20214 assert((VT == MVT::v16i8 || VT == MVT::v16i16) && "Unexpected VT."); in SplitAndExtendv16i1()
20222 return DAG.getNode(ISD::TRUNCATE, dl, VT, Res); in SplitAndExtendv16i1()
20228 MVT VT = Op->getSimpleValueType(0); in LowerZERO_EXTEND_Mask() local
20232 unsigned NumElts = VT.getVectorNumElements(); in LowerZERO_EXTEND_Mask()
20236 if (VT.getVectorElementType() != MVT::i8) { in LowerZERO_EXTEND_Mask()
20237 SDValue Extend = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, In); in LowerZERO_EXTEND_Mask()
20238 return DAG.getNode(ISD::SRL, DL, VT, Extend, in LowerZERO_EXTEND_Mask()
20239 DAG.getConstant(VT.getScalarSizeInBits() - 1, DL, VT)); in LowerZERO_EXTEND_Mask()
20243 MVT ExtVT = VT; in LowerZERO_EXTEND_Mask()
20247 return SplitAndExtendv16i1(ISD::ZERO_EXTEND, VT, In, DL, DAG); in LowerZERO_EXTEND_Mask()
20269 if (VT != ExtVT) { in LowerZERO_EXTEND_Mask()
20275 if (WideVT != VT) in LowerZERO_EXTEND_Mask()
20276 SelectedVal = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, SelectedVal, in LowerZERO_EXTEND_Mask()
20620 MVT VT = Op.getSimpleValueType(); in LowerTruncateVecI1() local
20623 assert(VT.getVectorElementType() == MVT::i1 && "Unexpected vector type."); in LowerTruncateVecI1()
20639 return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, InVT), in LowerTruncateVecI1()
20673 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi); in LowerTruncateVecI1()
20692 return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, InVT), In, ISD::SETGT); in LowerTruncateVecI1()
20693 return DAG.getSetCC(DL, VT, In, DAG.getConstant(0, DL, InVT), ISD::SETNE); in LowerTruncateVecI1()
20698 MVT VT = Op.getSimpleValueType(); in LowerTRUNCATE() local
20701 assert(VT.getVectorNumElements() == InVT.getVectorNumElements() && in LowerTRUNCATE()
20706 if (!TLI.isTypeLegal(VT) || !TLI.isTypeLegal(InVT)) { in LowerTRUNCATE()
20708 VT.is128BitVector() && Subtarget.hasAVX512()) { in LowerTRUNCATE()
20718 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT); in LowerTRUNCATE()
20722 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi); in LowerTRUNCATE()
20727 (InVT.is512BitVector() && VT.is256BitVector())) in LowerTRUNCATE()
20729 LowerTruncateVecPackWithSignBits(VT, In, DL, Subtarget, DAG)) in LowerTRUNCATE()
20734 return LowerTruncateVecPack(VT, In, DL, Subtarget, DAG); in LowerTRUNCATE()
20740 if (VT.getVectorElementType() == MVT::i1) in LowerTRUNCATE()
20747 LowerTruncateVecPackWithSignBits(VT, In, DL, Subtarget, DAG)) in LowerTRUNCATE()
20753 assert(VT == MVT::v32i8 && "Unexpected VT!"); in LowerTRUNCATE()
20767 assert(VT.is128BitVector() && InVT.is256BitVector() && "Unexpected types!"); in LowerTRUNCATE()
20769 if ((VT == MVT::v4i32) && (InVT == MVT::v4i64)) { in LowerTRUNCATE()
20775 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, In, in LowerTRUNCATE()
20784 return DAG.getVectorShuffle(VT, DL, DAG.getBitcast(MVT::v4i32, OpLo), in LowerTRUNCATE()
20788 if ((VT == MVT::v8i16) && (InVT == MVT::v8i32)) { in LowerTRUNCATE()
20808 ? truncateVectorWithPACKUS(VT, In, DL, Subtarget, DAG) in LowerTRUNCATE()
20809 : truncateVectorWithPACKSS(VT, In, DL, Subtarget, DAG); in LowerTRUNCATE()
20812 if (VT == MVT::v16i8 && InVT == MVT::v16i16) in LowerTRUNCATE()
20813 return truncateVectorWithPACKUS(VT, In, DL, Subtarget, DAG); in LowerTRUNCATE()
20820 static SDValue expandFP_TO_UINT_SSE(MVT VT, SDValue Src, const SDLoc &dl, in expandFP_TO_UINT_SSE() argument
20824 unsigned DstBits = VT.getScalarSizeInBits(); in expandFP_TO_UINT_SSE()
20829 SDValue Small = DAG.getNode(X86ISD::CVTTP2SI, dl, VT, Src); in expandFP_TO_UINT_SSE()
20831 DAG.getNode(X86ISD::CVTTP2SI, dl, VT, in expandFP_TO_UINT_SSE()
20844 if (VT == MVT::v8i32 && !Subtarget.hasAVX2()) { in expandFP_TO_UINT_SSE()
20845 SDValue Overflow = DAG.getNode(ISD::OR, dl, VT, Small, Big); in expandFP_TO_UINT_SSE()
20846 return DAG.getNode(X86ISD::BLENDV, dl, VT, Small, Overflow, Small); in expandFP_TO_UINT_SSE()
20850 DAG.getNode(X86ISD::VSRAI, dl, VT, Small, in expandFP_TO_UINT_SSE()
20852 return DAG.getNode(ISD::OR, dl, VT, Small, in expandFP_TO_UINT_SSE()
20853 DAG.getNode(ISD::AND, dl, VT, Big, IsOverflown)); in expandFP_TO_UINT_SSE()
20860 MVT VT = Op->getSimpleValueType(0); in LowerFP_TO_INT() local
20868 MVT NVT = VT.isVector() ? VT.changeVectorElementType(MVT::f32) : MVT::f32; in LowerFP_TO_INT()
20870 return DAG.getNode(Op.getOpcode(), dl, {VT, MVT::Other}, in LowerFP_TO_INT()
20873 return DAG.getNode(Op.getOpcode(), dl, VT, in LowerFP_TO_INT()
20875 } else if (isTypeLegal(SrcVT) && isLegalConversion(VT, IsSigned, Subtarget)) { in LowerFP_TO_INT()
20879 if (VT.isVector()) { in LowerFP_TO_INT()
20880 if (VT == MVT::v2i1 && SrcVT == MVT::v2f64) { in LowerFP_TO_INT()
20919 if (VT == MVT::v8i16 || VT == MVT::v16i16 || VT == MVT::v32i16) in LowerFP_TO_INT()
20922 MVT ResVT = VT; in LowerFP_TO_INT()
20923 MVT EleVT = VT.getVectorElementType(); in LowerFP_TO_INT()
20951 if (ResVT != VT) in LowerFP_TO_INT()
20952 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res, in LowerFP_TO_INT()
20961 if (VT.getVectorElementType() == MVT::i16) { in LowerFP_TO_INT()
20965 MVT NVT = VT.changeVectorElementType(MVT::i32); in LowerFP_TO_INT()
20977 Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res); in LowerFP_TO_INT()
20985 if (VT == MVT::v8i32 && SrcVT == MVT::v8f64) { in LowerFP_TO_INT()
20992 if ((VT == MVT::v4i32 || VT == MVT::v8i32) && in LowerFP_TO_INT()
21015 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res, in LowerFP_TO_INT()
21024 if ((VT == MVT::v2i64 || VT == MVT::v4i64) && in LowerFP_TO_INT()
21045 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res, in LowerFP_TO_INT()
21053 if (VT == MVT::v2i64 && SrcVT == MVT::v2f32) { in LowerFP_TO_INT()
21077 return DAG.getNode(Opc, dl, {VT, MVT::Other}, {Op->getOperand(0), Tmp}); in LowerFP_TO_INT()
21080 return DAG.getNode(Opc, dl, VT, Tmp); in LowerFP_TO_INT()
21085 if ((VT == MVT::v4i32 && SrcVT == MVT::v4f32) || in LowerFP_TO_INT()
21086 (VT == MVT::v4i32 && SrcVT == MVT::v4f64) || in LowerFP_TO_INT()
21087 (VT == MVT::v8i32 && SrcVT == MVT::v8f32)) { in LowerFP_TO_INT()
21089 return expandFP_TO_UINT_SSE(VT, Src, dl, DAG, Subtarget); in LowerFP_TO_INT()
21095 assert(!VT.isVector()); in LowerFP_TO_INT()
21106 if (!IsStrict && ((VT == MVT::i32 && !Subtarget.is64Bit()) || in LowerFP_TO_INT()
21107 (VT == MVT::i64 && Subtarget.is64Bit()))) { in LowerFP_TO_INT()
21108 unsigned DstBits = VT.getScalarSizeInBits(); in LowerFP_TO_INT()
21111 DAG.getConstant(UIntLimit, dl, VT)); in LowerFP_TO_INT()
21118 DAG.getNode(X86ISD::CVTTS2SI, dl, VT, in LowerFP_TO_INT()
21121 X86ISD::CVTTS2SI, dl, VT, in LowerFP_TO_INT()
21132 ISD::SRA, dl, VT, Small, DAG.getConstant(DstBits - 1, dl, MVT::i8)); in LowerFP_TO_INT()
21133 return DAG.getNode(ISD::OR, dl, VT, Small, in LowerFP_TO_INT()
21134 DAG.getNode(ISD::AND, dl, VT, Big, IsOverflown)); in LowerFP_TO_INT()
21138 if (VT == MVT::i64) in LowerFP_TO_INT()
21141 assert(VT == MVT::i32 && "Unexpected VT!"); in LowerFP_TO_INT()
21154 Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res); in LowerFP_TO_INT()
21169 if (VT == MVT::i16 && (UseSSEReg || SrcVT == MVT::f128)) { in LowerFP_TO_INT()
21178 Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res); in LowerFP_TO_INT()
21192 LC = RTLIB::getFPTOSINT(SrcVT, VT); in LowerFP_TO_INT()
21194 LC = RTLIB::getFPTOUINT(SrcVT, VT); in LowerFP_TO_INT()
21198 makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain); in LowerFP_TO_INT()
21434 MVT VT = Op.getSimpleValueType(); in LowerFP_EXTEND() local
21441 if (VT == MVT::f128 || (SVT == MVT::f16 && VT == MVT::f80 && in LowerFP_EXTEND()
21453 if (VT != MVT::f32) { in LowerFP_EXTEND()
21456 ISD::STRICT_FP_EXTEND, DL, {VT, MVT::Other}, in LowerFP_EXTEND()
21460 return DAG.getNode(ISD::FP_EXTEND, DL, VT, in LowerFP_EXTEND()
21468 assert(VT == MVT::f32 && SVT == MVT::f16 && "unexpected extend libcall"); in LowerFP_EXTEND()
21487 CallingConv::C, EVT(VT).getTypeForEVT(*DAG.getContext()), Callee, in LowerFP_EXTEND()
21531 return DAG.getNode(X86ISD::STRICT_VFPEXT, DL, {VT, MVT::Other}, in LowerFP_EXTEND()
21533 return DAG.getNode(X86ISD::VFPEXT, DL, VT, Res); in LowerFP_EXTEND()
21534 } else if (VT == MVT::v4f64 || VT == MVT::v8f64) { in LowerFP_EXTEND()
21543 return DAG.getNode(X86ISD::STRICT_VFPEXT, DL, {VT, MVT::Other}, in LowerFP_EXTEND()
21545 return DAG.getNode(X86ISD::VFPEXT, DL, VT, Res); in LowerFP_EXTEND()
21554 MVT VT = Op.getSimpleValueType(); in LowerFP_ROUND() local
21557 if (SVT == MVT::f128 || (VT == MVT::f16 && SVT == MVT::f80)) in LowerFP_ROUND()
21560 if (VT == MVT::f16 && (SVT == MVT::f64 || SVT == MVT::f32) && in LowerFP_ROUND()
21596 if (VT.getScalarType() == MVT::bf16) { in LowerFP_ROUND()
21604 if (VT.getScalarType() == MVT::f16 && !Subtarget.hasFP16()) { in LowerFP_ROUND()
21608 if (VT.isVector()) in LowerFP_ROUND()
21812 MVT VT = Op.getSimpleValueType(); in LowerFROUND() local
21815 const fltSemantics &Sem = SelectionDAG::EVTToAPFloatSemantics(VT); in LowerFROUND()
21821 SDValue Adder = DAG.getNode(ISD::FCOPYSIGN, dl, VT, in LowerFROUND()
21822 DAG.getConstantFP(Point5Pred, dl, VT), N0); in LowerFROUND()
21823 N0 = DAG.getNode(ISD::FADD, dl, VT, N0, Adder); in LowerFROUND()
21826 return DAG.getNode(ISD::FTRUNC, dl, VT, N0); in LowerFROUND()
21845 MVT VT = Op.getSimpleValueType(); in LowerFABSorFNEG() local
21847 bool IsF128 = (VT == MVT::f128); in LowerFABSorFNEG()
21848 assert(VT.isFloatingPoint() && VT != MVT::f80 && in LowerFABSorFNEG()
21849 DAG.getTargetLoweringInfo().isTypeLegal(VT) && in LowerFABSorFNEG()
21860 bool IsFakeVector = !VT.isVector() && !IsF128; in LowerFABSorFNEG()
21861 MVT LogicVT = VT; in LowerFABSorFNEG()
21863 LogicVT = (VT == MVT::f64) ? MVT::v2f64 in LowerFABSorFNEG()
21864 : (VT == MVT::f32) ? MVT::v4f32 in LowerFABSorFNEG()
21867 unsigned EltBits = VT.getScalarSizeInBits(); in LowerFABSorFNEG()
21871 const fltSemantics &Sem = SelectionDAG::EVTToAPFloatSemantics(VT); in LowerFABSorFNEG()
21881 if (VT.isVector() || IsF128) in LowerFABSorFNEG()
21888 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, LogicNode, in LowerFABSorFNEG()
21898 MVT VT = Op.getSimpleValueType(); in LowerFCOPYSIGN() local
21899 if (Sign.getSimpleValueType().bitsLT(VT)) in LowerFCOPYSIGN()
21900 Sign = DAG.getNode(ISD::FP_EXTEND, dl, VT, Sign); in LowerFCOPYSIGN()
21903 if (Sign.getSimpleValueType().bitsGT(VT)) in LowerFCOPYSIGN()
21904 Sign = DAG.getNode(ISD::FP_ROUND, dl, VT, Sign, in LowerFCOPYSIGN()
21909 bool IsF128 = (VT == MVT::f128); in LowerFCOPYSIGN()
21910 assert(VT.isFloatingPoint() && VT != MVT::f80 && in LowerFCOPYSIGN()
21911 DAG.getTargetLoweringInfo().isTypeLegal(VT) && in LowerFCOPYSIGN()
21914 const fltSemantics &Sem = SelectionDAG::EVTToAPFloatSemantics(VT); in LowerFCOPYSIGN()
21921 bool IsFakeVector = !VT.isVector() && !IsF128; in LowerFCOPYSIGN()
21922 MVT LogicVT = VT; in LowerFCOPYSIGN()
21924 LogicVT = (VT == MVT::f64) ? MVT::v2f64 in LowerFCOPYSIGN()
21925 : (VT == MVT::f32) ? MVT::v4f32 in LowerFCOPYSIGN()
21929 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in LowerFCOPYSIGN()
21957 return !IsFakeVector ? Or : DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Or, in LowerFCOPYSIGN()
21964 MVT VT = Op.getSimpleValueType(); in LowerFGETSIGN() local
21974 Res = DAG.getZExtOrTrunc(Res, dl, VT); in LowerFGETSIGN()
21975 Res = DAG.getNode(ISD::AND, dl, VT, Res, DAG.getConstant(1, dl, VT)); in LowerFGETSIGN()
22068 static SDValue combineVectorSizedSetCCEquality(EVT VT, SDValue X, SDValue Y, in combineVectorSizedSetCCEquality() argument
22191 return DAG.getSetCC(DL, VT, DAG.getBitcast(KRegVT, Cmp), in combineVectorSizedSetCCEquality()
22200 return DAG.getNode(ISD::TRUNCATE, DL, VT, X86SetCC.getValue(0)); in combineVectorSizedSetCCEquality()
22209 return DAG.getSetCC(DL, VT, MovMsk, FFFFs, CC); in combineVectorSizedSetCCEquality()
22224 EVT VT = MVT::Other; in matchScalarReduction() local
22256 VT = Src.getValueType(); in matchScalarReduction()
22258 if (!SrcOpMap.empty() && VT != SrcOpMap.begin()->first.getValueType()) in matchScalarReduction()
22260 unsigned NumElts = VT.getVectorNumElements(); in matchScalarReduction()
22292 EVT VT = LHS.getValueType(); in LowerVectorAllEqual() local
22293 unsigned ScalarSize = VT.getScalarSizeInBits(); in LowerVectorAllEqual()
22300 if (!llvm::has_single_bit<uint32_t>(VT.getSizeInBits())) in LowerVectorAllEqual()
22304 if (VT.isFloatingPoint()) in LowerVectorAllEqual()
22321 if (VT.getSizeInBits() < 128) { in LowerVectorAllEqual()
22322 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); in LowerVectorAllEqual()
22358 VT = EVT::getVectorVT(*DAG.getContext(), MVT::i64, VT.getSizeInBits() / 64); in LowerVectorAllEqual()
22359 LHS = DAG.getBitcast(VT, LHS); in LowerVectorAllEqual()
22360 RHS = DAG.getBitcast(VT, RHS); in LowerVectorAllEqual()
22364 if (VT.getSizeInBits() > TestSize) { in LowerVectorAllEqual()
22368 while (VT.getSizeInBits() > TestSize) { in LowerVectorAllEqual()
22370 VT = Split.first.getValueType(); in LowerVectorAllEqual()
22371 LHS = DAG.getNode(ISD::AND, DL, VT, Split.first, Split.second); in LowerVectorAllEqual()
22373 RHS = DAG.getAllOnesConstant(DL, VT); in LowerVectorAllEqual()
22378 VT = MVT::getVectorVT(SVT, VT.getSizeInBits() / SVT.getSizeInBits()); in LowerVectorAllEqual()
22379 LHS = DAG.getBitcast(VT, MaskBits(LHS)); in LowerVectorAllEqual()
22380 RHS = DAG.getBitcast(VT, MaskBits(RHS)); in LowerVectorAllEqual()
22381 EVT BoolVT = VT.changeVectorElementType(MVT::i1); in LowerVectorAllEqual()
22383 V = DAG.getSExtOrTrunc(V, DL, VT); in LowerVectorAllEqual()
22384 while (VT.getSizeInBits() > TestSize) { in LowerVectorAllEqual()
22386 VT = Split.first.getValueType(); in LowerVectorAllEqual()
22387 V = DAG.getNode(ISD::AND, DL, VT, Split.first, Split.second); in LowerVectorAllEqual()
22389 V = DAG.getNOT(DL, V, VT); in LowerVectorAllEqual()
22395 SDValue V = DAG.getNode(ISD::XOR, DL, VT, LHS, RHS); in LowerVectorAllEqual()
22396 while (VT.getSizeInBits() > TestSize) { in LowerVectorAllEqual()
22398 VT = Split.first.getValueType(); in LowerVectorAllEqual()
22399 V = DAG.getNode(ISD::OR, DL, VT, Split.first, Split.second); in LowerVectorAllEqual()
22402 RHS = DAG.getConstant(0, DL, VT); in LowerVectorAllEqual()
22406 if (UseKORTEST && VT.is512BitVector()) { in LowerVectorAllEqual()
22407 MVT TestVT = MVT::getVectorVT(MVT::i32, VT.getSizeInBits() / 32); in LowerVectorAllEqual()
22416 MVT TestVT = MVT::getVectorVT(MVT::i64, VT.getSizeInBits() / 64); in LowerVectorAllEqual()
22423 assert(VT.getSizeInBits() == 128 && "Failure to split to 128-bits"); in LowerVectorAllEqual()
22481 EVT VT = VecIns[0].getValueType(); in MatchVectorAllEqualTest() local
22483 [VT](SDValue V) { return VT == V.getValueType(); }) && in MatchVectorAllEqualTest()
22487 if (!llvm::has_single_bit<uint32_t>(VT.getSizeInBits())) in MatchVectorAllEqualTest()
22498 VecIns.push_back(DAG.getNode(LogicOp, DL, VT, LHS, RHS)); in MatchVectorAllEqualTest()
22502 CmpNull ? DAG.getConstant(0, DL, VT) in MatchVectorAllEqualTest()
22503 : DAG.getAllOnesConstant(DL, VT), in MatchVectorAllEqualTest()
22785 EVT VT) const { in isXAndYEqZeroPreferableToXAndYEqY()
22786 return !VT.isVector() || Cond != ISD::CondCode::SETEQ; in isXAndYEqZeroPreferableToXAndYEqY()
22809 EVT VT = Op.getValueType(); in isFsqrtCheap() local
22812 if (VT.getScalarType() == MVT::f16) in isFsqrtCheap()
22816 if (DAG.doesNodeExist(X86ISD::FRSQRT, DAG.getVTList(VT), Op)) in isFsqrtCheap()
22819 if (VT.isVector()) in isFsqrtCheap()
22832 EVT VT = Op.getValueType(); in getSqrtEstimate() local
22842 if ((VT == MVT::f32 && Subtarget.hasSSE1()) || in getSqrtEstimate()
22843 (VT == MVT::v4f32 && Subtarget.hasSSE1() && Reciprocal) || in getSqrtEstimate()
22844 (VT == MVT::v4f32 && Subtarget.hasSSE2() && !Reciprocal) || in getSqrtEstimate()
22845 (VT == MVT::v8f32 && Subtarget.hasAVX()) || in getSqrtEstimate()
22846 (VT == MVT::v16f32 && Subtarget.useAVX512Regs())) { in getSqrtEstimate()
22852 unsigned Opcode = VT == MVT::v16f32 ? X86ISD::RSQRT14 : X86ISD::FRSQRT; in getSqrtEstimate()
22853 SDValue Estimate = DAG.getNode(Opcode, DL, VT, Op); in getSqrtEstimate()
22855 Estimate = DAG.getNode(ISD::FMUL, DL, VT, Op, Estimate); in getSqrtEstimate()
22859 if (VT.getScalarType() == MVT::f16 && isTypeLegal(VT) && in getSqrtEstimate()
22865 if (VT == MVT::f16) { in getSqrtEstimate()
22873 return DAG.getNode(X86ISD::RSQRT14, DL, VT, Op); in getSqrtEstimate()
22884 EVT VT = Op.getValueType(); in getRecipEstimate() local
22893 if ((VT == MVT::f32 && Subtarget.hasSSE1()) || in getRecipEstimate()
22894 (VT == MVT::v4f32 && Subtarget.hasSSE1()) || in getRecipEstimate()
22895 (VT == MVT::v8f32 && Subtarget.hasAVX()) || in getRecipEstimate()
22896 (VT == MVT::v16f32 && Subtarget.useAVX512Regs())) { in getRecipEstimate()
22900 if (VT == MVT::f32 && Enabled == ReciprocalEstimate::Unspecified) in getRecipEstimate()
22907 unsigned Opcode = VT == MVT::v16f32 ? X86ISD::RCP14 : X86ISD::FRCP; in getRecipEstimate()
22908 return DAG.getNode(Opcode, DL, VT, Op); in getRecipEstimate()
22911 if (VT.getScalarType() == MVT::f16 && isTypeLegal(VT) && in getRecipEstimate()
22916 if (VT == MVT::f16) { in getRecipEstimate()
22924 return DAG.getNode(X86ISD::RCP14, DL, VT, Op); in getRecipEstimate()
22956 EVT VT = N->getValueType(0); in BuildSDIVPow2() local
22958 if (VT != MVT::i16 && VT != MVT::i32 && in BuildSDIVPow2()
22959 !(Subtarget.is64Bit() && VT == MVT::i64)) in BuildSDIVPow2()
23109 static SDValue splitIntVSETCC(EVT VT, SDValue LHS, SDValue RHS, in splitIntVSETCC() argument
23112 assert(VT.isInteger() && VT == LHS.getValueType() && in splitIntVSETCC()
23113 VT == RHS.getValueType() && "Unsupported VTs!"); in splitIntVSETCC()
23127 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT); in splitIntVSETCC()
23128 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, in splitIntVSETCC()
23138 MVT VT = Op.getSimpleValueType(); in LowerIntVSETCC_AVX512() local
23139 assert(VT.getVectorElementType() == MVT::i1 && in LowerIntVSETCC_AVX512()
23150 return DAG.getSetCC(dl, VT, Op0, Op1, SetCCOpcode); in LowerIntVSETCC_AVX512()
23163 MVT VT = V.getSimpleValueType(); in incDecVectorConstant() local
23164 MVT EltVT = VT.getVectorElementType(); in incDecVectorConstant()
23165 unsigned NumElts = VT.getVectorNumElements(); in incDecVectorConstant()
23184 return DAG.getBuildVector(VT, DL, NewVecC); in incDecVectorConstant()
23191 static SDValue LowerVSETCCWithSUBUS(SDValue Op0, SDValue Op1, MVT VT, in LowerVSETCCWithSUBUS() argument
23198 MVT VET = VT.getVectorElementType(); in LowerVSETCCWithSUBUS()
23241 SDValue Result = DAG.getNode(ISD::USUBSAT, dl, VT, Op0, Op1); in LowerVSETCCWithSUBUS()
23242 return DAG.getNode(X86ISD::PCMPEQ, dl, VT, Result, in LowerVSETCCWithSUBUS()
23243 DAG.getConstant(0, dl, VT)); in LowerVSETCCWithSUBUS()
23253 MVT VT = Op->getSimpleValueType(0); in LowerVSETCC() local
23272 if (Subtarget.hasAVX512() && VT.getVectorElementType() == MVT::i1 && in LowerVSETCC()
23276 unsigned Num = VT.getVectorNumElements(); in LowerVSETCC()
23285 VT = Op0.getSimpleValueType(); in LowerVSETCC()
23305 Opc, dl, {VT, MVT::Other}, in LowerVSETCC()
23335 Opc, dl, {VT, MVT::Other}, in LowerVSETCC()
23338 Opc, dl, {VT, MVT::Other}, in LowerVSETCC()
23344 Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(CC0, dl, MVT::i8)); in LowerVSETCC()
23346 Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(CC1, dl, MVT::i8)); in LowerVSETCC()
23348 Cmp = DAG.getNode(CombineOpc, dl, VT, Cmp0, Cmp1); in LowerVSETCC()
23352 Opc, dl, {VT, MVT::Other}, in LowerVSETCC()
23357 Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8)); in LowerVSETCC()
23365 Opc, dl, {VT, MVT::Other}, in LowerVSETCC()
23370 Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8)); in LowerVSETCC()
23373 if (VT.getFixedSizeInBits() > in LowerVSETCC()
23377 EVT CastVT = EVT(VT).changeVectorElementTypeToInteger(); in LowerVSETCC()
23399 assert(VT.getVectorNumElements() == VTOp0.getVectorNumElements() && in LowerVSETCC()
23404 assert((Subtarget.hasAVX512() || (VT == VTOp0)) && in LowerVSETCC()
23408 if (VT.getVectorElementType() == MVT::i1) { in LowerVSETCC()
23417 if (VT.is128BitVector() && Subtarget.hasXOP()) { in LowerVSETCC()
23440 return DAG.getNode(Opc, dl, VT, Op0, Op1, in LowerVSETCC()
23452 BC0.getOperand(1), VT.getScalarSizeInBits(), UndefElts, EltBits, in LowerVSETCC()
23456 Op1 = DAG.getBitcast(VT, BC0.getOperand(1)); in LowerVSETCC()
23467 unsigned BitWidth = VT.getScalarSizeInBits(); in LowerVSETCC()
23471 Result = DAG.getNode(ISD::SHL, dl, VT, Result, in LowerVSETCC()
23472 DAG.getConstant(ShiftAmt, dl, VT)); in LowerVSETCC()
23473 Result = DAG.getNode(ISD::SRA, dl, VT, Result, in LowerVSETCC()
23474 DAG.getConstant(BitWidth - 1, dl, VT)); in LowerVSETCC()
23480 if (VT.is256BitVector() && !Subtarget.hasInt256()) in LowerVSETCC()
23481 return splitIntVSETCC(VT, Op0, Op1, Cond, DAG, dl); in LowerVSETCC()
23485 if (VT.is512BitVector()) in LowerVSETCC()
23486 return splitIntVSETCC(VT, Op0, Op1, Cond, DAG, dl); in LowerVSETCC()
23515 TLI.isOperationLegal(ISD::UMIN, VT)) { in LowerVSETCC()
23546 SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1); in LowerVSETCC()
23547 Result = DAG.getNode(X86ISD::PCMPEQ, dl, VT, Op0, Result); in LowerVSETCC()
23551 Result = DAG.getNOT(dl, Result, VT); in LowerVSETCC()
23559 LowerVSETCCWithSUBUS(Op0, Op1, VT, Cond, dl, Subtarget, DAG)) in LowerVSETCC()
23577 if (VT == MVT::v2i64) { in LowerVSETCC()
23591 return DAG.getBitcast(VT, Result); in LowerVSETCC()
23602 return DAG.getBitcast(VT, Result); in LowerVSETCC()
23616 return DAG.getBitcast(VT, Result); in LowerVSETCC()
23650 return DAG.getBitcast(VT, Result); in LowerVSETCC()
23673 return DAG.getBitcast(VT, Result); in LowerVSETCC()
23680 MVT EltVT = VT.getVectorElementType(); in LowerVSETCC()
23682 VT); in LowerVSETCC()
23683 Op0 = DAG.getNode(ISD::XOR, dl, VT, Op0, SM); in LowerVSETCC()
23684 Op1 = DAG.getNode(ISD::XOR, dl, VT, Op1, SM); in LowerVSETCC()
23687 SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1); in LowerVSETCC()
23691 Result = DAG.getNOT(dl, Result, VT); in LowerVSETCC()
23708 MVT VT = Op0.getSimpleValueType(); in EmitAVX512Test() local
23709 if (!(Subtarget.hasAVX512() && VT == MVT::v16i1) && in EmitAVX512Test()
23710 !(Subtarget.hasDQI() && VT == MVT::v8i1) && in EmitAVX512Test()
23711 !(Subtarget.hasBWI() && (VT == MVT::v32i1 || VT == MVT::v64i1))) in EmitAVX512Test()
23725 if (Subtarget.hasDQI() && (VT == MVT::v8i1 || VT == MVT::v16i1)) in EmitAVX512Test()
23727 if (Subtarget.hasBWI() && (VT == MVT::v32i1 || VT == MVT::v64i1)) in EmitAVX512Test()
23804 EVT VT = Op0.getValueType(); in emitFlagsForSetcc() local
23805 if (VT == MVT::i32 || VT == MVT::i64 || Op0->hasOneUse()) { in emitFlagsForSetcc()
23806 SDVTList CmpVTs = DAG.getVTList(VT, MVT::i32); in emitFlagsForSetcc()
23810 DAG.getConstant(0, dl, VT), Op0); in emitFlagsForSetcc()
23845 MVT VT = Op->getSimpleValueType(0); in LowerSETCC() local
23847 if (VT.isVector()) return LowerVSETCC(Op, Subtarget, DAG); in LowerSETCC()
23849 assert(VT == MVT::i8 && "SetCC type must be 8-bit integer"); in LowerSETCC()
24047 MVT VT = Op1.getSimpleValueType(); in LowerSELECT() local
24050 if (isSoftF16(VT, Subtarget)) { in LowerSELECT()
24051 MVT NVT = VT.changeTypeToInteger(); in LowerSELECT()
24052 return DAG.getBitcast(VT, DAG.getNode(ISD::SELECT, DL, NVT, Cond, in LowerSELECT()
24060 if (Cond.getOpcode() == ISD::SETCC && isScalarFPTypeInSSEReg(VT) && in LowerSELECT()
24061 VT == Cond.getOperand(0).getSimpleValueType() && Cond->hasOneUse()) { in LowerSELECT()
24072 assert(!VT.isVector() && "Not a scalar type?"); in LowerSELECT()
24073 return DAG.getNode(X86ISD::SELECTS, DL, VT, Cmp, Op1, Op2); in LowerSELECT()
24077 SDValue Cmp = DAG.getNode(X86ISD::FSETCC, DL, VT, CondOp0, CondOp1, in LowerSELECT()
24096 MVT VecVT = VT == MVT::f32 ? MVT::v4f32 : MVT::v2f64; in LowerSELECT()
24101 MVT VCmpVT = VT == MVT::f32 ? MVT::v4i32 : MVT::v2i64; in LowerSELECT()
24106 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, in LowerSELECT()
24109 SDValue AndN = DAG.getNode(X86ISD::FANDN, DL, VT, Cmp, Op2); in LowerSELECT()
24110 SDValue And = DAG.getNode(X86ISD::FAND, DL, VT, Cmp, Op1); in LowerSELECT()
24111 return DAG.getNode(X86ISD::FOR, DL, VT, AndN, And); in LowerSELECT()
24116 if (isScalarFPTypeInSSEReg(VT) && Subtarget.hasAVX512()) { in LowerSELECT()
24118 return DAG.getNode(X86ISD::SELECTS, DL, VT, Cmp, Op1, Op2); in LowerSELECT()
24157 if (Subtarget.canUseCMOV() && (VT == MVT::i32 || VT == MVT::i64) && in LowerSELECT()
24181 SDValue SBB = DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, in LowerSELECT()
24184 return DAG.getNode(ISD::OR, DL, VT, SBB, Y); in LowerSELECT()
24208 if (CmpSz > VT.getSizeInBits()) in LowerSELECT()
24209 Neg = DAG.getNode(ISD::TRUNCATE, DL, VT, CmpOp0); in LowerSELECT()
24210 else if (CmpSz < VT.getSizeInBits()) in LowerSELECT()
24211 Neg = DAG.getNode(ISD::AND, DL, VT, in LowerSELECT()
24212 DAG.getNode(ISD::ANY_EXTEND, DL, VT, CmpOp0.getOperand(0)), in LowerSELECT()
24213 DAG.getConstant(1, DL, VT)); in LowerSELECT()
24216 SDValue Mask = DAG.getNegative(Neg, DL, VT); // -(and (x, 0x1)) in LowerSELECT()
24217 SDValue And = DAG.getNode(ISD::AND, DL, VT, Mask, Src1); // Mask & z in LowerSELECT()
24218 return DAG.getNode(Op2.getOpcode(), DL, VT, And, Src2); // And Op y in LowerSELECT()
24220 } else if ((VT == MVT::i32 || VT == MVT::i64) && isNullConstant(Op2) && in LowerSELECT()
24230 unsigned ShCt = VT.getSizeInBits() - 1; in LowerSELECT()
24231 SDValue ShiftAmt = DAG.getConstant(ShCt, DL, VT); in LowerSELECT()
24232 SDValue Shift = DAG.getNode(ISD::SRA, DL, VT, Op1, ShiftAmt); in LowerSELECT()
24234 Shift = DAG.getNOT(DL, Shift, VT); in LowerSELECT()
24235 return DAG.getNode(ISD::AND, DL, VT, Shift, Op1); in LowerSELECT()
24254 if (VT.isFloatingPoint() && !VT.isVector() && in LowerSELECT()
24255 !isScalarFPTypeInSSEReg(VT) && Subtarget.canUseCMOV()) // FPStack? in LowerSELECT()
24355 MVT VT = Op->getSimpleValueType(0); in LowerSIGN_EXTEND_Mask() local
24359 MVT VTElt = VT.getVectorElementType(); in LowerSIGN_EXTEND_Mask()
24360 unsigned NumElts = VT.getVectorNumElements(); in LowerSIGN_EXTEND_Mask()
24363 MVT ExtVT = VT; in LowerSIGN_EXTEND_Mask()
24367 return SplitAndExtendv16i1(Op.getOpcode(), VT, In, dl, DAG); in LowerSIGN_EXTEND_Mask()
24394 if (VT != ExtVT) { in LowerSIGN_EXTEND_Mask()
24400 if (WideVT != VT) in LowerSIGN_EXTEND_Mask()
24401 V = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, V, in LowerSIGN_EXTEND_Mask()
24428 MVT VT = Op->getSimpleValueType(0); in LowerEXTEND_VECTOR_INREG() local
24431 MVT SVT = VT.getVectorElementType(); in LowerEXTEND_VECTOR_INREG()
24439 if (!(VT.is128BitVector() && Subtarget.hasSSE2()) && in LowerEXTEND_VECTOR_INREG()
24440 !(VT.is256BitVector() && Subtarget.hasAVX()) && in LowerEXTEND_VECTOR_INREG()
24441 !(VT.is512BitVector() && Subtarget.hasAVX512())) in LowerEXTEND_VECTOR_INREG()
24446 unsigned NumElts = VT.getVectorNumElements(); in LowerEXTEND_VECTOR_INREG()
24462 assert(VT.getSizeInBits() > 128 && "Unexpected 128-bit vector extension"); in LowerEXTEND_VECTOR_INREG()
24465 return DAG.getNode(Op.getOpcode(), dl, VT, In); in LowerEXTEND_VECTOR_INREG()
24472 return DAG.getNode(ExtOpc, dl, VT, In); in LowerEXTEND_VECTOR_INREG()
24477 assert(VT.is256BitVector() && "256-bit vector expected"); in LowerEXTEND_VECTOR_INREG()
24478 MVT HalfVT = VT.getHalfNumVectorElementsVT(); in LowerEXTEND_VECTOR_INREG()
24489 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi); in LowerEXTEND_VECTOR_INREG()
24494 assert(VT.is128BitVector() && InVT.is128BitVector() && "Unexpected VTs"); in LowerEXTEND_VECTOR_INREG()
24505 return DAG.getBitcast(VT, in LowerEXTEND_VECTOR_INREG()
24516 MVT DestVT = VT == MVT::v2i64 ? MVT::v4i32 : VT; in LowerEXTEND_VECTOR_INREG()
24536 if (VT == MVT::v2i64) { in LowerEXTEND_VECTOR_INREG()
24541 SignExt = DAG.getBitcast(VT, SignExt); in LowerEXTEND_VECTOR_INREG()
24549 MVT VT = Op->getSimpleValueType(0); in LowerSIGN_EXTEND() local
24557 assert(VT.isVector() && InVT.isVector() && "Expected vector type"); in LowerSIGN_EXTEND()
24558 assert(VT.getVectorNumElements() == InVT.getVectorNumElements() && in LowerSIGN_EXTEND()
24560 assert((VT.getVectorElementType() == MVT::i16 || in LowerSIGN_EXTEND()
24561 VT.getVectorElementType() == MVT::i32 || in LowerSIGN_EXTEND()
24562 VT.getVectorElementType() == MVT::i64) && in LowerSIGN_EXTEND()
24569 if (VT == MVT::v32i16 && !Subtarget.hasBWI()) { in LowerSIGN_EXTEND()
24585 MVT HalfVT = VT.getHalfNumVectorElementsVT(); in LowerSIGN_EXTEND()
24596 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpHi); in LowerSIGN_EXTEND()
24940 EVT VT = Node->getValueType(0); in LowerDYNAMIC_STACKALLOC() local
24967 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT); in LowerDYNAMIC_STACKALLOC()
24969 Result = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value in LowerDYNAMIC_STACKALLOC()
24973 DAG.getNode(ISD::AND, dl, VT, Result, in LowerDYNAMIC_STACKALLOC()
24974 DAG.getConstant(~(Alignment->value() - 1ULL), dl, VT)); in LowerDYNAMIC_STACKALLOC()
25006 SP = DAG.getNode(ISD::AND, dl, VT, SP.getValue(0), in LowerDYNAMIC_STACKALLOC()
25007 DAG.getConstant(~(Alignment->value() - 1ULL), dl, VT)); in LowerDYNAMIC_STACKALLOC()
25181 static SDValue getTargetVShiftByConstNode(unsigned Opc, const SDLoc &dl, MVT VT, in getTargetVShiftByConstNode() argument
25184 MVT ElementType = VT.getVectorElementType(); in getTargetVShiftByConstNode()
25188 if (VT != SrcOp.getSimpleValueType()) in getTargetVShiftByConstNode()
25189 SrcOp = DAG.getBitcast(VT, SrcOp); in getTargetVShiftByConstNode()
25200 return DAG.getConstant(0, dl, VT); in getTargetVShiftByConstNode()
25223 SDValue Amt = DAG.getConstant(ShiftAmt, dl, VT); in getTargetVShiftByConstNode()
25224 if (SDValue C = DAG.FoldConstantArithmetic(ShiftOpc, dl, VT, {SrcOp, Amt})) in getTargetVShiftByConstNode()
25228 return DAG.getNode(Opc, dl, VT, SrcOp, in getTargetVShiftByConstNode()
25233 static SDValue getTargetVShiftNode(unsigned Opc, const SDLoc &dl, MVT VT, in getTargetVShiftNode() argument
25321 MVT EltVT = VT.getVectorElementType(); in getTargetVShiftNode()
25325 return DAG.getNode(Opc, dl, VT, SrcOp, ShAmt); in getTargetVShiftNode()
25368 MVT VT = Op.getSimpleValueType(); in getVectorMaskingNode() local
25369 MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorNumElements()); in getVectorMaskingNode()
25379 PreservedSrc = getZeroVector(VT, Subtarget, DAG, dl); in getVectorMaskingNode()
25380 return DAG.getNode(OpcodeSelect, dl, VT, VMask, Op, PreservedSrc); in getVectorMaskingNode()
25399 MVT VT = Op.getSimpleValueType(); in getScalarMaskingNode() local
25409 return DAG.getNode(ISD::AND, dl, VT, Op, IMask); in getScalarMaskingNode()
25412 PreservedSrc = getZeroVector(VT, Subtarget, DAG, dl); in getScalarMaskingNode()
25413 return DAG.getNode(X86ISD::SELECTS, dl, VT, IMask, Op, PreservedSrc); in getScalarMaskingNode()
25516 MVT VT = Op.getSimpleValueType(); in LowerINTRINSIC_WO_CHAIN() local
25650 DAG.getNode(IntrData->Opc0, dl, VT, Src), Mask, PassThru, in LowerINTRINSIC_WO_CHAIN()
25667 return getVectorMaskingNode(DAG.getNode(Opc, dl, VT, Src), Mask, PassThru, in LowerINTRINSIC_WO_CHAIN()
25686 DAG.getNode(IntrWithRoundingModeOpcode, dl, VT, Src1, Src2, in LowerINTRINSIC_WO_CHAIN()
25692 return getScalarMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT, Src1, in LowerINTRINSIC_WO_CHAIN()
25708 return getScalarMaskingNode(DAG.getNode(Opc, dl, VT, Src1, in LowerINTRINSIC_WO_CHAIN()
25722 NewOp = DAG.getNode(IntrData->Opc0, dl, VT, Src1, Src2); in LowerINTRINSIC_WO_CHAIN()
25724 NewOp = DAG.getNode(IntrData->Opc1, dl, VT, Src1, Src2, in LowerINTRINSIC_WO_CHAIN()
25745 return getScalarMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2), in LowerINTRINSIC_WO_CHAIN()
25758 NewOp = DAG.getNode(IntrData->Opc1, dl, VT, Src1, Src2, in LowerINTRINSIC_WO_CHAIN()
25764 NewOp = DAG.getNode(IntrData->Opc0, dl, VT, Src1, Src2); in LowerINTRINSIC_WO_CHAIN()
25782 return getVectorMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2), in LowerINTRINSIC_WO_CHAIN()
25800 return getScalarMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2, Src3), in LowerINTRINSIC_WO_CHAIN()
25818 return getVectorMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2, Src3), in LowerINTRINSIC_WO_CHAIN()
25830 return DAG.getNode(IntrData->Opc0, dl, VT, Src3, Src2, Src1); in LowerINTRINSIC_WO_CHAIN()
25837 return DAG.getNode(IntrData->Opc0, dl, VT,Src2, Src1); in LowerINTRINSIC_WO_CHAIN()
25845 MVT VT = Op.getSimpleValueType(); in LowerINTRINSIC_WO_CHAIN() local
25849 PassThru = getZeroVector(VT, Subtarget, DAG, dl); in LowerINTRINSIC_WO_CHAIN()
25859 NewOp = DAG.getNode(IntrData->Opc1, dl, VT, Src1, Src2, Src3, in LowerINTRINSIC_WO_CHAIN()
25865 NewOp = DAG.getNode(IntrData->Opc0, dl, VT, Src1, Src2, Src3); in LowerINTRINSIC_WO_CHAIN()
26019 PassThru = getZeroVector(VT, Subtarget, DAG, dl); in LowerINTRINSIC_WO_CHAIN()
26021 return DAG.getNode(IntrData->Opc0, dl, VT, DataToCompress, PassThru, in LowerINTRINSIC_WO_CHAIN()
26033 : getZeroVector(VT, Subtarget, DAG, dl); in LowerINTRINSIC_WO_CHAIN()
26044 SDValue FixupImm = DAG.getNode(Opc, dl, VT, Src1, Src2, Src3, Imm); in LowerINTRINSIC_WO_CHAIN()
26351 return DAG.getNode(getGlobalWrapperKind(nullptr, /*OpFlags=*/0), dl, VT, in LowerINTRINSIC_WO_CHAIN()
26365 SDValue Result = DAG.getMCSymbol(LSDASym, VT); in LowerINTRINSIC_WO_CHAIN()
26366 return DAG.getNode(X86ISD::Wrapper, dl, VT, Result); in LowerINTRINSIC_WO_CHAIN()
26395 return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg, VT); in LowerINTRINSIC_WO_CHAIN()
26531 MVT VT = Op.getSimpleValueType(); in getGatherNode() local
26541 VT.getVectorNumElements()); in getGatherNode()
27001 MVT VT = Op.getSimpleValueType(); in LowerINTRINSIC_W_CHAIN() local
27011 {Chain, Op1, Op2}, VT, MMO); in LowerINTRINSIC_W_CHAIN()
27013 Res = DAG.getZExtOrTrunc(getSETCC(X86::COND_B, Res, DL, DAG), DL, VT); in LowerINTRINSIC_W_CHAIN()
27020 MVT VT = Op.getSimpleValueType(); in LowerINTRINSIC_W_CHAIN() local
27027 SDValue Size = DAG.getConstant(VT.getScalarSizeInBits(), DL, MVT::i32); in LowerINTRINSIC_W_CHAIN()
27031 {Chain, Op1, Op2, Size}, VT, MMO); in LowerINTRINSIC_W_CHAIN()
27033 Res = DAG.getZExtOrTrunc(getSETCC(X86::COND_B, Res, DL, DAG), DL, VT); in LowerINTRINSIC_W_CHAIN()
27036 Res = DAG.getNode(ISD::SHL, DL, VT, Res, in LowerINTRINSIC_W_CHAIN()
27037 DAG.getShiftAmountConstant(Imm, VT, DL)); in LowerINTRINSIC_W_CHAIN()
27066 MVT VT = Op2.getSimpleValueType(); in LowerINTRINSIC_W_CHAIN() local
27090 {Chain, Op1, Op2}, VT, MMO); in LowerINTRINSIC_W_CHAIN()
27102 MVT VT = Op2.getSimpleValueType(); in LowerINTRINSIC_W_CHAIN() local
27126 {Chain, Op1, Op2}, VT, MMO); in LowerINTRINSIC_W_CHAIN()
27319 EVT VT = Op.getValueType(); in LowerFRAMEADDR() local
27335 return DAG.getFrameIndex(FrameAddrIndex, VT); in LowerFRAMEADDR()
27342 assert(((FrameReg == X86::RBP && VT == MVT::i64) || in LowerFRAMEADDR()
27343 (FrameReg == X86::EBP && VT == MVT::i32)) && in LowerFRAMEADDR()
27345 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, VT); in LowerFRAMEADDR()
27347 FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr, in LowerFRAMEADDR()
27354 Register X86TargetLowering::getRegisterByName(const char* RegName, LLT VT, in getRegisterByName() argument
27645 MVT VT = Op.getSimpleValueType(); in LowerGET_ROUNDING() local
27679 RetVal = DAG.getZExtOrTrunc(RetVal, DL, VT); in LowerGET_ROUNDING()
27924 MVT VT = Op.getSimpleValueType(); in LowerVectorCTLZ_AVX512CDI() local
27925 MVT EltVT = VT.getVectorElementType(); in LowerVectorCTLZ_AVX512CDI()
27926 unsigned NumElems = VT.getVectorNumElements(); in LowerVectorCTLZ_AVX512CDI()
27943 SDValue TruncNode = DAG.getNode(ISD::TRUNCATE, dl, VT, CtlzNode); in LowerVectorCTLZ_AVX512CDI()
27944 SDValue Delta = DAG.getConstant(32 - EltVT.getSizeInBits(), dl, VT); in LowerVectorCTLZ_AVX512CDI()
27946 return DAG.getNode(ISD::SUB, dl, VT, TruncNode, Delta); in LowerVectorCTLZ_AVX512CDI()
27953 MVT VT = Op.getSimpleValueType(); in LowerVectorCTLZInRegLUT() local
27954 int NumElts = VT.getVectorNumElements(); in LowerVectorCTLZInRegLUT()
27955 int NumBytes = NumElts * (VT.getScalarSizeInBits() / 8); in LowerVectorCTLZInRegLUT()
27999 while (CurrVT != VT) { in LowerVectorCTLZInRegLUT()
28035 MVT VT = Op.getSimpleValueType(); in LowerVectorCTLZ() local
28039 (Subtarget.canExtendTo512DQ() || VT.getVectorElementType() != MVT::i8)) in LowerVectorCTLZ()
28043 if (VT.is256BitVector() && !Subtarget.hasInt256()) in LowerVectorCTLZ()
28047 if (VT.is512BitVector() && !Subtarget.hasBWI()) in LowerVectorCTLZ()
28056 MVT VT = Op.getSimpleValueType(); in LowerCTLZ() local
28057 MVT OpVT = VT; in LowerCTLZ()
28058 unsigned NumBits = VT.getSizeInBits(); in LowerCTLZ()
28062 if (VT.isVector()) in LowerCTLZ()
28066 if (VT == MVT::i8) { in LowerCTLZ()
28088 if (VT == MVT::i8) in LowerCTLZ()
28095 MVT VT = Op.getSimpleValueType(); in LowerCTTZ() local
28096 unsigned NumBits = VT.getScalarSizeInBits(); in LowerCTTZ()
28100 assert(!VT.isVector() && Op.getOpcode() == ISD::CTTZ && in LowerCTTZ()
28104 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in LowerCTTZ()
28112 SDValue Ops[] = {Op, DAG.getConstant(NumBits, dl, VT), in LowerCTTZ()
28115 return DAG.getNode(X86ISD::CMOV, dl, VT, Ops); in LowerCTTZ()
28120 MVT VT = Op.getSimpleValueType(); in lowerAddSub() local
28123 if (VT == MVT::i16 || VT == MVT::i32) in lowerAddSub()
28126 if (VT == MVT::v32i16 || VT == MVT::v64i8) in lowerAddSub()
28137 MVT VT = Op.getSimpleValueType(); in LowerADDSAT_SUBSAT() local
28142 if (VT == MVT::v32i16 || VT == MVT::v64i8 || in LowerADDSAT_SUBSAT()
28143 (VT.is256BitVector() && !Subtarget.hasInt256())) { in LowerADDSAT_SUBSAT()
28152 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in LowerADDSAT_SUBSAT()
28154 unsigned BitWidth = VT.getScalarSizeInBits(); in LowerADDSAT_SUBSAT()
28156 if (!TLI.isOperationLegal(ISD::UMAX, VT) || useVPTERNLOG(Subtarget, VT)) { in LowerADDSAT_SUBSAT()
28164 SDValue SignMask = DAG.getConstant(C->getAPIntValue(), DL, VT); in LowerADDSAT_SUBSAT()
28165 SDValue ShiftAmt = DAG.getConstant(BitWidth - 1, DL, VT); in LowerADDSAT_SUBSAT()
28166 SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, X, SignMask); in LowerADDSAT_SUBSAT()
28167 SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, X, ShiftAmt); in LowerADDSAT_SUBSAT()
28168 return DAG.getNode(ISD::AND, DL, VT, Xor, Sra); in LowerADDSAT_SUBSAT()
28171 if (!TLI.isOperationLegal(ISD::UMAX, VT)) { in LowerADDSAT_SUBSAT()
28173 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, X, Y); in LowerADDSAT_SUBSAT()
28176 if (SetCCResultType == VT && in LowerADDSAT_SUBSAT()
28177 DAG.ComputeNumSignBits(Cmp) == VT.getScalarSizeInBits()) in LowerADDSAT_SUBSAT()
28178 return DAG.getNode(ISD::AND, DL, VT, Cmp, Sub); in LowerADDSAT_SUBSAT()
28179 return DAG.getSelect(DL, VT, Cmp, Sub, DAG.getConstant(0, DL, VT)); in LowerADDSAT_SUBSAT()
28184 (!VT.isVector() || VT == MVT::v2i64)) { in LowerADDSAT_SUBSAT()
28187 SDValue Zero = DAG.getConstant(0, DL, VT); in LowerADDSAT_SUBSAT()
28190 DAG.getVTList(VT, SetCCResultType), X, Y); in LowerADDSAT_SUBSAT()
28193 SDValue SatMin = DAG.getConstant(MinVal, DL, VT); in LowerADDSAT_SUBSAT()
28194 SDValue SatMax = DAG.getConstant(MaxVal, DL, VT); in LowerADDSAT_SUBSAT()
28197 Result = DAG.getSelect(DL, VT, SumNeg, SatMax, SatMin); in LowerADDSAT_SUBSAT()
28198 return DAG.getSelect(DL, VT, Overflow, Result, SumDiff); in LowerADDSAT_SUBSAT()
28207 MVT VT = Op.getSimpleValueType(); in LowerABS() local
28210 if (VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64) { in LowerABS()
28214 SDValue Neg = DAG.getNode(X86ISD::SUB, DL, DAG.getVTList(VT, MVT::i32), in LowerABS()
28215 DAG.getConstant(0, DL, VT), N0); in LowerABS()
28218 return DAG.getNode(X86ISD::CMOV, DL, VT, Ops); in LowerABS()
28222 if ((VT == MVT::v2i64 || VT == MVT::v4i64) && Subtarget.hasSSE41()) { in LowerABS()
28224 SDValue Neg = DAG.getNegative(Src, DL, VT); in LowerABS()
28225 return DAG.getNode(X86ISD::BLENDV, DL, VT, Src, Neg, Src); in LowerABS()
28228 if (VT.is256BitVector() && !Subtarget.hasInt256()) { in LowerABS()
28229 assert(VT.isInteger() && in LowerABS()
28234 if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI()) in LowerABS()
28243 MVT VT = Op.getSimpleValueType(); in LowerAVG() local
28247 if (VT.is256BitVector() && !Subtarget.hasInt256()) in LowerAVG()
28250 if (VT == MVT::v32i16 || VT == MVT::v64i8) in LowerAVG()
28259 MVT VT = Op.getSimpleValueType(); in LowerMINMAX() local
28263 if (VT.is256BitVector() && !Subtarget.hasInt256()) in LowerMINMAX()
28266 if (VT == MVT::v32i16 || VT == MVT::v64i8) in LowerMINMAX()
28278 EVT VT = Op.getValueType(); in LowerFMINIMUM_FMAXIMUM() local
28282 uint64_t SizeInBits = VT.getScalarSizeInBits(); in LowerFMINIMUM_FMAXIMUM()
28285 EVT IVT = VT.changeTypeToInteger(); in LowerFMINIMUM_FMAXIMUM()
28295 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in LowerFMINIMUM_FMAXIMUM()
28353 } else if (!VT.isVector() && (VT == MVT::f16 || Subtarget.hasDQI()) && in LowerFMINIMUM_FMAXIMUM()
28359 MVT VectorType = MVT::getVectorVT(VT.getSimpleVT(), 128 / SizeInBits); in LowerFMINIMUM_FMAXIMUM()
28371 NewX = DAG.getSelect(DL, VT, NeedSwap, Y, X); in LowerFMINIMUM_FMAXIMUM()
28372 NewY = DAG.getSelect(DL, VT, NeedSwap, X, Y); in LowerFMINIMUM_FMAXIMUM()
28373 return DAG.getNode(MinMaxOp, DL, VT, NewX, NewY, Op->getFlags()); in LowerFMINIMUM_FMAXIMUM()
28376 if (Subtarget.is64Bit() || VT != MVT::f64) { in LowerFMINIMUM_FMAXIMUM()
28381 assert(VT == MVT::f64); in LowerFMINIMUM_FMAXIMUM()
28395 NewX = DAG.getSelect(DL, VT, IsXSigned, X, Y); in LowerFMINIMUM_FMAXIMUM()
28396 NewY = DAG.getSelect(DL, VT, IsXSigned, Y, X); in LowerFMINIMUM_FMAXIMUM()
28398 NewX = DAG.getSelect(DL, VT, IsXSigned, Y, X); in LowerFMINIMUM_FMAXIMUM()
28399 NewY = DAG.getSelect(DL, VT, IsXSigned, X, Y); in LowerFMINIMUM_FMAXIMUM()
28412 SDValue MinMax = DAG.getNode(MinMaxOp, DL, VT, NewX, NewY, Op->getFlags()); in LowerFMINIMUM_FMAXIMUM()
28418 return DAG.getSelect(DL, VT, IsNaN, NewX, MinMax); in LowerFMINIMUM_FMAXIMUM()
28423 MVT VT = Op.getSimpleValueType(); in LowerABD() local
28427 if (VT.is256BitVector() && !Subtarget.hasInt256()) in LowerABD()
28430 if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.useBWIRegs()) in LowerABD()
28437 if (VT.isScalarInteger()) { in LowerABD()
28438 unsigned WideBits = std::max<unsigned>(2 * VT.getScalarSizeInBits(), 32u); in LowerABD()
28448 return DAG.getNode(ISD::TRUNCATE, dl, VT, AbsDiff); in LowerABD()
28459 MVT VT = Op.getSimpleValueType(); in LowerMUL() local
28462 if (VT.is256BitVector() && !Subtarget.hasInt256()) in LowerMUL()
28465 if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI()) in LowerMUL()
28473 if (VT == MVT::v16i8 || VT == MVT::v32i8 || VT == MVT::v64i8) { in LowerMUL()
28474 unsigned NumElts = VT.getVectorNumElements(); in LowerMUL()
28475 unsigned NumLanes = VT.getSizeInBits() / 128; in LowerMUL()
28478 if ((VT == MVT::v16i8 && Subtarget.hasInt256()) || in LowerMUL()
28479 (VT == MVT::v32i8 && Subtarget.canExtendTo512BW())) { in LowerMUL()
28480 MVT ExVT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements()); in LowerMUL()
28482 ISD::TRUNCATE, dl, VT, in LowerMUL()
28505 SDValue Mask = DAG.getBitcast(VT, DAG.getConstant(0x00FF, dl, ExVT)); in LowerMUL()
28506 SDValue BLo = DAG.getNode(ISD::AND, dl, VT, Mask, B); in LowerMUL()
28507 SDValue BHi = DAG.getNode(X86ISD::ANDNP, dl, VT, Mask, B); in LowerMUL()
28510 RLo = DAG.getNode(ISD::AND, dl, VT, DAG.getBitcast(VT, RLo), Mask); in LowerMUL()
28513 return DAG.getNode(ISD::OR, dl, VT, RLo, DAG.getBitcast(VT, RHi)); in LowerMUL()
28521 SDValue Undef = DAG.getUNDEF(VT); in LowerMUL()
28522 SDValue ALo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, A, Undef)); in LowerMUL()
28523 SDValue AHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, A, Undef)); in LowerMUL()
28541 BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, B, Undef)); in LowerMUL()
28542 BHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, B, Undef)); in LowerMUL()
28548 return getPack(DAG, Subtarget, dl, VT, RLo, RHi); in LowerMUL()
28552 if (VT == MVT::v4i32) { in LowerMUL()
28558 SDValue Aodds = DAG.getVectorShuffle(VT, dl, A, A, UnpackMask); in LowerMUL()
28559 SDValue Bodds = DAG.getVectorShuffle(VT, dl, B, B, UnpackMask); in LowerMUL()
28570 Evens = DAG.getBitcast(VT, Evens); in LowerMUL()
28571 Odds = DAG.getBitcast(VT, Odds); in LowerMUL()
28576 return DAG.getVectorShuffle(VT, dl, Evens, Odds, ShufMask); in LowerMUL()
28579 assert((VT == MVT::v2i64 || VT == MVT::v4i64 || VT == MVT::v8i64) && in LowerMUL()
28603 SDValue Zero = DAG.getConstant(0, dl, VT); in LowerMUL()
28608 AloBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, B); in LowerMUL()
28612 SDValue Bhi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, B, 32, DAG); in LowerMUL()
28613 AloBhi = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, Bhi); in LowerMUL()
28618 SDValue Ahi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, A, 32, DAG); in LowerMUL()
28619 AhiBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, Ahi, B); in LowerMUL()
28622 SDValue Hi = DAG.getNode(ISD::ADD, dl, VT, AloBhi, AhiBlo); in LowerMUL()
28623 Hi = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Hi, 32, DAG); in LowerMUL()
28625 return DAG.getNode(ISD::ADD, dl, VT, AloBlo, Hi); in LowerMUL()
28629 MVT VT, bool IsSigned, in LowervXi8MulWithUNPCK() argument
28633 unsigned NumElts = VT.getVectorNumElements(); in LowervXi8MulWithUNPCK()
28648 SDValue Zero = DAG.getConstant(0, dl, VT); in LowervXi8MulWithUNPCK()
28652 ALo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, Zero, A)); in LowervXi8MulWithUNPCK()
28653 AHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, Zero, A)); in LowervXi8MulWithUNPCK()
28655 ALo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, A, Zero)); in LowervXi8MulWithUNPCK()
28656 AHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, A, Zero)); in LowervXi8MulWithUNPCK()
28688 BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, Zero, B)); in LowervXi8MulWithUNPCK()
28689 BHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, Zero, B)); in LowervXi8MulWithUNPCK()
28691 BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, B, Zero)); in LowervXi8MulWithUNPCK()
28692 BHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, B, Zero)); in LowervXi8MulWithUNPCK()
28702 *Low = getPack(DAG, Subtarget, dl, VT, RLo, RHi); in LowervXi8MulWithUNPCK()
28704 return getPack(DAG, Subtarget, dl, VT, RLo, RHi, /*PackHiHalf*/ true); in LowervXi8MulWithUNPCK()
28710 MVT VT = Op.getSimpleValueType(); in LowerMULH() local
28712 unsigned NumElts = VT.getVectorNumElements(); in LowerMULH()
28717 if (VT.is256BitVector() && !Subtarget.hasInt256()) in LowerMULH()
28720 if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !Subtarget.hasBWI()) in LowerMULH()
28723 if (VT == MVT::v4i32 || VT == MVT::v8i32 || VT == MVT::v16i32) { in LowerMULH()
28724 assert((VT == MVT::v4i32 && Subtarget.hasSSE2()) || in LowerMULH()
28725 (VT == MVT::v8i32 && Subtarget.hasInt256()) || in LowerMULH()
28726 (VT == MVT::v16i32 && Subtarget.hasAVX512())); in LowerMULH()
28744 DAG.getVectorShuffle(VT, dl, A, A, ArrayRef(&Mask[0], NumElts)); in LowerMULH()
28747 DAG.getVectorShuffle(VT, dl, B, B, ArrayRef(&Mask[0], NumElts)); in LowerMULH()
28756 SDValue Mul1 = DAG.getBitcast(VT, DAG.getNode(Opcode, dl, MulVT, in LowerMULH()
28761 SDValue Mul2 = DAG.getBitcast(VT, DAG.getNode(Opcode, dl, MulVT, in LowerMULH()
28770 SDValue Res = DAG.getVectorShuffle(VT, dl, Mul1, Mul2, ShufMask); in LowerMULH()
28775 SDValue Zero = DAG.getConstant(0, dl, VT); in LowerMULH()
28776 SDValue T1 = DAG.getNode(ISD::AND, dl, VT, in LowerMULH()
28777 DAG.getSetCC(dl, VT, Zero, A, ISD::SETGT), B); in LowerMULH()
28778 SDValue T2 = DAG.getNode(ISD::AND, dl, VT, in LowerMULH()
28779 DAG.getSetCC(dl, VT, Zero, B, ISD::SETGT), A); in LowerMULH()
28781 SDValue Fixup = DAG.getNode(ISD::ADD, dl, VT, T1, T2); in LowerMULH()
28782 Res = DAG.getNode(ISD::SUB, dl, VT, Res, Fixup); in LowerMULH()
28789 assert((VT == MVT::v16i8 || (VT == MVT::v32i8 && Subtarget.hasInt256()) || in LowerMULH()
28790 (VT == MVT::v64i8 && Subtarget.hasBWI())) && in LowerMULH()
28799 if ((VT == MVT::v16i8 && Subtarget.hasInt256()) || in LowerMULH()
28800 (VT == MVT::v32i8 && Subtarget.canExtendTo512BW())) { in LowerMULH()
28807 return DAG.getNode(ISD::TRUNCATE, dl, VT, Mul); in LowerMULH()
28810 return LowervXi8MulWithUNPCK(A, B, dl, VT, IsSigned, Subtarget, DAG); in LowerMULH()
28816 MVT VT = Op.getSimpleValueType(); in LowerMULO() local
28819 if (!VT.isVector()) in LowerMULO()
28828 if ((VT == MVT::v32i8 && !Subtarget.hasInt256()) || in LowerMULO()
28829 (VT == MVT::v64i8 && !Subtarget.hasBWI())) { in LowerMULO()
28848 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi); in LowerMULO()
28857 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); in LowerMULO()
28859 if ((VT == MVT::v16i8 && Subtarget.hasInt256()) || in LowerMULO()
28860 (VT == MVT::v32i8 && Subtarget.canExtendTo512BW())) { in LowerMULO()
28861 unsigned NumElts = VT.getVectorNumElements(); in LowerMULO()
28868 SDValue Low = DAG.getNode(ISD::TRUNCATE, dl, VT, Mul); in LowerMULO()
28892 High = DAG.getNode(ISD::TRUNCATE, dl, VT, High); in LowerMULO()
28894 DAG.getNode(ISD::SRA, dl, VT, Low, DAG.getConstant(7, dl, VT)); in LowerMULO()
28911 High = DAG.getNode(ISD::TRUNCATE, dl, VT, High); in LowerMULO()
28926 LowervXi8MulWithUNPCK(A, B, dl, VT, IsSigned, Subtarget, DAG, &Low); in LowerMULO()
28932 DAG.getNode(ISD::SRA, dl, VT, Low, DAG.getConstant(7, dl, VT)); in LowerMULO()
28937 DAG.getSetCC(dl, SetccVT, High, DAG.getConstant(0, dl, VT), ISD::SETNE); in LowerMULO()
28947 EVT VT = Op.getValueType(); in LowerWin64_i128OP() local
28948 assert(VT.isInteger() && VT.getSizeInBits() == 128 && in LowerWin64_i128OP()
28954 return DAG.getNode(ISD::BUILD_PAIR, SDLoc(Op), VT, Result[0], Result[1]); in LowerWin64_i128OP()
29007 return DAG.getBitcast(VT, CallInfo.first); in LowerWin64_i128OP()
29014 EVT VT = Op.getValueType(); in LowerWin64_FP_TO_INT128() local
29020 assert(VT.isInteger() && VT.getSizeInBits() == 128 && in LowerWin64_FP_TO_INT128()
29026 LC = RTLIB::getFPTOSINT(ArgVT, VT); in LowerWin64_FP_TO_INT128()
29028 LC = RTLIB::getFPTOUINT(ArgVT, VT); in LowerWin64_FP_TO_INT128()
29040 Result = DAG.getBitcast(VT, Result); in LowerWin64_FP_TO_INT128()
29047 EVT VT = Op.getValueType(); in LowerWin64_INT128_TO_FP() local
29059 LC = RTLIB::getSINTTOFP(ArgVT, VT); in LowerWin64_INT128_TO_FP()
29061 LC = RTLIB::getUINTTOFP(ArgVT, VT); in LowerWin64_INT128_TO_FP()
29077 makeLibCall(DAG, LC, VT, StackPtr, CallOptions, dl, Chain); in LowerWin64_INT128_TO_FP()
29105 SDValue getGFNICtrlMask(unsigned Opcode, SelectionDAG &DAG, const SDLoc &DL, MVT VT, in getGFNICtrlMask() argument
29107 assert(VT.getVectorElementType() == MVT::i8 && in getGFNICtrlMask()
29108 (VT.getSizeInBits() % 64) == 0 && "Illegal GFNI control type"); in getGFNICtrlMask()
29111 for (unsigned I = 0, E = VT.getSizeInBits(); I != E; I += 8) { in getGFNICtrlMask()
29115 return DAG.getBuildVector(VT, DL, MaskBits); in getGFNICtrlMask()
29120 static bool supportedVectorShiftWithImm(EVT VT, const X86Subtarget &Subtarget, in supportedVectorShiftWithImm() argument
29125 if (!VT.isSimple()) in supportedVectorShiftWithImm()
29128 if (!(VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector())) in supportedVectorShiftWithImm()
29131 if (VT.getScalarSizeInBits() < 16) in supportedVectorShiftWithImm()
29134 if (VT.is512BitVector() && Subtarget.useAVX512Regs() && in supportedVectorShiftWithImm()
29135 (VT.getScalarSizeInBits() > 16 || Subtarget.hasBWI())) in supportedVectorShiftWithImm()
29138 bool LShift = (VT.is128BitVector() && Subtarget.hasSSE2()) || in supportedVectorShiftWithImm()
29139 (VT.is256BitVector() && Subtarget.hasInt256()); in supportedVectorShiftWithImm()
29142 (VT != MVT::v2i64 && VT != MVT::v4i64)); in supportedVectorShiftWithImm()
29149 bool supportedVectorShiftWithBaseAmnt(EVT VT, const X86Subtarget &Subtarget, in supportedVectorShiftWithBaseAmnt() argument
29151 return supportedVectorShiftWithImm(VT, Subtarget, Opcode); in supportedVectorShiftWithBaseAmnt()
29156 static bool supportedVectorVarShift(EVT VT, const X86Subtarget &Subtarget, in supportedVectorVarShift() argument
29161 if (!VT.isSimple()) in supportedVectorVarShift()
29164 if (!(VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector())) in supportedVectorVarShift()
29167 if (!Subtarget.hasInt256() || VT.getScalarSizeInBits() < 16) in supportedVectorVarShift()
29171 if (VT.getScalarSizeInBits() == 16 && !Subtarget.hasBWI()) in supportedVectorVarShift()
29175 (Subtarget.useAVX512Regs() || !VT.is512BitVector())) in supportedVectorVarShift()
29178 bool LShift = VT.is128BitVector() || VT.is256BitVector(); in supportedVectorVarShift()
29179 bool AShift = LShift && VT != MVT::v2i64 && VT != MVT::v4i64; in supportedVectorVarShift()
29185 MVT VT = Op.getSimpleValueType(); in LowerShiftByScalarImmediate() local
29190 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in LowerShiftByScalarImmediate()
29193 assert((VT == MVT::v2i64 || VT == MVT::v4i64) && "Unexpected SRA type"); in LowerShiftByScalarImmediate()
29194 MVT ExVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() * 2); in LowerShiftByScalarImmediate()
29199 assert((VT != MVT::v4i64 || Subtarget.hasInt256()) && in LowerShiftByScalarImmediate()
29201 return DAG.getNode(X86ISD::PCMPGT, dl, VT, DAG.getConstant(0, dl, VT), R); in LowerShiftByScalarImmediate()
29210 if (VT == MVT::v2i64) in LowerShiftByScalarImmediate()
29212 if (VT == MVT::v4i64) in LowerShiftByScalarImmediate()
29220 getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, R, ShiftAmt, DAG); in LowerShiftByScalarImmediate()
29222 if (VT == MVT::v2i64) in LowerShiftByScalarImmediate()
29224 if (VT == MVT::v4i64) in LowerShiftByScalarImmediate()
29228 return DAG.getBitcast(VT, Ex); in LowerShiftByScalarImmediate()
29238 return DAG.getUNDEF(VT); in LowerShiftByScalarImmediate()
29242 if (supportedVectorShiftWithImm(VT, Subtarget, Op.getOpcode())) { in LowerShiftByScalarImmediate()
29253 return DAG.getNode(ISD::ADD, dl, VT, R, R); in LowerShiftByScalarImmediate()
29256 return getTargetVShiftByConstNode(X86Opc, dl, VT, R, ShiftAmt, DAG); in LowerShiftByScalarImmediate()
29260 if (((!Subtarget.hasXOP() && VT == MVT::v2i64) || in LowerShiftByScalarImmediate()
29261 (Subtarget.hasInt256() && VT == MVT::v4i64)) && in LowerShiftByScalarImmediate()
29269 SDValue Mask = DAG.getAllOnesConstant(dl, VT); in LowerShiftByScalarImmediate()
29270 Mask = DAG.getNode(Op.getOpcode(), dl, VT, Mask, Amt); in LowerShiftByScalarImmediate()
29271 return DAG.getNode(ISD::AND, dl, VT, R, Mask); in LowerShiftByScalarImmediate()
29274 if (VT == MVT::v16i8 || (Subtarget.hasInt256() && VT == MVT::v32i8) || in LowerShiftByScalarImmediate()
29275 (Subtarget.hasBWI() && VT == MVT::v64i8)) { in LowerShiftByScalarImmediate()
29276 unsigned NumElts = VT.getVectorNumElements(); in LowerShiftByScalarImmediate()
29287 return DAG.getNode(ISD::ADD, dl, VT, R, R); in LowerShiftByScalarImmediate()
29292 SDValue Zeros = DAG.getConstant(0, dl, VT); in LowerShiftByScalarImmediate()
29293 if (VT.is512BitVector()) { in LowerShiftByScalarImmediate()
29294 assert(VT == MVT::v64i8 && "Unexpected element type!"); in LowerShiftByScalarImmediate()
29296 return DAG.getNode(ISD::SIGN_EXTEND, dl, VT, CMP); in LowerShiftByScalarImmediate()
29298 return DAG.getNode(X86ISD::PCMPGT, dl, VT, Zeros, R); in LowerShiftByScalarImmediate()
29302 if (VT == MVT::v16i8 && Subtarget.hasXOP()) in LowerShiftByScalarImmediate()
29306 SDValue Mask = getGFNICtrlMask(Op.getOpcode(), DAG, dl, VT, ShiftAmt); in LowerShiftByScalarImmediate()
29307 return DAG.getNode(X86ISD::GF2P8AFFINEQB, dl, VT, R, Mask, in LowerShiftByScalarImmediate()
29315 SHL = DAG.getBitcast(VT, SHL); in LowerShiftByScalarImmediate()
29318 return DAG.getNode(ISD::AND, dl, VT, SHL, DAG.getConstant(Mask, dl, VT)); in LowerShiftByScalarImmediate()
29324 SRL = DAG.getBitcast(VT, SRL); in LowerShiftByScalarImmediate()
29327 return DAG.getNode(ISD::AND, dl, VT, SRL, DAG.getConstant(Mask, dl, VT)); in LowerShiftByScalarImmediate()
29331 SDValue Res = DAG.getNode(ISD::SRL, dl, VT, R, Amt); in LowerShiftByScalarImmediate()
29333 SDValue Mask = DAG.getConstant(128 >> ShiftAmt, dl, VT); in LowerShiftByScalarImmediate()
29334 Res = DAG.getNode(ISD::XOR, dl, VT, Res, Mask); in LowerShiftByScalarImmediate()
29335 Res = DAG.getNode(ISD::SUB, dl, VT, Res, Mask); in LowerShiftByScalarImmediate()
29346 MVT VT = Op.getSimpleValueType(); in LowerShiftByScalarVariable() local
29355 if (supportedVectorShiftWithBaseAmnt(VT, Subtarget, Opcode)) in LowerShiftByScalarVariable()
29356 return getTargetVShiftNode(X86OpcI, dl, VT, R, BaseShAmt, BaseShAmtIdx, in LowerShiftByScalarVariable()
29360 if (((VT == MVT::v16i8 && !Subtarget.canExtendTo512DQ()) || in LowerShiftByScalarVariable()
29361 (VT == MVT::v32i8 && !Subtarget.canExtendTo512BW()) || in LowerShiftByScalarVariable()
29362 VT == MVT::v64i8) && in LowerShiftByScalarVariable()
29364 unsigned NumElts = VT.getVectorNumElements(); in LowerShiftByScalarVariable()
29378 BitMask = DAG.getBitcast(VT, BitMask); in LowerShiftByScalarVariable()
29379 BitMask = DAG.getVectorShuffle(VT, dl, BitMask, BitMask, in LowerShiftByScalarVariable()
29385 Res = DAG.getBitcast(VT, Res); in LowerShiftByScalarVariable()
29386 Res = DAG.getNode(ISD::AND, dl, VT, Res, BitMask); in LowerShiftByScalarVariable()
29395 SignMask = DAG.getBitcast(VT, SignMask); in LowerShiftByScalarVariable()
29396 Res = DAG.getNode(ISD::XOR, dl, VT, Res, SignMask); in LowerShiftByScalarVariable()
29397 Res = DAG.getNode(ISD::SUB, dl, VT, Res, SignMask); in LowerShiftByScalarVariable()
29411 MVT VT = Amt.getSimpleValueType(); in convertShiftLeftToScale() local
29412 if (!(VT == MVT::v8i16 || VT == MVT::v4i32 || in convertShiftLeftToScale()
29413 (Subtarget.hasInt256() && VT == MVT::v16i16) || in convertShiftLeftToScale()
29414 (Subtarget.hasAVX512() && VT == MVT::v32i16) || in convertShiftLeftToScale()
29415 (!Subtarget.hasAVX512() && VT == MVT::v16i8) || in convertShiftLeftToScale()
29416 (Subtarget.hasInt256() && VT == MVT::v32i8) || in convertShiftLeftToScale()
29417 (Subtarget.hasBWI() && VT == MVT::v64i8))) in convertShiftLeftToScale()
29420 MVT SVT = VT.getVectorElementType(); in convertShiftLeftToScale()
29422 unsigned NumElems = VT.getVectorNumElements(); in convertShiftLeftToScale()
29435 return DAG.getBuildVector(VT, dl, Elts); in convertShiftLeftToScale()
29440 if (VT == MVT::v4i32) { in convertShiftLeftToScale()
29441 Amt = DAG.getNode(ISD::SHL, dl, VT, Amt, DAG.getConstant(23, dl, VT)); in convertShiftLeftToScale()
29442 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, in convertShiftLeftToScale()
29443 DAG.getConstant(0x3f800000U, dl, VT)); in convertShiftLeftToScale()
29445 return DAG.getNode(ISD::FP_TO_SINT, dl, VT, Amt); in convertShiftLeftToScale()
29449 if (VT == MVT::v8i16 && !Subtarget.hasAVX2()) { in convertShiftLeftToScale()
29450 SDValue Z = DAG.getConstant(0, dl, VT); in convertShiftLeftToScale()
29451 SDValue Lo = DAG.getBitcast(MVT::v4i32, getUnpackl(DAG, dl, VT, Amt, Z)); in convertShiftLeftToScale()
29452 SDValue Hi = DAG.getBitcast(MVT::v4i32, getUnpackh(DAG, dl, VT, Amt, Z)); in convertShiftLeftToScale()
29456 return DAG.getNode(X86ISD::PACKUS, dl, VT, Lo, Hi); in convertShiftLeftToScale()
29457 return getPack(DAG, Subtarget, dl, VT, Lo, Hi); in convertShiftLeftToScale()
29465 MVT VT = Op.getSimpleValueType(); in LowerShift() local
29469 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in LowerShift()
29476 assert(VT.isVector() && "Custom lowering only for vector shifts!"); in LowerShift()
29485 if (supportedVectorVarShift(VT, Subtarget, Opc)) in LowerShift()
29491 if (((VT == MVT::v2i64 && !Subtarget.hasXOP()) || in LowerShift()
29492 (VT == MVT::v4i64 && Subtarget.hasInt256())) && in LowerShift()
29494 SDValue S = DAG.getConstant(APInt::getSignMask(64), dl, VT); in LowerShift()
29495 SDValue M = DAG.getNode(ISD::SRL, dl, VT, S, Amt); in LowerShift()
29496 R = DAG.getNode(ISD::SRL, dl, VT, R, Amt); in LowerShift()
29497 R = DAG.getNode(ISD::XOR, dl, VT, R, M); in LowerShift()
29498 R = DAG.getNode(ISD::SUB, dl, VT, R, M); in LowerShift()
29504 if (Subtarget.hasXOP() && (VT == MVT::v2i64 || VT == MVT::v4i32 || in LowerShift()
29505 VT == MVT::v8i16 || VT == MVT::v16i8)) { in LowerShift()
29507 Amt = DAG.getNegative(Amt, dl, VT); in LowerShift()
29509 return DAG.getNode(X86ISD::VPSHL, dl, VT, R, Amt); in LowerShift()
29511 return DAG.getNode(X86ISD::VPSHA, dl, VT, R, Amt); in LowerShift()
29516 if (VT == MVT::v2i64 && Opc != ISD::SRA) { in LowerShift()
29518 SDValue Amt0 = DAG.getVectorShuffle(VT, dl, Amt, Amt, {0, 0}); in LowerShift()
29519 SDValue Amt1 = DAG.getVectorShuffle(VT, dl, Amt, Amt, {1, 1}); in LowerShift()
29520 SDValue R0 = DAG.getNode(Opc, dl, VT, R, Amt0); in LowerShift()
29521 SDValue R1 = DAG.getNode(Opc, dl, VT, R, Amt1); in LowerShift()
29522 return DAG.getVectorShuffle(VT, dl, R0, R1, {0, 3}); in LowerShift()
29535 if (ConstantAmt && (VT == MVT::v8i16 || VT == MVT::v4i32 || in LowerShift()
29536 (VT == MVT::v16i16 && Subtarget.hasInt256()))) { in LowerShift()
29538 unsigned NumElts = VT.getVectorNumElements(); in LowerShift()
29561 (VT != MVT::v16i16 || in LowerShift()
29562 is128BitLaneRepeatedShuffleMask(VT, ShuffleMask)) && in LowerShift()
29563 (VT == MVT::v4i32 || Subtarget.hasSSE41() || Opc != ISD::SHL || in LowerShift()
29569 SDValue Shift1 = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, in LowerShift()
29571 SDValue Shift2 = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, in LowerShift()
29573 return DAG.getVectorShuffle(VT, dl, Shift1, Shift2, ShuffleMask); in LowerShift()
29581 if (Opc == ISD::SHL && !(VT == MVT::v32i8 && (Subtarget.hasXOP() || in LowerShift()
29584 return DAG.getNode(ISD::MUL, dl, VT, R, Scale); in LowerShift()
29589 (VT == MVT::v8i16 || (VT == MVT::v16i16 && Subtarget.hasInt256()))) { in LowerShift()
29590 SDValue EltBits = DAG.getConstant(EltSizeInBits, dl, VT); in LowerShift()
29591 SDValue RAmt = DAG.getNode(ISD::SUB, dl, VT, EltBits, Amt); in LowerShift()
29593 SDValue Zero = DAG.getConstant(0, dl, VT); in LowerShift()
29594 SDValue ZAmt = DAG.getSetCC(dl, VT, Amt, Zero, ISD::SETEQ); in LowerShift()
29595 SDValue Res = DAG.getNode(ISD::MULHU, dl, VT, R, Scale); in LowerShift()
29596 return DAG.getSelect(dl, VT, ZAmt, R, Res); in LowerShift()
29605 (VT == MVT::v8i16 || (VT == MVT::v16i16 && Subtarget.hasInt256())) && in LowerShift()
29609 SDValue EltBits = DAG.getConstant(EltSizeInBits, dl, VT); in LowerShift()
29610 SDValue RAmt = DAG.getNode(ISD::SUB, dl, VT, EltBits, Amt); in LowerShift()
29613 DAG.getSetCC(dl, VT, Amt, DAG.getConstant(0, dl, VT), ISD::SETEQ); in LowerShift()
29615 DAG.getSetCC(dl, VT, Amt, DAG.getConstant(1, dl, VT), ISD::SETEQ); in LowerShift()
29617 getTargetVShiftByConstNode(X86ISD::VSRAI, dl, VT, R, 1, DAG); in LowerShift()
29618 SDValue Res = DAG.getNode(ISD::MULHS, dl, VT, R, Scale); in LowerShift()
29619 Res = DAG.getSelect(dl, VT, Amt0, R, Res); in LowerShift()
29620 return DAG.getSelect(dl, VT, Amt1, Sra1, Res); in LowerShift()
29629 if (VT == MVT::v4i32) { in LowerShift()
29632 Amt0 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {0, 0, 0, 0}); in LowerShift()
29633 Amt1 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {1, 1, 1, 1}); in LowerShift()
29634 Amt2 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {2, 2, 2, 2}); in LowerShift()
29635 Amt3 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {3, 3, 3, 3}); in LowerShift()
29642 SDValue Z = DAG.getConstant(0, dl, VT); in LowerShift()
29643 Amt0 = DAG.getVectorShuffle(VT, dl, Amt, Z, {0, 4, -1, -1}); in LowerShift()
29644 Amt1 = DAG.getVectorShuffle(VT, dl, Amt, Z, {1, 5, -1, -1}); in LowerShift()
29645 Amt2 = DAG.getVectorShuffle(VT, dl, Amt, Z, {2, 6, -1, -1}); in LowerShift()
29646 Amt3 = DAG.getVectorShuffle(VT, dl, Amt, Z, {3, 7, -1, -1}); in LowerShift()
29661 SDValue R0 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt0)); in LowerShift()
29662 SDValue R1 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt1)); in LowerShift()
29663 SDValue R2 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt2)); in LowerShift()
29664 SDValue R3 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt3)); in LowerShift()
29669 SDValue R02 = DAG.getVectorShuffle(VT, dl, R0, R2, {0, -1, 6, -1}); in LowerShift()
29670 SDValue R13 = DAG.getVectorShuffle(VT, dl, R1, R3, {-1, 1, -1, 7}); in LowerShift()
29671 return DAG.getVectorShuffle(VT, dl, R02, R13, {0, 5, 2, 7}); in LowerShift()
29673 SDValue R01 = DAG.getVectorShuffle(VT, dl, R0, R1, {0, -1, -1, 5}); in LowerShift()
29674 SDValue R23 = DAG.getVectorShuffle(VT, dl, R2, R3, {2, -1, -1, 7}); in LowerShift()
29675 return DAG.getVectorShuffle(VT, dl, R01, R23, {0, 3, 4, 7}); in LowerShift()
29682 if ((Subtarget.hasInt256() && VT == MVT::v8i16) || in LowerShift()
29683 (Subtarget.canExtendTo512DQ() && VT == MVT::v16i16) || in LowerShift()
29684 (Subtarget.canExtendTo512DQ() && VT == MVT::v16i8) || in LowerShift()
29685 (Subtarget.canExtendTo512BW() && VT == MVT::v32i8) || in LowerShift()
29686 (Subtarget.hasBWI() && Subtarget.hasVLX() && VT == MVT::v16i8)) { in LowerShift()
29687 assert((!Subtarget.hasBWI() || VT == MVT::v32i8 || VT == MVT::v16i8) && in LowerShift()
29690 MVT ExtVT = MVT::getVectorVT(EvtSVT, VT.getVectorNumElements()); in LowerShift()
29694 return DAG.getNode(ISD::TRUNCATE, dl, VT, in LowerShift()
29701 (VT == MVT::v16i8 || (VT == MVT::v32i8 && Subtarget.hasInt256()) || in LowerShift()
29702 (VT == MVT::v64i8 && Subtarget.hasBWI())) && in LowerShift()
29704 int NumElts = VT.getVectorNumElements(); in LowerShift()
29716 if (VT == MVT::v16i8 && Subtarget.hasInt256()) { in LowerShift()
29721 return DAG.getZExtOrTrunc(R, dl, VT); in LowerShift()
29736 SDValue LoR = DAG.getBitcast(VT16, getUnpackl(DAG, dl, VT, R, R)); in LowerShift()
29737 SDValue HiR = DAG.getBitcast(VT16, getUnpackh(DAG, dl, VT, R, R)); in LowerShift()
29744 return DAG.getNode(X86ISD::PACKUS, dl, VT, LoR, HiR); in LowerShift()
29747 if (VT == MVT::v16i8 || in LowerShift()
29748 (VT == MVT::v32i8 && Subtarget.hasInt256() && !Subtarget.hasXOP()) || in LowerShift()
29749 (VT == MVT::v64i8 && Subtarget.hasBWI())) { in LowerShift()
29750 MVT ExtVT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements() / 2); in LowerShift()
29753 if (VT.is512BitVector()) { in LowerShift()
29757 MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorNumElements()); in LowerShift()
29758 V0 = DAG.getBitcast(VT, V0); in LowerShift()
29759 V1 = DAG.getBitcast(VT, V1); in LowerShift()
29760 Sel = DAG.getBitcast(VT, Sel); in LowerShift()
29761 Sel = DAG.getSetCC(dl, MaskVT, DAG.getConstant(0, dl, VT), Sel, in LowerShift()
29763 return DAG.getBitcast(SelVT, DAG.getSelect(dl, VT, Sel, V0, V1)); in LowerShift()
29767 V0 = DAG.getBitcast(VT, V0); in LowerShift()
29768 V1 = DAG.getBitcast(VT, V1); in LowerShift()
29769 Sel = DAG.getBitcast(VT, Sel); in LowerShift()
29771 DAG.getNode(X86ISD::BLENDV, dl, VT, Sel, V0, V1)); in LowerShift()
29786 Amt = DAG.getBitcast(VT, Amt); in LowerShift()
29790 SDValue M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(4, dl, VT)); in LowerShift()
29791 R = SignBitSelect(VT, Amt, M, R); in LowerShift()
29794 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt); in LowerShift()
29797 M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(2, dl, VT)); in LowerShift()
29798 R = SignBitSelect(VT, Amt, M, R); in LowerShift()
29801 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt); in LowerShift()
29804 M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(1, dl, VT)); in LowerShift()
29805 R = SignBitSelect(VT, Amt, M, R); in LowerShift()
29813 SDValue ALo = getUnpackl(DAG, dl, VT, DAG.getUNDEF(VT), Amt); in LowerShift()
29814 SDValue AHi = getUnpackh(DAG, dl, VT, DAG.getUNDEF(VT), Amt); in LowerShift()
29815 SDValue RLo = getUnpackl(DAG, dl, VT, DAG.getUNDEF(VT), R); in LowerShift()
29816 SDValue RHi = getUnpackh(DAG, dl, VT, DAG.getUNDEF(VT), R); in LowerShift()
29852 return DAG.getNode(X86ISD::PACKUS, dl, VT, RLo, RHi); in LowerShift()
29856 if (Subtarget.hasInt256() && !Subtarget.hasXOP() && VT == MVT::v16i16) { in LowerShift()
29858 SDValue Z = DAG.getConstant(0, dl, VT); in LowerShift()
29859 SDValue ALo = getUnpackl(DAG, dl, VT, Amt, Z); in LowerShift()
29860 SDValue AHi = getUnpackh(DAG, dl, VT, Amt, Z); in LowerShift()
29861 SDValue RLo = getUnpackl(DAG, dl, VT, Z, R); in LowerShift()
29862 SDValue RHi = getUnpackh(DAG, dl, VT, Z, R); in LowerShift()
29871 return DAG.getNode(X86ISD::PACKUS, dl, VT, Lo, Hi); in LowerShift()
29874 if (VT == MVT::v8i16) { in LowerShift()
29884 MVT ExtVT = MVT::getVectorVT(MVT::i8, VT.getVectorNumElements() * 2); in LowerShift()
29889 VT, DAG.getNode(X86ISD::BLENDV, dl, ExtVT, Sel, V0, V1)); in LowerShift()
29895 getTargetVShiftByConstNode(X86ISD::VSRAI, dl, VT, Sel, 15, DAG); in LowerShift()
29896 return DAG.getSelect(dl, VT, C, V0, V1); in LowerShift()
29904 ISD::OR, dl, VT, in LowerShift()
29905 getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Amt, 4, DAG), in LowerShift()
29906 getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Amt, 12, DAG)); in LowerShift()
29908 Amt = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Amt, 12, DAG); in LowerShift()
29912 SDValue M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 8, DAG); in LowerShift()
29916 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt); in LowerShift()
29919 M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 4, DAG); in LowerShift()
29923 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt); in LowerShift()
29926 M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 2, DAG); in LowerShift()
29930 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt); in LowerShift()
29933 M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 1, DAG); in LowerShift()
29939 if (VT.is256BitVector()) in LowerShift()
29942 if (VT == MVT::v32i16 || VT == MVT::v64i8) in LowerShift()
29950 MVT VT = Op.getSimpleValueType(); in LowerFunnelShift() local
29958 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in LowerFunnelShift()
29961 if (VT.isVector()) { in LowerFunnelShift()
29964 unsigned NumElts = VT.getVectorNumElements(); in LowerFunnelShift()
29973 return getAVX512Node(IsFSHR ? X86ISD::VSHRD : X86ISD::VSHLD, DL, VT, in LowerFunnelShift()
29976 return getAVX512Node(IsFSHR ? X86ISD::VSHRDV : X86ISD::VSHLDV, DL, VT, in LowerFunnelShift()
29979 assert((VT == MVT::v16i8 || VT == MVT::v32i8 || VT == MVT::v64i8 || in LowerFunnelShift()
29980 VT == MVT::v8i16 || VT == MVT::v16i16 || VT == MVT::v32i16 || in LowerFunnelShift()
29981 VT == MVT::v4i32 || VT == MVT::v8i32 || VT == MVT::v16i32) && in LowerFunnelShift()
29998 (useVPTERNLOG(Subtarget, VT) && in LowerFunnelShift()
30011 ShX = DAG.getNode(ISD::AND, DL, VT, DAG.getBitcast(VT, ShX), in LowerFunnelShift()
30012 DAG.getConstant(MaskX, DL, VT)); in LowerFunnelShift()
30013 ShY = DAG.getNode(ISD::AND, DL, VT, DAG.getBitcast(VT, ShY), in LowerFunnelShift()
30014 DAG.getConstant(MaskY, DL, VT)); in LowerFunnelShift()
30015 return DAG.getNode(ISD::OR, DL, VT, ShX, ShY); in LowerFunnelShift()
30018 SDValue ShX = DAG.getNode(ISD::SHL, DL, VT, Op0, in LowerFunnelShift()
30019 DAG.getShiftAmountConstant(ShXAmt, VT, DL)); in LowerFunnelShift()
30020 SDValue ShY = DAG.getNode(ISD::SRL, DL, VT, Op1, in LowerFunnelShift()
30021 DAG.getShiftAmountConstant(ShYAmt, VT, DL)); in LowerFunnelShift()
30022 return DAG.getNode(ISD::OR, DL, VT, ShX, ShY); in LowerFunnelShift()
30025 SDValue AmtMask = DAG.getConstant(EltSizeInBits - 1, DL, VT); in LowerFunnelShift()
30026 SDValue AmtMod = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask); in LowerFunnelShift()
30039 if ((VT.is256BitVector() && ((Subtarget.hasXOP() && EltSizeInBits < 16) || in LowerFunnelShift()
30041 (VT.is512BitVector() && !Subtarget.useBWIRegs() && in LowerFunnelShift()
30044 Op = DAG.getNode(Op.getOpcode(), DL, VT, Op0, Op1, AmtMod); in LowerFunnelShift()
30056 SDValue Lo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, Op1, Op0)); in LowerFunnelShift()
30057 SDValue Hi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, Op1, Op0)); in LowerFunnelShift()
30062 return getPack(DAG, Subtarget, DL, VT, Lo, Hi, !IsFSHR); in LowerFunnelShift()
30071 if (supportedVectorVarShift(VT, Subtarget, ShiftOpc) || Subtarget.hasXOP()) in LowerFunnelShift()
30089 return DAG.getNode(ISD::TRUNCATE, DL, VT, Res); in LowerFunnelShift()
30095 SDValue Z = DAG.getConstant(0, DL, VT); in LowerFunnelShift()
30096 SDValue RLo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, Op1, Op0)); in LowerFunnelShift()
30097 SDValue RHi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, Op1, Op0)); in LowerFunnelShift()
30098 SDValue ALo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, AmtMod, Z)); in LowerFunnelShift()
30099 SDValue AHi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, AmtMod, Z)); in LowerFunnelShift()
30102 return getPack(DAG, Subtarget, DL, VT, Lo, Hi, !IsFSHR); in LowerFunnelShift()
30109 (VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64) && in LowerFunnelShift()
30118 if ((VT == MVT::i8 || (ExpandFunnel && VT == MVT::i16)) && in LowerFunnelShift()
30133 return DAG.getZExtOrTrunc(Res, DL, VT); in LowerFunnelShift()
30136 if (VT == MVT::i8 || ExpandFunnel) in LowerFunnelShift()
30140 if (VT == MVT::i16) { in LowerFunnelShift()
30144 return DAG.getNode(FSHOp, DL, VT, Op0, Op1, Amt); in LowerFunnelShift()
30152 MVT VT = Op.getSimpleValueType(); in LowerRotate() local
30153 assert(VT.isVector() && "Custom lowering only for vector rotates!"); in LowerRotate()
30159 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in LowerRotate()
30160 int NumElts = VT.getVectorNumElements(); in LowerRotate()
30179 return DAG.getNode(RotOpc, DL, VT, R, in LowerRotate()
30190 return DAG.getNode(FunnelOpc, DL, VT, R, R, Amt); in LowerRotate()
30193 SDValue Z = DAG.getConstant(0, DL, VT); in LowerRotate()
30198 if (SDValue NegAmt = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {Z, Amt})) in LowerRotate()
30199 return DAG.getNode(ISD::ROTL, DL, VT, R, NegAmt); in LowerRotate()
30203 return DAG.getNode(ISD::ROTL, DL, VT, R, in LowerRotate()
30204 DAG.getNode(ISD::SUB, DL, VT, Z, Amt)); in LowerRotate()
30208 if (IsCstSplat && Subtarget.hasGFNI() && VT.getScalarType() == MVT::i8 && in LowerRotate()
30209 DAG.getTargetLoweringInfo().isTypeLegal(VT)) { in LowerRotate()
30211 SDValue Mask = getGFNICtrlMask(Opcode, DAG, DL, VT, RotAmt); in LowerRotate()
30212 return DAG.getNode(X86ISD::GF2P8AFFINEQB, DL, VT, R, Mask, in LowerRotate()
30217 if (VT.is256BitVector() && (Subtarget.hasXOP() || !Subtarget.hasAVX2())) in LowerRotate()
30225 assert(VT.is128BitVector() && "Only rotate 128-bit vectors!"); in LowerRotate()
30230 return DAG.getNode(X86ISD::VROTLI, DL, VT, R, in LowerRotate()
30245 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, R, in LowerRotate()
30246 DAG.getShiftAmountConstant(ShlAmt, VT, DL)); in LowerRotate()
30247 SDValue Srl = DAG.getNode(ISD::SRL, DL, VT, R, in LowerRotate()
30248 DAG.getShiftAmountConstant(SrlAmt, VT, DL)); in LowerRotate()
30249 return DAG.getNode(ISD::OR, DL, VT, Shl, Srl); in LowerRotate()
30253 if (VT.is512BitVector() && !Subtarget.useBWIRegs()) in LowerRotate()
30257 (VT == MVT::v4i32 || VT == MVT::v8i16 || VT == MVT::v16i8 || in LowerRotate()
30258 ((VT == MVT::v8i32 || VT == MVT::v16i16 || VT == MVT::v32i8) && in LowerRotate()
30260 ((VT == MVT::v32i16 || VT == MVT::v64i8) && Subtarget.useBWIRegs())) && in LowerRotate()
30266 SDValue AmtMask = DAG.getConstant(EltSizeInBits - 1, DL, VT); in LowerRotate()
30267 SDValue AmtMod = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask); in LowerRotate()
30277 return DAG.getNode(FunnelOpc, DL, VT, R, R, Amt); in LowerRotate()
30280 SDValue Lo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, R, R)); in LowerRotate()
30281 SDValue Hi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, R, R)); in LowerRotate()
30286 return getPack(DAG, Subtarget, DL, VT, Lo, Hi, IsROTL); in LowerRotate()
30298 !supportedVectorVarShift(VT, Subtarget, ShiftOpc) && in LowerRotate()
30300 SDValue RLo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, R, R)); in LowerRotate()
30301 SDValue RHi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, R, R)); in LowerRotate()
30302 SDValue ALo = DAG.getBitcast(ExtVT, getUnpackl(DAG, DL, VT, AmtMod, Z)); in LowerRotate()
30303 SDValue AHi = DAG.getBitcast(ExtVT, getUnpackh(DAG, DL, VT, AmtMod, Z)); in LowerRotate()
30306 return getPack(DAG, Subtarget, DL, VT, Lo, Hi, IsROTL); in LowerRotate()
30333 return DAG.getNode(ISD::TRUNCATE, DL, VT, R); in LowerRotate()
30341 V0 = DAG.getBitcast(VT, V0); in LowerRotate()
30342 V1 = DAG.getBitcast(VT, V1); in LowerRotate()
30343 Sel = DAG.getBitcast(VT, Sel); in LowerRotate()
30345 DAG.getNode(X86ISD::BLENDV, DL, VT, Sel, V0, V1)); in LowerRotate()
30356 if (!IsROTL && !useVPTERNLOG(Subtarget, VT)) { in LowerRotate()
30357 Amt = DAG.getNode(ISD::SUB, DL, VT, Z, Amt); in LowerRotate()
30369 Amt = DAG.getBitcast(VT, Amt); in LowerRotate()
30374 ISD::OR, DL, VT, in LowerRotate()
30375 DAG.getNode(ShiftLHS, DL, VT, R, DAG.getConstant(4, DL, VT)), in LowerRotate()
30376 DAG.getNode(ShiftRHS, DL, VT, R, DAG.getConstant(4, DL, VT))); in LowerRotate()
30377 R = SignBitSelect(VT, Amt, M, R); in LowerRotate()
30380 Amt = DAG.getNode(ISD::ADD, DL, VT, Amt, Amt); in LowerRotate()
30384 ISD::OR, DL, VT, in LowerRotate()
30385 DAG.getNode(ShiftLHS, DL, VT, R, DAG.getConstant(2, DL, VT)), in LowerRotate()
30386 DAG.getNode(ShiftRHS, DL, VT, R, DAG.getConstant(6, DL, VT))); in LowerRotate()
30387 R = SignBitSelect(VT, Amt, M, R); in LowerRotate()
30390 Amt = DAG.getNode(ISD::ADD, DL, VT, Amt, Amt); in LowerRotate()
30394 ISD::OR, DL, VT, in LowerRotate()
30395 DAG.getNode(ShiftLHS, DL, VT, R, DAG.getConstant(1, DL, VT)), in LowerRotate()
30396 DAG.getNode(ShiftRHS, DL, VT, R, DAG.getConstant(7, DL, VT))); in LowerRotate()
30397 return SignBitSelect(VT, Amt, M, R); in LowerRotate()
30401 bool LegalVarShifts = supportedVectorVarShift(VT, Subtarget, ISD::SHL) && in LowerRotate()
30402 supportedVectorVarShift(VT, Subtarget, ISD::SRL); in LowerRotate()
30407 Amt = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask); in LowerRotate()
30408 SDValue AmtR = DAG.getConstant(EltSizeInBits, DL, VT); in LowerRotate()
30409 AmtR = DAG.getNode(ISD::SUB, DL, VT, AmtR, Amt); in LowerRotate()
30410 SDValue SHL = DAG.getNode(IsROTL ? ISD::SHL : ISD::SRL, DL, VT, R, Amt); in LowerRotate()
30411 SDValue SRL = DAG.getNode(IsROTL ? ISD::SRL : ISD::SHL, DL, VT, R, AmtR); in LowerRotate()
30412 return DAG.getNode(ISD::OR, DL, VT, SHL, SRL); in LowerRotate()
30417 Amt = DAG.getNode(ISD::SUB, DL, VT, Z, Amt); in LowerRotate()
30422 Amt = DAG.getNode(ISD::AND, DL, VT, Amt, AmtMask); in LowerRotate()
30434 SDValue Lo = DAG.getNode(ISD::MUL, DL, VT, R, Scale); in LowerRotate()
30435 SDValue Hi = DAG.getNode(ISD::MULHU, DL, VT, R, Scale); in LowerRotate()
30436 return DAG.getNode(ISD::OR, DL, VT, Lo, Hi); in LowerRotate()
30442 assert(VT == MVT::v4i32 && "Only v4i32 vector rotate expected"); in LowerRotate()
30444 SDValue R13 = DAG.getVectorShuffle(VT, DL, R, R, OddMask); in LowerRotate()
30445 SDValue Scale13 = DAG.getVectorShuffle(VT, DL, Scale, Scale, OddMask); in LowerRotate()
30453 Res02 = DAG.getBitcast(VT, Res02); in LowerRotate()
30454 Res13 = DAG.getBitcast(VT, Res13); in LowerRotate()
30456 return DAG.getNode(ISD::OR, DL, VT, in LowerRotate()
30457 DAG.getVectorShuffle(VT, DL, Res02, Res13, {0, 4, 2, 6}), in LowerRotate()
30458 DAG.getVectorShuffle(VT, DL, Res02, Res13, {1, 5, 3, 7})); in LowerRotate()
31208 static SDValue LowerHorizontalByteSum(SDValue V, MVT VT, in LowerHorizontalByteSum() argument
31213 MVT EltVT = VT.getVectorElementType(); in LowerHorizontalByteSum()
31218 unsigned VecSize = VT.getSizeInBits(); in LowerHorizontalByteSum()
31227 return DAG.getBitcast(VT, V); in LowerHorizontalByteSum()
31236 SDValue Zeros = DAG.getConstant(0, DL, VT); in LowerHorizontalByteSum()
31237 SDValue V32 = DAG.getBitcast(VT, V); in LowerHorizontalByteSum()
31238 SDValue Low = getUnpackl(DAG, DL, VT, V32, Zeros); in LowerHorizontalByteSum()
31239 SDValue High = getUnpackh(DAG, DL, VT, V32, Zeros); in LowerHorizontalByteSum()
31255 return DAG.getBitcast(VT, V); in LowerHorizontalByteSum()
31265 SDValue ShifterV = DAG.getConstant(8, DL, VT); in LowerHorizontalByteSum()
31266 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, DAG.getBitcast(VT, V), ShifterV); in LowerHorizontalByteSum()
31269 return DAG.getNode(ISD::SRL, DL, VT, DAG.getBitcast(VT, V), ShifterV); in LowerHorizontalByteSum()
31275 MVT VT = Op.getSimpleValueType(); in LowerVectorCTPOPInRegLUT() local
31276 MVT EltVT = VT.getVectorElementType(); in LowerVectorCTPOPInRegLUT()
31277 int NumElts = VT.getVectorNumElements(); in LowerVectorCTPOPInRegLUT()
31299 SDValue InRegLUT = DAG.getBuildVector(VT, DL, LUTVec); in LowerVectorCTPOPInRegLUT()
31300 SDValue M0F = DAG.getConstant(0x0F, DL, VT); in LowerVectorCTPOPInRegLUT()
31303 SDValue FourV = DAG.getConstant(4, DL, VT); in LowerVectorCTPOPInRegLUT()
31304 SDValue HiNibbles = DAG.getNode(ISD::SRL, DL, VT, Op, FourV); in LowerVectorCTPOPInRegLUT()
31307 SDValue LoNibbles = DAG.getNode(ISD::AND, DL, VT, Op, M0F); in LowerVectorCTPOPInRegLUT()
31312 SDValue HiPopCnt = DAG.getNode(X86ISD::PSHUFB, DL, VT, InRegLUT, HiNibbles); in LowerVectorCTPOPInRegLUT()
31313 SDValue LoPopCnt = DAG.getNode(X86ISD::PSHUFB, DL, VT, InRegLUT, LoNibbles); in LowerVectorCTPOPInRegLUT()
31314 return DAG.getNode(ISD::ADD, DL, VT, HiPopCnt, LoPopCnt); in LowerVectorCTPOPInRegLUT()
31322 MVT VT = Op.getSimpleValueType(); in LowerVectorCTPOP() local
31323 assert((VT.is512BitVector() || VT.is256BitVector() || VT.is128BitVector()) && in LowerVectorCTPOP()
31329 unsigned NumElems = VT.getVectorNumElements(); in LowerVectorCTPOP()
31330 assert((VT.getVectorElementType() == MVT::i8 || in LowerVectorCTPOP()
31331 VT.getVectorElementType() == MVT::i16) && "Unexpected type"); in LowerVectorCTPOP()
31336 return DAG.getNode(ISD::TRUNCATE, DL, VT, Op); in LowerVectorCTPOP()
31341 if (VT.is256BitVector() && !Subtarget.hasInt256()) in LowerVectorCTPOP()
31345 if (VT.is512BitVector() && !Subtarget.hasBWI()) in LowerVectorCTPOP()
31349 if (VT.getScalarType() != MVT::i8) { in LowerVectorCTPOP()
31350 MVT ByteVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8); in LowerVectorCTPOP()
31353 return LowerHorizontalByteSum(PopCnt8, VT, Subtarget, DAG); in LowerVectorCTPOP()
31365 MVT VT = N.getSimpleValueType(); in LowerCTPOP() local
31369 if (VT.isScalarInteger()) { in LowerCTPOP()
31383 Op = DAG.getNode(ISD::SRL, DL, VT, Op, in LowerCTPOP()
31384 DAG.getShiftAmountConstant(TZ, VT, DL)); in LowerCTPOP()
31388 DAG.getShiftAmountConstant(1, VT, DL))); in LowerCTPOP()
31389 return DAG.getZExtOrTrunc(Op, DL, VT); in LowerCTPOP()
31395 Op = DAG.getNode(ISD::SRL, DL, VT, Op, in LowerCTPOP()
31396 DAG.getShiftAmountConstant(TZ, VT, DL)); in LowerCTPOP()
31399 DAG.getShiftAmountConstant(1, VT, DL)); in LowerCTPOP()
31404 return DAG.getZExtOrTrunc(Op, DL, VT); in LowerCTPOP()
31412 Op = DAG.getNode(ISD::SRL, DL, VT, Op, in LowerCTPOP()
31413 DAG.getShiftAmountConstant(TZ, VT, DL)); in LowerCTPOP()
31421 return DAG.getZExtOrTrunc(Op, DL, VT); in LowerCTPOP()
31428 Op = DAG.getNode(ISD::SRL, DL, VT, Op, in LowerCTPOP()
31429 DAG.getShiftAmountConstant(TZ, VT, DL)); in LowerCTPOP()
31439 return DAG.getZExtOrTrunc(Op, DL, VT); in LowerCTPOP()
31445 assert(VT.isVector() && in LowerCTPOP()
31451 MVT VT = Op.getSimpleValueType(); in LowerBITREVERSE_XOP() local
31457 if (!VT.isVector()) { in LowerBITREVERSE_XOP()
31458 MVT VecVT = MVT::getVectorVT(VT, 128 / VT.getSizeInBits()); in LowerBITREVERSE_XOP()
31461 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Res, in LowerBITREVERSE_XOP()
31465 int NumElts = VT.getVectorNumElements(); in LowerBITREVERSE_XOP()
31466 int ScalarSizeInBytes = VT.getScalarSizeInBits() / 8; in LowerBITREVERSE_XOP()
31469 if (VT.is256BitVector()) in LowerBITREVERSE_XOP()
31472 assert(VT.is128BitVector() && in LowerBITREVERSE_XOP()
31492 return DAG.getBitcast(VT, Res); in LowerBITREVERSE_XOP()
31497 MVT VT = Op.getSimpleValueType(); in LowerBITREVERSE() local
31499 if (Subtarget.hasXOP() && !VT.is512BitVector()) in LowerBITREVERSE()
31508 if (VT.is512BitVector() && !Subtarget.hasBWI()) in LowerBITREVERSE()
31512 if (VT.is256BitVector() && !Subtarget.hasInt256()) in LowerBITREVERSE()
31516 if (!VT.isVector()) { in LowerBITREVERSE()
31518 (VT == MVT::i32 || VT == MVT::i64 || VT == MVT::i16 || VT == MVT::i8) && in LowerBITREVERSE()
31520 MVT VecVT = MVT::getVectorVT(VT, 128 / VT.getSizeInBits()); in LowerBITREVERSE()
31524 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, in LowerBITREVERSE()
31526 return (VT == MVT::i8) ? Res : DAG.getNode(ISD::BSWAP, DL, VT, Res); in LowerBITREVERSE()
31529 assert(VT.isVector() && VT.getSizeInBits() >= 128); in LowerBITREVERSE()
31532 if (VT.getScalarType() != MVT::i8) { in LowerBITREVERSE()
31533 MVT ByteVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8); in LowerBITREVERSE()
31534 SDValue Res = DAG.getNode(ISD::BSWAP, DL, VT, In); in LowerBITREVERSE()
31537 return DAG.getBitcast(VT, Res); in LowerBITREVERSE()
31539 assert(VT.isVector() && VT.getScalarType() == MVT::i8 && in LowerBITREVERSE()
31542 unsigned NumElts = VT.getVectorNumElements(); in LowerBITREVERSE()
31546 SDValue Matrix = getGFNICtrlMask(ISD::BITREVERSE, DAG, DL, VT); in LowerBITREVERSE()
31547 return DAG.getNode(X86ISD::GF2P8AFFINEQB, DL, VT, In, Matrix, in LowerBITREVERSE()
31554 SDValue NibbleMask = DAG.getConstant(0xF, DL, VT); in LowerBITREVERSE()
31555 SDValue Lo = DAG.getNode(ISD::AND, DL, VT, In, NibbleMask); in LowerBITREVERSE()
31556 SDValue Hi = DAG.getNode(ISD::SRL, DL, VT, In, DAG.getConstant(4, DL, VT)); in LowerBITREVERSE()
31575 SDValue LoMask = DAG.getBuildVector(VT, DL, LoMaskElts); in LowerBITREVERSE()
31576 SDValue HiMask = DAG.getBuildVector(VT, DL, HiMaskElts); in LowerBITREVERSE()
31577 Lo = DAG.getNode(X86ISD::PSHUFB, DL, VT, LoMask, Lo); in LowerBITREVERSE()
31578 Hi = DAG.getNode(X86ISD::PSHUFB, DL, VT, HiMask, Hi); in LowerBITREVERSE()
31579 return DAG.getNode(ISD::OR, DL, VT, Lo, Hi); in LowerBITREVERSE()
31586 MVT VT = Op.getSimpleValueType(); in LowerPARITY() local
31589 if (VT == MVT::i8 || in LowerPARITY()
31590 DAG.MaskedValueIsZero(X, APInt::getBitsSetFrom(VT.getSizeInBits(), 8))) { in LowerPARITY()
31597 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Setnp); in LowerPARITY()
31604 if (VT == MVT::i64) { in LowerPARITY()
31613 if (VT != MVT::i16) { in LowerPARITY()
31635 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Setnp); in LowerPARITY()
31677 MVT VT = N->getSimpleValueType(0); in lowerAtomicArith() local
31690 return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, VT, Chain, LHS, in lowerAtomicArith()
31691 DAG.getNegative(RHS, DL, VT), AN->getMemOperand()); in lowerAtomicArith()
31720 DAG.getUNDEF(VT), NewChain); in lowerAtomicArith()
31727 DAG.getUNDEF(VT), NewChain); in lowerAtomicArith()
31735 DAG.getUNDEF(VT), LockOp.getValue(1)); in lowerAtomicArith()
31742 EVT VT = Node->getMemoryVT(); in LowerATOMIC_STORE() local
31746 bool IsTypeLegal = DAG.getTargetLoweringInfo().isTypeLegal(VT); in LowerATOMIC_STORE()
31759 if (VT == MVT::i128 && Subtarget.is64Bit() && Subtarget.hasAVX()) { in LowerATOMIC_STORE()
31767 if (VT == MVT::i64) { in LowerATOMIC_STORE()
31822 MVT VT = N->getSimpleValueType(0); in LowerADDSUBO_CARRY() local
31826 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT)) in LowerADDSUBO_CARRY()
31829 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in LowerADDSUBO_CARRY()
31964 MVT VT = Src.getSimpleValueType(); in LowerMSCATTER() local
31965 assert(VT.getScalarSizeInBits() >= 32 && "Unsupported scatter op"); in LowerMSCATTER()
31974 if (VT == MVT::v2f32 || VT == MVT::v2i32) { in LowerMSCATTER()
31979 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); in LowerMSCATTER()
31980 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT, Src, DAG.getUNDEF(VT)); in LowerMSCATTER()
31998 if (!Subtarget.hasVLX() && !VT.is512BitVector() && in LowerMSCATTER()
32001 unsigned Factor = std::min(512/VT.getSizeInBits(), in LowerMSCATTER()
32003 unsigned NumElts = VT.getVectorNumElements() * Factor; in LowerMSCATTER()
32005 VT = MVT::getVectorVT(VT.getVectorElementType(), NumElts); in LowerMSCATTER()
32009 Src = ExtendToType(Src, VT, DAG); in LowerMSCATTER()
32024 MVT VT = Op.getSimpleValueType(); in LowerMLOAD() local
32025 MVT ScalarVT = VT.getScalarType(); in LowerMLOAD()
32038 VT, dl, N->getChain(), N->getBasePtr(), N->getOffset(), Mask, in LowerMLOAD()
32039 getZeroVector(VT, Subtarget, DAG, dl), N->getMemoryVT(), in LowerMLOAD()
32043 SDValue Select = DAG.getNode(ISD::VSELECT, dl, VT, Mask, NewLoad, PassThru); in LowerMLOAD()
32053 assert(Subtarget.hasAVX512() && !Subtarget.hasVLX() && !VT.is512BitVector() && in LowerMLOAD()
32063 unsigned NumEltsInWideVec = 512 / VT.getScalarSizeInBits(); in LowerMLOAD()
32080 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, NewLoad.getValue(0), in LowerMLOAD()
32090 MVT VT = DataToStore.getSimpleValueType(); in LowerMSTORE() local
32091 MVT ScalarVT = VT.getScalarType(); in LowerMSTORE()
32101 assert(Subtarget.hasAVX512() && !Subtarget.hasVLX() && !VT.is512BitVector() && in LowerMSTORE()
32111 unsigned NumEltsInWideVec = 512/VT.getScalarSizeInBits(); in LowerMSTORE()
32135 MVT VT = Op.getSimpleValueType(); in LowerMGATHER() local
32141 assert(VT.getScalarSizeInBits() >= 32 && "Unsupported gather op"); in LowerMGATHER()
32149 MVT OrigVT = VT; in LowerMGATHER()
32150 if (Subtarget.hasAVX512() && !Subtarget.hasVLX() && !VT.is512BitVector() && in LowerMGATHER()
32153 unsigned Factor = std::min(512/VT.getSizeInBits(), in LowerMGATHER()
32156 unsigned NumElts = VT.getVectorNumElements() * Factor; in LowerMGATHER()
32158 VT = MVT::getVectorVT(VT.getVectorElementType(), NumElts); in LowerMGATHER()
32162 PassThru = ExtendToType(PassThru, VT, DAG); in LowerMGATHER()
32169 PassThru = getZeroVector(VT, Subtarget, DAG, dl); in LowerMGATHER()
32174 X86ISD::MGATHER, dl, DAG.getVTList(VT, MVT::Other), Ops, N->getMemoryVT(), in LowerMGATHER()
32224 EVT VT = Op.getValueType(); in LowerCVTPS2PH() local
32228 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT); in LowerCVTPS2PH()
32232 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi); in LowerCVTPS2PH()
32522 EVT VT = N->getValueType(0); in ReplaceNodeResults() local
32526 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT); in ReplaceNodeResults()
32529 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi); in ReplaceNodeResults()
32534 EVT VT = N->getValueType(0); in ReplaceNodeResults() local
32538 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT); in ReplaceNodeResults()
32545 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi); in ReplaceNodeResults()
32588 EVT VT = N->getValueType(0); in ReplaceNodeResults() local
32589 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && in ReplaceNodeResults()
32590 VT.getVectorElementType() == MVT::i8 && "Unexpected VT!"); in ReplaceNodeResults()
32593 MVT MulVT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements()); in ReplaceNodeResults()
32597 Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res); in ReplaceNodeResults()
32598 unsigned NumConcats = 16 / VT.getVectorNumElements(); in ReplaceNodeResults()
32599 SmallVector<SDValue, 8> ConcatOps(NumConcats, DAG.getUNDEF(VT)); in ReplaceNodeResults()
32607 EVT VT = N->getValueType(0); in ReplaceNodeResults() local
32608 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && in ReplaceNodeResults()
32609 VT == MVT::v2i32 && "Unexpected VT!"); in ReplaceNodeResults()
32619 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Hi, in ReplaceNodeResults()
32623 Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res); in ReplaceNodeResults()
32628 HiCmp = DAG.getNode(ISD::SRA, dl, VT, Res, DAG.getConstant(31, dl, VT)); in ReplaceNodeResults()
32631 HiCmp = DAG.getConstant(0, dl, VT); in ReplaceNodeResults()
32637 DAG.getUNDEF(VT)); in ReplaceNodeResults()
32646 EVT VT = N->getValueType(0); in ReplaceNodeResults() local
32648 assert(VT.getSizeInBits() < 128 && 128 % VT.getSizeInBits() == 0 && in ReplaceNodeResults()
32650 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && in ReplaceNodeResults()
32658 VT.getVectorElementType(), in ReplaceNodeResults()
32659 NumConcat * VT.getVectorNumElements()); in ReplaceNodeResults()
32676 EVT VT = N->getValueType(0); in ReplaceNodeResults() local
32677 assert(VT == MVT::v2f32 && "Unexpected type (!= v2f32) on FMIN/FMAX."); in ReplaceNodeResults()
32678 SDValue UNDEF = DAG.getUNDEF(VT); in ReplaceNodeResults()
32690 EVT VT = N->getValueType(0); in ReplaceNodeResults() local
32691 if (VT.isVector()) { in ReplaceNodeResults()
32692 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && in ReplaceNodeResults()
32699 unsigned NumConcats = 128 / VT.getSizeInBits(); in ReplaceNodeResults()
32700 SmallVector<SDValue, 8> Ops0(NumConcats, DAG.getUNDEF(VT)); in ReplaceNodeResults()
32702 EVT ResVT = getTypeToTransformTo(*DAG.getContext(), VT); in ReplaceNodeResults()
32716 MVT VT = N->getSimpleValueType(0); in ReplaceNodeResults() local
32717 if (getTypeAction(*DAG.getContext(), VT) != TypeWidenVector) in ReplaceNodeResults()
32723 MVT WidenVT = getTypeToTransformTo(*DAG.getContext(), VT).getSimpleVT(); in ReplaceNodeResults()
32727 EVT EltVT = VT.getVectorElementType(); in ReplaceNodeResults()
32728 unsigned MinElts = VT.getVectorNumElements(); in ReplaceNodeResults()
32735 matchTruncateWithPACK(PackOpcode, VT, In, dl, DAG, Subtarget)) { in ReplaceNodeResults()
32736 if (SDValue Res = truncateVectorWithPACK(PackOpcode, VT, Src, in ReplaceNodeResults()
32772 if (InVT == MVT::v4i64 && VT == MVT::v4i8 && isTypeLegal(MVT::v8i64)) { in ReplaceNodeResults()
32779 if (Subtarget.hasVLX() && InVT == MVT::v8i64 && VT == MVT::v8i8 && in ReplaceNodeResults()
32819 EVT VT = N->getValueType(0); in ReplaceNodeResults() local
32822 if (!Subtarget.hasSSE41() && VT == MVT::v4i64 && in ReplaceNodeResults()
32846 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi); in ReplaceNodeResults()
32851 if (VT == MVT::v16i32 || VT == MVT::v8i64) { in ReplaceNodeResults()
32884 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi); in ReplaceNodeResults()
32896 EVT VT = N->getValueType(0); in ReplaceNodeResults() local
32903 EVT NVT = VT.isVector() ? VT.changeVectorElementType(MVT::f32) : MVT::f32; in ReplaceNodeResults()
32906 DAG.getNode(N->getOpcode(), dl, {VT, MVT::Other}, in ReplaceNodeResults()
32911 Res = DAG.getNode(N->getOpcode(), dl, VT, in ReplaceNodeResults()
32921 if (VT.isVector() && Subtarget.hasFP16() && in ReplaceNodeResults()
32923 EVT EleVT = VT.getVectorElementType(); in ReplaceNodeResults()
32965 if (VT.isVector() && VT.getScalarSizeInBits() < 32) { in ReplaceNodeResults()
32966 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && in ReplaceNodeResults()
32970 unsigned NewEltWidth = std::min(128 / VT.getVectorNumElements(), 32U); in ReplaceNodeResults()
32972 VT.getVectorNumElements()); in ReplaceNodeResults()
32990 DAG.getValueType(VT.getVectorElementType())); in ReplaceNodeResults()
32997 Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res); in ReplaceNodeResults()
33000 unsigned NumConcats = 128 / VT.getSizeInBits(); in ReplaceNodeResults()
33001 MVT ConcatVT = MVT::getVectorVT(VT.getSimpleVT().getVectorElementType(), in ReplaceNodeResults()
33002 VT.getVectorNumElements() * NumConcats); in ReplaceNodeResults()
33003 SmallVector<SDValue, 8> ConcatOps(NumConcats, DAG.getUNDEF(VT)); in ReplaceNodeResults()
33013 if (VT == MVT::v2i32) { in ReplaceNodeResults()
33017 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && in ReplaceNodeResults()
33078 assert(!VT.isVector() && "Vectors should have been handled above!"); in ReplaceNodeResults()
33080 if ((Subtarget.hasDQI() && VT == MVT::i64 && in ReplaceNodeResults()
33109 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Res, ZeroIdx); in ReplaceNodeResults()
33116 if (VT == MVT::i128 && Subtarget.isTargetWin64()) { in ReplaceNodeResults()
33146 EVT VT = N->getValueType(0); in ReplaceNodeResults() local
33148 if (VT.getVectorElementType() == MVT::f16 && Subtarget.hasFP16() && in ReplaceNodeResults()
33153 if (VT == MVT::v2f16 && Src.getValueType() == MVT::v2i32) in ReplaceNodeResults()
33170 if (VT != MVT::v2f32) in ReplaceNodeResults()
33275 EVT VT = N->getValueType(0); in ReplaceNodeResults() local
33277 if (VT == MVT::v2f16 && Src.getValueType() == MVT::v2f32) { in ReplaceNodeResults()
33282 if (!Subtarget.hasFP16() && VT.getVectorElementType() == MVT::f16) { in ReplaceNodeResults()
33300 EVT NewVT = VT.getVectorElementType() == MVT::f16 ? MVT::v8f16 : MVT::v4f32; in ReplaceNodeResults()
33563 EVT VT = N->getValueType(0); in ReplaceNodeResults() local
33564 if ((VT == MVT::v2f32 || VT == MVT::v2i32) && in ReplaceNodeResults()
33570 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && in ReplaceNodeResults()
33572 EVT WideVT = getTypeToTransformTo(*DAG.getContext(), VT); in ReplaceNodeResults()
33577 DAG.getUNDEF(VT)); in ReplaceNodeResults()
33600 MVT VT = N->getSimpleValueType(0); in ReplaceNodeResults() local
33601 assert(VT.isVector() && VT.getSizeInBits() == 64 && "Unexpected VT"); in ReplaceNodeResults()
33602 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && in ReplaceNodeResults()
33608 MVT LdVT = Subtarget.is64Bit() && VT.isInteger() ? MVT::i64 : MVT::f64; in ReplaceNodeResults()
33615 EVT WideVT = getTypeToTransformTo(*DAG.getContext(), VT); in ReplaceNodeResults()
34340 EVT VT) const { in isFMAFasterThanFMulAndFAdd()
34344 VT = VT.getScalarType(); in isFMAFasterThanFMulAndFAdd()
34346 if (!VT.isSimple()) in isFMAFasterThanFMulAndFAdd()
34349 switch (VT.getSimpleVT().SimpleTy) { in isFMAFasterThanFMulAndFAdd()
34368 EVT VT) const { in shouldFoldSelectWithIdentityConstant()
34373 if (!Subtarget.hasVLX() && !VT.is512BitVector()) in shouldFoldSelectWithIdentityConstant()
34375 if (!VT.isVector() || VT.getScalarType() == MVT::i1) in shouldFoldSelectWithIdentityConstant()
34385 bool X86TargetLowering::isShuffleMaskLegal(ArrayRef<int> Mask, EVT VT) const { in isShuffleMaskLegal()
34386 if (!VT.isSimple()) in isShuffleMaskLegal()
34390 if (VT.getSimpleVT().getScalarType() == MVT::i1) in isShuffleMaskLegal()
34394 if (VT.getSimpleVT().getSizeInBits() == 64) in isShuffleMaskLegal()
34399 return isTypeLegal(VT.getSimpleVT()); in isShuffleMaskLegal()
34403 EVT VT) const { in isVectorClearMaskLegal()
34407 if (VT == MVT::v32i8 || VT == MVT::v16i16) in isVectorClearMaskLegal()
34411 return isShuffleMaskLegal(Mask, VT); in isVectorClearMaskLegal()
37018 EVT VT = Op.getValueType(); in targetShrinkDemandedConstant() local
37020 unsigned EltSize = VT.getScalarSizeInBits(); in targetShrinkDemandedConstant()
37022 if (VT.isVector()) { in targetShrinkDemandedConstant()
37042 if (EltSize > ActiveBits && EltSize > 1 && isTypeLegal(VT) && in targetShrinkDemandedConstant()
37047 VT.getVectorNumElements()); in targetShrinkDemandedConstant()
37049 TLO.DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(Op), VT, in targetShrinkDemandedConstant()
37052 TLO.DAG.getNode(Opcode, SDLoc(Op), VT, Op.getOperand(0), NewC); in targetShrinkDemandedConstant()
37100 SDValue NewC = TLO.DAG.getConstant(ZeroExtendMask, DL, VT); in targetShrinkDemandedConstant()
37101 SDValue NewOp = TLO.DAG.getNode(ISD::AND, DL, VT, Op.getOperand(0), NewC); in targetShrinkDemandedConstant()
37205 EVT VT = Op.getValueType(); in computeKnownBitsForTargetNode() local
37246 if (ShAmt >= VT.getScalarSizeInBits()) { in computeKnownBitsForTargetNode()
37254 ShAmt = VT.getScalarSizeInBits() - 1; in computeKnownBitsForTargetNode()
37277 getPackDemandedElts(VT, DemandedElts, DemandedLHS, DemandedRHS); in computeKnownBitsForTargetNode()
37348 assert(VT.getScalarType() == MVT::i64 && in computeKnownBitsForTargetNode()
37375 assert(VT.getVectorElementType() == MVT::i32 && in computeKnownBitsForTargetNode()
37385 assert(VT.getVectorElementType() == MVT::i16 && in computeKnownBitsForTargetNode()
37544 assert(VT.getScalarType() == MVT::i32 && in computeKnownBitsForTargetNode()
37556 assert(VT.getScalarType() == MVT::i16 && in computeKnownBitsForTargetNode()
37568 assert(VT.getScalarType() == MVT::i64 && in computeKnownBitsForTargetNode()
37587 unsigned NumElts = VT.getVectorNumElements(); in computeKnownBitsForTargetNode()
37610 if (Ops[OpIdx].getValueType() != VT) { in computeKnownBitsForTargetNode()
37633 EVT VT = Op.getValueType(); in ComputeNumSignBitsForTargetNode() local
37634 unsigned VTBits = VT.getScalarSizeInBits(); in ComputeNumSignBitsForTargetNode()
37719 if (VT == MVT::f32 || VT == MVT::f64 || in ComputeNumSignBitsForTargetNode()
37720 ((VT == MVT::v4f32 || VT == MVT::v2f64) && DemandedElts == 1)) in ComputeNumSignBitsForTargetNode()
37756 unsigned NumElts = VT.getVectorNumElements(); in ComputeNumSignBitsForTargetNode()
37776 if (Ops[OpIdx].getValueType() != VT) { in ComputeNumSignBitsForTargetNode()
37807 static SDValue narrowLoadToVZLoad(LoadSDNode *LN, MVT MemVT, MVT VT, in narrowLoadToVZLoad() argument
37813 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in narrowLoadToVZLoad()
38516 auto CanonicalizeShuffleInput = [&](MVT VT, SDValue Op) { in combineX86ShuffleChain() argument
38517 if (VT.getSizeInBits() > Op.getValueSizeInBits()) in combineX86ShuffleChain()
38518 Op = widenSubVector(Op, false, Subtarget, DAG, DL, VT.getSizeInBits()); in combineX86ShuffleChain()
38519 else if (VT.getSizeInBits() < Op.getValueSizeInBits()) in combineX86ShuffleChain()
38520 Op = extractSubVector(Op, 0, DAG, DL, VT.getSizeInBits()); in combineX86ShuffleChain()
38521 return DAG.getBitcast(VT, Op); in combineX86ShuffleChain()
39577 static SDValue combineX86ShufflesConstants(MVT VT, ArrayRef<SDValue> Ops, in combineX86ShufflesConstants() argument
39582 unsigned SizeInBits = VT.getSizeInBits(); in combineX86ShufflesConstants()
39645 return getZeroVector(VT, Subtarget, DAG, DL); in combineX86ShufflesConstants()
39649 if (VT.isFloatingPoint() && (MaskSizeInBits == 32 || MaskSizeInBits == 64)) in combineX86ShufflesConstants()
39659 return DAG.getBitcast(VT, CstOp); in combineX86ShufflesConstants()
39722 EVT VT = Op.getValueType(); in combineX86ShufflesRecursively() local
39723 if (!VT.isVector() || !VT.isSimple()) in combineX86ShufflesRecursively()
39727 if (VT.getVectorElementType() == MVT::f16) in combineX86ShufflesRecursively()
39730 assert((RootSizeInBits % VT.getSizeInBits()) == 0 && in combineX86ShufflesRecursively()
39740 if (RootSizeInBits != VT.getSizeInBits()) { in combineX86ShufflesRecursively()
39742 unsigned Scale = RootSizeInBits / VT.getSizeInBits(); in combineX86ShufflesRecursively()
39752 APIntOps::ScaleBitMask(OpDemandedElts, VT.getVectorNumElements()); in combineX86ShufflesRecursively()
39763 if (llvm::any_of(OpInputs, [VT](SDValue OpInput) { in combineX86ShufflesRecursively()
39764 return OpInput.getValueSizeInBits() > VT.getSizeInBits(); in combineX86ShufflesRecursively()
39772 unsigned NumElts = VT.getVectorNumElements(); in combineX86ShufflesRecursively()
39783 if (RootSizeInBits > VT.getSizeInBits()) { in combineX86ShufflesRecursively()
39784 unsigned NumSubVecs = RootSizeInBits / VT.getSizeInBits(); in combineX86ShufflesRecursively()
40151 MVT VT = N.getSimpleValueType(); in getPSHUFShuffleMask() local
40160 if (VT.getSizeInBits() > 128) { in getPSHUFShuffleMask()
40161 int LaneElts = 128 / VT.getScalarSizeInBits(); in getPSHUFShuffleMask()
40163 for (int i = 1, NumLanes = VT.getSizeInBits() / 128; i < NumLanes; ++i) in getPSHUFShuffleMask()
40322 static SDValue combineCommutableSHUFP(SDValue N, MVT VT, const SDLoc &DL, in combineCommutableSHUFP() argument
40325 if (VT != MVT::v4f32 && VT != MVT::v8f32 && VT != MVT::v16f32) in combineCommutableSHUFP()
40329 auto commuteSHUFP = [&VT, &DL, &DAG](SDValue Parent, SDValue V) { in combineCommutableSHUFP()
40340 return DAG.getNode(X86ISD::SHUFP, DL, VT, N1, N0, in combineCommutableSHUFP()
40348 return DAG.getNode(X86ISD::VPERMILPI, DL, VT, NewSHUFP, in combineCommutableSHUFP()
40358 return DAG.getNode(X86ISD::SHUFP, DL, VT, NewSHUFP, NewSHUFP, in combineCommutableSHUFP()
40361 return DAG.getNode(X86ISD::SHUFP, DL, VT, NewSHUFP, N1, in combineCommutableSHUFP()
40364 return DAG.getNode(X86ISD::SHUFP, DL, VT, N0, NewSHUFP, in combineCommutableSHUFP()
40377 combineBlendOfPermutes(MVT VT, SDValue N0, SDValue N1, ArrayRef<int> BlendMask, in combineBlendOfPermutes() argument
40384 unsigned NumElts = VT.getVectorNumElements(); in combineBlendOfPermutes()
40446 if (VT == MVT::v16i16) { in combineBlendOfPermutes()
40447 if (!is128BitLaneRepeatedShuffleMask(VT, NewBlendMask) && in combineBlendOfPermutes()
40456 if (VT.is256BitVector() && !Subtarget.hasAVX2() && in combineBlendOfPermutes()
40457 isLaneCrossingShuffleMask(128, VT.getScalarSizeInBits(), in combineBlendOfPermutes()
40463 DAG.getVectorShuffle(VT, DL, DAG.getBitcast(VT, Ops0[0]), in combineBlendOfPermutes()
40464 DAG.getBitcast(VT, Ops1[0]), NewBlendMask); in combineBlendOfPermutes()
40465 return DAG.getVectorShuffle(VT, DL, NewBlend, DAG.getUNDEF(VT), in combineBlendOfPermutes()
40645 MVT VT = V.getSimpleValueType(); in canonicalizeLaneShuffleWithRepeatedOps() local
40663 return DAG.getBitcast(VT, Res); in canonicalizeLaneShuffleWithRepeatedOps()
40683 return DAG.getBitcast(VT, Res); in canonicalizeLaneShuffleWithRepeatedOps()
40696 MVT VT = N.getSimpleValueType(); in combineTargetShuffle() local
40701 if (SDValue R = combineCommutableSHUFP(N, VT, DL, DAG)) in combineTargetShuffle()
40709 if (VT == MVT::v2f64 && Src.hasOneUse() && in combineTargetShuffle()
40732 VT.getScalarSizeInBits() % BCVT.getScalarSizeInBits() == 0) { in combineTargetShuffle()
40733 unsigned Scale = VT.getScalarSizeInBits() / BCVT.getScalarSizeInBits(); in combineTargetShuffle()
40743 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, in combineTargetShuffle()
40755 VT.getVectorNumElements()); in combineTargetShuffle()
40756 return DAG.getBitcast(VT, DAG.getNode(X86ISD::VBROADCAST, DL, NewVT, BC)); in combineTargetShuffle()
40766 (VT.getScalarSizeInBits() % BCVT.getScalarSizeInBits()) == 0 && in combineTargetShuffle()
40767 (VT.getSizeInBits() % BCVT.getSizeInBits()) == 0) { in combineTargetShuffle()
40770 VT.getSizeInBits() / BCVT.getScalarSizeInBits()); in combineTargetShuffle()
40771 return DAG.getBitcast(VT, DAG.getNode(X86ISD::VBROADCAST, DL, NewVT, BC)); in combineTargetShuffle()
40776 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, in combineTargetShuffle()
40782 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, Src.getOperand(0)); in combineTargetShuffle()
40790 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, Src.getOperand(0)); in combineTargetShuffle()
40798 VT.getFixedSizeInBits()) { in combineTargetShuffle()
40800 VT.getSizeInBits()); in combineTargetShuffle()
40805 if (!SrcVT.isVector() && (Src.hasOneUse() || VT.isFloatingPoint()) && in combineTargetShuffle()
40808 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in combineTargetShuffle()
40840 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in combineTargetShuffle()
40858 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in combineTargetShuffle()
40883 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in combineTargetShuffle()
40903 if (LN->getMemoryVT().getSizeInBits() == VT.getScalarSizeInBits()) { in combineTargetShuffle()
40904 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in combineTargetShuffle()
40923 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in combineTargetShuffle()
40946 narrowLoadToVZLoad(LN, VT.getVectorElementType(), VT, DAG)) { in combineTargetShuffle()
40959 if (VT.getScalarSizeInBits() == LN->getMemoryVT().getSizeInBits()) { in combineTargetShuffle()
40960 SDVTList Tys = DAG.getVTList(VT, MVT::Other); in combineTargetShuffle()
40982 MVT VecVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() * 2); in combineTargetShuffle()
40985 return DAG.getBitcast(VT, Movl); in combineTargetShuffle()
40997 unsigned NumElts = VT.getVectorNumElements(); in combineTargetShuffle()
41008 return DAG.getLoad(VT, DL, DAG.getEntryNode(), CP, MPI, Alignment, in combineTargetShuffle()
41023 MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), in combineTargetShuffle()
41025 VT.getScalarSizeInBits()); in combineTargetShuffle()
41028 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, in combineTargetShuffle()
41029 getZeroVector(VT, Subtarget, DAG, DL), Movl, in combineTargetShuffle()
41039 unsigned EltBits = VT.getScalarSizeInBits(); in combineTargetShuffle()
41048 unsigned Size = VT.getVectorNumElements(); in combineTargetShuffle()
41053 VT, DAG.getNode(X86ISD::BLENDI, DL, SrcVT, N0.getOperand(0), in combineTargetShuffle()
41087 return DAG.getNode(X86ISD::BLENDI, DL, VT, in combineTargetShuffle()
41088 DAG.getBitcast(VT, NewLHS), in combineTargetShuffle()
41089 DAG.getBitcast(VT, NewRHS), N.getOperand(2)); in combineTargetShuffle()
41100 if (VT == MVT::v4f32) { in combineTargetShuffle()
41116 Ops[i] = DAG.getBitcast(VT, SubOps[0]); in combineTargetShuffle()
41125 return DAG.getNode(X86ISD::SHUFP, DL, VT, Ops); in combineTargetShuffle()
41135 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in combineTargetShuffle()
41141 return DAG.getBitcast(VT, Res); in combineTargetShuffle()
41148 if (VT.is512BitVector()) { in combineTargetShuffle()
41167 return DAG.getNode(X86ISD::SHUF128, DL, VT, NewLHS ? NewLHS : LHS, in combineTargetShuffle()
41181 return DAG.getBitcast(VT, DAG.getNode(X86ISD::VPERM2X128, DL, SrcVT, in combineTargetShuffle()
41212 MVT SubVT = VT.getHalfNumVectorElementsVT(); in combineTargetShuffle()
41215 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, SubLo, SubHi); in combineTargetShuffle()
41237 if (InnerVT.getScalarSizeInBits() <= VT.getScalarSizeInBits()) { in combineTargetShuffle()
41238 SDValue Res = DAG.getNode(Opcode, DL, VT, in combineTargetShuffle()
41239 DAG.getBitcast(VT, V.getOperand(0)), N1); in combineTargetShuffle()
41242 return DAG.getBitcast(VT, Res); in combineTargetShuffle()
41271 MVT SVT = VT.getVectorElementType(); in combineTargetShuffle()
41276 SDValue SclVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Scl); in combineTargetShuffle()
41277 return DAG.getNode(Opcode, DL, VT, N0, SclVec); in combineTargetShuffle()
41284 assert(VT == MVT::v4f32 && "INSERTPS ValueType must be MVT::v4f32"); in combineTargetShuffle()
41294 return DAG.getNode(X86ISD::INSERTPS, DL, VT, DAG.getUNDEF(VT), Op1, in combineTargetShuffle()
41299 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, DAG.getUNDEF(VT), in combineTargetShuffle()
41311 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, DAG.getUNDEF(VT), in combineTargetShuffle()
41319 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, Op1, in combineTargetShuffle()
41366 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, Op1, in combineTargetShuffle()
41379 SDValue Insert = DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, in combineTargetShuffle()
41380 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, in combineTargetShuffle()
41405 VT.getScalarType(), NVT.getSizeInBits() / VT.getScalarSizeInBits()); in combineTargetShuffle()
41410 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Perm, in combineTargetShuffle()
41431 assert(VT.getVectorElementType() == MVT::i16 && "Bad word shuffle type!"); in combineTargetShuffle()
41441 MVT DVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() / 2); in combineTargetShuffle()
41445 return DAG.getBitcast(VT, V); in combineTargetShuffle()
41474 V = DAG.getBitcast(VT, D.getOperand(0)); in combineTargetShuffle()
41477 DL, VT, V, V); in combineTargetShuffle()
41538 EVT VT = N->getValueType(0); in isAddSubOrSubAdd() local
41540 if (!Subtarget.hasSSE3() || !TLI.isTypeLegal(VT) || in isAddSubOrSubAdd()
41541 !VT.getSimpleVT().isFloatingPoint()) in isAddSubOrSubAdd()
41603 MVT VT = N->getSimpleValueType(0); in combineShuffleToFMAddSub() local
41605 if (!Subtarget.hasAnyFMA() || !TLI.isTypeLegal(VT)) in combineShuffleToFMAddSub()
41630 return DAG.getNode(Opcode, DL, VT, FMAdd.getOperand(0), FMAdd.getOperand(1), in combineShuffleToFMAddSub()
41647 MVT VT = N->getSimpleValueType(0); in combineShuffleToAddSubOrFMAddSub() local
41653 return DAG.getNode(Opc, DL, VT, Opnd0, Opnd1, Opnd2); in combineShuffleToAddSubOrFMAddSub()
41662 if (VT.is512BitVector()) in combineShuffleToAddSubOrFMAddSub()
41668 if (VT.getVectorElementType() == MVT::f16) in combineShuffleToAddSubOrFMAddSub()
41671 return DAG.getNode(X86ISD::ADDSUB, DL, VT, Opnd0, Opnd1); in combineShuffleToAddSubOrFMAddSub()
41683 EVT VT = N->getValueType(0); in combineShuffleOfConcatUndef() local
41686 if (!VT.is128BitVector() && !VT.is256BitVector()) in combineShuffleOfConcatUndef()
41689 if (VT.getVectorElementType() != MVT::i32 && in combineShuffleOfConcatUndef()
41690 VT.getVectorElementType() != MVT::i64 && in combineShuffleOfConcatUndef()
41691 VT.getVectorElementType() != MVT::f32 && in combineShuffleOfConcatUndef()
41692 VT.getVectorElementType() != MVT::f64) in combineShuffleOfConcatUndef()
41708 int NumElts = VT.getVectorNumElements(); in combineShuffleOfConcatUndef()
41714 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, N0.getOperand(0), in combineShuffleOfConcatUndef()
41716 return DAG.getVectorShuffle(VT, DL, Concat, DAG.getUNDEF(VT), Mask); in combineShuffleOfConcatUndef()
41723 EVT VT = Shuf->getValueType(0); in narrowShuffle() local
41726 if (!VT.is256BitVector() && !VT.is512BitVector()) in narrowShuffle()
41762 EVT VT = N->getValueType(0); in combineShuffle() local
41764 if (TLI.isTypeLegal(VT) && !isSoftF16(VT, Subtarget)) in combineShuffle()
41771 VT, SDValue(N, 0), dl, DAG, Subtarget, /*IsAfterLegalize*/ true)) in combineShuffle()
41798 APInt DemandedElts = APInt::getAllOnes(VT.getVectorNumElements()); in combineShuffle()
41887 EVT VT = Op.getValueType(); in SimplifyDemandedVectorEltsForTargetNode() local
41938 assert(VT.getScalarType() == MVT::i64 && in SimplifyDemandedVectorEltsForTargetNode()
41955 Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewLHS, NewRHS)); in SimplifyDemandedVectorEltsForTargetNode()
41997 Op, getZeroVector(VT.getSimpleVT(), Subtarget, TLO.DAG, SDLoc(Op))); in SimplifyDemandedVectorEltsForTargetNode()
42004 Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc, Op.getOperand(1))); in SimplifyDemandedVectorEltsForTargetNode()
42023 Op, getZeroVector(VT.getSimpleVT(), Subtarget, TLO.DAG, SDLoc(Op))); in SimplifyDemandedVectorEltsForTargetNode()
42071 Op, TLO.DAG.getNode(NewOpc, dl, VT, Src.getOperand(0), NewSA)); in SimplifyDemandedVectorEltsForTargetNode()
42110 Op, TLO.DAG.getNode(NewOpc, dl, VT, Src.getOperand(0), NewSA)); in SimplifyDemandedVectorEltsForTargetNode()
42132 int NumElts = VT.getVectorNumElements(); in SimplifyDemandedVectorEltsForTargetNode()
42133 int EltSizeInBits = VT.getScalarSizeInBits(); in SimplifyDemandedVectorEltsForTargetNode()
42180 Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewLHS, NewRHS)); in SimplifyDemandedVectorEltsForTargetNode()
42204 getPackDemandedElts(VT, DemandedElts, DemandedLHS, DemandedRHS); in SimplifyDemandedVectorEltsForTargetNode()
42228 TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewN0, NewN1)); in SimplifyDemandedVectorEltsForTargetNode()
42241 getHorizDemandedElts(VT, DemandedElts, DemandedLHS, DemandedRHS); in SimplifyDemandedVectorEltsForTargetNode()
42265 TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewN0, NewN1)); in SimplifyDemandedVectorEltsForTargetNode()
42288 VT.getSimpleVT(), Op.getOperand(0), Op.getOperand(1), BlendMask, in SimplifyDemandedVectorEltsForTargetNode()
42326 MVT SVT = VT.getSimpleVT().getVectorElementType(); in SimplifyDemandedVectorEltsForTargetNode()
42332 SDValue Vec = TLO.DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Elt); in SimplifyDemandedVectorEltsForTargetNode()
42333 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Vec)); in SimplifyDemandedVectorEltsForTargetNode()
42344 if (Src.getValueType() != VT) in SimplifyDemandedVectorEltsForTargetNode()
42345 Src = widenSubVector(VT.getSimpleVT(), Src, false, Subtarget, TLO.DAG, in SimplifyDemandedVectorEltsForTargetNode()
42358 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc)); in SimplifyDemandedVectorEltsForTargetNode()
42384 if ((VT.is256BitVector() || VT.is512BitVector()) && in SimplifyDemandedVectorEltsForTargetNode()
42386 unsigned SizeInBits = VT.getSizeInBits(); in SimplifyDemandedVectorEltsForTargetNode()
42390 if (VT.is512BitVector() && DemandedElts.lshr(NumElts / 4) == 0) in SimplifyDemandedVectorEltsForTargetNode()
42400 EVT BcstVT = EVT::getVectorVT(*TLO.DAG.getContext(), VT.getScalarType(), in SimplifyDemandedVectorEltsForTargetNode()
42401 ExtSizeInBits / VT.getScalarSizeInBits()); in SimplifyDemandedVectorEltsForTargetNode()
42403 return TLO.CombineTo(Op, insertSubVector(TLO.DAG.getUNDEF(VT), Bcst, 0, in SimplifyDemandedVectorEltsForTargetNode()
42409 EVT BcstVT = EVT::getVectorVT(*TLO.DAG.getContext(), VT.getScalarType(), in SimplifyDemandedVectorEltsForTargetNode()
42410 ExtSizeInBits / VT.getScalarSizeInBits()); in SimplifyDemandedVectorEltsForTargetNode()
42418 return TLO.CombineTo(Op, insertSubVector(TLO.DAG.getUNDEF(VT), Bcst, 0, in SimplifyDemandedVectorEltsForTargetNode()
42432 return TLO.CombineTo(Op, insertSubVector(TLO.DAG.getUNDEF(VT), Ld, 0, in SimplifyDemandedVectorEltsForTargetNode()
42436 EVT BcstVT = EVT::getVectorVT(*TLO.DAG.getContext(), VT.getScalarType(), in SimplifyDemandedVectorEltsForTargetNode()
42437 ExtSizeInBits / VT.getScalarSizeInBits()); in SimplifyDemandedVectorEltsForTargetNode()
42441 insertSubVector(TLO.DAG.getUNDEF(VT), BcstLd, 0, in SimplifyDemandedVectorEltsForTargetNode()
42462 SDValue UndefVec = TLO.DAG.getUNDEF(VT); in SimplifyDemandedVectorEltsForTargetNode()
42470 if (VT == MVT::v4f64 || VT == MVT::v4i64) { in SimplifyDemandedVectorEltsForTargetNode()
42476 SDValue UndefVec = TLO.DAG.getUNDEF(VT); in SimplifyDemandedVectorEltsForTargetNode()
42489 Op, getZeroVector(VT.getSimpleVT(), Subtarget, TLO.DAG, DL)); in SimplifyDemandedVectorEltsForTargetNode()
42494 SDValue UndefVec = TLO.DAG.getUNDEF(VT); in SimplifyDemandedVectorEltsForTargetNode()
42544 MVT ExtVT = VT.getSimpleVT(); in SimplifyDemandedVectorEltsForTargetNode()
42548 SDValue UndefVec = TLO.DAG.getUNDEF(VT); in SimplifyDemandedVectorEltsForTargetNode()
42572 llvm::any_of(OpInputs, [VT](SDValue V) { in SimplifyDemandedVectorEltsForTargetNode()
42573 return VT.getSizeInBits() != V.getValueSizeInBits() || in SimplifyDemandedVectorEltsForTargetNode()
42589 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT)); in SimplifyDemandedVectorEltsForTargetNode()
42594 Op, getZeroVector(VT.getSimpleVT(), Subtarget, TLO.DAG, SDLoc(Op))); in SimplifyDemandedVectorEltsForTargetNode()
42598 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, OpInputs[Src])); in SimplifyDemandedVectorEltsForTargetNode()
42603 if (OpInputs[Src].getValueType() != VT) in SimplifyDemandedVectorEltsForTargetNode()
42653 EVT VT = Op.getValueType(); in SimplifyDemandedBitsForTargetNode() local
42700 SDValue Mask = TLO.DAG.getConstant(DemandedMask, DL, VT); in SimplifyDemandedBitsForTargetNode()
42701 return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, DL, VT, LHS, Mask)); in SimplifyDemandedBitsForTargetNode()
42713 Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, DemandedLHS, DemandedRHS)); in SimplifyDemandedBitsForTargetNode()
42762 NewOpc, SDLoc(Op), VT, Op0.getOperand(0), in SimplifyDemandedBitsForTargetNode()
42845 Op, TLO.DAG.getNode(X86ISD::VSRLI, SDLoc(Op), VT, Op0, Op1)); in SimplifyDemandedBitsForTargetNode()
42869 return TLO.CombineTo(Op, TLO.DAG.getNode(X86ISD::BLENDV, SDLoc(Op), VT, in SimplifyDemandedBitsForTargetNode()
42889 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT)); in SimplifyDemandedBitsForTargetNode()
42905 Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, V, Op.getOperand(1))); in SimplifyDemandedBitsForTargetNode()
42949 getPackDemandedElts(VT, OriginalDemandedElts, DemandedLHS, DemandedRHS); in SimplifyDemandedBitsForTargetNode()
42968 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, Op0, Op1)); in SimplifyDemandedBitsForTargetNode()
42990 MVT NewVT = MVT::getVectorVT(NewSrcVT, VT.getVectorNumElements() * 2); in SimplifyDemandedBitsForTargetNode()
42993 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, NewBcst)); in SimplifyDemandedBitsForTargetNode()
43012 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT)); in SimplifyDemandedBitsForTargetNode()
43018 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc)); in SimplifyDemandedBitsForTargetNode()
43046 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc)); in SimplifyDemandedBitsForTargetNode()
43092 Op, TLO.DAG.getNode(X86ISD::BEXTR, DL, VT, Op0, in SimplifyDemandedBitsForTargetNode()
43093 TLO.DAG.getConstant(MaskedVal1, DL, VT))); in SimplifyDemandedBitsForTargetNode()
43124 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT)); in SimplifyDemandedBitsForTargetNode()
43168 EVT VT = Op.getValueType(); in SimplifyMultipleUseDemandedBitsForTargetNode() local
43246 llvm::all_of(ShuffleOps, [VT](SDValue V) { in SimplifyMultipleUseDemandedBitsForTargetNode()
43247 return VT.getSizeInBits() == V.getValueSizeInBits(); in SimplifyMultipleUseDemandedBitsForTargetNode()
43251 return DAG.getUNDEF(VT); in SimplifyMultipleUseDemandedBitsForTargetNode()
43253 return getZeroVector(VT.getSimpleVT(), Subtarget, DAG, SDLoc(Op)); in SimplifyMultipleUseDemandedBitsForTargetNode()
43275 return DAG.getBitcast(VT, ShuffleOps[IdentityOp.countr_zero()]); in SimplifyMultipleUseDemandedBitsForTargetNode()
43470 static SDValue combineBitcastvxi1(SelectionDAG &DAG, EVT VT, SDValue Src, in combineBitcastvxi1() argument
43483 return DAG.getZExtOrTrunc(V, DL, VT); in combineBitcastvxi1()
43522 EVT SubVT = VT.getIntegerVT( in combineBitcastvxi1()
43525 EVT IntVT = VT.getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); in combineBitcastvxi1()
43526 return DAG.getBitcast(VT, DAG.getNode(ISD::ANY_EXTEND, DL, IntVT, V)); in combineBitcastvxi1()
43615 return DAG.getBitcast(VT, V); in combineBitcastvxi1()
43772 static SDValue combineBitcastToBoolVector(EVT VT, SDValue V, const SDLoc &DL, in combineBitcastToBoolVector() argument
43787 return DAG.getBitcast(VT, Src); in combineBitcastToBoolVector()
43793 return DAG.getConstant(0, DL, VT); in combineBitcastToBoolVector()
43795 return DAG.getAllOnesConstant(DL, VT); in combineBitcastToBoolVector()
43806 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, N0, in combineBitcastToBoolVector()
43819 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, in combineBitcastToBoolVector()
43820 Opc == ISD::ANY_EXTEND ? DAG.getUNDEF(VT) in combineBitcastToBoolVector()
43821 : DAG.getConstant(0, DL, VT), in combineBitcastToBoolVector()
43829 if (SDValue N0 = combineBitcastToBoolVector(VT, V.getOperand(0), DL, DAG, in combineBitcastToBoolVector()
43831 if (SDValue N1 = combineBitcastToBoolVector(VT, V.getOperand(1), DL, DAG, in combineBitcastToBoolVector()
43833 return DAG.getNode(Opc, DL, VT, N0, N1); in combineBitcastToBoolVector()
43839 if ((VT == MVT::v8i1 && !Subtarget.hasDQI()) || in combineBitcastToBoolVector()
43840 ((VT == MVT::v32i1 || VT == MVT::v64i1) && !Subtarget.hasBWI())) in combineBitcastToBoolVector()
43844 if (SDValue N0 = combineBitcastToBoolVector(VT, Src0, DL, DAG, Subtarget, in combineBitcastToBoolVector()
43847 X86ISD::KSHIFTL, DL, VT, N0, in combineBitcastToBoolVector()
43855 if (SDNode *Alt = DAG.getNodeIfExists(ISD::BITCAST, DAG.getVTList(VT), {V})) in combineBitcastToBoolVector()
43865 EVT VT = N->getValueType(0); in combineBitcast() local
43877 if (SDValue V = combineBitcastvxi1(DAG, VT, N0, dl, Subtarget)) in combineBitcast()
43882 if ((VT == MVT::v4i1 || VT == MVT::v2i1) && SrcVT.isScalarInteger() && in combineBitcast()
43886 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, N0, in combineBitcast()
43892 if ((SrcVT == MVT::v4i1 || SrcVT == MVT::v2i1) && VT.isScalarInteger() && in combineBitcast()
43910 return DAG.getNode(ISD::TRUNCATE, dl, VT, N0); in combineBitcast()
43919 return DAG.getNode(ISD::TRUNCATE, dl, VT, N0); in combineBitcast()
43924 if (VT.isVector() && VT.getScalarType() == MVT::i1 && in combineBitcast()
43925 SrcVT.isScalarInteger() && TLI.isTypeLegal(VT)) { in combineBitcast()
43927 combineBitcastToBoolVector(VT, N0, SDLoc(N), DAG, Subtarget)) in combineBitcast()
43937 if (VT == MVT::i8 && SrcVT == MVT::v8i1 && Subtarget.hasAVX512() && in combineBitcast()
43941 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, in combineBitcast()
43948 VT.isFloatingPoint() != SrcVT.isFloatingPoint() && VT.isVector()) { in combineBitcast()
43954 MVT MemVT = VT.isFloatingPoint() ? MVT::getFloatingPointVT(MemSize) in combineBitcast()
43956 MVT LoadVT = VT.isFloatingPoint() ? MVT::getFloatingPointVT(SrcVTSize) in combineBitcast()
43966 return DAG.getBitcast(VT, ResNode); in combineBitcast()
43973 if (VT == MVT::x86mmx) { in combineBitcast()
43983 return DAG.getNode(X86ISD::MMX_MOVW2D, DL, VT, in combineBitcast()
43988 return DAG.getBitcast(VT, DAG.getConstantFP(F64, DL, MVT::f64)); in combineBitcast()
44006 return DAG.getNode(X86ISD::MMX_MOVW2D, dl, VT, N00); in combineBitcast()
44024 return DAG.getNode(X86ISD::MOVDQ2Q, SDLoc(N00), VT, in combineBitcast()
44033 return DAG.getNode(X86ISD::MOVDQ2Q, DL, VT, in combineBitcast()
44040 if (Subtarget.hasAVX512() && VT.isScalarInteger() && in combineBitcast()
44046 if (Subtarget.hasAVX512() && SrcVT.isScalarInteger() && VT.isVector() && in combineBitcast()
44047 VT.getVectorElementType() == MVT::i1) { in combineBitcast()
44050 return DAG.getConstant(1, SDLoc(N0), VT); in combineBitcast()
44052 return DAG.getConstant(0, SDLoc(N0), VT); in combineBitcast()
44060 VT.isVector() && VT.getVectorElementType() == MVT::i1 && in combineBitcast()
44061 isPowerOf2_32(VT.getVectorNumElements())) { in combineBitcast()
44062 unsigned NumElts = VT.getVectorNumElements(); in combineBitcast()
44084 if (EVT(CmpVT) == VT) in combineBitcast()
44092 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Ops); in combineBitcast()
44118 if (!((Subtarget.hasSSE1() && VT == MVT::f32) || in combineBitcast()
44119 (Subtarget.hasSSE2() && VT == MVT::f64) || in combineBitcast()
44120 (Subtarget.hasFP16() && VT == MVT::f16) || in combineBitcast()
44121 (Subtarget.hasSSE2() && VT.isInteger() && VT.isVector() && in combineBitcast()
44122 TLI.isTypeLegal(VT)))) in combineBitcast()
44132 LogicOp0.getOperand(0).getValueType() == VT && in combineBitcast()
44134 SDValue CastedOp1 = DAG.getBitcast(VT, LogicOp1); in combineBitcast()
44135 unsigned Opcode = VT.isFloatingPoint() ? FPOpcode : N0.getOpcode(); in combineBitcast()
44136 return DAG.getNode(Opcode, DL0, VT, LogicOp0.getOperand(0), CastedOp1); in combineBitcast()
44141 LogicOp1.getOperand(0).getValueType() == VT && in combineBitcast()
44143 SDValue CastedOp0 = DAG.getBitcast(VT, LogicOp0); in combineBitcast()
44144 unsigned Opcode = VT.isFloatingPoint() ? FPOpcode : N0.getOpcode(); in combineBitcast()
44145 return DAG.getNode(Opcode, DL0, VT, LogicOp1.getOperand(0), CastedOp0); in combineBitcast()
44237 MVT VT = MVT::getVectorVT(MVT::i32, Ops[0].getValueSizeInBits() / 32); in createVPDPBUSD() local
44238 return DAG.getNode(X86ISD::VPDPBUSD, DL, VT, Ops); in createVPDPBUSD()
44269 MVT VT = MVT::getVectorVT(MVT::i64, Ops[0].getValueSizeInBits() / 64); in createPSADBW() local
44270 return DAG.getNode(X86ISD::PSADBW, DL, VT, Ops); in createPSADBW()
44503 EVT VT = Extract->getOperand(0).getValueType(); in combineVPDPBUSDPattern() local
44504 if (!isPowerOf2_32(VT.getVectorNumElements())) in combineVPDPBUSDPattern()
44537 unsigned Stages = Log2_32(VT.getVectorNumElements()); in combineVPDPBUSDPattern()
44575 EVT VT = Extract->getOperand(0).getValueType(); in combineBasicSADPattern() local
44576 if (!isPowerOf2_32(VT.getVectorNumElements())) in combineBasicSADPattern()
44612 unsigned Stages = Log2_32(VT.getVectorNumElements()); in combineBasicSADPattern()
44653 EVT VT = N->getValueType(0); in combineExtractFromVectorLoad() local
44662 if (LoadVec && ISD::isNormalLoad(LoadVec) && VT.isInteger() && in combineExtractFromVectorLoad()
44663 VecVT.getVectorElementType() == VT && in combineExtractFromVectorLoad()
44668 unsigned PtrOff = VT.getSizeInBits() * Idx / 8; in combineExtractFromVectorLoad()
44672 DAG.getLoad(VT, dl, LoadVec->getChain(), NewPtr, MPI, Alignment, in combineExtractFromVectorLoad()
44693 EVT VT = N->getValueType(0); in combineExtractWithShuffle() local
44713 if (SrcOpVT.isScalarInteger() && VT.isInteger() && in combineExtractWithShuffle()
44720 SrcOp = DAG.getZExtOrTrunc(SrcOp, dl, VT); in combineExtractWithShuffle()
44732 VT.getSizeInBits() == SrcBCWidth && SrcEltBits == SrcBCWidth) { in combineExtractWithShuffle()
44733 SDValue Load = DAG.getLoad(VT, dl, MemIntr->getChain(), in combineExtractWithShuffle()
44745 if (SrcBC.getOpcode() == ISD::SCALAR_TO_VECTOR && VT.isInteger() && in combineExtractWithShuffle()
44760 Scl = DAG.getZExtOrTrunc(Scl, dl, VT); in combineExtractWithShuffle()
44772 return DAG.getNode(N->getOpcode(), dl, VT, DAG.getBitcast(ExtractVT, Src), in combineExtractWithShuffle()
44866 return DAG.getUNDEF(VT); in combineExtractWithShuffle()
44869 return VT.isFloatingPoint() ? DAG.getConstantFP(0.0, dl, VT) in combineExtractWithShuffle()
44870 : DAG.getConstant(0, dl, VT); in combineExtractWithShuffle()
44875 return DAG.getZExtOrTrunc(V, dl, VT); in combineExtractWithShuffle()
44892 EVT VT = ExtElt->getValueType(0); in scalarizeExtEltFP() local
44897 if (!Vec.hasOneUse() || !isNullConstant(Index) || VecVT.getScalarType() != VT) in scalarizeExtEltFP()
44902 if (Vec.getOpcode() == ISD::SETCC && VT == MVT::i1) { in scalarizeExtEltFP()
44913 return DAG.getNode(Vec.getOpcode(), DL, VT, Ext0, Ext1, Vec.getOperand(2)); in scalarizeExtEltFP()
44916 if (!(VT == MVT::f16 && Subtarget.hasFP16()) && VT != MVT::f32 && in scalarizeExtEltFP()
44917 VT != MVT::f64) in scalarizeExtEltFP()
44935 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, in scalarizeExtEltFP()
44937 SDValue Ext2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, in scalarizeExtEltFP()
44939 return DAG.getNode(ISD::SELECT, DL, VT, Ext0, Ext1, Ext2); in scalarizeExtEltFP()
44977 ExtOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op, Index)); in scalarizeExtEltFP()
44978 return DAG.getNode(Vec.getOpcode(), DL, VT, ExtOps); in scalarizeExtEltFP()
45006 EVT VT = ExtElt->getValueType(0); in combineArithReduction() local
45008 if (VecVT.getScalarType() != VT) in combineArithReduction()
45035 if (VT != MVT::i8 || NumElts < 4 || !isPowerOf2_32(NumElts)) in combineArithReduction()
45064 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index); in combineArithReduction()
45073 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index); in combineArithReduction()
45081 if (VT == MVT::i8) { in combineArithReduction()
45097 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index); in combineArithReduction()
45121 MVT VT = MVT::getVectorVT(MVT::i64, Ops[0].getValueSizeInBits() / 64); in combineArithReduction() local
45123 return DAG.getNode(X86ISD::PSADBW, DL, VT, Ops[0], Zero); in combineArithReduction()
45142 VecVT = MVT::getVectorVT(VT.getSimpleVT(), 128 / VT.getSizeInBits()); in combineArithReduction()
45144 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index); in combineArithReduction()
45174 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index); in combineArithReduction()
45192 EVT VT = N->getValueType(0); in combineExtractVectorElt() local
45196 unsigned NumEltBits = VT.getScalarSizeInBits(); in combineExtractVectorElt()
45200 return IsPextr ? DAG.getConstant(0, dl, VT) : DAG.getUNDEF(VT); in combineExtractVectorElt()
45203 if (CIdx && VT.isInteger()) { in combineExtractVectorElt()
45212 return IsPextr ? DAG.getConstant(0, dl, VT) : DAG.getUNDEF(VT); in combineExtractVectorElt()
45213 return DAG.getConstant(EltBits[Idx].zext(NumEltBits), dl, VT); in combineExtractVectorElt()
45225 return DAG.getBitcast(VT, Sub); in combineExtractVectorElt()
45243 return DAG.getZExtOrTrunc(Scl, dl, VT); in combineExtractVectorElt()
45253 if (VT == MVT::i64 && SrcVT == MVT::v1i64 && in combineExtractVectorElt()
45257 return DAG.getBitcast(VT, InputVector); in combineExtractVectorElt()
45260 if (VT == MVT::i32 && SrcVT == MVT::v2i32 && in combineExtractVectorElt()
45349 return DAG.getAnyExtOrTrunc(NewExt, dl, VT); in combineExtractVectorElt()
45359 unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N0, SelectionDAG &DAG, in combineToExtendBoolVectorInReg() argument
45369 EVT SVT = VT.getScalarType(); in combineToExtendBoolVectorInReg()
45375 if (!VT.isVector()) in combineToExtendBoolVectorInReg()
45389 unsigned NumElts = VT.getVectorNumElements(); in combineToExtendBoolVectorInReg()
45402 Vec = DAG.getBitcast(VT, Vec); in combineToExtendBoolVectorInReg()
45406 Vec = DAG.getVectorShuffle(VT, DL, Vec, Vec, ShuffleMask); in combineToExtendBoolVectorInReg()
45420 Vec = DAG.getBitcast(VT, Vec); in combineToExtendBoolVectorInReg()
45426 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Scl); in combineToExtendBoolVectorInReg()
45428 Vec = DAG.getVectorShuffle(VT, DL, Vec, Vec, ShuffleMask); in combineToExtendBoolVectorInReg()
45438 SDValue BitMask = DAG.getBuildVector(VT, DL, Bits); in combineToExtendBoolVectorInReg()
45439 Vec = DAG.getNode(ISD::AND, DL, VT, Vec, BitMask); in combineToExtendBoolVectorInReg()
45444 Vec = DAG.getSExtOrTrunc(Vec, DL, VT); in combineToExtendBoolVectorInReg()
45450 return DAG.getNode(ISD::SRL, DL, VT, Vec, in combineToExtendBoolVectorInReg()
45451 DAG.getConstant(EltSizeInBits - 1, DL, VT)); in combineToExtendBoolVectorInReg()
45464 EVT VT = LHS.getValueType(); in combineVSelectWithAllOnesOrZeros() local
45482 if (VT.isFloatingPoint()) in combineVSelectWithAllOnesOrZeros()
45483 return DAG.getConstantFP(0.0, DL, VT); in combineVSelectWithAllOnesOrZeros()
45484 return DAG.getConstant(0, DL, VT); in combineVSelectWithAllOnesOrZeros()
45492 if (CondVT.getScalarSizeInBits() != VT.getScalarSizeInBits()) in combineVSelectWithAllOnesOrZeros()
45502 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT) == in combineVSelectWithAllOnesOrZeros()
45524 return DAG.getBitcast(VT, Cond); in combineVSelectWithAllOnesOrZeros()
45533 return DAG.getBitcast(VT, Or); in combineVSelectWithAllOnesOrZeros()
45540 return DAG.getBitcast(VT, And); in combineVSelectWithAllOnesOrZeros()
45553 return DAG.getBitcast(VT, AndN); in combineVSelectWithAllOnesOrZeros()
45570 EVT VT = N->getValueType(0); in narrowVectorSelect() local
45571 if (!VT.is256BitVector()) in narrowVectorSelect()
45587 return SplitOpsAndApply(DAG, Subtarget, DL, VT, {Cond, TVal, FVal}, makeBlend, in narrowVectorSelect()
45603 EVT VT = N->getValueType(0); in combineSelectOfTwoConstants() local
45604 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT)) in combineSelectOfTwoConstants()
45635 ((VT == MVT::i32 || VT == MVT::i64) && in combineSelectOfTwoConstants()
45647 SDValue R = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond); in combineSelectOfTwoConstants()
45651 R = DAG.getNode(ISD::MUL, DL, VT, R, DAG.getConstant(AbsDiff, DL, VT)); in combineSelectOfTwoConstants()
45655 R = DAG.getNode(ISD::ADD, DL, VT, R, SDValue(FalseC, 0)); in combineSelectOfTwoConstants()
45680 EVT VT = N->getValueType(0); in combineVSelectToBLENDV() local
45692 if (!TLI.isOperationLegalOrCustom(ISD::VSELECT, VT)) in combineVSelectToBLENDV()
45697 if (VT.getVectorElementType() == MVT::i16) in combineVSelectToBLENDV()
45700 if (VT.is128BitVector() && !Subtarget.hasSSE41()) in combineVSelectToBLENDV()
45703 if (VT == MVT::v32i8 && !Subtarget.hasAVX2()) in combineVSelectToBLENDV()
45706 if (VT.is512BitVector()) in combineVSelectToBLENDV()
45777 EVT VT, SDValue Mask, SDValue X, SDValue Y, const SDLoc &DL, in combineLogicBlendIntoConditionalNegate() argument
45819 return DAG.getBitcast(VT, Res); in combineLogicBlendIntoConditionalNegate()
45873 EVT VT = LHS.getValueType(); in combineSelect() local
45882 CondVT.getScalarSizeInBits() == VT.getScalarSizeInBits() && in combineSelect()
45885 if (SDValue V = combineLogicBlendIntoConditionalNegate(VT, Cond, RHS, LHS, in combineSelect()
45895 return DAG.getVectorShuffle(VT, DL, LHS, RHS, Mask); in combineSelect()
45904 MVT SimpleVT = VT.getSimpleVT(); in combineSelect()
45910 int NumElts = VT.getVectorNumElements(); in combineSelect()
45922 LHS = DAG.getNode(X86ISD::PSHUFB, DL, VT, LHS.getOperand(0), in combineSelect()
45924 RHS = DAG.getNode(X86ISD::PSHUFB, DL, VT, RHS.getOperand(0), in combineSelect()
45926 return DAG.getNode(ISD::OR, DL, VT, LHS, RHS); in combineSelect()
45935 if (Cond.getOpcode() == ISD::SETCC && VT.isFloatingPoint() && in combineSelect()
45936 VT != MVT::f80 && VT != MVT::f128 && !isSoftF16(VT, Subtarget) && in combineSelect()
45937 (TLI.isTypeLegal(VT) || VT == MVT::v2f32) && in combineSelect()
45939 (Subtarget.hasSSE1() && VT.getScalarType() == MVT::f32))) { in combineSelect()
46089 Cond.getOpcode() == ISD::SETCC && (VT == MVT::f32 || VT == MVT::f64)) { in combineSelect()
46098 return DAG.getNode(ISD::SELECT, DL, VT, AndNode, RHS, LHS); in combineSelect()
46111 (VT.getVectorElementType() == MVT::i8 || in combineSelect()
46112 VT.getVectorElementType() == MVT::i16)) { in combineSelect()
46113 Cond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Cond); in combineSelect()
46114 return DAG.getNode(N->getOpcode(), DL, VT, Cond, LHS, RHS); in combineSelect()
46141 VT.getSizeInBits()); in combineSelect()
46143 VT.getSizeInBits()); in combineSelect()
46148 return extractSubVector(Res, 0, DAG, DL, VT.getSizeInBits()); in combineSelect()
46186 return DAG.getSelect(DL, VT, Cond, LHS, RHS); in combineSelect()
46191 return DAG.getSelect(DL, VT, Cond, LHS, RHS); in combineSelect()
46222 return DAG.getSelect(DL, VT, Cond, LHS, RHS.getOperand(2)); in combineSelect()
46238 return DAG.getSelect(DL, VT, CondNew, RHS, LHS); in combineSelect()
46245 TLI.isTypeLegal(VT.getScalarType())) { in combineSelect()
46246 EVT ExtCondVT = VT.changeVectorElementTypeToInteger(); in combineSelect()
46250 return DAG.getSelect(DL, VT, ExtCond, LHS, RHS); in combineSelect()
46267 supportedVectorVarShift(VT, Subtarget, LHS.getOpcode()) && in combineSelect()
46270 m_SpecificInt(VT.getScalarSizeInBits()), in combineSelect()
46274 DL, VT, LHS.getOperand(0), LHS.getOperand(1)); in combineSelect()
46279 supportedVectorVarShift(VT, Subtarget, RHS.getOpcode()) && in combineSelect()
46282 m_SpecificInt(VT.getScalarSizeInBits()), in combineSelect()
46286 DL, VT, RHS.getOperand(0), RHS.getOperand(1)); in combineSelect()
46291 if (!TLI.isTypeLegal(VT) || isSoftF16(VT, Subtarget)) in combineSelect()
46306 return DAG.getNode(N->getOpcode(), DL, VT, in combineSelect()
46316 return DAG.getNode(N->getOpcode(), DL, VT, Cond, RHS, LHS); in combineSelect()
46325 if (N->getOpcode() == ISD::SELECT && VT.isVector() && in combineSelect()
46326 VT.getVectorElementType() == MVT::i1 && in combineSelect()
46327 (DCI.isBeforeLegalize() || (VT != MVT::v64i1 || Subtarget.is64Bit()))) { in combineSelect()
46328 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), VT.getVectorNumElements()); in combineSelect()
46348 return DAG.getBitcast(VT, Select); in combineSelect()
46362 Cond.getOperand(0).getValueType() == VT) { in combineSelect()
46370 return DAG.getSelect(DL, VT, NotCond, RHS, LHS); in combineSelect()
46376 unsigned EltBitWidth = VT.getScalarSizeInBits(); in combineSelect()
46378 TLI.isTypeLegal(VT) && ((Subtarget.hasAVX() && EltBitWidth == 32) || in combineSelect()
46388 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) { in combineSelect()
46394 SDValue ShlAmt = getConstVector(ShlVals, VT.getSimpleVT(), DAG, DL); in combineSelect()
46395 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, And.getOperand(0), ShlAmt); in combineSelect()
46398 return DAG.getSelect(DL, VT, NewCond, RHS, LHS); in combineSelect()
46829 MVT VT = EFLAGS.getSimpleValueType(); in combinePTESTCC() local
46867 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT, in combinePTESTCC()
46878 EFLAGS.getOpcode(), DL, VT, DAG.getBitcast(OpVT, NotOp1), in combinePTESTCC()
46889 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT, in combinePTESTCC()
46899 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT, in combinePTESTCC()
46907 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT, in combinePTESTCC()
46920 assert(VT == MVT::i32 && "Expected i32 EFLAGS comparison result"); in combinePTESTCC()
46932 return DAG.getNode(X86ISD::TESTP, SDLoc(EFLAGS), VT, Res, Res); in combinePTESTCC()
46951 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT, Op1, Op1); in combinePTESTCC()
46955 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT, Op0, Op0); in combinePTESTCC()
46969 return DAG.getNode(EFLAGS.getOpcode(), SDLoc(EFLAGS), VT, in combinePTESTCC()
47475 EVT VT = N->getValueType(0); in combineCMov() local
47477 SDValue Diff = DAG.getNode(ISD::SUB, DL, VT, Const, Add.getOperand(1)); in combineCMov()
47479 DAG.getNode(X86ISD::CMOV, DL, VT, Diff, Add.getOperand(0), in combineCMov()
47481 return DAG.getNode(ISD::ADD, DL, VT, CMov, Add.getOperand(1)); in combineCMov()
47492 EVT VT = N->getOperand(0).getValueType(); in canReduceVMulWidth() local
47493 if (VT.getScalarSizeInBits() != 32) in canReduceVMulWidth()
47574 EVT VT = N->getOperand(0).getValueType(); in reduceVMULWidth() local
47575 unsigned NumElts = VT.getVectorNumElements(); in reduceVMULWidth()
47591 DL, VT, MulLo); in reduceVMULWidth()
47619 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ResLo, ResHi); in reduceVMULWidth()
47623 EVT VT, const SDLoc &DL) { in combineMulSpecial() argument
47626 SDValue Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0), in combineMulSpecial()
47627 DAG.getConstant(Mult, DL, VT)); in combineMulSpecial()
47628 Result = DAG.getNode(ISD::SHL, DL, VT, Result, in combineMulSpecial()
47630 Result = DAG.getNode(isAdd ? ISD::ADD : ISD::SUB, DL, VT, Result, in combineMulSpecial()
47636 SDValue Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0), in combineMulSpecial()
47637 DAG.getConstant(Mul1, DL, VT)); in combineMulSpecial()
47638 Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, Result, in combineMulSpecial()
47639 DAG.getConstant(Mul2, DL, VT)); in combineMulSpecial()
47640 Result = DAG.getNode(isAdd ? ISD::ADD : ISD::SUB, DL, VT, Result, in combineMulSpecial()
47659 return DAG.getNode(ISD::ADD, DL, VT, N->getOperand(0), in combineMulSpecial()
47684 return DAG.getNode(ISD::ADD, DL, VT, N->getOperand(0), in combineMulSpecial()
47697 SDValue Shift1 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMulSpecial()
47699 SDValue Shift2 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMulSpecial()
47701 return DAG.getNode(ISD::ADD, DL, VT, Shift1, Shift2); in combineMulSpecial()
47720 EVT VT = N->getValueType(0); in combineMulToPMADDWD() local
47723 if (!VT.isVector() || VT.getVectorElementType() != MVT::i32) in combineMulToPMADDWD()
47728 unsigned NumElts = VT.getVectorNumElements(); in combineMulToPMADDWD()
47774 return DAG.getNode(ISD::AND, DL, VT, Op, DAG.getConstant(0xFFFF, DL, VT)); in combineMulToPMADDWD()
47778 if (Src.getScalarValueSizeInBits() == 16 && VT.getSizeInBits() <= 128) in combineMulToPMADDWD()
47779 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Src); in combineMulToPMADDWD()
47783 EVT ExtVT = VT.changeVectorElementType(MVT::i16); in combineMulToPMADDWD()
47785 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Src); in combineMulToPMADDWD()
47793 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, Src); in combineMulToPMADDWD()
47798 return DAG.getNode(X86ISD::VSRLI, DL, VT, Op.getOperand(0), in combineMulToPMADDWD()
47819 return SplitOpsAndApply(DAG, Subtarget, DL, VT, {N0, N1}, PMADDWDBuilder); in combineMulToPMADDWD()
47827 EVT VT = N->getValueType(0); in combineMulToPMULDQ() local
47830 if (!VT.isVector() || VT.getVectorElementType() != MVT::i64 || in combineMulToPMULDQ()
47831 VT.getVectorNumElements() < 2 || in combineMulToPMULDQ()
47832 !isPowerOf2_32(VT.getVectorNumElements())) in combineMulToPMULDQ()
47846 return SplitOpsAndApply(DAG, Subtarget, DL, VT, {N0, N1}, PMULDQBuilder, in combineMulToPMULDQ()
47857 return SplitOpsAndApply(DAG, Subtarget, DL, VT, {N0, N1}, PMULUDQBuilder, in combineMulToPMULDQ()
47867 EVT VT = N->getValueType(0); in combineMul() local
47876 if (DCI.isBeforeLegalize() && VT.isVector()) in combineMul()
47891 if (VT != MVT::i64 && VT != MVT::i32 && in combineMul()
47892 (!VT.isVector() || !VT.isSimple() || !VT.isInteger())) in combineMul()
47899 if (VT.isVector()) in combineMul()
47905 if (!C || C->getBitWidth() != VT.getScalarSizeInBits()) in combineMul()
47919 if (VT == MVT::i64 || VT == MVT::i32) { in combineMul()
47921 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0), in combineMul()
47922 DAG.getConstant(AbsMulAmt, DL, VT)); in combineMul()
47924 NewMul = DAG.getNegative(NewMul, DL, VT); in combineMul()
47956 NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMul()
47959 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0), in combineMul()
47960 DAG.getConstant(MulAmt1, DL, VT)); in combineMul()
47963 NewMul = DAG.getNode(ISD::SHL, DL, VT, NewMul, in combineMul()
47966 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, NewMul, in combineMul()
47967 DAG.getConstant(MulAmt2, DL, VT)); in combineMul()
47971 NewMul = DAG.getNegative(NewMul, DL, VT); in combineMul()
47973 NewMul = combineMulSpecial(C->getZExtValue(), N, DAG, VT, DL); in combineMul()
47976 EVT ShiftVT = VT.isVector() ? VT : MVT::i8; in combineMul()
47978 C->getZExtValue() != maxUIntN(VT.getScalarSizeInBits()) && in combineMul()
47984 ISD::ADD, DL, VT, N->getOperand(0), in combineMul()
47985 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMul()
47988 NewMul = DAG.getNegative(NewMul, DL, VT); in combineMul()
47992 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMul()
47996 NewMul = DAG.getNode(ISD::SUB, DL, VT, N->getOperand(0), NewMul); in combineMul()
47998 NewMul = DAG.getNode(ISD::SUB, DL, VT, NewMul, N->getOperand(0)); in combineMul()
48000 (!VT.isVector() || Subtarget.fastImmVectorShift())) { in combineMul()
48003 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMul()
48006 ISD::ADD, DL, VT, NewMul, in combineMul()
48007 DAG.getNode(ISD::ADD, DL, VT, N->getOperand(0), N->getOperand(0))); in combineMul()
48009 (!VT.isVector() || Subtarget.fastImmVectorShift())) { in combineMul()
48012 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMul()
48015 ISD::SUB, DL, VT, NewMul, in combineMul()
48016 DAG.getNode(ISD::ADD, DL, VT, N->getOperand(0), N->getOperand(0))); in combineMul()
48017 } else if (SignMulAmt >= 0 && VT.isVector() && in combineMul()
48032 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMul()
48035 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), in combineMul()
48037 NewMul = DAG.getNode(*Opc, DL, VT, Shift1, Shift2); in combineMul()
48067 EVT VT = N->getValueType(0); in combineShiftToPMULH() local
48068 if (!VT.isVector() || VT.getVectorElementType().getSizeInBits() < 32) in combineShiftToPMULH()
48098 return DAG.getNode(ExtOpc, DL, VT, Mulh); in combineShiftToPMULH()
48107 EVT VT = N0.getValueType(); in combineShiftLeft() local
48108 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in combineShiftLeft()
48114 supportedVectorVarShift(VT, Subtarget, ISD::SHL)) { in combineShiftLeft()
48122 return DAG.getNode(X86ISD::VSHLV, DL, VT, N00, N1); in combineShiftLeft()
48128 return DAG.getNode(X86ISD::VSHLV, DL, VT, N01, N1); in combineShiftLeft()
48134 if (VT.isInteger() && !VT.isVector() && in combineShiftLeft()
48163 return DAG.getNode(ISD::AND, DL, VT, N00, DAG.getConstant(Mask, DL, VT)); in combineShiftLeft()
48174 EVT VT = N0.getValueType(); in combineShiftRightArithmetic() local
48175 unsigned Size = VT.getSizeInBits(); in combineShiftRightArithmetic()
48182 if (supportedVectorVarShift(VT, Subtarget, ISD::SRA)) { in combineShiftRightArithmetic()
48185 m_SpecificInt(VT.getScalarSizeInBits() - 1)))) in combineShiftRightArithmetic()
48186 return DAG.getNode(X86ISD::VSRAV, DL, VT, N0, ShrAmtVal); in combineShiftRightArithmetic()
48203 if (VT.isVector() || N1.getOpcode() != ISD::Constant || in combineShiftRightArithmetic()
48225 DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, N00, DAG.getValueType(SVT)); in combineShiftRightArithmetic()
48229 return DAG.getNode(ISD::SHL, DL, VT, NN, in combineShiftRightArithmetic()
48231 return DAG.getNode(ISD::SRA, DL, VT, NN, in combineShiftRightArithmetic()
48243 EVT VT = N0.getValueType(); in combineShiftRightLogical() local
48244 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in combineShiftRightLogical()
48253 supportedVectorVarShift(VT, Subtarget, ISD::SRL)) { in combineShiftRightLogical()
48261 return DAG.getNode(X86ISD::VSRLV, DL, VT, N00, N1); in combineShiftRightLogical()
48267 return DAG.getNode(X86ISD::VSRLV, DL, VT, N01, N1); in combineShiftRightLogical()
48306 SDValue NewMask = DAG.getConstant(NewMaskVal, DL, VT); in combineShiftRightLogical()
48307 SDValue NewShift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), N1); in combineShiftRightLogical()
48308 return DAG.getNode(ISD::AND, DL, VT, NewShift, NewMask); in combineShiftRightLogical()
48319 EVT VT = N->getValueType(0); in combineHorizOpWithShuffle() local
48334 if (VT.is128BitVector() && SrcVT.getScalarSizeInBits() <= 32) { in combineHorizOpWithShuffle()
48347 MVT ShufVT = VT.isFloatingPoint() ? MVT::v4f32 : MVT::v4i32; in combineHorizOpWithShuffle()
48351 SDValue Res = DAG.getNode(Opcode, DL, VT, Lo, Hi); in combineHorizOpWithShuffle()
48354 return DAG.getBitcast(VT, Res); in combineHorizOpWithShuffle()
48361 if (VT.is128BitVector() && SrcVT.getScalarSizeInBits() <= 32) { in combineHorizOpWithShuffle()
48408 MVT ShufVT = VT.isFloatingPoint() ? MVT::v4f32 : MVT::v4i32; in combineHorizOpWithShuffle()
48409 SDValue Res = DAG.getNode(Opcode, DL, VT, LHS, RHS); in combineHorizOpWithShuffle()
48412 return DAG.getBitcast(VT, Res); in combineHorizOpWithShuffle()
48418 if (VT.is256BitVector() && Subtarget.hasInt256()) { in combineHorizOpWithShuffle()
48444 MVT ShufVT = VT.isFloatingPoint() ? MVT::v4f64 : MVT::v4i64; in combineHorizOpWithShuffle()
48445 SDValue Res = DAG.getNode(Opcode, DL, VT, DAG.getBitcast(SrcVT, Op00), in combineHorizOpWithShuffle()
48449 return DAG.getBitcast(VT, Res); in combineHorizOpWithShuffle()
48464 EVT VT = N->getValueType(0); in combineVectorPack() local
48467 unsigned NumDstElts = VT.getVectorNumElements(); in combineVectorPack()
48468 unsigned DstBitsPerElt = VT.getScalarSizeInBits(); in combineVectorPack()
48487 unsigned NumLanes = VT.getSizeInBits() / 128; in combineVectorPack()
48527 return getConstVector(Bits, Undefs, VT.getSimpleVT(), DAG, SDLoc(N)); in combineVectorPack()
48545 DAG.getNode(X86ISD::PACKSS, DL, VT, DAG.getBitcast(SrcVT, Not0), in combineVectorPack()
48547 return DAG.getNOT(DL, Pack, VT); in combineVectorPack()
48554 N0.getOpcode() == ISD::TRUNCATE && N1.isUndef() && VT == MVT::v16i8 && in combineVectorPack()
48560 return DAG.getNode(X86ISD::VTRUNC, SDLoc(N), VT, N0.getOperand(0)); in combineVectorPack()
48566 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Concat); in combineVectorPack()
48571 if (VT.is128BitVector()) { in combineVectorPack()
48588 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Src0, Src1); in combineVectorPack()
48596 return getEXTEND_VECTOR_INREG(ExtOpc, SDLoc(N), VT, N0.getOperand(0), in combineVectorPack()
48616 MVT VT = N->getSimpleValueType(0); in combineVectorHADDSUB() local
48635 MVT ShufVT = MVT::getVectorVT(MVT::i32, VT.getSizeInBits() / 32); in combineVectorHADDSUB()
48643 return DAG.getNode(N->getOpcode(), DL, VT, DAG.getBitcast(VT, NewLHS), in combineVectorHADDSUB()
48644 DAG.getBitcast(VT, NewRHS)); in combineVectorHADDSUB()
48662 EVT VT = N->getValueType(0); in combineVectorShiftVar() local
48668 return DAG.getConstant(0, SDLoc(N), VT); in combineVectorShiftVar()
48677 return getTargetVShiftByConstNode(X86Opc, SDLoc(N), VT.getSimpleVT(), N0, in combineVectorShiftVar()
48682 APInt DemandedElts = APInt::getAllOnes(VT.getVectorNumElements()); in combineVectorShiftVar()
48697 EVT VT = N->getValueType(0); in combineVectorShiftImm() local
48700 unsigned NumBitsPerElt = VT.getScalarSizeInBits(); in combineVectorShiftImm()
48701 assert(VT == N0.getValueType() && (NumBitsPerElt % 8) == 0 && in combineVectorShiftImm()
48707 return DAG.getConstant(0, SDLoc(N), VT); in combineVectorShiftImm()
48714 return DAG.getConstant(0, SDLoc(N), VT); in combineVectorShiftImm()
48726 return DAG.getConstant(0, SDLoc(N), VT); in combineVectorShiftImm()
48732 return DAG.getConstant(-1, SDLoc(N), VT); in combineVectorShiftImm()
48740 return DAG.getConstant(0, SDLoc(N), VT); in combineVectorShiftImm()
48743 return DAG.getNode(Opcode, SDLoc(N), VT, N0.getOperand(0), in combineVectorShiftImm()
48777 Src = DAG.getBitcast(VT, Src); in combineVectorShiftImm()
48778 Src = DAG.getNode(X86ISD::PSHUFD, DL, VT, Src, in combineVectorShiftImm()
48780 Src = DAG.getNode(X86ISD::VSHLI, DL, VT, Src, N1); in combineVectorShiftImm()
48781 Src = DAG.getNode(X86ISD::VSRAI, DL, VT, Src, N1); in combineVectorShiftImm()
48793 assert(EltBits.size() == VT.getVectorNumElements() && in combineVectorShiftImm()
48811 return getConstVector(EltBits, UndefElts, VT.getSimpleVT(), DAG, SDLoc(N)); in combineVectorShiftImm()
48827 SDValue LHS = DAG.getNode(Opcode, DL, VT, in combineVectorShiftImm()
48828 DAG.getBitcast(VT, BC.getOperand(0)), N1); in combineVectorShiftImm()
48829 return DAG.getNode(BC.getOpcode(), DL, VT, LHS, RHS); in combineVectorShiftImm()
48845 EVT VT = N->getValueType(0); in combineVectorInsert() local
48847 assert(((Opcode == X86ISD::PINSRB && VT == MVT::v16i8) || in combineVectorInsert()
48848 (Opcode == X86ISD::PINSRW && VT == MVT::v8i16) || in combineVectorInsert()
48858 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), VT, Scl); in combineVectorInsert()
48861 unsigned NumBitsPerElt = VT.getScalarSizeInBits(); in combineVectorInsert()
48869 if (VT.isSimple() && DCI.isAfterLegalizeDAG()) { in combineVectorInsert()
48901 EVT VT = CMP00.getValueType(); in combineCompareEqual() local
48903 if (VT == MVT::f32 || VT == MVT::f64 || in combineCompareEqual()
48904 (VT == MVT::f16 && Subtarget.hasFP16())) { in combineCompareEqual()
48991 MVT VT = N->getSimpleValueType(0); in combineAndNotIntoANDNP() local
48992 if (!VT.is128BitVector() && !VT.is256BitVector() && !VT.is512BitVector()) in combineAndNotIntoANDNP()
49008 X = DAG.getBitcast(VT, X); in combineAndNotIntoANDNP()
49009 Y = DAG.getBitcast(VT, Y); in combineAndNotIntoANDNP()
49010 return DAG.getNode(X86ISD::ANDNP, SDLoc(N), VT, X, Y); in combineAndNotIntoANDNP()
49023 EVT VT = N->getValueType(0); in combineAndShuffleNot() local
49026 if (!((VT.is128BitVector() && Subtarget.hasSSE2()) || in combineAndShuffleNot()
49027 ((VT.is256BitVector() || VT.is512BitVector()) && Subtarget.hasAVX()))) in combineAndShuffleNot()
49072 X = DAG.getBitcast(VT, X); in combineAndShuffleNot()
49073 Y = DAG.getBitcast(VT, Y); in combineAndShuffleNot()
49078 if (!Subtarget.useAVX512Regs() && VT.is512BitVector() && in combineAndShuffleNot()
49079 TLI.isTypeLegal(VT.getHalfNumVectorElementsVT(*DAG.getContext()))) { in combineAndShuffleNot()
49087 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, {LoV, HiV}); in combineAndShuffleNot()
49090 if (TLI.isTypeLegal(VT)) in combineAndShuffleNot()
49091 return DAG.getNode(X86ISD::ANDNP, DL, VT, {X, Y}); in combineAndShuffleNot()
49105 static SDValue PromoteMaskArithmetic(SDValue N, const SDLoc &DL, EVT VT, in PromoteMaskArithmetic() argument
49118 if (!TLI.isOperationLegalOrPromote(N.getOpcode(), VT)) in PromoteMaskArithmetic()
49121 if (SDValue NN0 = PromoteMaskArithmetic(N0, DL, VT, DAG, Depth + 1)) in PromoteMaskArithmetic()
49129 if (N0.getOperand(0).getValueType() != VT) in PromoteMaskArithmetic()
49135 if (SDValue NN1 = PromoteMaskArithmetic(N1, DL, VT, DAG, Depth + 1)) in PromoteMaskArithmetic()
49140 N1.getOperand(0).getValueType() == VT; in PromoteMaskArithmetic()
49144 DAG.FoldConstantArithmetic(ISD::ZERO_EXTEND, DL, VT, {N1})) in PromoteMaskArithmetic()
49150 return DAG.getNode(N.getOpcode(), DL, VT, N0, N1); in PromoteMaskArithmetic()
49162 EVT VT = N.getValueType(); in PromoteMaskArithmetic() local
49163 assert(VT.isVector() && "Expected vector type"); in PromoteMaskArithmetic()
49172 SDValue Op = PromoteMaskArithmetic(Narrow, DL, VT, DAG, 0); in PromoteMaskArithmetic()
49182 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, in PromoteMaskArithmetic()
49206 EVT VT = N->getValueType(0); in convertIntLogicToFPLogic() local
49229 return DAG.getBitcast(VT, FPLogic); in convertIntLogicToFPLogic()
49232 if (VT != MVT::i1 || N0.getOpcode() != ISD::SETCC || !N0.hasOneUse() || in convertIntLogicToFPLogic()
49262 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Logic, ZeroIndex); in convertIntLogicToFPLogic()
49309 EVT VT = N->getValueType(0); in combineBitOpWithShift() local
49335 return DAG.getBitcast(VT, Shift); in combineBitOpWithShift()
49352 EVT VT = N->getValueType(0); in combineBitOpWithPACK() local
49382 return DAG.getBitcast(VT, DAG.getNode(X86ISD::PACKSS, DL, DstVT, LHS, RHS)); in combineBitOpWithPACK()
49392 EVT VT = Op0.getValueType(); in combineAndMaskToShift() local
49393 if (VT != Op1.getValueType() || !VT.isSimple() || !VT.isInteger()) in combineAndMaskToShift()
49405 if (N->getValueType(0) == VT && in combineAndMaskToShift()
49406 supportedVectorShiftWithImm(VT, Subtarget, ISD::SRA)) { in combineAndMaskToShift()
49420 getTargetVShiftByConstNode(X86ISD::VSRAI, DL, VT.getSimpleVT(), X, in combineAndMaskToShift()
49421 VT.getScalarSizeInBits() - 1, DAG); in combineAndMaskToShift()
49422 return DAG.getNode(X86ISD::ANDNP, DL, VT, Sra, Y); in combineAndMaskToShift()
49434 if (!supportedVectorShiftWithImm(VT, Subtarget, ISD::SRL)) in combineAndMaskToShift()
49437 unsigned EltBitWidth = VT.getScalarSizeInBits(); in combineAndMaskToShift()
49444 SDValue Shift = DAG.getNode(X86ISD::VSRLI, DL, VT, Op0, ShAmt); in combineAndMaskToShift()
49468 static bool hasBZHI(const X86Subtarget &Subtarget, MVT VT) { in hasBZHI() argument
49470 (VT == MVT::i32 || (VT == MVT::i64 && Subtarget.is64Bit())); in hasBZHI()
49485 MVT VT = Node->getSimpleValueType(0); in combineAndLoadToBZHI() local
49489 if (!hasBZHI(Subtarget, VT)) in combineAndLoadToBZHI()
49515 VT.getSizeInBits() || in combineAndLoadToBZHI()
49538 SDValue SizeC = DAG.getConstant(VT.getSizeInBits(), dl, MVT::i32); in combineAndLoadToBZHI()
49549 SDValue AllOnes = DAG.getAllOnesConstant(dl, VT); in combineAndLoadToBZHI()
49550 SDValue LShr = DAG.getNode(ISD::SRL, dl, VT, AllOnes, Sub); in combineAndLoadToBZHI()
49552 return DAG.getNode(ISD::AND, dl, VT, Inp, LShr); in combineAndLoadToBZHI()
49569 EVT VT = N->getValueType(0); in combineScalarAndWithMaskSetcc() local
49578 assert(!VT.isVector() && "Expected scalar VT!"); in combineScalarAndWithMaskSetcc()
49642 return DAG.getZExtOrTrunc(DAG.getBitcast(IntVT, Concat), dl, VT); in combineScalarAndWithMaskSetcc()
49692 EVT VT = N->getValueType(0); in combineBMILogicOp() local
49694 if (!Subtarget.hasBMI() || !VT.isScalarInteger() || in combineBMILogicOp()
49695 (VT != MVT::i32 && VT != MVT::i64)) in combineBMILogicOp()
49856 EVT VT = N->getValueType(0); in combineAnd() local
49861 if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32) { in combineAnd()
49869 if (VT == MVT::i64 && Subtarget.is64Bit() && !isa<ConstantSDNode>(N1)) { in combineAnd()
49882 if (VT == MVT::i1) { in combineAnd()
49923 SDValue Neg = DAG.getNegative(N0.getOperand(0), dl, VT); in combineAnd()
49927 SDValue Shift = DAG.getNode(ISD::SHL, dl, VT, Neg, in combineAnd()
49928 DAG.getConstant(MulCLowBitLog, dl, VT)); in combineAnd()
49929 return DAG.getNode(ISD::AND, dl, VT, Shift, N1); in combineAnd()
49973 if (VT.isVector() && getTargetConstantFromNode(N1)) { in combineAnd()
49977 DAG.ComputeNumSignBits(N1) == VT.getScalarSizeInBits() && in combineAnd()
49979 SDValue MaskMul = DAG.getNode(ISD::AND, dl, VT, N0.getOperand(1), N1); in combineAnd()
49980 return DAG.getNode(Opc0, dl, VT, N0.getOperand(0), MaskMul); in combineAnd()
50011 if (!(Subtarget.hasBMI2() && !ContainsNOT && VT.getSizeInBits() >= 32)) in combineAnd()
50013 return DAG.getZExtOrTrunc(getSETCC(X86CC, BT, dl, DAG), dl, VT); in combineAnd()
50017 if (VT.isVector() && (VT.getScalarSizeInBits() % 8) == 0) { in combineAnd()
50028 int NumElts = VT.getVectorNumElements(); in combineAnd()
50029 int EltSizeInBits = VT.getScalarSizeInBits(); in combineAnd()
50067 return DAG.getNode(ISD::AND, dl, VT, NewN0 ? NewN0 : N0, in combineAnd()
50072 if ((VT.getScalarSizeInBits() % 8) == 0 && in combineAnd()
50082 if (VT == SrcVecVT.getScalarType() && N0->isOnlyUserOf(SrcVec.getNode()) && in combineAnd()
50105 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Shuffle, in combineAnd()
50121 MVT VT = N->getSimpleValueType(0); in canonicalizeBitSelect() local
50122 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in canonicalizeBitSelect()
50123 if (!VT.isVector() || (EltSizeInBits % 8) != 0) in canonicalizeBitSelect()
50133 if (!(Subtarget.hasXOP() || useVPTERNLOG(Subtarget, VT) || in canonicalizeBitSelect()
50159 if (useVPTERNLOG(Subtarget, VT)) { in canonicalizeBitSelect()
50164 MVT::getVectorVT(OpSVT, VT.getSizeInBits() / OpSVT.getSizeInBits()); in canonicalizeBitSelect()
50171 return DAG.getBitcast(VT, Res); in canonicalizeBitSelect()
50176 DAG.getNode(X86ISD::ANDNP, DL, VT, DAG.getBitcast(VT, N0.getOperand(1)), in canonicalizeBitSelect()
50177 DAG.getBitcast(VT, N1.getOperand(0))); in canonicalizeBitSelect()
50178 return DAG.getNode(ISD::OR, DL, VT, X, Y); in canonicalizeBitSelect()
50225 EVT VT = N->getValueType(0); in combineLogicBlendIntoPBLENDV() local
50226 if (!((VT.is128BitVector() && Subtarget.hasSSE2()) || in combineLogicBlendIntoPBLENDV()
50227 (VT.is256BitVector() && Subtarget.hasInt256()))) in combineLogicBlendIntoPBLENDV()
50249 if (SDValue Res = combineLogicBlendIntoConditionalNegate(VT, Mask, X, Y, DL, in combineLogicBlendIntoPBLENDV()
50261 MVT BlendVT = VT.is256BitVector() ? MVT::v32i8 : MVT::v16i8; in combineLogicBlendIntoPBLENDV()
50267 return DAG.getBitcast(VT, Mask); in combineLogicBlendIntoPBLENDV()
50278 EVT VT = Cmp.getOperand(0).getValueType(); in lowerX86CmpEqZeroToCtlzSrl() local
50279 unsigned Log2b = Log2_32(VT.getSizeInBits()); in lowerX86CmpEqZeroToCtlzSrl()
50281 SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Cmp->getOperand(0)); in lowerX86CmpEqZeroToCtlzSrl()
50385 EVT VT = And1_L->getValueType(0); in foldMaskedMergeImpl() local
50386 SDValue Freeze_And0_R = DAG.getNode(ISD::FREEZE, SDLoc(), VT, And0_R); in foldMaskedMergeImpl()
50387 SDValue Xor0 = DAG.getNode(ISD::XOR, DL, VT, And1_R, Freeze_And0_R); in foldMaskedMergeImpl()
50388 SDValue And = DAG.getNode(ISD::AND, DL, VT, Xor0, NotOp); in foldMaskedMergeImpl()
50389 SDValue Xor1 = DAG.getNode(ISD::XOR, DL, VT, And, Freeze_And0_R); in foldMaskedMergeImpl()
50429 static SDValue combineAddOrSubToADCOrSBB(bool IsSub, const SDLoc &DL, EVT VT, in combineAddOrSubToADCOrSBB() argument
50433 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT)) in combineAddOrSubToADCOrSBB()
50462 return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, in combineAddOrSubToADCOrSBB()
50479 return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, in combineAddOrSubToADCOrSBB()
50490 DAG.getVTList(VT, MVT::i32), X, in combineAddOrSubToADCOrSBB()
50491 DAG.getConstant(0, DL, VT), EFLAGS); in combineAddOrSubToADCOrSBB()
50512 DAG.getVTList(VT, MVT::i32), X, in combineAddOrSubToADCOrSBB()
50513 DAG.getConstant(0, DL, VT), NewEFLAGS); in combineAddOrSubToADCOrSBB()
50521 DAG.getVTList(VT, MVT::i32), X, in combineAddOrSubToADCOrSBB()
50522 DAG.getConstant(-1, DL, VT), EFLAGS); in combineAddOrSubToADCOrSBB()
50542 DAG.getVTList(VT, MVT::i32), X, in combineAddOrSubToADCOrSBB()
50543 DAG.getConstant(-1, DL, VT), NewEFLAGS); in combineAddOrSubToADCOrSBB()
50570 return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, in combineAddOrSubToADCOrSBB()
50584 return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, in combineAddOrSubToADCOrSBB()
50596 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in combineAddOrSubToADCOrSBB()
50602 DAG.getConstant(-1ULL, DL, VT), Cmp1.getValue(1)); in combineAddOrSubToADCOrSBB()
50607 DAG.getConstant(0, DL, VT), Cmp1.getValue(1)); in combineAddOrSubToADCOrSBB()
50618 EVT VT = N->getValueType(0); in combineAddOrSubToADCOrSBB() local
50620 if (SDValue ADCOrSBB = combineAddOrSubToADCOrSBB(IsSub, DL, VT, X, Y, DAG)) in combineAddOrSubToADCOrSBB()
50624 if (SDValue ADCOrSBB = combineAddOrSubToADCOrSBB(IsSub, DL, VT, Y, X, DAG)) { in combineAddOrSubToADCOrSBB()
50626 ADCOrSBB = DAG.getNegative(ADCOrSBB, DL, VT); in combineAddOrSubToADCOrSBB()
50651 EVT VT = N->getValueType(0); in combineOrXorWithSETCC() local
50652 if (SDValue R = combineAddOrSubToADCOrSBB(IsSub, DL, VT, N1, N0, DAG)) in combineOrXorWithSETCC()
50663 MVT VT = N->getSimpleValueType(0); in combineOrXorWithSETCC() local
50667 VT.getScalarSizeInBits(), UndefElts, in combineOrXorWithSETCC()
50674 return DAG.getNode(X86ISD::PCMPEQ, SDLoc(N), VT, N0.getOperand(0), in combineOrXorWithSETCC()
50687 EVT VT = N->getValueType(0); in combineOr() local
50692 if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32) { in combineOr()
50701 if (VT == MVT::i1) { in combineOr()
50750 if ((VT == MVT::i32 || VT == MVT::i64) && in combineOr()
50765 SDValue R = DAG.getZExtOrTrunc(NotCond, dl, VT); in combineOr()
50766 R = DAG.getNode(ISD::MUL, dl, VT, R, DAG.getConstant(Val + 1, dl, VT)); in combineOr()
50767 R = DAG.getNode(ISD::SUB, dl, VT, R, DAG.getConstant(1, dl, VT)); in combineOr()
50779 unsigned NumElts = VT.getVectorNumElements(); in combineOr()
50786 ISD::CONCAT_VECTORS, dl, VT, in combineOr()
50794 ISD::CONCAT_VECTORS, dl, VT, in combineOr()
50800 if (VT.isVector() && (VT.getScalarSizeInBits() % 8) == 0) { in combineOr()
50811 int NumElts = VT.getVectorNumElements(); in combineOr()
50812 int EltSizeInBits = VT.getScalarSizeInBits(); in combineOr()
50831 if (!Subtarget.hasBMI() && VT.isScalarInteger() && VT != MVT::i1) in combineOr()
50902 EVT VT = N->getValueType(0); in foldVectorXorShiftIntoCmp() local
50903 if (!VT.isSimple()) in foldVectorXorShiftIntoCmp()
50906 switch (VT.getSimpleVT().SimpleTy) { in foldVectorXorShiftIntoCmp()
50937 return DAG.getSetCC(SDLoc(N), VT, Shift.getOperand(0), Ones, ISD::SETGT); in foldVectorXorShiftIntoCmp()
50956 static SDValue detectUSatPattern(SDValue In, EVT VT, SelectionDAG &DAG, in detectUSatPattern() argument
50961 assert(InVT.getScalarSizeInBits() > VT.getScalarSizeInBits() && in detectUSatPattern()
50976 if (C2.isMask(VT.getScalarSizeInBits())) in detectUSatPattern()
50981 if (C1.isNonNegative() && C2.isMask(VT.getScalarSizeInBits())) in detectUSatPattern()
50986 if (C1.isNonNegative() && C2.isMask(VT.getScalarSizeInBits()) && in detectUSatPattern()
51003 static SDValue detectSSatPattern(SDValue In, EVT VT, bool MatchPackUS = false) { in detectSSatPattern() argument
51004 unsigned NumDstBits = VT.getScalarSizeInBits(); in detectSSatPattern()
51037 static SDValue combineTruncateWithSat(SDValue In, EVT VT, const SDLoc &DL, in combineTruncateWithSat() argument
51040 if (!Subtarget.hasSSE2() || !VT.isVector()) in combineTruncateWithSat()
51043 EVT SVT = VT.getVectorElementType(); in combineTruncateWithSat()
51052 InVT == MVT::v16i32 && VT == MVT::v16i8) { in combineTruncateWithSat()
51053 if (SDValue USatVal = detectSSatPattern(In, VT, true)) { in combineTruncateWithSat()
51058 return DAG.getNode(X86ISD::VTRUNCUS, DL, VT, Mid); in combineTruncateWithSat()
51072 !(!Subtarget.useAVX512Regs() && VT.getSizeInBits() >= 256); in combineTruncateWithSat()
51074 if (!PreferAVX512 && VT.getVectorNumElements() > 1 && in combineTruncateWithSat()
51075 isPowerOf2_32(VT.getVectorNumElements()) && in combineTruncateWithSat()
51078 if (SDValue USatVal = detectSSatPattern(In, VT, true)) { in combineTruncateWithSat()
51081 EVT MidVT = VT.changeVectorElementType(MVT::i16); in combineTruncateWithSat()
51085 SDValue V = truncateVectorWithPACK(X86ISD::PACKUS, VT, Mid, DL, DAG, in combineTruncateWithSat()
51090 return truncateVectorWithPACK(X86ISD::PACKUS, VT, USatVal, DL, DAG, in combineTruncateWithSat()
51093 if (SDValue SSatVal = detectSSatPattern(In, VT)) in combineTruncateWithSat()
51094 return truncateVectorWithPACK(X86ISD::PACKSS, VT, SSatVal, DL, DAG, in combineTruncateWithSat()
51104 if (SDValue SSatVal = detectSSatPattern(In, VT)) { in combineTruncateWithSat()
51107 } else if (SDValue USatVal = detectUSatPattern(In, VT, DAG, DL)) { in combineTruncateWithSat()
51112 unsigned ResElts = VT.getVectorNumElements(); in combineTruncateWithSat()
51129 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res, in combineTruncateWithSat()
51402 EVT VT = ML->getValueType(0); in reduceMaskedLoadToScalarLoad() local
51403 EVT EltVT = VT.getVectorElementType(); in reduceMaskedLoadToScalarLoad()
51405 EVT CastVT = VT; in reduceMaskedLoadToScalarLoad()
51408 CastVT = VT.changeVectorElementType(EltVT); in reduceMaskedLoadToScalarLoad()
51421 Insert = DAG.getBitcast(VT, Insert); in reduceMaskedLoadToScalarLoad()
51433 EVT VT = ML->getValueType(0); in combineMaskedLoadConstantMask() local
51438 unsigned NumElts = VT.getVectorNumElements(); in combineMaskedLoadConstantMask()
51443 SDValue VecLd = DAG.getLoad(VT, DL, ML->getChain(), ML->getBasePtr(), in combineMaskedLoadConstantMask()
51445 SDValue Blend = DAG.getSelect(DL, VT, ML->getMask(), VecLd, in combineMaskedLoadConstantMask()
51465 VT, DL, ML->getChain(), ML->getBasePtr(), ML->getOffset(), ML->getMask(), in combineMaskedLoadConstantMask()
51466 DAG.getUNDEF(VT), ML->getMemoryVT(), ML->getMemOperand(), in combineMaskedLoadConstantMask()
51468 SDValue Blend = DAG.getSelect(DL, VT, ML->getMask(), NewML, in combineMaskedLoadConstantMask()
51498 EVT VT = Mld->getValueType(0); in combineMaskedLoad() local
51500 APInt DemandedBits(APInt::getSignMask(VT.getScalarSizeInBits())); in combineMaskedLoad()
51509 VT, SDLoc(N), Mld->getChain(), Mld->getBasePtr(), Mld->getOffset(), in combineMaskedLoad()
51537 EVT VT = Value.getValueType(); in reduceMaskedStoreToScalarStore() local
51538 EVT EltVT = VT.getVectorElementType(); in reduceMaskedStoreToScalarStore()
51541 EVT CastVT = VT.changeVectorElementType(EltVT); in reduceMaskedStoreToScalarStore()
51560 EVT VT = Mst->getValue().getValueType(); in combineMaskedStore() local
51574 APInt DemandedBits(APInt::getSignMask(VT.getScalarSizeInBits())); in combineMaskedStore()
51608 EVT VT = StoredVal.getValueType(); in combineStore() local
51612 if (!Subtarget.hasAVX512() && VT == StVT && VT.isVector() && in combineStore()
51613 VT.getVectorElementType() == MVT::i1) { in combineStore()
51615 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), VT.getVectorNumElements()); in combineStore()
51625 if (VT == MVT::v1i1 && VT == StVT && Subtarget.hasAVX512() && in combineStore()
51638 if ((VT == MVT::v1i1 || VT == MVT::v2i1 || VT == MVT::v4i1) && VT == StVT && in combineStore()
51640 unsigned NumConcats = 8 / VT.getVectorNumElements(); in combineStore()
51642 SmallVector<SDValue, 4> Ops(NumConcats, DAG.getConstant(0, dl, VT)); in combineStore()
51651 if ((VT == MVT::v8i1 || VT == MVT::v16i1 || VT == MVT::v32i1 || in combineStore()
51652 VT == MVT::v64i1) && VT == StVT && TLI.isTypeLegal(VT) && in combineStore()
51655 if (!DCI.isBeforeLegalize() && VT == MVT::v64i1 && !Subtarget.is64Bit()) { in combineStore()
51687 if (VT.is256BitVector() && StVT == VT && in combineStore()
51688 TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT, in combineStore()
51691 unsigned NumElems = VT.getVectorNumElements(); in combineStore()
51699 if (St->isNonTemporal() && StVT == VT && in combineStore()
51700 St->getAlign().value() < VT.getStoreSize()) { in combineStore()
51703 if (VT.is256BitVector() || VT.is512BitVector()) { in combineStore()
51704 unsigned NumElems = VT.getVectorNumElements(); in combineStore()
51712 if (VT.is128BitVector() && Subtarget.hasSSE2()) { in combineStore()
51722 if (!St->isTruncatingStore() && VT == MVT::v16i8 && !Subtarget.hasBWI() && in combineStore()
51738 TLI.isTruncStoreLegal(StoredVal.getOperand(0).getValueType(), VT)) { in combineStore()
51742 VT, St->getMemOperand(), DAG); in combineStore()
51766 if (NumTruncBits == VT.getSizeInBits() && in combineStore()
51778 if (St->isTruncatingStore() && VT.isVector()) { in combineStore()
51779 if (TLI.isTruncStoreLegal(VT, StVT)) { in combineStore()
51815 if (VT.getSizeInBits() != 64) in combineStore()
51826 if (VT == MVT::i64 && isa<LoadSDNode>(St->getValue()) && in combineStore()
51856 if (VT == MVT::i64 && in combineStore()
51879 MVT VT = StoredVal.getSimpleValueType(); in combineVEXTRACT_STORE() local
51883 unsigned StElts = MemVT.getSizeInBits() / VT.getScalarSizeInBits(); in combineVEXTRACT_STORE()
51884 APInt DemandedElts = APInt::getLowBitsSet(VT.getVectorNumElements(), StElts); in combineVEXTRACT_STORE()
51927 MVT VT = LHS.getSimpleValueType(); in isHorizontalBinOp() local
51928 assert((VT.is128BitVector() || VT.is256BitVector()) && in isHorizontalBinOp()
51930 unsigned NumElts = VT.getVectorNumElements(); in isHorizontalBinOp()
52026 unsigned Num128BitChunks = VT.getSizeInBits() / 128; in isHorizontalBinOp()
52070 if (!IsIdentityPostShuffle && !Subtarget.hasAVX2() && VT.isFloatingPoint() && in isHorizontalBinOp()
52071 isMultiLaneShuffleMask(128, VT.getScalarSizeInBits(), PostShuffleMask)) in isHorizontalBinOp()
52077 return User->getOpcode() == HOpcode && User->getValueType(0) == VT; in isHorizontalBinOp()
52091 LHS = DAG.getBitcast(VT, NewLHS); in isHorizontalBinOp()
52092 RHS = DAG.getBitcast(VT, NewRHS); in isHorizontalBinOp()
52099 EVT VT = N->getValueType(0); in combineToHorizontalAddSub() local
52114 if ((Subtarget.hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) || in combineToHorizontalAddSub()
52115 (Subtarget.hasAVX() && (VT == MVT::v8f32 || VT == MVT::v4f64))) { in combineToHorizontalAddSub()
52121 SDValue HorizBinOp = DAG.getNode(HorizOpcode, SDLoc(N), VT, LHS, RHS); in combineToHorizontalAddSub()
52123 HorizBinOp = DAG.getVectorShuffle(VT, SDLoc(HorizBinOp), HorizBinOp, in combineToHorizontalAddSub()
52124 DAG.getUNDEF(VT), PostShuffleMask); in combineToHorizontalAddSub()
52131 if (Subtarget.hasSSSE3() && (VT == MVT::v8i16 || VT == MVT::v4i32 || in combineToHorizontalAddSub()
52132 VT == MVT::v16i16 || VT == MVT::v8i32)) { in combineToHorizontalAddSub()
52142 SDValue HorizBinOp = SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT, in combineToHorizontalAddSub()
52145 HorizBinOp = DAG.getVectorShuffle(VT, SDLoc(HorizBinOp), HorizBinOp, in combineToHorizontalAddSub()
52146 DAG.getUNDEF(VT), PostShuffleMask); in combineToHorizontalAddSub()
52171 EVT VT = N->getValueType(0); in combineFMulcFCMulc() local
52189 SDValue I2F = DAG.getBitcast(VT, LHS.getOperand(0).getOperand(0)); in combineFMulcFCMulc()
52190 SDValue FCMulC = DAG.getNode(CombineOpcode, SDLoc(N), VT, RHS, I2F); in combineFMulcFCMulc()
52191 r = DAG.getBitcast(VT, FCMulC); in combineFMulcFCMulc()
52232 EVT VT = N->getValueType(0); in combineFaddCFmul() local
52233 if (VT != MVT::v8f16 && VT != MVT::v16f16 && VT != MVT::v32f16) in combineFaddCFmul()
52274 MVT CVT = MVT::getVectorVT(MVT::f32, VT.getVectorNumElements() / 2); in combineFaddCFmul()
52281 return DAG.getBitcast(VT, CFmul); in combineFaddCFmul()
52298 EVT VT = N->getValueType(0); in combineLRINT_LLRINT() local
52303 if (!Subtarget.hasDQI() || !Subtarget.hasVLX() || VT != MVT::v2i64 || in combineLRINT_LLRINT()
52307 return DAG.getNode(X86ISD::CVTP2SI, DL, VT, in combineLRINT_LLRINT()
52325 EVT VT = N->getValueType(0); in combineTruncatedArithmetic() local
52328 auto IsFreeTruncation = [VT](SDValue Op) { in combineTruncatedArithmetic()
52329 unsigned TruncSizeInBits = VT.getScalarSizeInBits(); in combineTruncatedArithmetic()
52348 SDValue Trunc0 = DAG.getNode(ISD::TRUNCATE, DL, VT, N0); in combineTruncatedArithmetic()
52349 SDValue Trunc1 = DAG.getNode(ISD::TRUNCATE, DL, VT, N1); in combineTruncatedArithmetic()
52350 return DAG.getNode(SrcOpcode, DL, VT, Trunc0, Trunc1); in combineTruncatedArithmetic()
52359 if (!VT.isVector()) in combineTruncatedArithmetic()
52370 TLI.isOperationLegal(SrcOpcode, VT) && in combineTruncatedArithmetic()
52381 if (TLI.isOperationLegal(SrcOpcode, VT) && in combineTruncatedArithmetic()
52398 static SDValue combinePMULH(SDValue Src, EVT VT, const SDLoc &DL, in combinePMULH() argument
52410 if (!VT.isVector() || VT.getVectorElementType() != MVT::i16) in combinePMULH()
52457 !(Subtarget.hasAVX512() && !Subtarget.hasBWI() && VT.is256BitVector()) && in combinePMULH()
52463 return DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getBitcast(InVT, Res)); in combinePMULH()
52467 LHS = DAG.getNode(ISD::TRUNCATE, DL, VT, LHS); in combinePMULH()
52468 RHS = DAG.getNode(ISD::TRUNCATE, DL, VT, RHS); in combinePMULH()
52471 return DAG.getNode(Opc, DL, VT, LHS, RHS); in combinePMULH()
52482 static SDValue detectPMADDUBSW(SDValue In, EVT VT, SelectionDAG &DAG, in detectPMADDUBSW() argument
52485 if (!VT.isVector() || !Subtarget.hasSSSE3()) in detectPMADDUBSW()
52488 unsigned NumElems = VT.getVectorNumElements(); in detectPMADDUBSW()
52489 EVT ScalarVT = VT.getVectorElementType(); in detectPMADDUBSW()
52493 SDValue SSatVal = detectSSatPattern(In, VT); in detectPMADDUBSW()
52620 return SplitOpsAndApply(DAG, Subtarget, DL, VT, { ZExtIn, SExtIn }, in detectPMADDUBSW()
52626 EVT VT = N->getValueType(0); in combineTruncate() local
52635 if (SDValue PMAdd = detectPMADDUBSW(Src, VT, DAG, Subtarget, DL)) in combineTruncate()
52639 if (SDValue Val = combineTruncateWithSat(Src, VT, DL, DAG, Subtarget)) in combineTruncate()
52643 if (SDValue V = combinePMULH(Src, VT, DL, DAG, Subtarget)) in combineTruncate()
52648 if (Src.getOpcode() == ISD::BITCAST && VT == MVT::i32) { in combineTruncate()
52655 if (Src.getOpcode() == ISD::LRINT && VT.getScalarType() == MVT::i32 && in combineTruncate()
52657 return DAG.getNode(ISD::LRINT, DL, VT, Src.getOperand(0)); in combineTruncate()
52664 EVT VT = N->getValueType(0); in combineVTRUNC() local
52668 if (SDValue SSatVal = detectSSatPattern(In, VT)) in combineVTRUNC()
52669 return DAG.getNode(X86ISD::VTRUNCS, DL, VT, SSatVal); in combineVTRUNC()
52670 if (SDValue USatVal = detectUSatPattern(In, VT, DAG, DL)) in combineVTRUNC()
52671 return DAG.getNode(X86ISD::VTRUNCUS, DL, VT, USatVal); in combineVTRUNC()
52674 APInt DemandedMask(APInt::getAllOnes(VT.getScalarSizeInBits())); in combineVTRUNC()
52701 EVT VT = Op->getValueType(0); in isFNEG() local
52704 if (VT.getScalarSizeInBits() != ScalarSize) in isFNEG()
52715 if (NegOp0.getValueType() == VT) // FIXME: Can we do better? in isFNEG()
52716 return DAG.getVectorShuffle(VT, SDLoc(Op), NegOp0, DAG.getUNDEF(VT), in isFNEG()
52728 if (NegInsVal.getValueType() == VT.getVectorElementType()) // FIXME in isFNEG()
52729 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(Op), VT, InsVector, in isFNEG()
52845 EVT VT = Arg.getValueType(); in combineFneg() local
52846 EVT SVT = VT.getScalarType(); in combineFneg()
52850 if (!TLI.isTypeLegal(VT)) in combineFneg()
52858 SDValue Zero = DAG.getConstantFP(0.0, DL, VT); in combineFneg()
52859 SDValue NewNode = DAG.getNode(X86ISD::FNMSUB, DL, VT, Arg.getOperand(0), in combineFneg()
52884 EVT VT = Op.getValueType(); in getNegatedExpression() local
52885 EVT SVT = VT.getScalarType(); in getNegatedExpression()
52897 if (!Op.hasOneUse() || !Subtarget.hasAnyFMA() || !isTypeLegal(VT) || in getNegatedExpression()
52899 !isOperationLegal(ISD::FMA, VT)) in getNegatedExpression()
52926 return DAG.getNode(NewOpc, SDLoc(Op), VT, NewOps); in getNegatedExpression()
52932 return DAG.getNode(Opc, SDLoc(Op), VT, NegOp0); in getNegatedExpression()
52942 MVT VT = N->getSimpleValueType(0); in lowerX86FPLogicOp() local
52944 if (!VT.isVector() || !Subtarget.hasSSE2()) in lowerX86FPLogicOp()
52949 unsigned IntBits = VT.getScalarSizeInBits(); in lowerX86FPLogicOp()
52951 MVT IntVT = MVT::getVectorVT(IntSVT, VT.getSizeInBits() / IntBits); in lowerX86FPLogicOp()
52966 return DAG.getBitcast(VT, IntOp); in lowerX86FPLogicOp()
52992 EVT VT = N->getValueType(0); in combineXorSubCTLZ() local
52993 if (VT != MVT::i8 && VT != MVT::i16 && VT != MVT::i32 && in combineXorSubCTLZ()
52994 (VT != MVT::i64 || !Subtarget.is64Bit())) in combineXorSubCTLZ()
53025 EVT OpVT = VT; in combineXorSubCTLZ()
53027 if (VT == MVT::i8) { in combineXorSubCTLZ()
53035 if (VT == MVT::i8) in combineXorSubCTLZ()
53046 EVT VT = N->getValueType(0); in combineXor() local
53050 if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32) { in combineXor()
53094 VT, DAG.getNOT(DL, N0.getOperand(0), N0.getOperand(0).getValueType())); in combineXor()
53099 if (ISD::isBuildVectorAllOnes(N1.getNode()) && VT.isVector() && in combineXor()
53100 VT.getVectorElementType() == MVT::i1 && in combineXor()
53104 ISD::INSERT_SUBVECTOR, DL, VT, N0.getOperand(0), in combineXor()
53118 SDValue LHS = DAG.getZExtOrTrunc(TruncExtSrc.getOperand(0), DL, VT); in combineXor()
53119 SDValue RHS = DAG.getZExtOrTrunc(TruncExtSrc.getOperand(1), DL, VT); in combineXor()
53120 return DAG.getNode(ISD::XOR, DL, VT, LHS, in combineXor()
53121 DAG.getNode(ISD::XOR, DL, VT, RHS, N1)); in combineXor()
53135 EVT VT = N->getValueType(0); in combineBITREVERSE() local
53138 if (VT.isInteger() && N0.getOpcode() == ISD::BITCAST && N0.hasOneUse()) { in combineBITREVERSE()
53151 return DAG.getBitcast(VT, Rev); in combineBITREVERSE()
53165 EVT VT = N->getValueType(0); in combineAVG() local
53166 EVT SVT = VT.getScalarType(); in combineAVG()
53171 if (Opcode == ISD::AVGCEILS && VT.isVector() && SVT == MVT::i8) { in combineAVG()
53172 APInt SignBit = APInt::getSignMask(VT.getScalarSizeInBits()); in combineAVG()
53173 SDValue SignMask = DAG.getConstant(SignBit, DL, VT); in combineAVG()
53174 N0 = DAG.getNode(ISD::XOR, DL, VT, N0, SignMask); in combineAVG()
53175 N1 = DAG.getNode(ISD::XOR, DL, VT, N1, SignMask); in combineAVG()
53176 return DAG.getNode(ISD::XOR, DL, VT, in combineAVG()
53177 DAG.getNode(ISD::AVGCEILU, DL, VT, N0, N1), SignMask); in combineAVG()
53186 EVT VT = N->getValueType(0); in combineBEXTR() local
53187 unsigned NumBits = VT.getSizeInBits(); in combineBEXTR()
53225 EVT VT = N->getValueType(0); in combineFAndFNotToFAndn() local
53229 if (!((VT == MVT::f32 && Subtarget.hasSSE1()) || in combineFAndFNotToFAndn()
53230 (VT == MVT::f64 && Subtarget.hasSSE2()) || in combineFAndFNotToFAndn()
53231 (VT == MVT::v4f32 && Subtarget.hasSSE1() && !Subtarget.hasSSE2()))) in combineFAndFNotToFAndn()
53243 return DAG.getNode(X86ISD::FANDN, DL, VT, N0.getOperand(0), N1); in combineFAndFNotToFAndn()
53247 return DAG.getNode(X86ISD::FANDN, DL, VT, N1.getOperand(0), N0); in combineFAndFNotToFAndn()
53327 EVT VT = N->getValueType(0); in combineFMinNumFMaxNum() local
53328 if (Subtarget.useSoftFloat() || isSoftF16(VT, Subtarget)) in combineFMinNumFMaxNum()
53333 if (!((Subtarget.hasSSE1() && VT == MVT::f32) || in combineFMinNumFMaxNum()
53334 (Subtarget.hasSSE2() && VT == MVT::f64) || in combineFMinNumFMaxNum()
53335 (Subtarget.hasFP16() && VT == MVT::f16) || in combineFMinNumFMaxNum()
53336 (VT.isVector() && TLI.isTypeLegal(VT)))) in combineFMinNumFMaxNum()
53347 return DAG.getNode(MinMaxOp, DL, VT, Op0, Op1, N->getFlags()); in combineFMinNumFMaxNum()
53352 return DAG.getNode(MinMaxOp, DL, VT, Op0, Op1, N->getFlags()); in combineFMinNumFMaxNum()
53354 return DAG.getNode(MinMaxOp, DL, VT, Op1, Op0, N->getFlags()); in combineFMinNumFMaxNum()
53358 if (!VT.isVector() && DAG.getMachineFunction().getFunction().hasMinSize()) in combineFMinNumFMaxNum()
53362 VT); in combineFMinNumFMaxNum()
53383 SDValue MinOrMax = DAG.getNode(MinMaxOp, DL, VT, Op1, Op0); in combineFMinNumFMaxNum()
53388 return DAG.getSelect(DL, VT, IsOp0Nan, Op1, MinOrMax); in combineFMinNumFMaxNum()
53393 EVT VT = N->getValueType(0); in combineX86INT_TO_FP() local
53396 APInt DemandedElts = APInt::getAllOnes(VT.getVectorNumElements()); in combineX86INT_TO_FP()
53403 if (VT.getVectorNumElements() < InVT.getVectorNumElements() && in combineX86INT_TO_FP()
53407 unsigned NumBits = InVT.getScalarSizeInBits() * VT.getVectorNumElements(); in combineX86INT_TO_FP()
53412 SDValue Convert = DAG.getNode(N->getOpcode(), dl, VT, in combineX86INT_TO_FP()
53427 EVT VT = N->getValueType(0); in combineCVTP2I_CVTTP2I() local
53432 if (VT.getVectorNumElements() < InVT.getVectorNumElements() && in combineCVTP2I_CVTTP2I()
53436 unsigned NumBits = InVT.getScalarSizeInBits() * VT.getVectorNumElements(); in combineCVTP2I_CVTTP2I()
53443 DAG.getNode(N->getOpcode(), dl, {VT, MVT::Other}, in combineCVTP2I_CVTTP2I()
53448 DAG.getNode(N->getOpcode(), dl, VT, DAG.getBitcast(InVT, VZLoad)); in combineCVTP2I_CVTTP2I()
53466 MVT VT = N->getSimpleValueType(0); in combineAndnp() local
53467 int NumElts = VT.getVectorNumElements(); in combineAndnp()
53468 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in combineAndnp()
53474 return DAG.getConstant(0, DL, VT); in combineAndnp()
53482 return DAG.getConstant(0, DL, VT); in combineAndnp()
53486 return DAG.getNOT(DL, N0, VT); in combineAndnp()
53490 return DAG.getNode(ISD::AND, DL, VT, DAG.getBitcast(VT, Not), N1); in combineAndnp()
53497 DL, DAG.getNode(ISD::OR, DL, VT, N0, DAG.getBitcast(VT, Not)), VT); in combineAndnp()
53511 return getConstVector(ResultBits, VT, DAG, DL); in combineAndnp()
53522 SDValue Not = getConstVector(EltBits0, VT, DAG, DL); in combineAndnp()
53523 return DAG.getNode(ISD::AND, DL, VT, Not, N1); in combineAndnp()
53529 if (VT.isVector() && (VT.getScalarSizeInBits() % 8) == 0) { in combineAndnp()
53704 EVT VT = N->getValueType(0); in combineSignExtendInReg() local
53715 if (VT == MVT::v4i64 && (N0.getOpcode() == ISD::ANY_EXTEND || in combineSignExtendInReg()
53728 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, VT, Promote, N1); in combineSignExtendInReg()
53751 EVT VT = Ext->getValueType(0); in promoteExtBeforeAdd() local
53752 if (VT != MVT::i64) in promoteExtBeforeAdd()
53796 SDValue NewExt = DAG.getNode(Ext->getOpcode(), SDLoc(Ext), VT, AddOp0); in promoteExtBeforeAdd()
53797 SDValue NewConstant = DAG.getConstant(AddC, SDLoc(Add), VT); in promoteExtBeforeAdd()
53804 return DAG.getNode(ISD::ADD, SDLoc(Add), VT, NewExt, NewConstant, Flags); in promoteExtBeforeAdd()
53828 EVT VT = CMovN.getValueType(); in combineToExtendCMOV() local
53842 if (VT != MVT::i16 && !(ExtendOpcode == ISD::SIGN_EXTEND && VT == MVT::i32)) in combineToExtendCMOV()
53869 EVT VT = N->getValueType(0); in combineExtSetcc() local
53873 if (!Subtarget.hasAVX512() || !VT.isVector() || N0.getOpcode() != ISD::SETCC) in combineExtSetcc()
53877 EVT SVT = VT.getVectorElementType(); in combineExtSetcc()
53886 unsigned Size = VT.getSizeInBits(); in combineExtSetcc()
53902 SDValue Res = DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC); in combineExtSetcc()
53914 EVT VT = N->getValueType(0); in combineSext() local
53920 SDValue Setcc = DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, N0->getOperand(0), in combineSext()
53943 if (SDValue V = combineToExtendBoolVectorInReg(N->getOpcode(), DL, VT, N0, in combineSext()
53947 if (VT.isVector()) { in combineSext()
53952 return DAG.getNode(N0.getOpcode(), DL, VT, N0.getOperand(0)); in combineSext()
53980 EVT VT = V.getValueType(); in getInvertedVectorForFMA() local
53981 EVT EltVT = VT.getVectorElementType(); in getInvertedVectorForFMA()
53991 SDNode *NV = DAG.getNodeIfExists(ISD::BUILD_VECTOR, DAG.getVTList(VT), Ops); in getInvertedVectorForFMA()
54018 EVT VT = N->getValueType(0); in combineFMA() local
54023 if (!TLI.isTypeLegal(VT)) in combineFMA()
54034 TLI.isOperationExpand(ISD::FMA, VT)) { in combineFMA()
54035 SDValue Fmul = DAG.getNode(ISD::FMUL, dl, VT, A, B, Flags); in combineFMA()
54036 return DAG.getNode(ISD::FADD, dl, VT, Fmul, C, Flags); in combineFMA()
54039 EVT ScalarVT = VT.getScalarType(); in combineFMA()
54091 return DAG.getNode(NewOpcode, dl, {VT, MVT::Other}, in combineFMA()
54095 return DAG.getNode(NewOpcode, dl, VT, A, B, C, N->getOperand(3)); in combineFMA()
54096 return DAG.getNode(NewOpcode, dl, VT, A, B, C); in combineFMA()
54105 EVT VT = N->getValueType(0); in combineFMADDSUB() local
54119 return DAG.getNode(NewOpcode, dl, VT, N->getOperand(0), N->getOperand(1), in combineFMADDSUB()
54121 return DAG.getNode(NewOpcode, dl, VT, N->getOperand(0), N->getOperand(1), in combineFMADDSUB()
54130 EVT VT = N->getValueType(0); in combineZext() local
54136 SDValue Setcc = DAG.getNode(X86ISD::SETCC_CARRY, dl, VT, N0->getOperand(0), in combineZext()
54157 if (SDValue V = combineToExtendBoolVectorInReg(N->getOpcode(), dl, VT, N0, in combineZext()
54161 if (VT.isVector()) in combineZext()
54173 VT.getScalarSizeInBits() == N0.getOperand(0).getScalarValueSizeInBits()) { in combineZext()
54190 static SDValue truncateAVX512SetCCNoBWI(EVT VT, EVT OpVT, SDValue LHS, in truncateAVX512SetCCNoBWI() argument
54194 if (Subtarget.hasAVX512() && !Subtarget.hasBWI() && VT.isVector() && in truncateAVX512SetCCNoBWI()
54195 VT.getVectorElementType() == MVT::i1 && in truncateAVX512SetCCNoBWI()
54199 return DAG.getNode(ISD::TRUNCATE, DL, VT, Setcc); in truncateAVX512SetCCNoBWI()
54210 EVT VT = N->getValueType(0); in combineSetCC() local
54215 if (SDValue V = combineVectorSizedSetCCEquality(VT, LHS, RHS, CC, DL, DAG, in combineSetCC()
54219 if (VT == MVT::i1) { in combineSetCC()
54223 return DAG.getNode(ISD::TRUNCATE, DL, VT, getSETCC(X86CC, V, DL, DAG)); in combineSetCC()
54241 return DAG.getSetCC(DL, VT, AndN, DAG.getConstant(0, DL, OpVT), CC); in combineSetCC()
54243 return DAG.getSetCC(DL, VT, AndN, DAG.getConstant(0, DL, OpVT), CC); in combineSetCC()
54259 return DAG.getSetCC(DL, VT, AndN, DAG.getConstant(0, DL, OpVT), CC); in combineSetCC()
54261 return DAG.getSetCC(DL, VT, AndN, DAG.getConstant(0, DL, OpVT), CC); in combineSetCC()
54275 return DAG.getSetCC(DL, VT, LHS.getOperand(0), in combineSetCC()
54293 SDValue SETCC0 = DAG.getSetCC(DL, VT, BaseOp, RHS, CC); in combineSetCC()
54295 DL, VT, BaseOp, DAG.getConstant(-CInt, DL, OpVT), CC); in combineSetCC()
54296 return DAG.getNode(CC == ISD::SETEQ ? ISD::OR : ISD::AND, DL, VT, in combineSetCC()
54304 if (VT.isVector() && VT.getVectorElementType() == MVT::i1 && in combineSetCC()
54323 assert(VT == Op0.getOperand(0).getValueType() && in combineSetCC()
54326 return DAG.getConstant(0, DL, VT); in combineSetCC()
54328 return DAG.getConstant(1, DL, VT); in combineSetCC()
54330 return DAG.getNOT(DL, Op0.getOperand(0), VT); in combineSetCC()
54342 if (VT.isVector() && OpVT.isVector() && OpVT.isInteger()) { in combineSetCC()
54399 if (SDValue R = truncateAVX512SetCCNoBWI(VT, OpVT, LHSOut, RHSOut, in combineSetCC()
54402 return DAG.getSetCC(DL, VT, LHSOut, RHSOut, NewCC); in combineSetCC()
54408 truncateAVX512SetCCNoBWI(VT, OpVT, LHS, RHS, CC, DL, DAG, Subtarget)) in combineSetCC()
54425 if (VT.isVector() && OpVT.isVector() && OpVT.isInteger() && in combineSetCC()
54466 DAG.getSetCC(DL, VT, LHS.getOperand(0), C0, ISD::SETEQ); in combineSetCC()
54468 DAG.getSetCC(DL, VT, LHS.getOperand(0), C1, ISD::SETEQ); in combineSetCC()
54469 return DAG.getNode(ISD::OR, DL, VT, NewLHS, NewRHS); in combineSetCC()
54476 if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32 && in combineSetCC()
54487 return DAG.getSetCC(DL, VT, LHS, SDValue(FNeg, 0), CC); in combineSetCC()
54498 MVT VT = N->getSimpleValueType(0); in combineMOVMSK() local
54499 unsigned NumBits = VT.getScalarSizeInBits(); in combineMOVMSK()
54502 assert(VT == MVT::i32 && NumElts <= NumBits && "Unexpected MOVMSK types"); in combineMOVMSK()
54515 return DAG.getConstant(Imm, SDLoc(N), VT); in combineMOVMSK()
54522 return DAG.getNode(X86ISD::MOVMSK, SDLoc(N), VT, Src.getOperand(0)); in combineMOVMSK()
54530 return DAG.getNode(ISD::XOR, DL, VT, in combineMOVMSK()
54531 DAG.getNode(X86ISD::MOVMSK, DL, VT, NotSrc), in combineMOVMSK()
54532 DAG.getConstant(NotMask, DL, VT)); in combineMOVMSK()
54541 return DAG.getNode(ISD::XOR, DL, VT, in combineMOVMSK()
54542 DAG.getNode(X86ISD::MOVMSK, DL, VT, Src.getOperand(0)), in combineMOVMSK()
54543 DAG.getConstant(NotMask, DL, VT)); in combineMOVMSK()
54575 return DAG.getNode(X86ISD::MOVMSK, DL, VT, DAG.getNOT(DL, Res, SrcVT)); in combineMOVMSK()
54594 SDValue NewMovMsk = DAG.getNode(X86ISD::MOVMSK, DL, VT, NewSrc); in combineMOVMSK()
54595 return DAG.getNode(SrcBC.getOpcode(), DL, VT, NewMovMsk, in combineMOVMSK()
54596 DAG.getConstant(Mask, DL, VT)); in combineMOVMSK()
54613 MVT VT = N->getSimpleValueType(0); in combineTESTP() local
54614 unsigned NumBits = VT.getScalarSizeInBits(); in combineTESTP()
54824 EVT VT = N->getValueType(0); in combineVectorCompareAndMaskUnaryOp() local
54826 unsigned NumEltBits = VT.getScalarSizeInBits(); in combineVectorCompareAndMaskUnaryOp()
54828 if (!VT.isVector() || Op0.getOpcode() != ISD::AND || in combineVectorCompareAndMaskUnaryOp()
54830 VT.getSizeInBits() != Op0.getValueSizeInBits()) in combineVectorCompareAndMaskUnaryOp()
54849 SourceConst = DAG.getNode(N->getOpcode(), DL, {VT, MVT::Other}, in combineVectorCompareAndMaskUnaryOp()
54852 SourceConst = DAG.getNode(N->getOpcode(), DL, VT, SDValue(BV, 0)); in combineVectorCompareAndMaskUnaryOp()
54857 SDValue Res = DAG.getBitcast(VT, NewAnd); in combineVectorCompareAndMaskUnaryOp()
54905 EVT VT = N->getValueType(0); in combineUIntToFP() local
54916 if (InVT.isVector() && VT.getVectorElementType() == MVT::f16) { in combineUIntToFP()
54930 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other}, in combineUIntToFP()
54932 return DAG.getNode(ISD::SINT_TO_FP, dl, VT, P); in combineUIntToFP()
54939 VT.getScalarType() != MVT::f16) { in combineUIntToFP()
54946 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other}, in combineUIntToFP()
54948 return DAG.getNode(ISD::SINT_TO_FP, dl, VT, P); in combineUIntToFP()
54957 return DAG.getNode(ISD::STRICT_SINT_TO_FP, SDLoc(N), {VT, MVT::Other}, in combineUIntToFP()
54959 return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, Op0); in combineUIntToFP()
54976 EVT VT = N->getValueType(0); in combineSIntToFP() local
54987 if (InVT.isVector() && VT.getVectorElementType() == MVT::f16) { in combineSIntToFP()
55001 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other}, in combineSIntToFP()
55003 return DAG.getNode(ISD::SINT_TO_FP, dl, VT, P); in combineSIntToFP()
55010 VT.getScalarType() != MVT::f16) { in combineSIntToFP()
55015 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other}, in combineSIntToFP()
55017 return DAG.getNode(ISD::SINT_TO_FP, dl, VT, P); in combineSIntToFP()
55034 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other}, in combineSIntToFP()
55036 return DAG.getNode(ISD::SINT_TO_FP, dl, VT, Trunc); in combineSIntToFP()
55045 return DAG.getNode(X86ISD::STRICT_CVTSI2P, dl, {VT, MVT::Other}, in combineSIntToFP()
55047 return DAG.getNode(X86ISD::CVTSI2P, dl, VT, Shuf); in combineSIntToFP()
55058 if (VT == MVT::f16 || VT == MVT::f128) in combineSIntToFP()
55063 if (Subtarget.hasDQI() && VT != MVT::f80) in combineSIntToFP()
55066 if (Ld->isSimple() && !VT.isVector() && ISD::isNormalLoad(Op0.getNode()) && in combineSIntToFP()
55070 VT, InVT, SDLoc(N), Ld->getChain(), Ld->getBasePtr(), in combineSIntToFP()
55161 EVT VT = Op.getValueType(); in combineCMP() local
55174 unsigned BitWidth = VT.getSizeInBits(); in combineCMP()
55182 Op = DAG.getNode(ISD::AND, dl, VT, Op.getOperand(0), in combineCMP()
55183 DAG.getConstant(Mask, dl, VT)); in combineCMP()
55185 DAG.getConstant(0, dl, VT)); in combineCMP()
55241 APInt::getBitsSetFrom(OpVT.getSizeInBits(), VT.getSizeInBits()); in combineCMP()
55279 SDValue Op0 = DAG.getNode(ISD::TRUNCATE, dl, VT, Op.getOperand(0)); in combineCMP()
55280 SDValue Op1 = DAG.getNode(ISD::TRUNCATE, dl, VT, Op.getOperand(1)); in combineCMP()
55283 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in combineCMP()
55289 DAG.getConstant(0, dl, VT)); in combineCMP()
55304 MVT VT = LHS.getSimpleValueType(); in combineX86AddSub() local
55314 SDValue Res = DAG.getNode(GenericOpc, DL, VT, LHS, RHS); in combineX86AddSub()
55325 Op = DAG.getNegative(Op, DL, VT); in combineX86AddSub()
55334 return combineAddOrSubToADCOrSBB(IsSub, DL, VT, LHS, RHS, DAG, in combineX86AddSub()
55344 MVT VT = N->getSimpleValueType(0); in combineSBB() local
55345 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in combineSBB()
55381 EVT VT = N->getValueType(0); in combineADC() local
55384 ISD::AND, DL, VT, in combineADC()
55385 DAG.getNode(X86ISD::SETCC_CARRY, DL, VT, in combineADC()
55387 DAG.getConstant(1, DL, VT)); in combineADC()
55403 MVT VT = N->getSimpleValueType(0); in combineADC() local
55404 SDVTList VTs = DAG.getVTList(VT, MVT::i32); in combineADC()
55419 const SDLoc &DL, EVT VT, in matchPMADDWD() argument
55439 if (!VT.isVector() || VT.getVectorElementType() != MVT::i32 || in matchPMADDWD()
55440 VT.getVectorNumElements() < 4 || in matchPMADDWD()
55441 !isPowerOf2_32(VT.getVectorNumElements())) in matchPMADDWD()
55456 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; i += 2) { in matchPMADDWD()
55508 VT.getVectorNumElements() * 2); in matchPMADDWD()
55520 return SplitOpsAndApply(DAG, Subtarget, DL, VT, { N0, N1 }, PMADDBuilder); in matchPMADDWD()
55527 const SDLoc &DL, EVT VT, in matchPMADDWD_2() argument
55535 if (!VT.isVector() || VT.getVectorElementType() != MVT::i32 || in matchPMADDWD_2()
55536 VT.getVectorNumElements() < 4 || in matchPMADDWD_2()
55537 !isPowerOf2_32(VT.getVectorNumElements())) in matchPMADDWD_2()
55621 if (In0.getValueSizeInBits() < VT.getSizeInBits() || in matchPMADDWD_2()
55622 In1.getValueSizeInBits() < VT.getSizeInBits()) in matchPMADDWD_2()
55649 VT.getVectorNumElements() * 2); in matchPMADDWD_2()
55658 return SplitOpsAndApply(DAG, Subtarget, DL, VT, { In0, In1 }, in matchPMADDWD_2()
55667 const SDLoc &DL, EVT VT) { in combineAddOfPMADDWD() argument
55672 if (VT.getSizeInBits() > 128) in combineAddOfPMADDWD()
55675 unsigned NumElts = VT.getVectorNumElements(); in combineAddOfPMADDWD()
55703 return DAG.getNode(X86ISD::VPMADDWD, DL, VT, LHS, RHS); in combineAddOfPMADDWD()
55742 EVT VT = N->getValueType(0); in pushAddIntoCmovOfConsts() local
55761 FalseOp = DAG.getNode(ISD::ADD, DL, VT, X, FalseOp); in pushAddIntoCmovOfConsts()
55762 TrueOp = DAG.getNode(ISD::ADD, DL, VT, X, TrueOp); in pushAddIntoCmovOfConsts()
55763 Cmov = DAG.getNode(X86ISD::CMOV, DL, VT, FalseOp, TrueOp, in pushAddIntoCmovOfConsts()
55765 return DAG.getNode(ISD::ADD, DL, VT, Cmov, Y); in pushAddIntoCmovOfConsts()
55769 FalseOp = DAG.getNode(ISD::ADD, DL, VT, OtherOp, FalseOp); in pushAddIntoCmovOfConsts()
55770 TrueOp = DAG.getNode(ISD::ADD, DL, VT, OtherOp, TrueOp); in pushAddIntoCmovOfConsts()
55771 return DAG.getNode(X86ISD::CMOV, DL, VT, FalseOp, TrueOp, Cmov.getOperand(2), in pushAddIntoCmovOfConsts()
55778 EVT VT = N->getValueType(0); in combineAdd() local
55786 if (SDValue MAdd = matchPMADDWD(DAG, Op0, Op1, DL, VT, Subtarget)) in combineAdd()
55788 if (SDValue MAdd = matchPMADDWD_2(DAG, Op0, Op1, DL, VT, Subtarget)) in combineAdd()
55790 if (SDValue MAdd = combineAddOfPMADDWD(DAG, Op0, Op1, DL, VT)) in combineAdd()
55806 return DAG.getNode(X86ISD::PSADBW, DL, VT, Sum, in combineAdd()
55816 if (VT.isVector()) { in combineAdd()
55821 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op0.getOperand(0)); in combineAdd()
55822 return DAG.getNode(ISD::SUB, DL, VT, Op1, SExt); in combineAdd()
55828 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op1.getOperand(0)); in combineAdd()
55829 return DAG.getNode(ISD::SUB, DL, VT, Op0, SExt); in combineAdd()
55878 MVT VT = N->getSimpleValueType(0); in combineSubABS() local
55879 SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, VT, TrueOp, FalseOp, in combineSubABS()
55882 return DAG.getNode(ISD::ADD, DL, VT, N0, Cmov); in combineSubABS()
55893 EVT VT = N->getValueType(0); in combineSubSetcc() local
55904 NewSetCC = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, NewSetCC); in combineSubSetcc()
55905 return DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(VT, VT), NewSetCC, in combineSubSetcc()
55906 DAG.getConstant(NewImm, DL, VT)); in combineSubSetcc()
55963 EVT VT = Op0.getValueType(); in combineSub() local
55964 SDValue NewXor = DAG.getNode(ISD::XOR, SDLoc(Op1), VT, Op1.getOperand(0), in combineSub()
55965 DAG.getNOT(SDLoc(Op1), Op1.getOperand(1), VT)); in combineSub()
55967 DAG.getNode(ISD::ADD, DL, VT, Op0, DAG.getConstant(1, DL, VT)); in combineSub()
55968 return DAG.getNode(ISD::ADD, DL, VT, NewXor, NewAdd); in combineSub()
56014 MVT VT = N->getSimpleValueType(0); in combineVectorCompare() local
56015 unsigned EltBits = VT.getScalarSizeInBits(); in combineVectorCompare()
56016 unsigned NumElts = VT.getVectorNumElements(); in combineVectorCompare()
56020 return (Opcode == X86ISD::PCMPEQ) ? DAG.getAllOnesConstant(DL, VT) in combineVectorCompare()
56021 : DAG.getConstant(0, DL, VT); in combineVectorCompare()
56043 return getConstVector(Results, LHSUndefs | RHSUndefs, VT, DAG, DL); in combineVectorCompare()
56044 return getConstVector(Results, VT, DAG, DL); in combineVectorCompare()
56053 CastIntSETCCtoFP(MVT VT, ISD::CondCode CC, unsigned NumSignificantBitsLHS, in CastIntSETCCtoFP() argument
56055 MVT SVT = VT.getScalarType(); in CastIntSETCCtoFP()
56075 static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT, in combineConcatVectorOps() argument
56080 unsigned EltSizeInBits = VT.getScalarSizeInBits(); in combineConcatVectorOps()
56083 return DAG.getUNDEF(VT); in combineConcatVectorOps()
56088 return getZeroVector(VT, Subtarget, DAG, DL); in combineConcatVectorOps()
56098 (VT.is256BitVector() || (VT.is512BitVector() && Subtarget.hasAVX512()))) { in combineConcatVectorOps()
56101 return DAG.getNode(Op0.getOpcode(), DL, VT, Op0.getOperand(0)); in combineConcatVectorOps()
56104 if (Op0.getOpcode() == X86ISD::MOVDDUP && VT == MVT::v4f64 && in combineConcatVectorOps()
56107 VT.getScalarType(), Subtarget))) in combineConcatVectorOps()
56108 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, in combineConcatVectorOps()
56118 Op0.getOperand(0).getValueType() == VT.getScalarType()) in combineConcatVectorOps()
56119 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, Op0.getOperand(0)); in combineConcatVectorOps()
56126 Op0.getOperand(0).getValueType() == VT) { in combineConcatVectorOps()
56139 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56140 DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, in combineConcatVectorOps()
56148 if (VT.is256BitVector() && NumOps == 2) { in combineConcatVectorOps()
56160 return DAG.getNode(X86ISD::VPERM2X128, DL, VT, in combineConcatVectorOps()
56161 DAG.getBitcast(VT, Src0.getOperand(0)), in combineConcatVectorOps()
56162 DAG.getBitcast(VT, Src1.getOperand(0)), in combineConcatVectorOps()
56174 auto ConcatSubOperand = [&](EVT VT, ArrayRef<SDValue> SubOps, unsigned I) { in combineConcatVectorOps() argument
56187 VT, DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT, Subs)); in combineConcatVectorOps()
56189 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Subs); in combineConcatVectorOps()
56191 auto IsConcatFree = [](MVT VT, ArrayRef<SDValue> SubOps, unsigned Op) { in combineConcatVectorOps() argument
56201 Sub.getOperand(0).getValueType() == VT && in combineConcatVectorOps()
56212 if (VT == MVT::v4f64 || VT == MVT::v4i64) in combineConcatVectorOps()
56213 return DAG.getNode(X86ISD::UNPCKL, DL, VT, in combineConcatVectorOps()
56214 ConcatSubOperand(VT, Ops, 0), in combineConcatVectorOps()
56215 ConcatSubOperand(VT, Ops, 0)); in combineConcatVectorOps()
56217 if (VT == MVT::v8f32 || (VT == MVT::v8i32 && Subtarget.hasInt256())) in combineConcatVectorOps()
56218 return DAG.getNode(VT == MVT::v8f32 ? X86ISD::VPERMILPI in combineConcatVectorOps()
56220 DL, VT, ConcatSubOperand(VT, Ops, 0), in combineConcatVectorOps()
56229 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56230 ConcatSubOperand(VT, Ops, 0)); in combineConcatVectorOps()
56235 if (!IsSplat && VT.getScalarType() == MVT::f32 && in combineConcatVectorOps()
56239 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56240 ConcatSubOperand(VT, Ops, 0), in combineConcatVectorOps()
56241 ConcatSubOperand(VT, Ops, 1), Op0.getOperand(2)); in combineConcatVectorOps()
56249 ((VT.is256BitVector() && Subtarget.hasInt256()) || in combineConcatVectorOps()
56250 (VT.is512BitVector() && Subtarget.useAVX512Regs())) && in combineConcatVectorOps()
56257 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56258 ConcatSubOperand(VT, Ops, 0), in combineConcatVectorOps()
56259 ConcatSubOperand(VT, Ops, 1)); in combineConcatVectorOps()
56266 if (!IsSplat && NumOps == 2 && VT.is256BitVector() && in combineConcatVectorOps()
56268 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56269 ConcatSubOperand(VT, Ops, 0), Op0.getOperand(1)); in combineConcatVectorOps()
56274 (VT.is256BitVector() || in combineConcatVectorOps()
56275 (VT.is512BitVector() && Subtarget.useAVX512Regs())) && in combineConcatVectorOps()
56279 MVT FloatVT = VT.changeVectorElementType(MVT::f32); in combineConcatVectorOps()
56280 SDValue Res = DAG.getBitcast(FloatVT, ConcatSubOperand(VT, Ops, 0)); in combineConcatVectorOps()
56283 return DAG.getBitcast(VT, Res); in combineConcatVectorOps()
56285 if (!IsSplat && NumOps == 2 && VT == MVT::v4f64) { in combineConcatVectorOps()
56289 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56290 ConcatSubOperand(VT, Ops, 0), in combineConcatVectorOps()
56298 if (!IsSplat && ((VT.is256BitVector() && Subtarget.hasInt256()) || in combineConcatVectorOps()
56299 (VT.is512BitVector() && Subtarget.useBWIRegs()))) { in combineConcatVectorOps()
56303 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56310 (VT.is512BitVector() && Subtarget.useAVX512Regs())) { in combineConcatVectorOps()
56331 return DAG.getNode(X86ISD::VPERMV, DL, VT, Mask, Src); in combineConcatVectorOps()
56336 if (!IsSplat && NumOps == 2 && VT.is512BitVector()) { in combineConcatVectorOps()
56361 return DAG.getNode(X86ISD::VPERMV3, DL, VT, Src0, Mask, Src1); in combineConcatVectorOps()
56366 if (!IsSplat && VT.is512BitVector() && Subtarget.useAVX512Regs()) { in combineConcatVectorOps()
56374 MVT ShuffleVT = VT.isFloatingPoint() ? MVT::v8f64 : MVT::v8i64; in combineConcatVectorOps()
56383 return DAG.getBitcast(VT, Res); in combineConcatVectorOps()
56389 if (!IsSplat && NumOps == 2 && VT.is512BitVector()) { in combineConcatVectorOps()
56398 return DAG.getNode(X86ISD::SHUF128, DL, VT, LHS, RHS, in combineConcatVectorOps()
56404 if (!IsSplat && NumOps == 2 && VT.is256BitVector()) { in combineConcatVectorOps()
56412 return DAG.getNode(ISD::TRUNCATE, DL, VT, in combineConcatVectorOps()
56421 if (VT == MVT::v4i64 && !Subtarget.hasInt256() && in combineConcatVectorOps()
56425 SDValue Res = DAG.getBitcast(MVT::v8i32, ConcatSubOperand(VT, Ops, 0)); in combineConcatVectorOps()
56434 return DAG.getBitcast(VT, Res); in combineConcatVectorOps()
56441 if (((VT.is256BitVector() && Subtarget.hasInt256()) || in combineConcatVectorOps()
56442 (VT.is512BitVector() && Subtarget.useAVX512Regs() && in combineConcatVectorOps()
56447 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56448 ConcatSubOperand(VT, Ops, 0), Op0.getOperand(1)); in combineConcatVectorOps()
56454 if (VT.is512BitVector() && Subtarget.useAVX512Regs() && in combineConcatVectorOps()
56458 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56459 ConcatSubOperand(VT, Ops, 0), Op0.getOperand(1)); in combineConcatVectorOps()
56466 if (!IsSplat && ((VT.is256BitVector() && Subtarget.hasInt256()) || in combineConcatVectorOps()
56467 (VT.is512BitVector() && Subtarget.useAVX512Regs()))) { in combineConcatVectorOps()
56468 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56469 ConcatSubOperand(VT, Ops, 0), in combineConcatVectorOps()
56470 ConcatSubOperand(VT, Ops, 1)); in combineConcatVectorOps()
56475 if (!IsSplat && VT.is256BitVector() && in combineConcatVectorOps()
56476 (Subtarget.hasInt256() || VT == MVT::v8i32) && in combineConcatVectorOps()
56477 (IsConcatFree(VT, Ops, 0) || IsConcatFree(VT, Ops, 1))) { in combineConcatVectorOps()
56479 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56480 ConcatSubOperand(VT, Ops, 0), in combineConcatVectorOps()
56481 ConcatSubOperand(VT, Ops, 1)); in combineConcatVectorOps()
56503 MVT FpVT = VT.changeVectorElementType(FpSVT); in combineConcatVectorOps()
56507 SDValue LHS = ConcatSubOperand(VT, Ops, 0); in combineConcatVectorOps()
56508 SDValue RHS = ConcatSubOperand(VT, Ops, 1); in combineConcatVectorOps()
56516 VT, DAG.getNode(X86ISD::CMPP, DL, FpVT, LHS, RHS, in combineConcatVectorOps()
56526 if (!IsSplat && ((VT.is256BitVector() && Subtarget.hasInt256()) || in combineConcatVectorOps()
56527 (VT.is512BitVector() && Subtarget.useBWIRegs()))) { in combineConcatVectorOps()
56528 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56529 ConcatSubOperand(VT, Ops, 0)); in combineConcatVectorOps()
56534 (VT.is256BitVector() || in combineConcatVectorOps()
56535 (VT.is512BitVector() && Subtarget.useAVX512Regs())) && in combineConcatVectorOps()
56539 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56540 ConcatSubOperand(VT, Ops, 0), in combineConcatVectorOps()
56541 ConcatSubOperand(VT, Ops, 1), Op0.getOperand(2)); in combineConcatVectorOps()
56547 if (!IsSplat && ((VT.is256BitVector() && Subtarget.hasInt256()) || in combineConcatVectorOps()
56548 (VT.is512BitVector() && Subtarget.useAVX512Regs() && in combineConcatVectorOps()
56550 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56551 ConcatSubOperand(VT, Ops, 0), in combineConcatVectorOps()
56552 ConcatSubOperand(VT, Ops, 1)); in combineConcatVectorOps()
56561 if (!IsSplat && (IsConcatFree(VT, Ops, 0) || IsConcatFree(VT, Ops, 1)) && in combineConcatVectorOps()
56562 (VT.is256BitVector() || in combineConcatVectorOps()
56563 (VT.is512BitVector() && Subtarget.useAVX512Regs()))) { in combineConcatVectorOps()
56564 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56565 ConcatSubOperand(VT, Ops, 0), in combineConcatVectorOps()
56566 ConcatSubOperand(VT, Ops, 1)); in combineConcatVectorOps()
56570 if (!IsSplat && (VT.is256BitVector() || in combineConcatVectorOps()
56571 (VT.is512BitVector() && Subtarget.useAVX512Regs()))) { in combineConcatVectorOps()
56572 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56573 ConcatSubOperand(VT, Ops, 0), in combineConcatVectorOps()
56574 ConcatSubOperand(VT, Ops, 1)); in combineConcatVectorOps()
56581 if (!IsSplat && VT.is256BitVector() && in combineConcatVectorOps()
56582 (VT.isFloatingPoint() || Subtarget.hasInt256())) { in combineConcatVectorOps()
56583 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56584 ConcatSubOperand(VT, Ops, 0), in combineConcatVectorOps()
56585 ConcatSubOperand(VT, Ops, 1)); in combineConcatVectorOps()
56590 if (!IsSplat && ((VT.is256BitVector() && Subtarget.hasInt256()) || in combineConcatVectorOps()
56591 (VT.is512BitVector() && Subtarget.useBWIRegs()))) { in combineConcatVectorOps()
56595 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56602 ((VT.is256BitVector() && Subtarget.hasInt256()) || in combineConcatVectorOps()
56603 (VT.is512BitVector() && Subtarget.useBWIRegs())) && in combineConcatVectorOps()
56607 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56608 ConcatSubOperand(VT, Ops, 0), in combineConcatVectorOps()
56609 ConcatSubOperand(VT, Ops, 1), Op0.getOperand(2)); in combineConcatVectorOps()
56613 if (NumOps == 2 && VT.is512BitVector() && Subtarget.useBWIRegs()) { in combineConcatVectorOps()
56621 uint64_t Mask = (Mask1 << (VT.getVectorNumElements() / 2)) | Mask0; in combineConcatVectorOps()
56622 MVT MaskSVT = MVT::getIntegerVT(VT.getVectorNumElements()); in combineConcatVectorOps()
56623 MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorNumElements()); in combineConcatVectorOps()
56626 return DAG.getSelect(DL, VT, Sel, ConcatSubOperand(VT, Ops, 1), in combineConcatVectorOps()
56627 ConcatSubOperand(VT, Ops, 0)); in combineConcatVectorOps()
56632 (VT.is256BitVector() || in combineConcatVectorOps()
56633 (VT.is512BitVector() && Subtarget.useAVX512Regs())) && in combineConcatVectorOps()
56640 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56642 ConcatSubOperand(VT, Ops, 1), in combineConcatVectorOps()
56643 ConcatSubOperand(VT, Ops, 2)); in combineConcatVectorOps()
56648 if (!IsSplat && VT.is256BitVector() && NumOps == 2 && in combineConcatVectorOps()
56650 IsConcatFree(VT, Ops, 1) && IsConcatFree(VT, Ops, 2)) { in combineConcatVectorOps()
56654 return DAG.getNode(Op0.getOpcode(), DL, VT, in combineConcatVectorOps()
56656 ConcatSubOperand(VT, Ops, 1), in combineConcatVectorOps()
56657 ConcatSubOperand(VT, Ops, 2)); in combineConcatVectorOps()
56668 if (TLI->allowsMemoryAccess(Ctx, DAG.getDataLayout(), VT, in combineConcatVectorOps()
56672 EltsFromConsecutiveLoads(VT, Ops, DL, DAG, Subtarget, false)) in combineConcatVectorOps()
56680 APInt UndefElts = APInt::getZero(VT.getVectorNumElements()); in combineConcatVectorOps()
56691 if (EltBits.size() == VT.getVectorNumElements()) { in combineConcatVectorOps()
56692 Constant *C = getConstantVector(VT, EltBits, UndefElts, Ctx); in combineConcatVectorOps()
56697 SDValue Ld = DAG.getLoad(VT, DL, DAG.getEntryNode(), CV, MPI); in combineConcatVectorOps()
56708 (VT.is256BitVector() || (VT.is512BitVector() && Subtarget.hasAVX512()))) { in combineConcatVectorOps()
56717 getBROADCAST_LOAD(Opc, DL, VT, Mem->getMemoryVT(), Mem, 0, DAG)) { in combineConcatVectorOps()
56727 if (IsSplat && NumOps == 4 && VT.is512BitVector() && in combineConcatVectorOps()
56729 MVT ShuffleVT = VT.isFloatingPoint() ? MVT::v8f64 : MVT::v8i64; in combineConcatVectorOps()
56734 return DAG.getBitcast(VT, Res); in combineConcatVectorOps()
56743 EVT VT = N->getValueType(0); in combineCONCAT_VECTORS() local
56748 if (VT.getVectorElementType() == MVT::i1) { in combineCONCAT_VECTORS()
56751 APInt Constant = APInt::getZero(VT.getSizeInBits()); in combineCONCAT_VECTORS()
56757 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); in combineCONCAT_VECTORS()
56759 return DAG.getBitcast(VT, DAG.getConstant(Constant, SDLoc(N), IntVT)); in combineCONCAT_VECTORS()
56767 if (Subtarget.hasAVX() && TLI.isTypeLegal(VT) && TLI.isTypeLegal(SrcVT)) { in combineCONCAT_VECTORS()
56768 if (SDValue R = combineConcatVectorOps(SDLoc(N), VT.getSimpleVT(), Ops, DAG, in combineCONCAT_VECTORS()
56946 MVT VT = Ext->getSimpleValueType(0); in narrowExtractedVectorSelect() local
56947 if (!VT.is128BitVector()) in narrowExtractedVectorSelect()
56977 unsigned NarrowingFactor = WideVT.getSizeInBits() / VT.getSizeInBits(); in narrowExtractedVectorSelect()
56984 return DAG.getBitcast(VT, NarrowSel); in narrowExtractedVectorSelect()
57003 MVT VT = N->getSimpleValueType(0); in combineEXTRACT_SUBVECTOR() local
57008 unsigned SizeInBits = VT.getSizeInBits(); in combineEXTRACT_SUBVECTOR()
57010 unsigned NumSubElts = VT.getVectorNumElements(); in combineEXTRACT_SUBVECTOR()
57028 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, in combineEXTRACT_SUBVECTOR()
57040 return getZeroVector(VT, Subtarget, DAG, DL); in combineEXTRACT_SUBVECTOR()
57043 if (VT.getScalarType() == MVT::i1) in combineEXTRACT_SUBVECTOR()
57044 return DAG.getConstant(1, DL, VT); in combineEXTRACT_SUBVECTOR()
57045 return getOnesVector(VT, DAG, DL); in combineEXTRACT_SUBVECTOR()
57049 return DAG.getBuildVector(VT, DL, InVec->ops().slice(IdxVal, NumSubElts)); in combineEXTRACT_SUBVECTOR()
57055 if (VT.getVectorElementType() != MVT::i1 && in combineEXTRACT_SUBVECTOR()
57059 SDValue NewExt = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, in combineEXTRACT_SUBVECTOR()
57062 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, NewExt, in combineEXTRACT_SUBVECTOR()
57077 cast<MemIntrinsicSDNode>(InVec)->getMemoryVT() == VT) in combineEXTRACT_SUBVECTOR()
57091 return DAG.getUNDEF(VT); in combineEXTRACT_SUBVECTOR()
57093 return getZeroVector(VT, Subtarget, DAG, DL); in combineEXTRACT_SUBVECTOR()
57117 if (IdxVal == 0 && VT == MVT::v2f64 && InVecVT == MVT::v4f64) { in combineEXTRACT_SUBVECTOR()
57121 return DAG.getNode(X86ISD::CVTSI2P, DL, VT, InVec.getOperand(0)); in combineEXTRACT_SUBVECTOR()
57126 return DAG.getNode(X86ISD::CVTUI2P, DL, VT, InVec.getOperand(0)); in combineEXTRACT_SUBVECTOR()
57131 return DAG.getNode(X86ISD::VFPEXT, DL, VT, InVec.getOperand(0)); in combineEXTRACT_SUBVECTOR()
57135 if (InOpcode == ISD::FP_TO_SINT && VT == MVT::v4i32) { in combineEXTRACT_SUBVECTOR()
57138 return DAG.getNode(InOpcode, DL, VT, in combineEXTRACT_SUBVECTOR()
57149 return DAG.getNode(ExtOp, DL, VT, Ext); in combineEXTRACT_SUBVECTOR()
57158 return DAG.getNode(InOpcode, DL, VT, Ext0, Ext1, Ext2); in combineEXTRACT_SUBVECTOR()
57165 return DAG.getNode(InOpcode, DL, VT, Ext); in combineEXTRACT_SUBVECTOR()
57177 return DAG.getNode(InOpcode, DL, VT, Ext0, Ext1, InVec.getOperand(2)); in combineEXTRACT_SUBVECTOR()
57178 return DAG.getNode(InOpcode, DL, VT, Ext0, Ext1); in combineEXTRACT_SUBVECTOR()
57184 return DAG.getNode(InOpcode, DL, VT, Ext0); in combineEXTRACT_SUBVECTOR()
57195 return DAG.getNode(InOpcode, DL, VT, Ext, InVec.getOperand(1)); in combineEXTRACT_SUBVECTOR()
57202 EVT VT = N->getValueType(0); in combineScalarToVector() local
57210 if (VT == MVT::v1i1 && Src.getOpcode() == ISD::AND && Src.hasOneUse() && in combineScalarToVector()
57215 if (VT == MVT::v1i1 && Src.getOpcode() == ISD::EXTRACT_VECTOR_ELT && in combineScalarToVector()
57219 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src.getOperand(0), in combineScalarToVector()
57224 if ((VT == MVT::v2i64 || VT == MVT::v2f64) && Src.hasOneUse()) { in combineScalarToVector()
57247 VT, DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i32, in combineScalarToVector()
57252 VT, in combineScalarToVector()
57259 if (VT == MVT::v2i64 && Src.getOpcode() == ISD::BITCAST && in combineScalarToVector()
57261 return DAG.getNode(X86ISD::MOVQ2DQ, DL, VT, Src.getOperand(0)); in combineScalarToVector()
57265 if (VT.getScalarType() == Src.getValueType()) in combineScalarToVector()
57269 unsigned SizeInBits = VT.getFixedSizeInBits(); in combineScalarToVector()
57339 MVT VT = N->getSimpleValueType(0); in combineVPMADD() local
57351 return DAG.getConstant(0, SDLoc(N), VT); in combineVPMADD()
57357 unsigned DstEltBits = VT.getScalarSizeInBits(); in combineVPMADD()
57371 return getConstVector(Result, VT, DAG, SDLoc(N)); in combineVPMADD()
57375 APInt DemandedElts = APInt::getAllOnes(VT.getVectorNumElements()); in combineVPMADD()
57385 EVT VT = N->getValueType(0); in combineEXTEND_VECTOR_INREG() local
57401 EVT MemVT = VT.changeVectorElementType(SVT); in combineEXTEND_VECTOR_INREG()
57402 if (TLI.isLoadExtLegal(Ext, VT, MemVT)) { in combineEXTEND_VECTOR_INREG()
57404 Ext, DL, VT, Ld->getChain(), Ld->getBasePtr(), Ld->getPointerInfo(), in combineEXTEND_VECTOR_INREG()
57414 return DAG.getNode(Opcode, DL, VT, In.getOperand(0)); in combineEXTEND_VECTOR_INREG()
57423 return DAG.getNode(Opcode, DL, VT, In.getOperand(0).getOperand(0)); in combineEXTEND_VECTOR_INREG()
57429 In.getValueSizeInBits() == VT.getSizeInBits()) { in combineEXTEND_VECTOR_INREG()
57430 unsigned NumElts = VT.getVectorNumElements(); in combineEXTEND_VECTOR_INREG()
57431 unsigned Scale = VT.getScalarSizeInBits() / In.getScalarValueSizeInBits(); in combineEXTEND_VECTOR_INREG()
57436 return DAG.getBitcast(VT, DAG.getBuildVector(In.getValueType(), DL, Elts)); in combineEXTEND_VECTOR_INREG()
57442 if (TLI.isTypeLegal(VT) && TLI.isTypeLegal(In.getValueType())) in combineEXTEND_VECTOR_INREG()
57452 EVT VT = N->getValueType(0); in combineKSHIFT() local
57455 return DAG.getConstant(0, SDLoc(N), VT); in combineKSHIFT()
57458 APInt DemandedElts = APInt::getAllOnes(VT.getVectorNumElements()); in combineKSHIFT()
57493 EVT VT = N->getValueType(0); in combineFP_EXTEND() local
57501 !IsStrict && Src.getOperand(0).getValueType() == VT) in combineFP_EXTEND()
57508 if (VT.getVectorElementType() == MVT::f64) { in combineFP_EXTEND()
57509 EVT TmpVT = VT.changeVectorElementType(MVT::f32); in combineFP_EXTEND()
57510 return DAG.getNode(ISD::FP_EXTEND, dl, VT, in combineFP_EXTEND()
57513 assert(VT.getVectorElementType() == MVT::f32 && "Unexpected fpext"); in combineFP_EXTEND()
57518 return DAG.getBitcast(VT, Src); in combineFP_EXTEND()
57530 if (VT.getVectorElementType() != MVT::f32 && in combineFP_EXTEND()
57531 VT.getVectorElementType() != MVT::f64) in combineFP_EXTEND()
57534 unsigned NumElts = VT.getVectorNumElements(); in combineFP_EXTEND()
57572 if (Cvt.getValueType() != VT) { in combineFP_EXTEND()
57573 Cvt = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {VT, MVT::Other}, in combineFP_EXTEND()
57581 return DAG.getNode(ISD::FP_EXTEND, dl, VT, Cvt); in combineFP_EXTEND()
57601 EVT VT = N->getSimpleValueType(0); in combineBROADCAST_LOAD() local
57613 User->getValueSizeInBits(0).getFixedValue() > VT.getFixedSizeInBits()) { in combineBROADCAST_LOAD()
57615 VT.getSizeInBits()); in combineBROADCAST_LOAD()
57616 Extract = DAG.getBitcast(VT, Extract); in combineBROADCAST_LOAD()
57629 EVT VT = N->getValueType(0); in combineFP_ROUND() local
57633 if (!VT.isVector() || VT.getVectorElementType() != MVT::f16 || in combineFP_ROUND()
57640 unsigned NumElts = VT.getVectorNumElements(); in combineFP_ROUND()
57701 EVT IntVT = VT.changeVectorElementTypeToInteger(); in combineFP_ROUND()
57706 Cvt = DAG.getBitcast(VT, Cvt); in combineFP_ROUND()
57941 bool X86TargetLowering::preferABDSToABSWithNSW(EVT VT) const { in preferABDSToABSWithNSW()
57946 bool X86TargetLowering::preferSextInRegOfTruncate(EVT TruncVT, EVT VT, in preferSextInRegOfTruncate() argument
57948 return Subtarget.hasAVX512() || !VT.isVector(); in preferSextInRegOfTruncate()
57951 bool X86TargetLowering::isTypeDesirableForOp(unsigned Opc, EVT VT) const { in isTypeDesirableForOp()
57952 if (!isTypeLegal(VT)) in isTypeDesirableForOp()
57956 if (Opc == ISD::SHL && VT.isVector() && VT.getVectorElementType() == MVT::i8) in isTypeDesirableForOp()
57967 if ((Opc == ISD::MUL || Opc == ISD::SHL) && VT == MVT::i8) in isTypeDesirableForOp()
57972 if (VT == MVT::i16) { in isTypeDesirableForOp()
58023 EVT VT = LogicOp->getValueType(0); in isDesirableToCombineLogicOpOfSETCC() local
58025 if (!VT.isInteger()) in isDesirableToCombineLogicOpOfSETCC()
58028 if (VT.isVector()) in isDesirableToCombineLogicOpOfSETCC()
58043 EVT VT = Op.getValueType(); in IsDesirableToPromoteOp() local
58044 bool Is8BitMulByConstant = VT == MVT::i8 && Op.getOpcode() == ISD::MUL && in IsDesirableToPromoteOp()
58051 if (VT != MVT::i16 && !Is8BitMulByConstant) in IsDesirableToPromoteOp()
58741 MVT VT) const { in getRegForInlineAsmConstraint()
58761 if (VT == MVT::v1i1 || VT == MVT::i1) in getRegForInlineAsmConstraint()
58763 if (VT == MVT::v8i1 || VT == MVT::i8) in getRegForInlineAsmConstraint()
58765 if (VT == MVT::v16i1 || VT == MVT::i16) in getRegForInlineAsmConstraint()
58769 if (VT == MVT::v32i1 || VT == MVT::i32) in getRegForInlineAsmConstraint()
58771 if (VT == MVT::v64i1 || VT == MVT::i64) in getRegForInlineAsmConstraint()
58777 if (VT == MVT::i8 || VT == MVT::i1) in getRegForInlineAsmConstraint()
58781 if (VT == MVT::i16) in getRegForInlineAsmConstraint()
58785 if (VT == MVT::i32 || VT == MVT::f32) in getRegForInlineAsmConstraint()
58789 if (VT != MVT::f80 && !VT.isVector()) in getRegForInlineAsmConstraint()
58798 if (VT == MVT::i8 || VT == MVT::i1) in getRegForInlineAsmConstraint()
58800 if (VT == MVT::i16) in getRegForInlineAsmConstraint()
58802 if (VT == MVT::i32 || VT == MVT::f32 || in getRegForInlineAsmConstraint()
58803 (!VT.isVector() && !Subtarget.is64Bit())) in getRegForInlineAsmConstraint()
58805 if (VT != MVT::f80 && !VT.isVector()) in getRegForInlineAsmConstraint()
58810 if (VT == MVT::i8 || VT == MVT::i1) in getRegForInlineAsmConstraint()
58814 if (VT == MVT::i16) in getRegForInlineAsmConstraint()
58818 if (VT == MVT::i32 || VT == MVT::f32 || in getRegForInlineAsmConstraint()
58819 (!VT.isVector() && !Subtarget.is64Bit())) in getRegForInlineAsmConstraint()
58823 if (VT != MVT::f80 && !VT.isVector()) in getRegForInlineAsmConstraint()
58829 if (VT == MVT::i8 || VT == MVT::i1) in getRegForInlineAsmConstraint()
58831 if (VT == MVT::i16) in getRegForInlineAsmConstraint()
58833 if (VT == MVT::i32 || VT == MVT::f32 || in getRegForInlineAsmConstraint()
58834 (!VT.isVector() && !Subtarget.is64Bit())) in getRegForInlineAsmConstraint()
58836 if (VT != MVT::f80 && !VT.isVector()) in getRegForInlineAsmConstraint()
58842 if (VT == MVT::f32 && !isScalarFPTypeInSSEReg(VT)) in getRegForInlineAsmConstraint()
58844 if (VT == MVT::f64 && !isScalarFPTypeInSSEReg(VT)) in getRegForInlineAsmConstraint()
58846 if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f80) in getRegForInlineAsmConstraint()
58857 switch (VT.SimpleTy) { in getRegForInlineAsmConstraint()
58960 return getRegForInlineAsmConstraint(TRI, "x", VT); in getRegForInlineAsmConstraint()
58966 switch (VT.SimpleTy) { in getRegForInlineAsmConstraint()
59035 if (VT == MVT::v1i1 || VT == MVT::i1) in getRegForInlineAsmConstraint()
59037 if (VT == MVT::v8i1 || VT == MVT::i8) in getRegForInlineAsmConstraint()
59039 if (VT == MVT::v16i1 || VT == MVT::i16) in getRegForInlineAsmConstraint()
59043 if (VT == MVT::v32i1 || VT == MVT::i32) in getRegForInlineAsmConstraint()
59045 if (VT == MVT::v64i1 || VT == MVT::i64) in getRegForInlineAsmConstraint()
59055 if (VT == MVT::i8 || VT == MVT::i1) in getRegForInlineAsmConstraint()
59057 if (VT == MVT::i16) in getRegForInlineAsmConstraint()
59059 if (VT == MVT::i32 || VT == MVT::f32) in getRegForInlineAsmConstraint()
59061 if (VT != MVT::f80 && !VT.isVector()) in getRegForInlineAsmConstraint()
59065 if (VT == MVT::i8 || VT == MVT::i1) in getRegForInlineAsmConstraint()
59067 if (VT == MVT::i16) in getRegForInlineAsmConstraint()
59069 if (VT == MVT::i32 || VT == MVT::f32) in getRegForInlineAsmConstraint()
59071 if (VT != MVT::f80 && !VT.isVector()) in getRegForInlineAsmConstraint()
59083 Res = TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT); in getRegForInlineAsmConstraint()
59089 if (VT == MVT::Other || VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f80) { in getRegForInlineAsmConstraint()
59116 VT == MVT::Other) in getRegForInlineAsmConstraint()
59121 if (StringRef("{fpsr}").equals_insensitive(Constraint) && VT == MVT::Other) in getRegForInlineAsmConstraint()
59146 if (TRI->isTypeLegalForClass(*Res.second, VT) || VT == MVT::Other) in getRegForInlineAsmConstraint()
59158 unsigned Size = VT.getSizeInBits(); in getRegForInlineAsmConstraint()
59205 if (VT == MVT::f16) in getRegForInlineAsmConstraint()
59207 else if (VT == MVT::f32 || VT == MVT::i32) in getRegForInlineAsmConstraint()
59209 else if (VT == MVT::f64 || VT == MVT::i64) in getRegForInlineAsmConstraint()
59211 else if (TRI->isTypeLegalForClass(X86::VR128XRegClass, VT)) in getRegForInlineAsmConstraint()
59213 else if (TRI->isTypeLegalForClass(X86::VR256XRegClass, VT)) in getRegForInlineAsmConstraint()
59215 else if (TRI->isTypeLegalForClass(X86::VR512RegClass, VT)) in getRegForInlineAsmConstraint()
59223 if (VT == MVT::v1i1 || VT == MVT::i1) in getRegForInlineAsmConstraint()
59225 else if (VT == MVT::v8i1 || VT == MVT::i8) in getRegForInlineAsmConstraint()
59227 else if (VT == MVT::v16i1 || VT == MVT::i16) in getRegForInlineAsmConstraint()
59229 else if (VT == MVT::v32i1 || VT == MVT::i32) in getRegForInlineAsmConstraint()
59231 else if (VT == MVT::v64i1 || VT == MVT::i64) in getRegForInlineAsmConstraint()
59243 bool X86TargetLowering::isIntDivCheap(EVT VT, AttributeList Attr) const { in isIntDivCheap() argument
59252 return OptSize && !VT.isVector(); in isIntDivCheap()