|
|
reference, declaration → definition
definition → references, declarations, derived classes, virtual overrides
reference to multiple definitions → definitions
unreferenced
|
References
lib/IR/AutoUpgrade.cpp 1647 Function *F = CI->getCalledFunction();
1648 LLVMContext &C = CI->getContext();
1650 Builder.SetInsertPoint(CI->getParent(), CI->getIterator());
1650 Builder.SetInsertPoint(CI->getParent(), CI->getIterator());
1675 Value *Arg0 = CI->getArgOperand(0);
1676 Value *Arg1 = CI->getArgOperand(1);
1690 CI->eraseFromParent();
1702 Value *Arg0 = CI->getArgOperand(0);
1703 Value *Arg1 = CI->getArgOperand(1);
1715 CI->eraseFromParent();
1720 Value *Arg0 = CI->getArgOperand(0);
1721 Value *Arg1 = CI->getArgOperand(1);
1732 CI->eraseFromParent();
1739 Value *Arg0 = CI->getArgOperand(0);
1740 Value *Arg1 = CI->getArgOperand(1);
1748 CI->eraseFromParent();
1753 Value *Mask = Builder.CreateAnd(CI->getArgOperand(2), Builder.getInt8(1));
1754 UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
1754 UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
1758 CI->eraseFromParent();
1765 UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
1765 UpgradeMaskedStore(Builder, CI->getArgOperand(0), CI->getArgOperand(1),
1766 CI->getArgOperand(2), Aligned);
1769 CI->eraseFromParent();
1780 CI->getArgOperand(0), CI->getArgOperand(1));
1780 CI->getArgOperand(0), CI->getArgOperand(1));
1781 Rep = Builder.CreateSExt(Rep, CI->getType(), "");
1784 if (CI->getOperand(0)->getType()->isIntegerTy(8))
1786 unsigned NumElts = CI->getType()->getPrimitiveSizeInBits() /
1788 Rep = Builder.CreateZExt(CI->getArgOperand(0), ExtTy);
1792 Value *Vec = CI->getArgOperand(0);
1803 CI->getType()),
1804 {CI->getArgOperand(0)});
1806 if (CI->getNumArgOperands() == 4 &&
1807 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
1808 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
1812 Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(3) };
1812 Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(3) };
1813 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),
1818 CI->getType()),
1819 {CI->getArgOperand(0)});
1821 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
1822 CI->getArgOperand(1));
1825 Value *Op0 = CI->getArgOperand(0);
1826 Value *Op1 = CI->getArgOperand(1);
1827 Value *Mask = CI->getArgOperand(2);
1837 CI->getArgOperand(1)->getType()->getVectorNumElements();
1838 Rep = Builder.CreateVectorSplat(NumElts, CI->getArgOperand(0));
1839 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
1840 CI->getArgOperand(1));
1842 unsigned NumElts = CI->getType()->getScalarSizeInBits();
1843 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), NumElts);
1844 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), NumElts);
1859 Rep = Builder.CreateBitCast(Rep, CI->getType());
1861 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
1862 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
1864 Rep = Builder.CreateBitCast(Rep, CI->getType());
1866 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
1867 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
1870 Rep = Builder.CreateBitCast(Rep, CI->getType());
1872 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
1873 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
1875 Rep = Builder.CreateBitCast(Rep, CI->getType());
1877 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
1878 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
1880 Rep = Builder.CreateBitCast(Rep, CI->getType());
1882 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
1883 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
1886 Rep = Builder.CreateBitCast(Rep, CI->getType());
1888 Rep = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
1890 Rep = Builder.CreateBitCast(Rep, CI->getType());
1893 Value *LHS = getX86MaskVec(Builder, CI->getArgOperand(0), 16);
1894 Value *RHS = getX86MaskVec(Builder, CI->getArgOperand(1), 16);
1909 Value *Elt0 = Builder.CreateExtractElement(CI->getArgOperand(0),
1911 Value *Elt1 = Builder.CreateExtractElement(CI->getArgOperand(1),
1922 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), EltOp,
1927 Rep = upgradeMaskedCompare(Builder, *CI, CmpEq ? 0 : 6, true);
1929 Type *OpTy = CI->getArgOperand(0)->getType();
1940 { CI->getOperand(0), CI->getArgOperand(1) });
1940 { CI->getOperand(0), CI->getArgOperand(1) });
1941 Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
1943 Type *OpTy = CI->getArgOperand(0)->getType();
1963 { CI->getOperand(0), CI->getArgOperand(1) });
1963 { CI->getOperand(0), CI->getArgOperand(1) });
1964 Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
1966 Type *OpTy = CI->getArgOperand(0)->getType();
1986 Args.push_back(CI->getArgOperand(0));
1987 Args.push_back(CI->getArgOperand(1));
1988 Args.push_back(CI->getArgOperand(2));
1989 if (CI->getNumArgOperands() == 5)
1990 Args.push_back(CI->getArgOperand(4));
1994 Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(3));
1998 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
1999 Rep = upgradeMaskedCompare(Builder, *CI, Imm, true);
2001 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2002 Rep = upgradeMaskedCompare(Builder, *CI, Imm, false);
2007 Value *Op = CI->getArgOperand(0);
2016 Rep = upgradeAbs(Builder, *CI);
2022 Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_SGT);
2028 Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_UGT);
2034 Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_SLT);
2040 Rep = upgradeIntMinMax(Builder, *CI, ICmpInst::ICMP_ULT);
2045 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/false);
2050 Rep = upgradePMULDQ(Builder, *CI, /*Signed*/true);
2055 Rep = Builder.CreateSIToFP(CI->getArgOperand(1),
2056 CI->getType()->getVectorElementType());
2057 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
2059 Rep = Builder.CreateUIToFP(CI->getArgOperand(1),
2060 CI->getType()->getVectorElementType());
2061 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
2063 Rep = Builder.CreateExtractElement(CI->getArgOperand(1), (uint64_t)0);
2064 Rep = Builder.CreateFPExt(Rep, CI->getType()->getVectorElementType());
2065 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep, (uint64_t)0);
2084 Type *DstTy = CI->getType();
2085 Rep = CI->getArgOperand(0);
2099 else if (CI->getNumArgOperands() == 4 &&
2100 (!isa<ConstantInt>(CI->getArgOperand(3)) ||
2101 cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
2104 Function *F = Intrinsic::getDeclaration(CI->getModule(), IID,
2106 Rep = Builder.CreateCall(F, { Rep, CI->getArgOperand(3) });
2112 if (CI->getNumArgOperands() >= 3)
2113 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2114 CI->getArgOperand(1));
2116 Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0),
2117 CI->getArgOperand(1), CI->getArgOperand(2),
2117 CI->getArgOperand(1), CI->getArgOperand(2),
2120 Rep = UpgradeMaskedLoad(Builder, CI->getArgOperand(0),
2121 CI->getArgOperand(1),CI->getArgOperand(2),
2121 CI->getArgOperand(1),CI->getArgOperand(2),
2124 Type *ResultTy = CI->getType();
2128 Value *Ptr = Builder.CreateBitCast(CI->getOperand(0),
2131 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
2137 Rep = Builder.CreateCall(ELd, { Ptr, MaskVec, CI->getOperand(1) });
2139 Type *ResultTy = CI->getArgOperand(1)->getType();
2143 Value *Ptr = Builder.CreateBitCast(CI->getOperand(0),
2146 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
2152 Rep = Builder.CreateCall(CSt, { CI->getArgOperand(1), Ptr, MaskVec });
2155 Type *ResultTy = CI->getType();
2157 Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
2164 Rep = Builder.CreateCall(Intr, { CI->getOperand(0), CI->getOperand(1),
2164 Rep = Builder.CreateCall(Intr, { CI->getOperand(0), CI->getOperand(1),
2178 if (CI->getNumArgOperands() == 3) {
2179 Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2202 Rep = upgradeX86vpcom(Builder, *CI, Imm, IsSigned);
2204 Value *Sel = CI->getArgOperand(2);
2206 Value *Sel0 = Builder.CreateAnd(CI->getArgOperand(0), Sel);
2207 Value *Sel1 = Builder.CreateAnd(CI->getArgOperand(1), NotSel);
2212 Rep = upgradeX86Rotate(Builder, *CI, false);
2215 Rep = upgradeX86Rotate(Builder, *CI, true);
2220 Rep = upgradeX86ConcatShift(Builder, *CI, false, ZeroMask);
2225 Rep = upgradeX86ConcatShift(Builder, *CI, true, ZeroMask);
2229 Value *Trunc0 = Builder.CreateTrunc(CI->getArgOperand(0), Type::getInt32Ty(C));
2230 Rep = Builder.CreateCall(CRC32, {Trunc0, CI->getArgOperand(1)});
2231 Rep = Builder.CreateZExt(Rep, CI->getType(), "");
2235 Type *VecTy = CI->getType();
2238 Value *Cast = Builder.CreateBitCast(CI->getArgOperand(0),
2252 VectorType *SrcTy = cast<VectorType>(CI->getArgOperand(0)->getType());
2253 VectorType *DstTy = cast<VectorType>(CI->getType());
2262 CI->getArgOperand(0), UndefValue::get(SrcTy), ShuffleMask);
2268 if (CI->getNumArgOperands() == 3)
2269 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2270 CI->getArgOperand(1));
2275 Type *Ty = CI->getArgOperand(1)->getType();
2276 Rep = Builder.CreateTrunc(CI->getArgOperand(0), Ty);
2277 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2278 CI->getArgOperand(1));
2282 Type *EltTy = CI->getType()->getVectorElementType();
2285 Value *Op = Builder.CreatePointerCast(CI->getArgOperand(0),
2296 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2297 Type *VT = CI->getType();
2312 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
2313 CI->getArgOperand(1), ShuffleMask);
2314 Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep,
2315 CI->getArgOperand(3));
2319 CI->getArgOperand(0)->getType()->getVectorNumElements();
2320 unsigned NumDstElts = CI->getType()->getVectorNumElements();
2326 Rep = Builder.CreateShuffleVector(CI->getArgOperand(0),
2327 CI->getArgOperand(0),
2329 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2330 CI->getArgOperand(1));
2336 Value *Op = CI->getArgOperand(0);
2337 unsigned NumElts = CI->getType()->getVectorNumElements();
2342 if (CI->getNumArgOperands() == 3)
2343 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2344 CI->getArgOperand(1));
2354 Rep = UpgradeX86AddSubSatIntrinsics(Builder, *CI, true, IsAdd);
2362 Rep = UpgradeX86AddSubSatIntrinsics(Builder, *CI, false, IsAdd);
2364 Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
2365 CI->getArgOperand(1),
2366 CI->getArgOperand(2),
2367 CI->getArgOperand(3),
2368 CI->getArgOperand(4),
2371 Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
2372 CI->getArgOperand(1),
2373 CI->getArgOperand(2),
2374 CI->getArgOperand(3),
2375 CI->getArgOperand(4),
2380 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2381 Rep = UpgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0),
2386 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2387 Rep = UpgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0),
2393 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2394 Rep = UpgradeX86PSLLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
2399 unsigned Shift = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2400 Rep = UpgradeX86PSRLDQIntrinsics(Builder, CI->getArgOperand(0), Shift);
2406 Value *Op0 = CI->getArgOperand(0);
2407 Value *Op1 = CI->getArgOperand(1);
2408 unsigned Imm = cast <ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2409 VectorType *VecTy = cast<VectorType>(CI->getType());
2420 Value *Op0 = CI->getArgOperand(0);
2421 Value *Op1 = CI->getArgOperand(1);
2422 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2423 unsigned DstNumElts = CI->getType()->getVectorNumElements();
2459 if (CI->getNumArgOperands() == 5)
2460 Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep,
2461 CI->getArgOperand(3));
2465 Value *Op0 = CI->getArgOperand(0);
2466 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2467 unsigned DstNumElts = CI->getType()->getVectorNumElements();
2482 if (CI->getNumArgOperands() == 4)
2483 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2484 CI->getArgOperand(2));
2489 Value *Op0 = CI->getArgOperand(0);
2490 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2491 VectorType *VecTy = cast<VectorType>(CI->getType());
2500 if (CI->getNumArgOperands() == 4)
2501 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2502 CI->getArgOperand(2));
2513 uint8_t Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2515 unsigned NumElts = CI->getType()->getVectorNumElements();
2520 Value *V0 = (Imm & 0x02) ? CI->getArgOperand(1) : CI->getArgOperand(0);
2520 Value *V0 = (Imm & 0x02) ? CI->getArgOperand(1) : CI->getArgOperand(0);
2521 Value *V1 = (Imm & 0x20) ? CI->getArgOperand(1) : CI->getArgOperand(0);
2521 Value *V1 = (Imm & 0x20) ? CI->getArgOperand(1) : CI->getArgOperand(0);
2524 V0 = (Imm & 0x08) ? ConstantAggregateZero::get(CI->getType()) : V0;
2525 V1 = (Imm & 0x80) ? ConstantAggregateZero::get(CI->getType()) : V1;
2543 Value *Op0 = CI->getArgOperand(0);
2544 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2545 VectorType *VecTy = cast<VectorType>(CI->getType());
2560 if (CI->getNumArgOperands() == 4)
2561 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2562 CI->getArgOperand(2));
2565 Value *Op0 = CI->getArgOperand(0);
2566 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2567 unsigned NumElts = CI->getType()->getVectorNumElements();
2579 if (CI->getNumArgOperands() == 4)
2580 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2581 CI->getArgOperand(2));
2584 Value *Op0 = CI->getArgOperand(0);
2585 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
2586 unsigned NumElts = CI->getType()->getVectorNumElements();
2598 if (CI->getNumArgOperands() == 4)
2599 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2600 CI->getArgOperand(2));
2602 Value *Op0 = CI->getArgOperand(0);
2603 Value *Op1 = CI->getArgOperand(1);
2604 unsigned Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
2605 unsigned NumElts = CI->getType()->getVectorNumElements();
2607 unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
2624 Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep,
2625 CI->getArgOperand(3));
2629 Value *Op0 = CI->getArgOperand(0);
2630 unsigned NumElts = CI->getType()->getVectorNumElements();
2631 unsigned NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
2646 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2647 CI->getArgOperand(1));
2650 Value *Op0 = CI->getArgOperand(0);
2651 Value *Op1 = CI->getArgOperand(1);
2652 int NumElts = CI->getType()->getVectorNumElements();
2653 int NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
2662 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2663 CI->getArgOperand(2));
2666 Value *Op0 = CI->getArgOperand(0);
2667 Value *Op1 = CI->getArgOperand(1);
2668 int NumElts = CI->getType()->getVectorNumElements();
2669 int NumLaneElts = 128/CI->getType()->getScalarSizeInBits();
2678 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2679 CI->getArgOperand(2));
2682 VectorType *FTy = cast<VectorType>(CI->getType());
2684 Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
2685 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
2687 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2688 CI->getArgOperand(2));
2691 VectorType *FTy = cast<VectorType>(CI->getType());
2693 Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy));
2695 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
2697 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2698 CI->getArgOperand(2));
2701 VectorType *FTy = cast<VectorType>(CI->getType());
2703 Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
2704 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
2706 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2707 CI->getArgOperand(2));
2710 VectorType *FTy = cast<VectorType>(CI->getType());
2712 Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
2713 Builder.CreateBitCast(CI->getArgOperand(1), ITy));
2715 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2716 CI->getArgOperand(2));
2718 Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1));
2718 Rep = Builder.CreateAdd(CI->getArgOperand(0), CI->getArgOperand(1));
2719 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2720 CI->getArgOperand(2));
2722 Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1));
2722 Rep = Builder.CreateSub(CI->getArgOperand(0), CI->getArgOperand(1));
2723 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2724 CI->getArgOperand(2));
2726 Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1));
2726 Rep = Builder.CreateMul(CI->getArgOperand(0), CI->getArgOperand(1));
2727 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2728 CI->getArgOperand(2));
2738 { CI->getArgOperand(0), CI->getArgOperand(1),
2738 { CI->getArgOperand(0), CI->getArgOperand(1),
2739 CI->getArgOperand(4) });
2741 Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1));
2741 Rep = Builder.CreateFAdd(CI->getArgOperand(0), CI->getArgOperand(1));
2743 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2744 CI->getArgOperand(2));
2754 { CI->getArgOperand(0), CI->getArgOperand(1),
2754 { CI->getArgOperand(0), CI->getArgOperand(1),
2755 CI->getArgOperand(4) });
2757 Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1));
2757 Rep = Builder.CreateFDiv(CI->getArgOperand(0), CI->getArgOperand(1));
2759 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2760 CI->getArgOperand(2));
2770 { CI->getArgOperand(0), CI->getArgOperand(1),
2770 { CI->getArgOperand(0), CI->getArgOperand(1),
2771 CI->getArgOperand(4) });
2773 Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1));
2773 Rep = Builder.CreateFMul(CI->getArgOperand(0), CI->getArgOperand(1));
2775 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2776 CI->getArgOperand(2));
2786 { CI->getArgOperand(0), CI->getArgOperand(1),
2786 { CI->getArgOperand(0), CI->getArgOperand(1),
2787 CI->getArgOperand(4) });
2789 Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1));
2789 Rep = Builder.CreateFSub(CI->getArgOperand(0), CI->getArgOperand(1));
2791 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2792 CI->getArgOperand(2));
2805 { CI->getArgOperand(0), CI->getArgOperand(1),
2805 { CI->getArgOperand(0), CI->getArgOperand(1),
2806 CI->getArgOperand(4) });
2807 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
2808 CI->getArgOperand(2));
2812 CI->getType()),
2813 { CI->getArgOperand(0), Builder.getInt1(false) });
2814 Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
2815 CI->getArgOperand(1));
2883 Rep = UpgradeX86MaskedShift(Builder, *CI, IID);
2951 Rep = UpgradeX86MaskedShift(Builder, *CI, IID);
3017 Rep = UpgradeX86MaskedShift(Builder, *CI, IID);
3019 Rep = upgradeMaskedMove(Builder, *CI);
3021 Rep = UpgradeMaskToInt(Builder, *CI);
3027 Value *Ptr = CI->getArgOperand(0);
3028 VectorType *VTy = cast<VectorType>(CI->getType());
3044 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3044 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3045 CI->getArgOperand(2) };
3060 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),
3066 Rep = Builder.CreateInsertElement(CI->getArgOperand(0), Rep,
3069 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3069 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3070 CI->getArgOperand(2) };
3076 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(),
3081 Rep = Builder.CreateInsertElement(Constant::getNullValue(CI->getType()),
3095 Value *A = CI->getArgOperand(0);
3096 Value *B = CI->getArgOperand(1);
3097 Value *C = CI->getArgOperand(2);
3110 if (!isa<ConstantInt>(CI->getArgOperand(4)) ||
3111 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4) {
3112 Value *Ops[] = { A, B, C, CI->getArgOperand(4) };
3119 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), IID);
3122 Function *FMA = Intrinsic::getDeclaration(CI->getModule(),
3134 PassThru = Builder.CreateExtractElement(CI->getArgOperand(2),
3137 Rep = EmitX86ScalarSelect(Builder, CI->getArgOperand(3),
3139 Rep = Builder.CreateInsertElement(CI->getArgOperand(IsMask3 ? 2 : 0),
3155 Value *A = CI->getArgOperand(0);
3156 Value *B = CI->getArgOperand(1);
3157 Value *C = CI->getArgOperand(2);
3166 if (CI->getNumArgOperands() == 5 &&
3167 (!isa<ConstantInt>(CI->getArgOperand(4)) ||
3168 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4)) {
3177 { A, B, C, CI->getArgOperand(4) });
3179 Function *FMA = Intrinsic::getDeclaration(CI->getModule(),
3185 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType()) :
3186 IsMask3 ? CI->getArgOperand(2) :
3187 CI->getArgOperand(0);
3189 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3193 int NumElts = CI->getType()->getVectorNumElements();
3195 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3195 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3196 CI->getArgOperand(2) };
3198 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::fma,
3221 if (CI->getNumArgOperands() == 5 &&
3222 (!isa<ConstantInt>(CI->getArgOperand(4)) ||
3223 cast<ConstantInt>(CI->getArgOperand(4))->getZExtValue() != 4)) {
3231 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3231 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3232 CI->getArgOperand(2), CI->getArgOperand(4) };
3232 CI->getArgOperand(2), CI->getArgOperand(4) };
3237 {CI->getArgOperand(0), CI->getArgOperand(1),
3237 {CI->getArgOperand(0), CI->getArgOperand(1),
3238 CI->getArgOperand(2), CI->getArgOperand(4)});
3238 CI->getArgOperand(2), CI->getArgOperand(4)});
3240 int NumElts = CI->getType()->getVectorNumElements();
3242 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3242 Value *Ops[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3243 CI->getArgOperand(2) };
3245 Function *FMA = Intrinsic::getDeclaration(CI->getModule(), Intrinsic::fma,
3261 Value *PassThru = IsMaskZ ? llvm::Constant::getNullValue(CI->getType()) :
3262 IsMask3 ? CI->getArgOperand(2) :
3263 CI->getArgOperand(0);
3265 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3269 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
3270 unsigned EltWidth = CI->getType()->getScalarSizeInBits();
3287 Value *Args[] = { CI->getArgOperand(0) , CI->getArgOperand(1),
3287 Value *Args[] = { CI->getArgOperand(0) , CI->getArgOperand(1),
3288 CI->getArgOperand(2), CI->getArgOperand(3) };
3288 CI->getArgOperand(2), CI->getArgOperand(3) };
3289 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
3291 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
3292 : CI->getArgOperand(0);
3293 Rep = EmitX86Select(Builder, CI->getArgOperand(4), Rep, PassThru);
3298 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
3315 Value *Args[] = { CI->getArgOperand(0) , CI->getArgOperand(1),
3315 Value *Args[] = { CI->getArgOperand(0) , CI->getArgOperand(1),
3316 CI->getArgOperand(2) };
3317 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
3319 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
3320 : CI->getArgOperand(0);
3321 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3327 Rep = UpgradeX86VPERMT2Intrinsics(Builder, *CI, ZeroMask, IndexForm);
3334 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
3351 Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3351 Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3352 CI->getArgOperand(2) };
3353 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
3355 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
3356 : CI->getArgOperand(0);
3357 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3364 unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
3381 Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3381 Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3382 CI->getArgOperand(2) };
3383 Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
3385 Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
3386 : CI->getArgOperand(0);
3387 Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
3404 Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3404 Value *Args[] = { CI->getArgOperand(0), CI->getArgOperand(1),
3405 CI->getArgOperand(2)};
3407 Intrinsic::getDeclaration(CI->getModule(), IID),
3413 Value *Ptr = Builder.CreateBitCast(CI->getArgOperand(3),
3419 CI->replaceAllUsesWith(CF);
3422 upgradeAVX512MaskToSelect(Name, Builder, *CI, Rep)) {
3425 Value *Arg = CI->getArgOperand(0);
3432 Value *Ptr = CI->getArgOperand(0);
3433 Value *Val = CI->getArgOperand(1);
3438 Value *Arg0 = CI->getArgOperand(0);
3439 Value *Arg1 = CI->getArgOperand(1);
3446 Value *Arg0 = CI->getArgOperand(0);
3447 Value *Arg1 = CI->getArgOperand(1);
3454 Value *Arg = CI->getArgOperand(0);
3463 Value *Arg = CI->getArgOperand(0);
3473 CI->getArgOperand(0), "h2f");
3479 CI->replaceAllUsesWith(Rep);
3480 CI->eraseFromParent();
3484 const auto &DefaultCase = [&NewFn, &CI]() -> void {
3487 (CI->getCalledFunction()->getName() != NewFn->getName()) &&
3489 CI->setCalledFunction(NewFn);
3499 if (CI->isFast())
3500 Args.push_back(ConstantFP::get(CI->getOperand(0)->getType(), 1.0));
3502 Args.push_back(CI->getOperand(0));
3503 Args.push_back(CI->getOperand(1));
3505 cast<Instruction>(NewCall)->copyFastMathFlags(CI);
3510 if (CI->isFast())
3511 Args.push_back(Constant::getNullValue(CI->getOperand(0)->getType()));
3513 Args.push_back(CI->getOperand(0));
3514 Args.push_back(CI->getOperand(1));
3516 cast<Instruction>(NewCall)->copyFastMathFlags(CI);
3533 SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
3534 CI->arg_operands().end());
3540 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
3545 assert(CI->getNumArgOperands() == 1 &&
3548 Builder.CreateCall(NewFn, {CI->getArgOperand(0), Builder.getFalse()});
3552 Value *NullIsUnknownSize = CI->getNumArgOperands() == 2
3554 : CI->getArgOperand(2);
3556 CI->getNumArgOperands() < 4 ? Builder.getFalse() : CI->getArgOperand(3);
3556 CI->getNumArgOperands() < 4 ? Builder.getFalse() : CI->getArgOperand(3);
3558 NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize, Dynamic});
3558 NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize, Dynamic});
3563 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
3567 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(0)});
3572 assert(CI->getNumArgOperands() == 4);
3574 if (auto *Offset = dyn_cast_or_null<Constant>(CI->getArgOperand(1)))
3578 {CI->getArgOperand(0), CI->getArgOperand(2), CI->getArgOperand(3)});
3578 {CI->getArgOperand(0), CI->getArgOperand(2), CI->getArgOperand(3)});
3578 {CI->getArgOperand(0), CI->getArgOperand(2), CI->getArgOperand(3)});
3581 CI->eraseFromParent();
3586 NewCall = Builder.CreateCall(NewFn, {CI->getArgOperand(1)});
3593 SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
3594 CI->arg_operands().end());
3609 Value *Arg0 = CI->getArgOperand(0);
3614 Value *Arg1 = CI->getArgOperand(1);
3628 if (CI->getNumOperands() == 0)
3635 Value *Ptr = Builder.CreateBitCast(CI->getArgOperand(0),
3641 std::string Name = CI->getName();
3643 CI->setName(Name + ".old");
3646 CI->replaceAllUsesWith(TSC);
3647 CI->eraseFromParent();
3659 SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
3660 CI->arg_operands().end());
3679 SmallVector<Value *, 4> Args(CI->arg_operands().begin(),
3680 CI->arg_operands().end());
3695 if (CI->getNumArgOperands() != 5) {
3701 Value *Args[4] = {CI->getArgOperand(0), CI->getArgOperand(1),
3701 Value *Args[4] = {CI->getArgOperand(0), CI->getArgOperand(1),
3702 CI->getArgOperand(2), CI->getArgOperand(4)};
3702 CI->getArgOperand(2), CI->getArgOperand(4)};
3706 const ConstantInt *Align = cast<ConstantInt>(CI->getArgOperand(3));
3716 std::string Name = CI->getName();
3718 CI->setName(Name + ".old");
3721 CI->replaceAllUsesWith(NewCall);
3722 CI->eraseFromParent();