Lines Matching refs:SCEV
136 typedef DenseMap<const SCEV *, RegSortData> RegUsesTy;
139 SmallVector<const SCEV *, 16> RegSequence;
142 void CountRegister(const SCEV *Reg, size_t LUIdx);
143 void DropRegister(const SCEV *Reg, size_t LUIdx);
146 bool isRegUsedByUsesOtherThan(const SCEV *Reg, size_t LUIdx) const;
148 const SmallBitVector &getUsedByIndices(const SCEV *Reg) const;
152 typedef SmallVectorImpl<const SCEV *>::iterator iterator;
153 typedef SmallVectorImpl<const SCEV *>::const_iterator const_iterator;
163 RegUseTracker::CountRegister(const SCEV *Reg, size_t LUIdx) { in CountRegister()
174 RegUseTracker::DropRegister(const SCEV *Reg, size_t LUIdx) { in DropRegister()
198 RegUseTracker::isRegUsedByUsesOtherThan(const SCEV *Reg, size_t LUIdx) const { in isRegUsedByUsesOtherThan()
209 const SmallBitVector &RegUseTracker::getUsedByIndices(const SCEV *Reg) const { in getUsedByIndices()
248 SmallVector<const SCEV *, 4> BaseRegs;
252 const SCEV *ScaledReg;
263 void InitialMatch(const SCEV *S, Loop *L, ScalarEvolution &SE);
274 void DeleteBaseReg(const SCEV *&S);
276 bool referencesReg(const SCEV *S) const;
287 static void DoInitialMatch(const SCEV *S, Loop *L, in DoInitialMatch()
288 SmallVectorImpl<const SCEV *> &Good, in DoInitialMatch()
289 SmallVectorImpl<const SCEV *> &Bad, in DoInitialMatch()
299 for (const SCEV *S : Add->operands()) in DoInitialMatch()
311 AR->getLoop(), SCEV::FlagAnyWrap), in DoInitialMatch()
319 SmallVector<const SCEV *, 4> Ops(Mul->op_begin()+1, Mul->op_end()); in DoInitialMatch()
320 const SCEV *NewMul = SE.getMulExpr(Ops); in DoInitialMatch()
322 SmallVector<const SCEV *, 4> MyGood; in DoInitialMatch()
323 SmallVector<const SCEV *, 4> MyBad; in DoInitialMatch()
325 const SCEV *NegOne = SE.getSCEV(ConstantInt::getAllOnesValue( in DoInitialMatch()
327 for (const SCEV *S : MyGood) in DoInitialMatch()
329 for (const SCEV *S : MyBad) in DoInitialMatch()
342 void Formula::InitialMatch(const SCEV *S, Loop *L, ScalarEvolution &SE) { in InitialMatch()
343 SmallVector<const SCEV *, 4> Good; in InitialMatch()
344 SmallVector<const SCEV *, 4> Bad; in InitialMatch()
347 const SCEV *Sum = SE.getAddExpr(Good); in InitialMatch()
353 const SCEV *Sum = SE.getAddExpr(Bad); in InitialMatch()
423 void Formula::DeleteBaseReg(const SCEV *&S) { in DeleteBaseReg()
430 bool Formula::referencesReg(const SCEV *S) const { in referencesReg()
442 for (const SCEV *BaseReg : BaseRegs) in hasRegsUsedByUsesOtherThan()
458 for (const SCEV *BaseReg : BaseRegs) { in print()
521 static const SCEV *getExactSDiv(const SCEV *LHS, const SCEV *RHS, in getExactSDiv()
555 const SCEV *Step = getExactSDiv(AR->getStepRecurrence(SE), RHS, SE, in getExactSDiv()
558 const SCEV *Start = getExactSDiv(AR->getStart(), RHS, SE, in getExactSDiv()
564 return SE.getAddRecExpr(Start, Step, AR->getLoop(), SCEV::FlagAnyWrap); in getExactSDiv()
572 SmallVector<const SCEV *, 8> Ops; in getExactSDiv()
573 for (const SCEV *S : Add->operands()) { in getExactSDiv()
574 const SCEV *Op = getExactSDiv(S, RHS, SE, IgnoreSignificantBits); in getExactSDiv()
586 SmallVector<const SCEV *, 4> Ops; in getExactSDiv()
588 for (const SCEV *S : Mul->operands()) { in getExactSDiv()
590 if (const SCEV *Q = getExactSDiv(S, RHS, SE, in getExactSDiv()
609 static int64_t ExtractImmediate(const SCEV *&S, ScalarEvolution &SE) { in ExtractImmediate()
616 SmallVector<const SCEV *, 8> NewOps(Add->op_begin(), Add->op_end()); in ExtractImmediate()
622 SmallVector<const SCEV *, 8> NewOps(AR->op_begin(), AR->op_end()); in ExtractImmediate()
627 SCEV::FlagAnyWrap); in ExtractImmediate()
636 static GlobalValue *ExtractSymbol(const SCEV *&S, ScalarEvolution &SE) { in ExtractSymbol()
643 SmallVector<const SCEV *, 8> NewOps(Add->op_begin(), Add->op_end()); in ExtractSymbol()
649 SmallVector<const SCEV *, 8> NewOps(AR->op_begin(), AR->op_end()); in ExtractSymbol()
654 SCEV::FlagAnyWrap); in ExtractSymbol()
735 static bool isHighCostExpansion(const SCEV *S, in isHighCostExpansion()
736 SmallPtrSetImpl<const SCEV*> &Processed, in isHighCostExpansion()
758 for (const SCEV *S : Add->operands()) { in isHighCostExpansion()
883 SmallPtrSetImpl<const SCEV *> &Regs,
884 const DenseSet<const SCEV *> &VisitedRegs,
889 SmallPtrSetImpl<const SCEV *> *LoserRegs = nullptr);
895 void RateRegister(const SCEV *Reg,
896 SmallPtrSetImpl<const SCEV *> &Regs,
899 void RatePrimaryRegister(const SCEV *Reg,
900 SmallPtrSetImpl<const SCEV *> &Regs,
903 SmallPtrSetImpl<const SCEV *> *LoserRegs);
909 void Cost::RateRegister(const SCEV *Reg, in RateRegister()
910 SmallPtrSetImpl<const SCEV *> &Regs, in RateRegister()
957 void Cost::RatePrimaryRegister(const SCEV *Reg, in RatePrimaryRegister()
958 SmallPtrSetImpl<const SCEV *> &Regs, in RatePrimaryRegister()
961 SmallPtrSetImpl<const SCEV *> *LoserRegs) { in RatePrimaryRegister()
975 SmallPtrSetImpl<const SCEV *> &Regs, in RateFormula()
976 const DenseSet<const SCEV *> &VisitedRegs, in RateFormula()
981 SmallPtrSetImpl<const SCEV *> *LoserRegs) { in RateFormula()
984 if (const SCEV *ScaledReg = F.ScaledReg) { in RateFormula()
993 for (const SCEV *BaseReg : F.BaseRegs) { in RateFormula()
1163 static SmallVector<const SCEV *, 4> getEmptyKey() { in getEmptyKey()
1164 SmallVector<const SCEV *, 4> V; in getEmptyKey()
1165 V.push_back(reinterpret_cast<const SCEV *>(-1)); in getEmptyKey()
1169 static SmallVector<const SCEV *, 4> getTombstoneKey() { in getTombstoneKey()
1170 SmallVector<const SCEV *, 4> V; in getTombstoneKey()
1171 V.push_back(reinterpret_cast<const SCEV *>(-2)); in getTombstoneKey()
1175 static unsigned getHashValue(const SmallVector<const SCEV *, 4> &V) { in getHashValue()
1179 static bool isEqual(const SmallVector<const SCEV *, 4> &LHS, in isEqual()
1180 const SmallVector<const SCEV *, 4> &RHS) { in isEqual()
1191 DenseSet<SmallVector<const SCEV *, 4>, UniquifierDenseMapInfo> Uniquifier;
1204 typedef PointerIntPair<const SCEV *, 2, KindType> SCEVUseKindPair;
1237 SmallPtrSet<const SCEV *, 4> Regs;
1260 SmallVector<const SCEV *, 4> Key = F.BaseRegs; in HasFormulaWithSameRegs()
1276 SmallVector<const SCEV *, 4> Key = F.BaseRegs; in InsertFormula()
1288 for (const SCEV *BaseReg : F.BaseRegs) in InsertFormula()
1313 SmallPtrSet<const SCEV *, 4> OldRegs = std::move(Regs); in RecomputeRegs()
1321 for (const SCEV *S : OldRegs) in RecomputeRegs()
1542 Type *AccessTy, const SCEV *S, bool HasBaseReg) { in isAlwaysFoldable()
1579 const SCEV *IncExpr;
1581 IVInc(Instruction *U, Value *O, const SCEV *E): in IVInc()
1589 const SCEV *ExprBase;
1593 IVChain(const IVInc &Head, const SCEV *Base) in IVChain()
1618 bool isProfitableIncrement(const SCEV *OperExpr,
1619 const SCEV *IncExpr,
1701 std::pair<size_t, int64_t> getUse(const SCEV *&Expr,
1709 void InsertInitialFormula(const SCEV *S, LSRUse &LU, size_t LUIdx);
1710 void InsertSupplementalFormula(const SCEV *S, LSRUse &LU, size_t LUIdx);
1751 const SmallPtrSet<const SCEV *, 16> &CurRegs,
1752 DenseSet<const SCEV *> &VisitedRegs) const;
1799 const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L); in OptimizeShadowIV()
1979 const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L); in OptimizeMax()
1982 const SCEV *One = SE.getConstant(BackedgeTakenCount->getType(), 1); in OptimizeMax()
1985 const SCEV *IterationCount = SE.getAddExpr(One, BackedgeTakenCount); in OptimizeMax()
2012 const SCEV *MaxLHS = Max->getOperand(0); in OptimizeMax()
2013 const SCEV *MaxRHS = Max->getOperand(1); in OptimizeMax()
2023 const SCEV *IV = SE.getSCEV(Cond->getOperand(0)); in OptimizeMax()
2133 const SCEV *A = IU.getStride(*CondUse, L); in OptimizeLoopTermCond()
2134 const SCEV *B = IU.getStride(*UI, L); in OptimizeLoopTermCond()
2264 LSRInstance::getUse(const SCEV *&Expr, in getUse()
2266 const SCEV *Copy = Expr; in getUse()
2356 SmallSetVector<const SCEV *, 4> Strides; in CollectInterestingTypesAndFactors()
2359 SmallVector<const SCEV *, 4> Worklist; in CollectInterestingTypesAndFactors()
2361 const SCEV *Expr = IU.getExpr(U); in CollectInterestingTypesAndFactors()
2369 const SCEV *S = Worklist.pop_back_val(); in CollectInterestingTypesAndFactors()
2381 for (SmallSetVector<const SCEV *, 4>::const_iterator in CollectInterestingTypesAndFactors()
2383 for (SmallSetVector<const SCEV *, 4>::const_iterator NewStrideIter = in CollectInterestingTypesAndFactors()
2385 const SCEV *OldStride = *I; in CollectInterestingTypesAndFactors()
2386 const SCEV *NewStride = *NewStrideIter; in CollectInterestingTypesAndFactors()
2465 static const SCEV *getExprBase(const SCEV *S) { in getExprBase()
2484 const SCEV *SubExpr = *I; in getExprBase()
2503 bool IVChain::isProfitableIncrement(const SCEV *OperExpr, in isProfitableIncrement()
2504 const SCEV *IncExpr, in isProfitableIncrement()
2513 const SCEV *HeadExpr = SE.getSCEV(getWideOperand(Incs[0].IVOperand)); in isProfitableIncrement()
2518 SmallPtrSet<const SCEV*, 8> Processed; in isProfitableIncrement()
2560 const SCEV *LastIncExpr = nullptr; in isProfitableChain()
2611 const SCEV *const OperExpr = SE.getSCEV(NextIV); in ChainInstruction()
2612 const SCEV *const OperExprBase = getExprBase(OperExpr); in ChainInstruction()
2617 const SCEV *LastIncExpr = nullptr; in ChainInstruction()
2637 const SCEV *PrevExpr = SE.getSCEV(PrevIV); in ChainInstruction()
2638 const SCEV *IncExpr = SE.getMinusSCEV(OperExpr, PrevExpr); in ChainInstruction()
2825 static bool canFoldIVIncExpr(const SCEV *IncExpr, Instruction *UserInst, in canFoldIVIncExpr()
2881 const SCEV *LeftOverExpr = nullptr; in GenerateIVChain()
2893 const SCEV *IncExpr = SE.getNoopOrSignExtend(Inc.IncExpr, IntTy); in GenerateIVChain()
2901 const SCEV *IVOperExpr = SE.getAddExpr(SE.getUnknown(IVSrc), in GenerateIVChain()
2970 const SCEV *S = IU.getExpr(U); in CollectFixupsAndInitialFormulae()
2991 const SCEV *N = SE.getSCEV(NV); in CollectFixupsAndInitialFormulae()
3034 LSRInstance::InsertInitialFormula(const SCEV *S, LSRUse &LU, size_t LUIdx) { in InsertInitialFormula()
3048 LSRInstance::InsertSupplementalFormula(const SCEV *S, in InsertSupplementalFormula()
3062 for (const SCEV *BaseReg : F.BaseRegs) in CountRegisters()
3086 SmallVector<const SCEV *, 8> Worklist(RegUses.begin(), RegUses.end()); in CollectLoopInvariantFixupsAndFormulae()
3087 SmallPtrSet<const SCEV *, 32> Visited; in CollectLoopInvariantFixupsAndFormulae()
3090 const SCEV *S = Worklist.pop_back_val(); in CollectLoopInvariantFixupsAndFormulae()
3130 const SCEV *UserS = SE.getSCEV(const_cast<Instruction *>(UserInst)); in CollectLoopInvariantFixupsAndFormulae()
3173 static const SCEV *CollectSubexprs(const SCEV *S, const SCEVConstant *C, in CollectSubexprs()
3174 SmallVectorImpl<const SCEV *> &Ops, in CollectSubexprs()
3184 for (const SCEV *S : Add->operands()) { in CollectSubexprs()
3185 const SCEV *Remainder = CollectSubexprs(S, C, Ops, L, SE, Depth+1); in CollectSubexprs()
3195 const SCEV *Remainder = CollectSubexprs(AR->getStart(), in CollectSubexprs()
3210 SCEV::FlagAnyWrap); in CollectSubexprs()
3219 const SCEV *Remainder = in CollectSubexprs()
3234 const SCEV *BaseReg = IsScaledReg ? Base.ScaledReg : Base.BaseRegs[Idx]; in GenerateReassociationsImpl()
3235 SmallVector<const SCEV *, 8> AddOps; in GenerateReassociationsImpl()
3236 const SCEV *Remainder = CollectSubexprs(BaseReg, nullptr, AddOps, L, SE); in GenerateReassociationsImpl()
3243 for (SmallVectorImpl<const SCEV *>::const_iterator J = AddOps.begin(), in GenerateReassociationsImpl()
3259 SmallVector<const SCEV *, 8> InnerAddOps( in GenerateReassociationsImpl()
3260 ((const SmallVector<const SCEV *, 8> &)AddOps).begin(), J); in GenerateReassociationsImpl()
3262 ((const SmallVector<const SCEV *, 8> &)AddOps).end()); in GenerateReassociationsImpl()
3271 const SCEV *InnerSum = SE.getAddExpr(InnerAddOps); in GenerateReassociationsImpl()
3342 SmallVector<const SCEV *, 4> Ops; in GenerateCombinations()
3343 for (const SCEV *BaseReg : Base.BaseRegs) { in GenerateCombinations()
3351 const SCEV *Sum = SE.getAddExpr(Ops); in GenerateCombinations()
3367 const SCEV *G = IsScaledReg ? Base.ScaledReg : Base.BaseRegs[Idx]; in GenerateSymbolicOffsetsImpl()
3399 const SCEV *G = IsScaledReg ? Base.ScaledReg : Base.BaseRegs[Idx]; in GenerateConstantOffsetsImpl()
3406 const SCEV *NewG = SE.getAddExpr(SE.getConstant(G->getType(), Offset), G); in GenerateConstantOffsetsImpl()
3506 const SCEV *FactorS = SE.getConstant(IntTy, Factor); in GenerateICmpZeroScales()
3581 const SCEV *FactorS = SE.getConstant(IntTy, Factor); in GenerateScales()
3586 if (const SCEV *Quotient = getExactSDiv(AR, FactorS, SE, true)) { in GenerateScales()
3617 for (const SCEV *&BaseReg : F.BaseRegs) in GenerateTruncates()
3638 const SCEV *OrigReg;
3640 WorkItem(size_t LI, int64_t I, const SCEV *R) in WorkItem()
3664 typedef std::map<int64_t, const SCEV *> ImmMapTy; in GenerateCrossUseConstantOffsets()
3665 DenseMap<const SCEV *, ImmMapTy> Map; in GenerateCrossUseConstantOffsets()
3666 DenseMap<const SCEV *, SmallBitVector> UsedByIndicesMap; in GenerateCrossUseConstantOffsets()
3667 SmallVector<const SCEV *, 8> Sequence; in GenerateCrossUseConstantOffsets()
3668 for (const SCEV *Use : RegUses) { in GenerateCrossUseConstantOffsets()
3669 const SCEV *Reg = Use; // Make a copy for ExtractImmediate to modify. in GenerateCrossUseConstantOffsets()
3683 for (const SCEV *Reg : Sequence) { in GenerateCrossUseConstantOffsets()
3698 const SCEV *OrigReg = J->second; in GenerateCrossUseConstantOffsets()
3741 const SCEV *OrigReg = WI.OrigReg; in GenerateCrossUseConstantOffsets()
3744 const SCEV *NegImmS = SE.getSCEV(ConstantInt::get(IntTy, -(uint64_t)Imm)); in GenerateCrossUseConstantOffsets()
3785 const SCEV *BaseReg = F.BaseRegs[N]; in GenerateCrossUseConstantOffsets()
3802 for (const SCEV *NewReg : NewF.BaseRegs) in GenerateCrossUseConstantOffsets()
3861 DenseSet<const SCEV *> VisitedRegs; in FilterOutUndesirableDedicatedRegisters()
3862 SmallPtrSet<const SCEV *, 16> Regs; in FilterOutUndesirableDedicatedRegisters()
3863 SmallPtrSet<const SCEV *, 16> LoserRegs; in FilterOutUndesirableDedicatedRegisters()
3870 typedef DenseMap<SmallVector<const SCEV *, 4>, size_t, UniquifierDenseMapInfo> in FilterOutUndesirableDedicatedRegisters()
3905 SmallVector<const SCEV *, 4> Key; in FilterOutUndesirableDedicatedRegisters()
3906 for (const SCEV *Reg : F.BaseRegs) { in FilterOutUndesirableDedicatedRegisters()
4001 for (SmallVectorImpl<const SCEV *>::const_iterator in NarrowSearchSpaceByDetectingSupersets()
4148 SmallPtrSet<const SCEV *, 4> Taken; in NarrowSearchSpaceByPickingWinnerRegs()
4156 const SCEV *Best = nullptr; in NarrowSearchSpaceByPickingWinnerRegs()
4158 for (const SCEV *Reg : RegUses) { in NarrowSearchSpaceByPickingWinnerRegs()
4221 const SmallPtrSet<const SCEV *, 16> &CurRegs, in SolveRecurse() argument
4222 DenseSet<const SCEV *> &VisitedRegs) const { in SolveRecurse()
4239 SmallSetVector<const SCEV *, 4> ReqRegs; in SolveRecurse()
4240 for (const SCEV *S : CurRegs) in SolveRecurse()
4244 SmallPtrSet<const SCEV *, 16> NewRegs; in SolveRecurse()
4251 for (const SCEV *Reg : ReqRegs) { in SolveRecurse()
4282 for (const SCEV *S : NewRegs) in SolveRecurse()
4301 SmallPtrSet<const SCEV *, 16> CurRegs; in Solve()
4302 DenseSet<const SCEV *> VisitedRegs; in Solve()
4478 SmallVector<const SCEV *, 8> Ops; in Expand()
4481 for (const SCEV *Reg : F.BaseRegs) { in Expand()
4496 const SCEV *ScaledS = F.ScaledReg; in Expand()
4584 const SCEV *FullS = Ops.empty() ? in Expand()