diff options
Diffstat (limited to 'llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp')
-rw-r--r-- | llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp | 172 |
1 files changed, 131 insertions, 41 deletions
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp index 854769d283f7..a103e8e4e6e0 100644 --- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp @@ -74,12 +74,35 @@ bool CombinerHelper::matchCombineCopy(MachineInstr &MI) { return false; Register DstReg = MI.getOperand(0).getReg(); Register SrcReg = MI.getOperand(1).getReg(); + + // Give up if either DstReg or SrcReg is a physical register. + if (Register::isPhysicalRegister(DstReg) || + Register::isPhysicalRegister(SrcReg)) + return false; + + // Give up the types don't match. LLT DstTy = MRI.getType(DstReg); LLT SrcTy = MRI.getType(SrcReg); - // Simple Copy Propagation. - // a(sx) = COPY b(sx) -> Replace all uses of a with b. - if (DstTy.isValid() && SrcTy.isValid() && DstTy == SrcTy) + // Give up if one has a valid LLT, but the other doesn't. + if (DstTy.isValid() != SrcTy.isValid()) + return false; + // Give up if the types don't match. + if (DstTy.isValid() && SrcTy.isValid() && DstTy != SrcTy) + return false; + + // Get the register banks and classes. + const RegisterBank *DstBank = MRI.getRegBankOrNull(DstReg); + const RegisterBank *SrcBank = MRI.getRegBankOrNull(SrcReg); + const TargetRegisterClass *DstRC = MRI.getRegClassOrNull(DstReg); + const TargetRegisterClass *SrcRC = MRI.getRegClassOrNull(SrcReg); + + // Replace if the register constraints match. + if ((SrcRC == DstRC) && (SrcBank == DstBank)) + return true; + // Replace if DstReg has no constraints. + if (!DstBank && !DstRC) return true; + return false; } void CombinerHelper::applyCombineCopy(MachineInstr &MI) { @@ -109,10 +132,7 @@ bool CombinerHelper::matchCombineConcatVectors(MachineInstr &MI, bool &IsUndef, // Walk over all the operands of concat vectors and check if they are // build_vector themselves or undef. // Then collect their operands in Ops. - for (const MachineOperand &MO : MI.operands()) { - // Skip the instruction definition. - if (MO.isDef()) - continue; + for (const MachineOperand &MO : MI.uses()) { Register Reg = MO.getReg(); MachineInstr *Def = MRI.getVRegDef(Reg); assert(Def && "Operand not defined"); @@ -121,12 +141,8 @@ bool CombinerHelper::matchCombineConcatVectors(MachineInstr &MI, bool &IsUndef, IsUndef = false; // Remember the operands of the build_vector to fold // them into the yet-to-build flattened concat vectors. - for (const MachineOperand &BuildVecMO : Def->operands()) { - // Skip the definition. - if (BuildVecMO.isDef()) - continue; + for (const MachineOperand &BuildVecMO : Def->uses()) Ops.push_back(BuildVecMO.getReg()); - } break; case TargetOpcode::G_IMPLICIT_DEF: { LLT OpType = MRI.getType(Reg); @@ -189,8 +205,11 @@ bool CombinerHelper::matchCombineShuffleVector(MachineInstr &MI, LLT DstType = MRI.getType(MI.getOperand(0).getReg()); Register Src1 = MI.getOperand(1).getReg(); LLT SrcType = MRI.getType(Src1); - unsigned DstNumElts = DstType.getNumElements(); - unsigned SrcNumElts = SrcType.getNumElements(); + // As bizarre as it may look, shuffle vector can actually produce + // scalar! This is because at the IR level a <1 x ty> shuffle + // vector is perfectly valid. + unsigned DstNumElts = DstType.isVector() ? DstType.getNumElements() : 1; + unsigned SrcNumElts = SrcType.isVector() ? SrcType.getNumElements() : 1; // If the resulting vector is smaller than the size of the source // vectors being concatenated, we won't be able to replace the @@ -199,7 +218,15 @@ bool CombinerHelper::matchCombineShuffleVector(MachineInstr &MI, // Note: We may still be able to produce a concat_vectors fed by // extract_vector_elt and so on. It is less clear that would // be better though, so don't bother for now. - if (DstNumElts < 2 * SrcNumElts) + // + // If the destination is a scalar, the size of the sources doesn't + // matter. we will lower the shuffle to a plain copy. This will + // work only if the source and destination have the same size. But + // that's covered by the next condition. + // + // TODO: If the size between the source and destination don't match + // we could still emit an extract vector element in that case. + if (DstNumElts < 2 * SrcNumElts && DstNumElts != 1) return false; // Check that the shuffle mask can be broken evenly between the @@ -212,8 +239,7 @@ bool CombinerHelper::matchCombineShuffleVector(MachineInstr &MI, // vectors. unsigned NumConcat = DstNumElts / SrcNumElts; SmallVector<int, 8> ConcatSrcs(NumConcat, -1); - SmallVector<int, 8> Mask; - ShuffleVectorInst::getShuffleMask(MI.getOperand(3).getShuffleMask(), Mask); + ArrayRef<int> Mask = MI.getOperand(3).getShuffleMask(); for (unsigned i = 0; i != DstNumElts; ++i) { int Idx = Mask[i]; // Undef value. @@ -254,7 +280,10 @@ void CombinerHelper::applyCombineShuffleVector(MachineInstr &MI, Builder.setInsertPt(*MI.getParent(), MI); Register NewDstReg = MRI.cloneVirtualRegister(DstReg); - Builder.buildConcatVectors(NewDstReg, Ops); + if (Ops.size() == 1) + Builder.buildCopy(NewDstReg, Ops[0]); + else + Builder.buildMerge(NewDstReg, Ops); MI.eraseFromParent(); replaceRegWith(MRI, DstReg, NewDstReg); @@ -571,7 +600,7 @@ bool CombinerHelper::findPostIndexCandidate(MachineInstr &MI, Register &Addr, LLVM_DEBUG(dbgs() << "Searching for post-indexing opportunity for: " << MI); for (auto &Use : MRI.use_instructions(Base)) { - if (Use.getOpcode() != TargetOpcode::G_GEP) + if (Use.getOpcode() != TargetOpcode::G_PTR_ADD) continue; Offset = Use.getOperand(2).getReg(); @@ -597,8 +626,8 @@ bool CombinerHelper::findPostIndexCandidate(MachineInstr &MI, Register &Addr, // forming an indexed one. bool MemOpDominatesAddrUses = true; - for (auto &GEPUse : MRI.use_instructions(Use.getOperand(0).getReg())) { - if (!dominates(MI, GEPUse)) { + for (auto &PtrAddUse : MRI.use_instructions(Use.getOperand(0).getReg())) { + if (!dominates(MI, PtrAddUse)) { MemOpDominatesAddrUses = false; break; } @@ -631,7 +660,7 @@ bool CombinerHelper::findPreIndexCandidate(MachineInstr &MI, Register &Addr, #endif Addr = MI.getOperand(1).getReg(); - MachineInstr *AddrDef = getOpcodeDef(TargetOpcode::G_GEP, Addr, MRI); + MachineInstr *AddrDef = getOpcodeDef(TargetOpcode::G_PTR_ADD, Addr, MRI); if (!AddrDef || MRI.hasOneUse(Addr)) return false; @@ -667,8 +696,8 @@ bool CombinerHelper::findPreIndexCandidate(MachineInstr &MI, Register &Addr, } } - // FIXME: check whether all uses of the base pointer are constant GEPs. That - // might allow us to end base's liveness here by adjusting the constant. + // FIXME: check whether all uses of the base pointer are constant PtrAdds. + // That might allow us to end base's liveness here by adjusting the constant. for (auto &UseMI : MRI.use_instructions(Addr)) { if (!dominates(MI, UseMI)) { @@ -681,18 +710,36 @@ bool CombinerHelper::findPreIndexCandidate(MachineInstr &MI, Register &Addr, } bool CombinerHelper::tryCombineIndexedLoadStore(MachineInstr &MI) { + IndexedLoadStoreMatchInfo MatchInfo; + if (matchCombineIndexedLoadStore(MI, MatchInfo)) { + applyCombineIndexedLoadStore(MI, MatchInfo); + return true; + } + return false; +} + +bool CombinerHelper::matchCombineIndexedLoadStore(MachineInstr &MI, IndexedLoadStoreMatchInfo &MatchInfo) { unsigned Opcode = MI.getOpcode(); if (Opcode != TargetOpcode::G_LOAD && Opcode != TargetOpcode::G_SEXTLOAD && Opcode != TargetOpcode::G_ZEXTLOAD && Opcode != TargetOpcode::G_STORE) return false; - bool IsStore = Opcode == TargetOpcode::G_STORE; - Register Addr, Base, Offset; - bool IsPre = findPreIndexCandidate(MI, Addr, Base, Offset); - if (!IsPre && !findPostIndexCandidate(MI, Addr, Base, Offset)) + MatchInfo.IsPre = findPreIndexCandidate(MI, MatchInfo.Addr, MatchInfo.Base, + MatchInfo.Offset); + if (!MatchInfo.IsPre && + !findPostIndexCandidate(MI, MatchInfo.Addr, MatchInfo.Base, + MatchInfo.Offset)) return false; + return true; +} +void CombinerHelper::applyCombineIndexedLoadStore( + MachineInstr &MI, IndexedLoadStoreMatchInfo &MatchInfo) { + MachineInstr &AddrDef = *MRI.getUniqueVRegDef(MatchInfo.Addr); + MachineIRBuilder MIRBuilder(MI); + unsigned Opcode = MI.getOpcode(); + bool IsStore = Opcode == TargetOpcode::G_STORE; unsigned NewOpcode; switch (Opcode) { case TargetOpcode::G_LOAD: @@ -711,25 +758,22 @@ bool CombinerHelper::tryCombineIndexedLoadStore(MachineInstr &MI) { llvm_unreachable("Unknown load/store opcode"); } - MachineInstr &AddrDef = *MRI.getUniqueVRegDef(Addr); - MachineIRBuilder MIRBuilder(MI); auto MIB = MIRBuilder.buildInstr(NewOpcode); if (IsStore) { - MIB.addDef(Addr); + MIB.addDef(MatchInfo.Addr); MIB.addUse(MI.getOperand(0).getReg()); } else { MIB.addDef(MI.getOperand(0).getReg()); - MIB.addDef(Addr); + MIB.addDef(MatchInfo.Addr); } - MIB.addUse(Base); - MIB.addUse(Offset); - MIB.addImm(IsPre); + MIB.addUse(MatchInfo.Base); + MIB.addUse(MatchInfo.Offset); + MIB.addImm(MatchInfo.IsPre); MI.eraseFromParent(); AddrDef.eraseFromParent(); LLVM_DEBUG(dbgs() << " Combinined to indexed operation"); - return true; } bool CombinerHelper::matchElideBrByInvertingCond(MachineInstr &MI) { @@ -1016,7 +1060,7 @@ bool CombinerHelper::optimizeMemset(MachineInstr &MI, Register Dst, Register Val if (DstOff != 0) { auto Offset = MIB.buildConstant(LLT::scalar(PtrTy.getSizeInBits()), DstOff); - Ptr = MIB.buildGEP(PtrTy, Dst, Offset).getReg(0); + Ptr = MIB.buildPtrAdd(PtrTy, Dst, Offset).getReg(0); } MIB.buildStore(Value, Ptr, *StoreMMO); @@ -1121,13 +1165,13 @@ bool CombinerHelper::optimizeMemcpy(MachineInstr &MI, Register Dst, if (CurrOffset != 0) { Offset = MIB.buildConstant(LLT::scalar(PtrTy.getSizeInBits()), CurrOffset) .getReg(0); - LoadPtr = MIB.buildGEP(PtrTy, Src, Offset).getReg(0); + LoadPtr = MIB.buildPtrAdd(PtrTy, Src, Offset).getReg(0); } auto LdVal = MIB.buildLoad(CopyTy, LoadPtr, *LoadMMO); // Create the store. Register StorePtr = - CurrOffset == 0 ? Dst : MIB.buildGEP(PtrTy, Dst, Offset).getReg(0); + CurrOffset == 0 ? Dst : MIB.buildPtrAdd(PtrTy, Dst, Offset).getReg(0); MIB.buildStore(LdVal, StorePtr, *StoreMMO); CurrOffset += CopyTy.getSizeInBytes(); Size -= CopyTy.getSizeInBytes(); @@ -1218,7 +1262,7 @@ bool CombinerHelper::optimizeMemmove(MachineInstr &MI, Register Dst, if (CurrOffset != 0) { auto Offset = MIB.buildConstant(LLT::scalar(PtrTy.getSizeInBits()), CurrOffset); - LoadPtr = MIB.buildGEP(PtrTy, Src, Offset).getReg(0); + LoadPtr = MIB.buildPtrAdd(PtrTy, Src, Offset).getReg(0); } LoadVals.push_back(MIB.buildLoad(CopyTy, LoadPtr, *LoadMMO).getReg(0)); CurrOffset += CopyTy.getSizeInBytes(); @@ -1235,7 +1279,7 @@ bool CombinerHelper::optimizeMemmove(MachineInstr &MI, Register Dst, if (CurrOffset != 0) { auto Offset = MIB.buildConstant(LLT::scalar(PtrTy.getSizeInBits()), CurrOffset); - StorePtr = MIB.buildGEP(PtrTy, Dst, Offset).getReg(0); + StorePtr = MIB.buildPtrAdd(PtrTy, Dst, Offset).getReg(0); } MIB.buildStore(LoadVals[I], StorePtr, *StoreMMO); CurrOffset += CopyTy.getSizeInBytes(); @@ -1295,6 +1339,52 @@ bool CombinerHelper::tryCombineMemCpyFamily(MachineInstr &MI, unsigned MaxLen) { return false; } +bool CombinerHelper::matchPtrAddImmedChain(MachineInstr &MI, + PtrAddChain &MatchInfo) { + // We're trying to match the following pattern: + // %t1 = G_PTR_ADD %base, G_CONSTANT imm1 + // %root = G_PTR_ADD %t1, G_CONSTANT imm2 + // --> + // %root = G_PTR_ADD %base, G_CONSTANT (imm1 + imm2) + + if (MI.getOpcode() != TargetOpcode::G_PTR_ADD) + return false; + + Register Add2 = MI.getOperand(1).getReg(); + Register Imm1 = MI.getOperand(2).getReg(); + auto MaybeImmVal = getConstantVRegValWithLookThrough(Imm1, MRI); + if (!MaybeImmVal) + return false; + + MachineInstr *Add2Def = MRI.getUniqueVRegDef(Add2); + if (!Add2Def || Add2Def->getOpcode() != TargetOpcode::G_PTR_ADD) + return false; + + Register Base = Add2Def->getOperand(1).getReg(); + Register Imm2 = Add2Def->getOperand(2).getReg(); + auto MaybeImm2Val = getConstantVRegValWithLookThrough(Imm2, MRI); + if (!MaybeImm2Val) + return false; + + // Pass the combined immediate to the apply function. + MatchInfo.Imm = MaybeImmVal->Value + MaybeImm2Val->Value; + MatchInfo.Base = Base; + return true; +} + +bool CombinerHelper::applyPtrAddImmedChain(MachineInstr &MI, + PtrAddChain &MatchInfo) { + assert(MI.getOpcode() == TargetOpcode::G_PTR_ADD && "Expected G_PTR_ADD"); + MachineIRBuilder MIB(MI); + LLT OffsetTy = MRI.getType(MI.getOperand(2).getReg()); + auto NewOffset = MIB.buildConstant(OffsetTy, MatchInfo.Imm); + Observer.changingInstr(MI); + MI.getOperand(1).setReg(MatchInfo.Base); + MI.getOperand(2).setReg(NewOffset.getReg(0)); + Observer.changedInstr(MI); + return true; +} + bool CombinerHelper::tryCombine(MachineInstr &MI) { if (tryCombineCopy(MI)) return true; |