diff options
Diffstat (limited to 'contrib/llvm-project/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp')
-rw-r--r-- | contrib/llvm-project/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 115 |
1 files changed, 94 insertions, 21 deletions
diff --git a/contrib/llvm-project/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/contrib/llvm-project/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp index 9b08a24e14d4..574d9174bebf 100644 --- a/contrib/llvm-project/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp +++ b/contrib/llvm-project/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp @@ -791,6 +791,38 @@ void OpenMPIRBuilder::emitOffloadingEntry(Constant *Addr, StringRef Name, Entry->setAlignment(Align(1)); } +OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitTargetKernel( + const LocationDescription &Loc, Value *&Return, Value *Ident, + Value *DeviceID, Value *NumTeams, Value *NumThreads, Value *HostPtr, + ArrayRef<Value *> KernelArgs, ArrayRef<Value *> NoWaitArgs) { + if (!updateToLocation(Loc)) + return Loc.IP; + + auto *KernelArgsPtr = + Builder.CreateAlloca(OpenMPIRBuilder::KernelArgs, nullptr, "kernel_args"); + for (unsigned I = 0, Size = KernelArgs.size(); I != Size; ++I) { + llvm::Value *Arg = + Builder.CreateStructGEP(OpenMPIRBuilder::KernelArgs, KernelArgsPtr, I); + Builder.CreateAlignedStore( + KernelArgs[I], Arg, + M.getDataLayout().getPrefTypeAlign(KernelArgs[I]->getType())); + } + + bool HasNoWait = !NoWaitArgs.empty(); + SmallVector<Value *> OffloadingArgs{Ident, DeviceID, NumTeams, + NumThreads, HostPtr, KernelArgsPtr}; + if (HasNoWait) + OffloadingArgs.append(NoWaitArgs.begin(), NoWaitArgs.end()); + + Return = Builder.CreateCall( + HasNoWait + ? getOrCreateRuntimeFunction(M, OMPRTL___tgt_target_kernel_nowait) + : getOrCreateRuntimeFunction(M, OMPRTL___tgt_target_kernel), + OffloadingArgs); + + return Builder.saveIP(); +} + void OpenMPIRBuilder::emitCancelationCheckImpl(Value *CancelFlag, omp::Directive CanceledDirective, FinalizeCallbackTy ExitCB) { @@ -1260,6 +1292,9 @@ OpenMPIRBuilder::createTask(const LocationDescription &Loc, if (!updateToLocation(Loc)) return InsertPointTy(); + uint32_t SrcLocStrSize; + Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize); + Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize); // The current basic block is split into four basic blocks. After outlining, // they will be mapped as follows: // ``` @@ -1285,7 +1320,7 @@ OpenMPIRBuilder::createTask(const LocationDescription &Loc, OI.EntryBB = TaskAllocaBB; OI.OuterAllocaBB = AllocaIP.getBlock(); OI.ExitBB = TaskExitBB; - OI.PostOutlineCB = [this, &Loc, Tied, Final](Function &OutlinedFn) { + OI.PostOutlineCB = [this, Ident, Tied, Final](Function &OutlinedFn) { // The input IR here looks like the following- // ``` // func @current_fn() { @@ -1324,9 +1359,6 @@ OpenMPIRBuilder::createTask(const LocationDescription &Loc, // Arguments - `loc_ref` (Ident) and `gtid` (ThreadID) // call. - uint32_t SrcLocStrSize; - Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize); - Value *Ident = getOrCreateIdent(SrcLocStr, SrcLocStrSize); Value *ThreadID = getOrCreateThreadID(Ident); // Argument - `flags` @@ -2834,7 +2866,8 @@ void OpenMPIRBuilder::unrollLoopHeuristic(DebugLoc, CanonicalLoopInfo *Loop) { }); } -void OpenMPIRBuilder::applySimd(DebugLoc, CanonicalLoopInfo *CanonicalLoop) { +void OpenMPIRBuilder::applySimd(CanonicalLoopInfo *CanonicalLoop, + ConstantInt *Simdlen) { LLVMContext &Ctx = Builder.getContext(); Function *F = CanonicalLoop->getFunction(); @@ -2879,6 +2912,11 @@ void OpenMPIRBuilder::applySimd(DebugLoc, CanonicalLoopInfo *CanonicalLoop) { AccessGroup}), MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"), BoolConst})}); + if (Simdlen != nullptr) + addLoopMetadata( + CanonicalLoop, + MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.width"), + ConstantAsMetadata::get(Simdlen)})); } /// Create the TargetMachine object to query the backend for optimization @@ -3962,6 +4000,8 @@ Value *OpenMPIRBuilder::emitRMWOpAsInstruction(Value *Src1, Value *Src2, case AtomicRMWInst::Min: case AtomicRMWInst::UMax: case AtomicRMWInst::UMin: + case AtomicRMWInst::FMax: + case AtomicRMWInst::FMin: llvm_unreachable("Unsupported atomic update operation"); } llvm_unreachable("Unsupported atomic update operation"); @@ -4126,20 +4166,37 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCompare( assert(X.Var->getType()->isPointerTy() && "OMP atomic expects a pointer to target memory"); - assert((X.ElemTy->isIntegerTy() || X.ElemTy->isPointerTy()) && - "OMP atomic compare expected a integer scalar type"); // compare capture if (V.Var) { assert(V.Var->getType()->isPointerTy() && "v.var must be of pointer type"); assert(V.ElemTy == X.ElemTy && "x and v must be of same type"); } + bool IsInteger = E->getType()->isIntegerTy(); + if (Op == OMPAtomicCompareOp::EQ) { AtomicOrdering Failure = AtomicCmpXchgInst::getStrongestFailureOrdering(AO); - AtomicCmpXchgInst *Result = - Builder.CreateAtomicCmpXchg(X.Var, E, D, MaybeAlign(), AO, Failure); + AtomicCmpXchgInst *Result = nullptr; + if (!IsInteger) { + unsigned Addrspace = + cast<PointerType>(X.Var->getType())->getAddressSpace(); + IntegerType *IntCastTy = + IntegerType::get(M.getContext(), X.ElemTy->getScalarSizeInBits()); + Value *XBCast = + Builder.CreateBitCast(X.Var, IntCastTy->getPointerTo(Addrspace)); + Value *EBCast = Builder.CreateBitCast(E, IntCastTy); + Value *DBCast = Builder.CreateBitCast(D, IntCastTy); + Result = Builder.CreateAtomicCmpXchg(XBCast, EBCast, DBCast, MaybeAlign(), + AO, Failure); + } else { + Result = + Builder.CreateAtomicCmpXchg(X.Var, E, D, MaybeAlign(), AO, Failure); + } + if (V.Var) { Value *OldValue = Builder.CreateExtractValue(Result, /*Idxs=*/0); + if (!IsInteger) + OldValue = Builder.CreateBitCast(OldValue, X.ElemTy); assert(OldValue->getType() == V.ElemTy && "OldValue and V must be of same type"); if (IsPostfixUpdate) { @@ -4213,19 +4270,29 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCompare( // x = x <= expr ? x : expr; AtomicRMWInst::BinOp NewOp; if (IsXBinopExpr) { - if (X.IsSigned) - NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::Min - : AtomicRMWInst::Max; - else - NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::UMin - : AtomicRMWInst::UMax; + if (IsInteger) { + if (X.IsSigned) + NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::Min + : AtomicRMWInst::Max; + else + NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::UMin + : AtomicRMWInst::UMax; + } else { + NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::FMin + : AtomicRMWInst::FMax; + } } else { - if (X.IsSigned) - NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::Max - : AtomicRMWInst::Min; - else - NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::UMax - : AtomicRMWInst::UMin; + if (IsInteger) { + if (X.IsSigned) + NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::Max + : AtomicRMWInst::Min; + else + NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::UMax + : AtomicRMWInst::UMin; + } else { + NewOp = Op == OMPAtomicCompareOp::MAX ? AtomicRMWInst::FMax + : AtomicRMWInst::FMin; + } } AtomicRMWInst *OldValue = @@ -4243,12 +4310,18 @@ OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createAtomicCompare( case AtomicRMWInst::UMax: Pred = CmpInst::ICMP_UGT; break; + case AtomicRMWInst::FMax: + Pred = CmpInst::FCMP_OGT; + break; case AtomicRMWInst::Min: Pred = CmpInst::ICMP_SLT; break; case AtomicRMWInst::UMin: Pred = CmpInst::ICMP_ULT; break; + case AtomicRMWInst::FMin: + Pred = CmpInst::FCMP_OLT; + break; default: llvm_unreachable("unexpected comparison op"); } |