aboutsummaryrefslogtreecommitdiff
path: root/contrib/llvm/lib/IR/AutoUpgrade.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'contrib/llvm/lib/IR/AutoUpgrade.cpp')
-rw-r--r--contrib/llvm/lib/IR/AutoUpgrade.cpp399
1 files changed, 291 insertions, 108 deletions
diff --git a/contrib/llvm/lib/IR/AutoUpgrade.cpp b/contrib/llvm/lib/IR/AutoUpgrade.cpp
index 27064154221f..a2d820352825 100644
--- a/contrib/llvm/lib/IR/AutoUpgrade.cpp
+++ b/contrib/llvm/lib/IR/AutoUpgrade.cpp
@@ -1,9 +1,8 @@
//===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
//
-// The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
@@ -199,14 +198,14 @@ static bool ShouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
Name.startswith("avx512.mask.pmull.") || // Added in 4.0
Name.startswith("avx512.mask.cvtdq2pd.") || // Added in 4.0
Name.startswith("avx512.mask.cvtudq2pd.") || // Added in 4.0
- Name == "avx512.mask.cvtudq2ps.128" || // Added in 7.0
- Name == "avx512.mask.cvtudq2ps.256" || // Added in 7.0
- Name == "avx512.mask.cvtqq2pd.128" || // Added in 7.0
- Name == "avx512.mask.cvtqq2pd.256" || // Added in 7.0
- Name == "avx512.mask.cvtuqq2pd.128" || // Added in 7.0
- Name == "avx512.mask.cvtuqq2pd.256" || // Added in 7.0
- Name == "avx512.mask.cvtdq2ps.128" || // Added in 7.0
- Name == "avx512.mask.cvtdq2ps.256" || // Added in 7.0
+ Name.startswith("avx512.mask.cvtudq2ps.") || // Added in 7.0 updated 9.0
+ Name.startswith("avx512.mask.cvtqq2pd.") || // Added in 7.0 updated 9.0
+ Name.startswith("avx512.mask.cvtuqq2pd.") || // Added in 7.0 updated 9.0
+ Name.startswith("avx512.mask.cvtdq2ps.") || // Added in 7.0 updated 9.0
+ Name == "avx512.mask.cvtqq2ps.256" || // Added in 9.0
+ Name == "avx512.mask.cvtqq2ps.512" || // Added in 9.0
+ Name == "avx512.mask.cvtuqq2ps.256" || // Added in 9.0
+ Name == "avx512.mask.cvtuqq2ps.512" || // Added in 9.0
Name == "avx512.mask.cvtpd2dq.256" || // Added in 7.0
Name == "avx512.mask.cvtpd2ps.256" || // Added in 7.0
Name == "avx512.mask.cvttpd2dq.256" || // Added in 7.0
@@ -216,7 +215,6 @@ static bool ShouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
Name == "avx512.mask.cvtps2pd.256" || // Added in 7.0
Name == "avx512.cvtusi2sd" || // Added in 7.0
Name.startswith("avx512.mask.permvar.") || // Added in 7.0
- Name.startswith("avx512.mask.permvar.") || // Added in 7.0
Name == "sse2.pmulu.dq" || // Added in 7.0
Name == "sse41.pmuldq" || // Added in 7.0
Name == "avx2.pmulu.dq" || // Added in 7.0
@@ -300,6 +298,11 @@ static bool ShouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
Name.startswith("avx512.mask.fpclass.p") || // Added in 7.0
Name.startswith("avx512.mask.vpshufbitqmb.") || // Added in 8.0
Name.startswith("avx512.mask.pmultishift.qb.") || // Added in 8.0
+ Name.startswith("avx512.mask.conflict.") || // Added in 9.0
+ Name == "avx512.mask.pmov.qd.256" || // Added in 9.0
+ Name == "avx512.mask.pmov.qd.512" || // Added in 9.0
+ Name == "avx512.mask.pmov.wb.256" || // Added in 9.0
+ Name == "avx512.mask.pmov.wb.512" || // Added in 9.0
Name == "sse.cvtsi2ss" || // Added in 7.0
Name == "sse.cvtsi642ss" || // Added in 7.0
Name == "sse2.cvtsi2sd" || // Added in 7.0
@@ -338,6 +341,16 @@ static bool ShouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
Name.startswith("avx512.mask.load.") || // Added in 3.9
Name.startswith("avx512.mask.expand.load.") || // Added in 7.0
Name.startswith("avx512.mask.compress.store.") || // Added in 7.0
+ Name.startswith("avx512.mask.expand.b") || // Added in 9.0
+ Name.startswith("avx512.mask.expand.w") || // Added in 9.0
+ Name.startswith("avx512.mask.expand.d") || // Added in 9.0
+ Name.startswith("avx512.mask.expand.q") || // Added in 9.0
+ Name.startswith("avx512.mask.expand.p") || // Added in 9.0
+ Name.startswith("avx512.mask.compress.b") || // Added in 9.0
+ Name.startswith("avx512.mask.compress.w") || // Added in 9.0
+ Name.startswith("avx512.mask.compress.d") || // Added in 9.0
+ Name.startswith("avx512.mask.compress.q") || // Added in 9.0
+ Name.startswith("avx512.mask.compress.p") || // Added in 9.0
Name == "sse42.crc32.64.8" || // Added in 3.4
Name.startswith("avx.vbroadcast.s") || // Added in 3.5
Name.startswith("avx512.vbroadcast.s") || // Added in 7.0
@@ -362,8 +375,7 @@ static bool ShouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
Name == "xop.vpcmov.256" || // Added in 5.0
Name.startswith("avx512.mask.move.s") || // Added in 4.0
Name.startswith("avx512.cvtmask2") || // Added in 5.0
- (Name.startswith("xop.vpcom") && // Added in 3.2
- F->arg_size() == 2) ||
+ Name.startswith("xop.vpcom") || // Added in 3.2, Updated in 9.0
Name.startswith("xop.vprot") || // Added in 8.0
Name.startswith("avx512.prol") || // Added in 8.0
Name.startswith("avx512.pror") || // Added in 8.0
@@ -373,8 +385,6 @@ static bool ShouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
Name.startswith("avx512.mask.prol.") || // Added in 8.0
Name.startswith("avx512.ptestm") || //Added in 6.0
Name.startswith("avx512.ptestnm") || //Added in 6.0
- Name.startswith("sse2.pavg") || // Added in 6.0
- Name.startswith("avx2.pavg") || // Added in 6.0
Name.startswith("avx512.mask.pavg")) // Added in 6.0
return true;
@@ -480,6 +490,12 @@ static bool UpgradeX86IntrinsicFunction(Function *F, StringRef Name,
static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
assert(F && "Illegal to upgrade a non-existent Function.");
+ // Upgrade intrinsics "clang.arc.use" which doesn't start with "llvm.".
+ if (F->getName() == "clang.arc.use") {
+ NewFn = nullptr;
+ return true;
+ }
+
// Quickly eliminate it, if it's not a candidate.
StringRef Name = F->getName();
if (Name.size() <= 8 || !Name.startswith("llvm."))
@@ -549,6 +565,17 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::thread_pointer);
return true;
}
+ if (Name.startswith("aarch64.neon.addp")) {
+ if (F->arg_size() != 2)
+ break; // Invalid IR.
+ auto fArgs = F->getFunctionType()->params();
+ VectorType *ArgTy = dyn_cast<VectorType>(fArgs[0]);
+ if (ArgTy && ArgTy->getElementType()->isFloatingPointTy()) {
+ NewFn = Intrinsic::getDeclaration(F->getParent(),
+ Intrinsic::aarch64_neon_faddp, fArgs);
+ return true;
+ }
+ }
break;
}
@@ -575,6 +602,26 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
}
break;
}
+ case 'e': {
+ SmallVector<StringRef, 2> Groups;
+ Regex R("^experimental.vector.reduce.([a-z]+)\\.[fi][0-9]+");
+ if (R.match(Name, &Groups)) {
+ Intrinsic::ID ID = Intrinsic::not_intrinsic;
+ if (Groups[1] == "fadd")
+ ID = Intrinsic::experimental_vector_reduce_v2_fadd;
+ if (Groups[1] == "fmul")
+ ID = Intrinsic::experimental_vector_reduce_v2_fmul;
+
+ if (ID != Intrinsic::not_intrinsic) {
+ rename(F);
+ auto Args = F->getFunctionType()->params();
+ Type *Tys[] = {F->getFunctionType()->getReturnType(), Args[1]};
+ NewFn = Intrinsic::getDeclaration(F->getParent(), ID, Tys);
+ return true;
+ }
+ }
+ break;
+ }
case 'i':
case 'l': {
bool IsLifetimeStart = Name.startswith("lifetime.start");
@@ -717,6 +764,8 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
.Cases("clz.ll", "popc.ll", "h2f", true)
.Cases("max.i", "max.ll", "max.ui", "max.ull", true)
.Cases("min.i", "min.ll", "min.ui", "min.ull", true)
+ .StartsWith("atomic.load.add.f32.p", true)
+ .StartsWith("atomic.load.add.f64.p", true)
.Default(false);
if (Expand) {
NewFn = nullptr;
@@ -730,7 +779,7 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
// address space.
if (Name.startswith("objectsize.")) {
Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };
- if (F->arg_size() == 2 ||
+ if (F->arg_size() == 2 || F->arg_size() == 3 ||
F->getName() != Intrinsic::getName(Intrinsic::objectsize, Tys)) {
rename(F);
NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::objectsize,
@@ -778,9 +827,35 @@ bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn) {
return Upgraded;
}
-bool llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
- // Nothing to do yet.
- return false;
+GlobalVariable *llvm::UpgradeGlobalVariable(GlobalVariable *GV) {
+ if (!(GV->hasName() && (GV->getName() == "llvm.global_ctors" ||
+ GV->getName() == "llvm.global_dtors")) ||
+ !GV->hasInitializer())
+ return nullptr;
+ ArrayType *ATy = dyn_cast<ArrayType>(GV->getValueType());
+ if (!ATy)
+ return nullptr;
+ StructType *STy = dyn_cast<StructType>(ATy->getElementType());
+ if (!STy || STy->getNumElements() != 2)
+ return nullptr;
+
+ LLVMContext &C = GV->getContext();
+ IRBuilder<> IRB(C);
+ auto EltTy = StructType::get(STy->getElementType(0), STy->getElementType(1),
+ IRB.getInt8PtrTy());
+ Constant *Init = GV->getInitializer();
+ unsigned N = Init->getNumOperands();
+ std::vector<Constant *> NewCtors(N);
+ for (unsigned i = 0; i != N; ++i) {
+ auto Ctor = cast<Constant>(Init->getOperand(i));
+ NewCtors[i] = ConstantStruct::get(
+ EltTy, Ctor->getAggregateElement(0u), Ctor->getAggregateElement(1),
+ Constant::getNullValue(IRB.getInt8PtrTy()));
+ }
+ Constant *NewInit = ConstantArray::get(ArrayType::get(EltTy, N), NewCtors);
+
+ return new GlobalVariable(NewInit->getType(), false, GV->getLinkage(),
+ NewInit, GV->getName());
}
// Handles upgrading SSE2/AVX2/AVX512BW PSLLDQ intrinsics by converting them
@@ -1054,6 +1129,45 @@ static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallInst &CI,
return Res;
}
+static Value *upgradeX86vpcom(IRBuilder<> &Builder, CallInst &CI, unsigned Imm,
+ bool IsSigned) {
+ Type *Ty = CI.getType();
+ Value *LHS = CI.getArgOperand(0);
+ Value *RHS = CI.getArgOperand(1);
+
+ CmpInst::Predicate Pred;
+ switch (Imm) {
+ case 0x0:
+ Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
+ break;
+ case 0x1:
+ Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
+ break;
+ case 0x2:
+ Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
+ break;
+ case 0x3:
+ Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
+ break;
+ case 0x4:
+ Pred = ICmpInst::ICMP_EQ;
+ break;
+ case 0x5:
+ Pred = ICmpInst::ICMP_NE;
+ break;
+ case 0x6:
+ return Constant::getNullValue(Ty); // FALSE
+ case 0x7:
+ return Constant::getAllOnesValue(Ty); // TRUE
+ default:
+ llvm_unreachable("Unknown XOP vpcom/vpcomu predicate");
+ }
+
+ Value *Cmp = Builder.CreateICmp(Pred, LHS, RHS);
+ Value *Ext = Builder.CreateSExt(Cmp, Ty);
+ return Ext;
+}
+
static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallInst &CI,
bool IsShiftRight, bool ZeroMask) {
Type *Ty = CI.getType();
@@ -1111,16 +1225,16 @@ static Value *UpgradeMaskedStore(IRBuilder<> &Builder,
static Value *UpgradeMaskedLoad(IRBuilder<> &Builder,
Value *Ptr, Value *Passthru, Value *Mask,
bool Aligned) {
+ Type *ValTy = Passthru->getType();
// Cast the pointer to the right type.
- Ptr = Builder.CreateBitCast(Ptr,
- llvm::PointerType::getUnqual(Passthru->getType()));
+ Ptr = Builder.CreateBitCast(Ptr, llvm::PointerType::getUnqual(ValTy));
unsigned Align =
Aligned ? cast<VectorType>(Passthru->getType())->getBitWidth() / 8 : 1;
// If the mask is all ones just emit a regular store.
if (const auto *C = dyn_cast<Constant>(Mask))
if (C->isAllOnesValue())
- return Builder.CreateAlignedLoad(Ptr, Align);
+ return Builder.CreateAlignedLoad(ValTy, Ptr, Align);
// Convert the mask from an integer type to a vector of i1.
unsigned NumElts = Passthru->getType()->getVectorNumElements();
@@ -1463,6 +1577,36 @@ static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder,
IID = Intrinsic::x86_avx512_pmultishift_qb_512;
else
llvm_unreachable("Unexpected intrinsic");
+ } else if (Name.startswith("conflict.")) {
+ if (Name[9] == 'd' && VecWidth == 128)
+ IID = Intrinsic::x86_avx512_conflict_d_128;
+ else if (Name[9] == 'd' && VecWidth == 256)
+ IID = Intrinsic::x86_avx512_conflict_d_256;
+ else if (Name[9] == 'd' && VecWidth == 512)
+ IID = Intrinsic::x86_avx512_conflict_d_512;
+ else if (Name[9] == 'q' && VecWidth == 128)
+ IID = Intrinsic::x86_avx512_conflict_q_128;
+ else if (Name[9] == 'q' && VecWidth == 256)
+ IID = Intrinsic::x86_avx512_conflict_q_256;
+ else if (Name[9] == 'q' && VecWidth == 512)
+ IID = Intrinsic::x86_avx512_conflict_q_512;
+ else
+ llvm_unreachable("Unexpected intrinsic");
+ } else if (Name.startswith("pavg.")) {
+ if (Name[5] == 'b' && VecWidth == 128)
+ IID = Intrinsic::x86_sse2_pavg_b;
+ else if (Name[5] == 'b' && VecWidth == 256)
+ IID = Intrinsic::x86_avx2_pavg_b;
+ else if (Name[5] == 'b' && VecWidth == 512)
+ IID = Intrinsic::x86_avx512_pavg_b_512;
+ else if (Name[5] == 'w' && VecWidth == 128)
+ IID = Intrinsic::x86_sse2_pavg_w;
+ else if (Name[5] == 'w' && VecWidth == 256)
+ IID = Intrinsic::x86_avx2_pavg_w;
+ else if (Name[5] == 'w' && VecWidth == 512)
+ IID = Intrinsic::x86_avx512_pavg_w_512;
+ else
+ llvm_unreachable("Unexpected intrinsic");
} else
return false;
@@ -1504,6 +1648,14 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
// Get the Function's name.
StringRef Name = F->getName();
+ // clang.arc.use is an old name for llvm.arc.clang.arc.use. It is dropped
+ // from upgrader because the optimizer now only recognizes intrinsics for
+ // ARC runtime calls.
+ if (Name == "clang.arc.use") {
+ CI->eraseFromParent();
+ return;
+ }
+
assert(Name.startswith("llvm.") && "Intrinsic doesn't start with 'llvm.'");
Name = Name.substr(5);
@@ -1918,38 +2070,47 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
Name == "avx.cvtdq2.ps.256" ||
Name.startswith("avx512.mask.cvtdq2pd.") ||
Name.startswith("avx512.mask.cvtudq2pd.") ||
- Name == "avx512.mask.cvtdq2ps.128" ||
- Name == "avx512.mask.cvtdq2ps.256" ||
- Name == "avx512.mask.cvtudq2ps.128" ||
- Name == "avx512.mask.cvtudq2ps.256" ||
- Name == "avx512.mask.cvtqq2pd.128" ||
- Name == "avx512.mask.cvtqq2pd.256" ||
- Name == "avx512.mask.cvtuqq2pd.128" ||
- Name == "avx512.mask.cvtuqq2pd.256" ||
+ Name.startswith("avx512.mask.cvtdq2ps.") ||
+ Name.startswith("avx512.mask.cvtudq2ps.") ||
+ Name.startswith("avx512.mask.cvtqq2pd.") ||
+ Name.startswith("avx512.mask.cvtuqq2pd.") ||
+ Name == "avx512.mask.cvtqq2ps.256" ||
+ Name == "avx512.mask.cvtqq2ps.512" ||
+ Name == "avx512.mask.cvtuqq2ps.256" ||
+ Name == "avx512.mask.cvtuqq2ps.512" ||
Name == "sse2.cvtps2pd" ||
Name == "avx.cvt.ps2.pd.256" ||
Name == "avx512.mask.cvtps2pd.128" ||
Name == "avx512.mask.cvtps2pd.256")) {
Type *DstTy = CI->getType();
Rep = CI->getArgOperand(0);
+ Type *SrcTy = Rep->getType();
unsigned NumDstElts = DstTy->getVectorNumElements();
- if (NumDstElts < Rep->getType()->getVectorNumElements()) {
+ if (NumDstElts < SrcTy->getVectorNumElements()) {
assert(NumDstElts == 2 && "Unexpected vector size");
uint32_t ShuffleMask[2] = { 0, 1 };
Rep = Builder.CreateShuffleVector(Rep, Rep, ShuffleMask);
}
- bool IsPS2PD = (StringRef::npos != Name.find("ps2"));
+ bool IsPS2PD = SrcTy->getVectorElementType()->isFloatTy();
bool IsUnsigned = (StringRef::npos != Name.find("cvtu"));
if (IsPS2PD)
Rep = Builder.CreateFPExt(Rep, DstTy, "cvtps2pd");
- else if (IsUnsigned)
- Rep = Builder.CreateUIToFP(Rep, DstTy, "cvt");
- else
- Rep = Builder.CreateSIToFP(Rep, DstTy, "cvt");
+ else if (CI->getNumArgOperands() == 4 &&
+ (!isa<ConstantInt>(CI->getArgOperand(3)) ||
+ cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
+ Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
+ : Intrinsic::x86_avx512_sitofp_round;
+ Function *F = Intrinsic::getDeclaration(CI->getModule(), IID,
+ { DstTy, SrcTy });
+ Rep = Builder.CreateCall(F, { Rep, CI->getArgOperand(3) });
+ } else {
+ Rep = IsUnsigned ? Builder.CreateUIToFP(Rep, DstTy, "cvt")
+ : Builder.CreateSIToFP(Rep, DstTy, "cvt");
+ }
- if (CI->getNumArgOperands() == 3)
+ if (CI->getNumArgOperands() >= 3)
Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
CI->getArgOperand(1));
} else if (IsX86 && (Name.startswith("avx512.mask.loadu."))) {
@@ -1990,52 +2151,56 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
Intrinsic::masked_compressstore,
ResultTy);
Rep = Builder.CreateCall(CSt, { CI->getArgOperand(1), Ptr, MaskVec });
+ } else if (IsX86 && (Name.startswith("avx512.mask.compress.") ||
+ Name.startswith("avx512.mask.expand."))) {
+ Type *ResultTy = CI->getType();
+
+ Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
+ ResultTy->getVectorNumElements());
+
+ bool IsCompress = Name[12] == 'c';
+ Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
+ : Intrinsic::x86_avx512_mask_expand;
+ Function *Intr = Intrinsic::getDeclaration(F->getParent(), IID, ResultTy);
+ Rep = Builder.CreateCall(Intr, { CI->getOperand(0), CI->getOperand(1),
+ MaskVec });
} else if (IsX86 && Name.startswith("xop.vpcom")) {
- Intrinsic::ID intID;
- if (Name.endswith("ub"))
- intID = Intrinsic::x86_xop_vpcomub;
- else if (Name.endswith("uw"))
- intID = Intrinsic::x86_xop_vpcomuw;
- else if (Name.endswith("ud"))
- intID = Intrinsic::x86_xop_vpcomud;
- else if (Name.endswith("uq"))
- intID = Intrinsic::x86_xop_vpcomuq;
- else if (Name.endswith("b"))
- intID = Intrinsic::x86_xop_vpcomb;
- else if (Name.endswith("w"))
- intID = Intrinsic::x86_xop_vpcomw;
- else if (Name.endswith("d"))
- intID = Intrinsic::x86_xop_vpcomd;
- else if (Name.endswith("q"))
- intID = Intrinsic::x86_xop_vpcomq;
+ bool IsSigned;
+ if (Name.endswith("ub") || Name.endswith("uw") || Name.endswith("ud") ||
+ Name.endswith("uq"))
+ IsSigned = false;
+ else if (Name.endswith("b") || Name.endswith("w") || Name.endswith("d") ||
+ Name.endswith("q"))
+ IsSigned = true;
else
llvm_unreachable("Unknown suffix");
- Name = Name.substr(9); // strip off "xop.vpcom"
unsigned Imm;
- if (Name.startswith("lt"))
- Imm = 0;
- else if (Name.startswith("le"))
- Imm = 1;
- else if (Name.startswith("gt"))
- Imm = 2;
- else if (Name.startswith("ge"))
- Imm = 3;
- else if (Name.startswith("eq"))
- Imm = 4;
- else if (Name.startswith("ne"))
- Imm = 5;
- else if (Name.startswith("false"))
- Imm = 6;
- else if (Name.startswith("true"))
- Imm = 7;
- else
- llvm_unreachable("Unknown condition");
+ if (CI->getNumArgOperands() == 3) {
+ Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
+ } else {
+ Name = Name.substr(9); // strip off "xop.vpcom"
+ if (Name.startswith("lt"))
+ Imm = 0;
+ else if (Name.startswith("le"))
+ Imm = 1;
+ else if (Name.startswith("gt"))
+ Imm = 2;
+ else if (Name.startswith("ge"))
+ Imm = 3;
+ else if (Name.startswith("eq"))
+ Imm = 4;
+ else if (Name.startswith("ne"))
+ Imm = 5;
+ else if (Name.startswith("false"))
+ Imm = 6;
+ else if (Name.startswith("true"))
+ Imm = 7;
+ else
+ llvm_unreachable("Unknown condition");
+ }
- Function *VPCOM = Intrinsic::getDeclaration(F->getParent(), intID);
- Rep =
- Builder.CreateCall(VPCOM, {CI->getArgOperand(0), CI->getArgOperand(1),
- Builder.getInt8(Imm)});
+ Rep = upgradeX86vpcom(Builder, *CI, Imm, IsSigned);
} else if (IsX86 && Name.startswith("xop.vpcmov")) {
Value *Sel = CI->getArgOperand(2);
Value *NotSel = Builder.CreateNot(Sel);
@@ -2104,6 +2269,14 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
if (CI->getNumArgOperands() == 3)
Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
CI->getArgOperand(1));
+ } else if (Name == "avx512.mask.pmov.qd.256" ||
+ Name == "avx512.mask.pmov.qd.512" ||
+ Name == "avx512.mask.pmov.wb.256" ||
+ Name == "avx512.mask.pmov.wb.512") {
+ Type *Ty = CI->getArgOperand(1)->getType();
+ Rep = Builder.CreateTrunc(CI->getArgOperand(0), Ty);
+ Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
+ CI->getArgOperand(1));
} else if (IsX86 && (Name.startswith("avx.vbroadcastf128") ||
Name == "avx2.vbroadcasti128")) {
// Replace vbroadcastf128/vbroadcasti128 with a vector load+shuffle.
@@ -2112,7 +2285,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
Type *VT = VectorType::get(EltTy, NumSrcElts);
Value *Op = Builder.CreatePointerCast(CI->getArgOperand(0),
PointerType::getUnqual(VT));
- Value *Load = Builder.CreateAlignedLoad(Op, 1);
+ Value *Load = Builder.CreateAlignedLoad(VT, Op, 1);
if (NumSrcElts == 2)
Rep = Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()),
{ 0, 1, 0, 1 });
@@ -2858,28 +3031,9 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
// Convert the type of the pointer to a pointer to the stored type.
Value *BC =
Builder.CreateBitCast(Ptr, PointerType::getUnqual(VTy), "cast");
- LoadInst *LI = Builder.CreateAlignedLoad(BC, VTy->getBitWidth() / 8);
+ LoadInst *LI = Builder.CreateAlignedLoad(VTy, BC, VTy->getBitWidth() / 8);
LI->setMetadata(M->getMDKindID("nontemporal"), Node);
Rep = LI;
- } else if (IsX86 &&
- (Name.startswith("sse2.pavg") || Name.startswith("avx2.pavg") ||
- Name.startswith("avx512.mask.pavg"))) {
- // llvm.x86.sse2.pavg.b/w, llvm.x86.avx2.pavg.b/w,
- // llvm.x86.avx512.mask.pavg.b/w
- Value *A = CI->getArgOperand(0);
- Value *B = CI->getArgOperand(1);
- VectorType *ZextType = VectorType::getExtendedElementVectorType(
- cast<VectorType>(A->getType()));
- Value *ExtendedA = Builder.CreateZExt(A, ZextType);
- Value *ExtendedB = Builder.CreateZExt(B, ZextType);
- Value *Sum = Builder.CreateAdd(ExtendedA, ExtendedB);
- Value *AddOne = Builder.CreateAdd(Sum, ConstantInt::get(ZextType, 1));
- Value *ShiftR = Builder.CreateLShr(AddOne, ConstantInt::get(ZextType, 1));
- Rep = Builder.CreateTrunc(ShiftR, A->getType());
- if (CI->getNumArgOperands() > 2) {
- Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
- CI->getArgOperand(2));
- }
} else if (IsX86 && (Name.startswith("fma.vfmadd.") ||
Name.startswith("fma.vfmsub.") ||
Name.startswith("fma.vfnmadd.") ||
@@ -3274,6 +3428,12 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
Value *Cmp = Builder.CreateICmpSGE(
Arg, llvm::Constant::getNullValue(Arg->getType()), "abs.cond");
Rep = Builder.CreateSelect(Cmp, Arg, Neg, "abs");
+ } else if (IsNVVM && (Name.startswith("atomic.load.add.f32.p") ||
+ Name.startswith("atomic.load.add.f64.p"))) {
+ Value *Ptr = CI->getArgOperand(0);
+ Value *Val = CI->getArgOperand(1);
+ Rep = Builder.CreateAtomicRMW(AtomicRMWInst::FAdd, Ptr, Val,
+ AtomicOrdering::SequentiallyConsistent);
} else if (IsNVVM && (Name == "max.i" || Name == "max.ll" ||
Name == "max.ui" || Name == "max.ull")) {
Value *Arg0 = CI->getArgOperand(0);
@@ -3335,7 +3495,28 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
DefaultCase();
return;
}
-
+ case Intrinsic::experimental_vector_reduce_v2_fmul: {
+ SmallVector<Value *, 2> Args;
+ if (CI->isFast())
+ Args.push_back(ConstantFP::get(CI->getOperand(0)->getType(), 1.0));
+ else
+ Args.push_back(CI->getOperand(0));
+ Args.push_back(CI->getOperand(1));
+ NewCall = Builder.CreateCall(NewFn, Args);
+ cast<Instruction>(NewCall)->copyFastMathFlags(CI);
+ break;
+ }
+ case Intrinsic::experimental_vector_reduce_v2_fadd: {
+ SmallVector<Value *, 2> Args;
+ if (CI->isFast())
+ Args.push_back(Constant::getNullValue(CI->getOperand(0)->getType()));
+ else
+ Args.push_back(CI->getOperand(0));
+ Args.push_back(CI->getOperand(1));
+ NewCall = Builder.CreateCall(NewFn, Args);
+ cast<Instruction>(NewCall)->copyFastMathFlags(CI);
+ break;
+ }
case Intrinsic::arm_neon_vld1:
case Intrinsic::arm_neon_vld2:
case Intrinsic::arm_neon_vld3:
@@ -3372,8 +3553,10 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
Value *NullIsUnknownSize = CI->getNumArgOperands() == 2
? Builder.getFalse()
: CI->getArgOperand(2);
+ Value *Dynamic =
+ CI->getNumArgOperands() < 4 ? Builder.getFalse() : CI->getArgOperand(3);
NewCall = Builder.CreateCall(
- NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize});
+ NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize, Dynamic});
break;
}
@@ -3650,8 +3833,8 @@ bool llvm::UpgradeDebugInfo(Module &M) {
bool llvm::UpgradeRetainReleaseMarker(Module &M) {
bool Changed = false;
- NamedMDNode *ModRetainReleaseMarker =
- M.getNamedMetadata("clang.arc.retainAutoreleasedReturnValueMarker");
+ const char *MarkerKey = "clang.arc.retainAutoreleasedReturnValueMarker";
+ NamedMDNode *ModRetainReleaseMarker = M.getNamedMetadata(MarkerKey);
if (ModRetainReleaseMarker) {
MDNode *Op = ModRetainReleaseMarker->getOperand(0);
if (Op) {
@@ -3661,11 +3844,11 @@ bool llvm::UpgradeRetainReleaseMarker(Module &M) {
ID->getString().split(ValueComp, "#");
if (ValueComp.size() == 2) {
std::string NewValue = ValueComp[0].str() + ";" + ValueComp[1].str();
- Metadata *Ops[1] = {MDString::get(M.getContext(), NewValue)};
- ModRetainReleaseMarker->setOperand(0,
- MDNode::get(M.getContext(), Ops));
- Changed = true;
+ ID = MDString::get(M.getContext(), NewValue);
}
+ M.addModuleFlag(Module::Error, MarkerKey, ID);
+ M.eraseNamedMetadata(ModRetainReleaseMarker);
+ Changed = true;
}
}
}
generated by cgit v1.2.3 (git 2.25.1) at 2025-03-03 17:20:26 +0000